diff options
Diffstat (limited to 'include/linux')
| -rw-r--r-- | include/linux/bio.h | 283 | ||||
| -rw-r--r-- | include/linux/blk-mq.h | 13 | ||||
| -rw-r--r-- | include/linux/blk_types.h | 24 | ||||
| -rw-r--r-- | include/linux/blkdev.h | 9 | ||||
| -rw-r--r-- | include/linux/ceph/messenger.h | 4 | ||||
| -rw-r--r-- | include/linux/cmdline-parser.h | 8 | ||||
| -rw-r--r-- | include/linux/dm-io.h | 4 |
7 files changed, 208 insertions, 137 deletions
diff --git a/include/linux/bio.h b/include/linux/bio.h index 060ff695085c..70654521dab6 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -61,25 +61,87 @@ * various member access, note that bio_data should of course not be used * on highmem page vectors */ -#define bio_iovec_idx(bio, idx) (&((bio)->bi_io_vec[(idx)])) -#define bio_iovec(bio) bio_iovec_idx((bio), (bio)->bi_idx) -#define bio_page(bio) bio_iovec((bio))->bv_page -#define bio_offset(bio) bio_iovec((bio))->bv_offset -#define bio_segments(bio) ((bio)->bi_vcnt - (bio)->bi_idx) -#define bio_sectors(bio) ((bio)->bi_size >> 9) -#define bio_end_sector(bio) ((bio)->bi_sector + bio_sectors((bio))) +#define __bvec_iter_bvec(bvec, iter) (&(bvec)[(iter).bi_idx]) + +#define bvec_iter_page(bvec, iter) \ + (__bvec_iter_bvec((bvec), (iter))->bv_page) + +#define bvec_iter_len(bvec, iter) \ + min((iter).bi_size, \ + __bvec_iter_bvec((bvec), (iter))->bv_len - (iter).bi_bvec_done) + +#define bvec_iter_offset(bvec, iter) \ + (__bvec_iter_bvec((bvec), (iter))->bv_offset + (iter).bi_bvec_done) + +#define bvec_iter_bvec(bvec, iter) \ +((struct bio_vec) { \ + .bv_page = bvec_iter_page((bvec), (iter)), \ + .bv_len = bvec_iter_len((bvec), (iter)), \ + .bv_offset = bvec_iter_offset((bvec), (iter)), \ +}) + +#define bio_iter_iovec(bio, iter) \ + bvec_iter_bvec((bio)->bi_io_vec, (iter)) + +#define bio_iter_page(bio, iter) \ + bvec_iter_page((bio)->bi_io_vec, (iter)) +#define bio_iter_len(bio, iter) \ + bvec_iter_len((bio)->bi_io_vec, (iter)) +#define bio_iter_offset(bio, iter) \ + bvec_iter_offset((bio)->bi_io_vec, (iter)) + +#define bio_page(bio) bio_iter_page((bio), (bio)->bi_iter) +#define bio_offset(bio) bio_iter_offset((bio), (bio)->bi_iter) +#define bio_iovec(bio) bio_iter_iovec((bio), (bio)->bi_iter) + +#define bio_multiple_segments(bio) \ + ((bio)->bi_iter.bi_size != bio_iovec(bio).bv_len) +#define bio_sectors(bio) ((bio)->bi_iter.bi_size >> 9) +#define bio_end_sector(bio) ((bio)->bi_iter.bi_sector + bio_sectors((bio))) + +/* + * Check whether this bio carries any data or not. A NULL bio is allowed. + */ +static inline bool bio_has_data(struct bio *bio) +{ + if (bio && + bio->bi_iter.bi_size && + !(bio->bi_rw & REQ_DISCARD)) + return true; + + return false; +} + +static inline bool bio_is_rw(struct bio *bio) +{ + if (!bio_has_data(bio)) + return false; + + if (bio->bi_rw & BIO_NO_ADVANCE_ITER_MASK) + return false; + + return true; +} + +static inline bool bio_mergeable(struct bio *bio) +{ + if (bio->bi_rw & REQ_NOMERGE_FLAGS) + return false; + + return true; +} static inline unsigned int bio_cur_bytes(struct bio *bio) { - if (bio->bi_vcnt) - return bio_iovec(bio)->bv_len; + if (bio_has_data(bio)) + return bio_iovec(bio).bv_len; else /* dataless requests such as discard */ - return bio->bi_size; + return bio->bi_iter.bi_size; } static inline void *bio_data(struct bio *bio) { - if (bio->bi_vcnt) + if (bio_has_data(bio)) return page_address(bio_page(bio)) + bio_offset(bio); return NULL; @@ -97,19 +159,16 @@ static inline void *bio_data(struct bio *bio) * permanent PIO fall back, user is probably better off disabling highmem * I/O completely on that queue (see ide-dma for example) */ -#define __bio_kmap_atomic(bio, idx) \ - (kmap_atomic(bio_iovec_idx((bio), (idx))->bv_page) + \ - bio_iovec_idx((bio), (idx))->bv_offset) +#define __bio_kmap_atomic(bio, iter) \ + (kmap_atomic(bio_iter_iovec((bio), (iter)).bv_page) + \ + bio_iter_iovec((bio), (iter)).bv_offset) -#define __bio_kunmap_atomic(addr) kunmap_atomic(addr) +#define __bio_kunmap_atomic(addr) kunmap_atomic(addr) /* * merge helpers etc */ -#define __BVEC_END(bio) bio_iovec_idx((bio), (bio)->bi_vcnt - 1) -#define __BVEC_START(bio) bio_iovec_idx((bio), (bio)->bi_idx) - /* Default implementation of BIOVEC_PHYS_MERGEABLE */ #define __BIOVEC_PHYS_MERGEABLE(vec1, vec2) \ ((bvec_to_phys((vec1)) + (vec1)->bv_len) == bvec_to_phys((vec2))) @@ -126,33 +185,76 @@ static inline void *bio_data(struct bio *bio) (((addr1) | (mask)) == (((addr2) - 1) | (mask))) #define BIOVEC_SEG_BOUNDARY(q, b1, b2) \ __BIO_SEG_BOUNDARY(bvec_to_phys((b1)), bvec_to_phys((b2)) + (b2)->bv_len, queue_segment_boundary((q))) -#define BIO_SEG_BOUNDARY(q, b1, b2) \ - BIOVEC_SEG_BOUNDARY((q), __BVEC_END((b1)), __BVEC_START((b2))) #define bio_io_error(bio) bio_endio((bio), -EIO) /* - * drivers should not use the __ version unless they _really_ know what - * they're doing - */ -#define __bio_for_each_segment(bvl, bio, i, start_idx) \ - for (bvl = bio_iovec_idx((bio), (start_idx)), i = (start_idx); \ - i < (bio)->bi_vcnt; \ - bvl++, i++) - -/* * drivers should _never_ use the all version - the bio may have been split * before it got to the driver and the driver won't own all of it */ #define bio_for_each_segment_all(bvl, bio, i) \ - for (i = 0; \ - bvl = bio_iovec_idx((bio), (i)), i < (bio)->bi_vcnt; \ - i++) + for (i = 0, bvl = (bio)->bi_io_vec; i < (bio)->bi_vcnt; i++, bvl++) + +static inline void bvec_iter_advance(struct bio_vec *bv, struct bvec_iter *iter, + unsigned bytes) +{ + WARN_ONCE(bytes > iter->bi_size, + "Attempted to advance past end of bvec iter\n"); + + while (bytes) { + unsigned len = min(bytes, bvec_iter_len(bv, *iter)); + + bytes -= len; + iter->bi_size -= len; + iter->bi_bvec_done += len; + + if (iter->bi_bvec_done == __bvec_iter_bvec(bv, *iter)->bv_len) { + iter->bi_bvec_done = 0; + iter->bi_idx++; + } + } +} + +#define for_each_bvec(bvl, bio_vec, iter, start) \ + for ((iter) = start; \ + (bvl) = bvec_iter_bvec((bio_vec), (iter)), \ + (iter).bi_size; \ + bvec_iter_advance((bio_vec), &(iter), (bvl).bv_len)) + + +static inline void bio_advance_iter(struct bio *bio, struct bvec_iter *iter, + unsigned bytes) +{ + iter->bi_sector += bytes >> 9; + + if (bio->bi_rw & BIO_NO_ADVANCE_ITER_MASK) + iter->bi_size -= bytes; + else + bvec_iter_advance(bio->bi_io_vec, iter, bytes); +} -#define bio_for_each_segment(bvl, bio, i) \ - for (i = (bio)->bi_idx; \ - bvl = bio_iovec_idx((bio), (i)), i < (bio)->bi_vcnt; \ - i++) +#define __bio_for_each_segment(bvl, bio, iter, start) \ + for (iter = (start); \ + (iter).bi_size && \ + ((bvl = bio_iter_iovec((bio), (iter))), 1); \ + bio_advance_iter((bio), &(iter), (bvl).bv_len)) + +#define bio_for_each_segment(bvl, bio, iter) \ + __bio_for_each_segment(bvl, bio, iter, (bio)->bi_iter) + +#define bio_iter_last(bvec, iter) ((iter).bi_size == (bvec).bv_len) + +static inline unsigned bio_segments(struct bio *bio) +{ + unsigned segs = 0; + struct bio_vec bv; + struct bvec_iter iter; + + bio_for_each_segment(bv, bio, iter) + segs++; + + return segs; +} /* * get a reference to a bio, so it won't disappear. the intended use is @@ -177,16 +279,15 @@ static inline void *bio_data(struct bio *bio) struct bio_integrity_payload { struct bio *bip_bio; /* parent bio */ - sector_t bip_sector; /* virtual start sector */ + struct bvec_iter bip_iter; + /* kill - should just use bip_vec */ void *bip_buf; /* generated integrity data */ - bio_end_io_t *bip_end_io; /* saved I/O completion fn */ - unsigned int bip_size; + bio_end_io_t *bip_end_io; /* saved I/O completion fn */ unsigned short bip_slab; /* slab the bip came from */ unsigned short bip_vcnt; /* # of integrity bio_vecs */ - unsigned short bip_idx; /* current bip_vec index */ unsigned bip_owns_buf:1; /* should free bip_buf */ struct work_struct bip_work; /* I/O completion */ @@ -196,29 +297,28 @@ struct bio_integrity_payload { }; #endif /* CONFIG_BLK_DEV_INTEGRITY */ -/* - * A bio_pair is used when we need to split a bio. - * This can only happen for a bio that refers to just one - * page of data, and in the unusual situation when the - * page crosses a chunk/device boundary +extern void bio_trim(struct bio *bio, int offset, int size); +extern struct bio *bio_split(struct bio *bio, int sectors, + gfp_t gfp, struct bio_set *bs); + +/** + * bio_next_split - get next @sectors from a bio, splitting if necessary + * @bio: bio to split + * @sectors: number of sectors to split from the front of @bio + * @gfp: gfp mask + * @bs: bio set to allocate from * - * The address of the master bio is stored in bio1.bi_private - * The address of the pool the pair was allocated from is stored - * in bio2.bi_private + * Returns a bio representing the next @sectors of @bio - if the bio is smaller + * than @sectors, returns the original bio unchanged. */ -struct bio_pair { - struct bio bio1, bio2; - struct bio_vec bv1, bv2; -#if defined(CONFIG_BLK_DEV_INTEGRITY) - struct bio_integrity_payload bip1, bip2; - struct bio_vec iv1, iv2; -#endif - atomic_t cnt; - int error; -}; -extern struct bio_pair *bio_split(struct bio *bi, int first_sectors); -extern void bio_pair_release(struct bio_pair *dbio); -extern void bio_trim(struct bio *bio, int offset, int size); +static inline struct bio *bio_next_split(struct bio *bio, int sectors, + gfp_t gfp, struct bio_set *bs) +{ + if (sectors >= bio_sectors(bio)) + return bio; + + return bio_split(bio, sectors, gfp, bs); +} extern struct bio_set *bioset_create(unsigned int, unsigned int); extern void bioset_free(struct bio_set *); @@ -227,7 +327,8 @@ extern mempool_t *biovec_create_pool(struct bio_set *bs, int pool_entries); extern struct bio *bio_alloc_bioset(gfp_t, int, struct bio_set *); extern void bio_put(struct bio *); -extern void __bio_clone(struct bio *, struct bio *); +extern void __bio_clone_fast(struct bio *, struct bio *); +extern struct bio *bio_clone_fast(struct bio *, gfp_t, struct bio_set *); extern struct bio *bio_clone_bioset(struct bio *, gfp_t, struct bio_set *bs); extern struct bio_set *fs_bio_set; @@ -254,6 +355,7 @@ static inline struct bio *bio_clone_kmalloc(struct bio *bio, gfp_t gfp_mask) } extern void bio_endio(struct bio *, int); +extern void bio_endio_nodec(struct bio *, int); struct request_queue; extern int bio_phys_segments(struct request_queue *, struct bio *); @@ -262,12 +364,12 @@ extern void bio_advance(struct bio *, unsigned); extern void bio_init(struct bio *); extern void bio_reset(struct bio *); +void bio_chain(struct bio *, struct bio *); extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int); extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *, unsigned int, unsigned int); extern int bio_get_nr_vecs(struct block_device *); -extern sector_t bio_sector_offset(struct bio *, unsigned short, unsigned int); extern struct bio *bio_map_user(struct request_queue *, struct block_device *, unsigned long, unsigned int, int, gfp_t); struct sg_iovec; @@ -357,48 +459,18 @@ static inline void bvec_kunmap_irq(char *buffer, unsigned long *flags) } #endif -static inline char *__bio_kmap_irq(struct bio *bio, unsigned short idx, +static inline char *__bio_kmap_irq(struct bio *bio, struct bvec_iter iter, unsigned long *flags) { - return bvec_kmap_irq(bio_iovec_idx(bio, idx), flags); + return bvec_kmap_irq(&bio_iter_iovec(bio, iter), flags); } #define __bio_kunmap_irq(buf, flags) bvec_kunmap_irq(buf, flags) #define bio_kmap_irq(bio, flags) \ - __bio_kmap_irq((bio), (bio)->bi_idx, (flags)) + __bio_kmap_irq((bio), (bio)->bi_iter, (flags)) #define bio_kunmap_irq(buf,flags) __bio_kunmap_irq(buf, flags) /* - * Check whether this bio carries any data or not. A NULL bio is allowed. - */ -static inline bool bio_has_data(struct bio *bio) -{ - if (bio && bio->bi_vcnt) - return true; - - return false; -} - -static inline bool bio_is_rw(struct bio *bio) -{ - if (!bio_has_data(bio)) - return false; - - if (bio->bi_rw & REQ_WRITE_SAME) - return false; - - return true; -} - -static inline bool bio_mergeable(struct bio *bio) -{ - if (bio->bi_rw & REQ_NOMERGE_FLAGS) - return false; - - return true; -} - -/* * BIO list management for use by remapping drivers (e.g. DM or MD) and loop. * * A bio_list anchors a singly-linked list of bios chained through the bi_next @@ -559,16 +631,12 @@ struct biovec_slab { #if defined(CONFIG_BLK_DEV_INTEGRITY) -#define bip_vec_idx(bip, idx) (&(bip->bip_vec[(idx)])) -#define bip_vec(bip) bip_vec_idx(bip, 0) -#define __bip_for_each_vec(bvl, bip, i, start_idx) \ - for (bvl = bip_vec_idx((bip), (start_idx)), i = (start_idx); \ - i < (bip)->bip_vcnt; \ - bvl++, i++) -#define bip_for_each_vec(bvl, bip, i) \ - __bip_for_each_vec(bvl, bip, i, (bip)->bip_idx) +#define bip_vec_idx(bip, idx) (&(bip->bip_vec[(idx)])) + +#define bip_for_each_vec(bvl, bip, iter) \ + for_each_bvec(bvl, (bip)->bip_vec, iter, (bip)->bip_iter) #define bio_for_each_integrity_vec(_bvl, _bio, _iter) \ for_each_bio(_bio) \ @@ -586,7 +654,6 @@ extern int bio_integrity_prep(struct bio *); extern void bio_integrity_endio(struct bio *, int); extern void bio_integrity_advance(struct bio *, unsigned int); extern void bio_integrity_trim(struct bio *, unsigned int, unsigned int); -extern void bio_integrity_split(struct bio *, struct bio_pair *, int); extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t); extern int bioset_integrity_create(struct bio_set *, int); extern void bioset_integrity_free(struct bio_set *); @@ -630,12 +697,6 @@ static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src, return 0; } -static inline void bio_integrity_split(struct bio *bio, struct bio_pair *bp, - int sectors) -{ - return; -} - static inline void bio_integrity_advance(struct bio *bio, unsigned int bytes_done) { diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index ab0e9b2025b3..161b23105b1e 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -113,7 +113,6 @@ enum { }; struct request_queue *blk_mq_init_queue(struct blk_mq_reg *, void *); -void blk_mq_free_queue(struct request_queue *); int blk_mq_register_disk(struct gendisk *); void blk_mq_unregister_disk(struct gendisk *); void blk_mq_init_commands(struct request_queue *, void (*init)(void *data, struct blk_mq_hw_ctx *, struct request *, unsigned int), void *data); @@ -159,16 +158,16 @@ static inline struct request *blk_mq_tag_to_rq(struct blk_mq_hw_ctx *hctx, } #define queue_for_each_hw_ctx(q, hctx, i) \ - for ((i) = 0, hctx = (q)->queue_hw_ctx[0]; \ - (i) < (q)->nr_hw_queues; (i)++, hctx = (q)->queue_hw_ctx[i]) + for ((i) = 0; (i) < (q)->nr_hw_queues && \ + ({ hctx = (q)->queue_hw_ctx[i]; 1; }); (i)++) #define queue_for_each_ctx(q, ctx, i) \ - for ((i) = 0, ctx = per_cpu_ptr((q)->queue_ctx, 0); \ - (i) < (q)->nr_queues; (i)++, ctx = per_cpu_ptr(q->queue_ctx, (i))) + for ((i) = 0; (i) < (q)->nr_queues && \ + ({ ctx = per_cpu_ptr((q)->queue_ctx, (i)); 1; }); (i)++) #define hctx_for_each_ctx(hctx, ctx, i) \ - for ((i) = 0, ctx = (hctx)->ctxs[0]; \ - (i) < (hctx)->nr_ctx; (i)++, ctx = (hctx)->ctxs[(i)]) + for ((i) = 0; (i) < (hctx)->nr_ctx && \ + ({ ctx = (hctx)->ctxs[(i)]; 1; }); (i)++) #define blk_ctx_sum(q, sum) \ ({ \ diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 238ef0ed62f8..bbc3a6c88fce 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -28,13 +28,22 @@ struct bio_vec { unsigned int bv_offset; }; +struct bvec_iter { + sector_t bi_sector; /* device address in 512 byte + sectors */ + unsigned int bi_size; /* residual I/O count */ + + unsigned int bi_idx; /* current index into bvl_vec */ + + unsigned int bi_bvec_done; /* number of bytes completed in + current bvec */ +}; + /* * main unit of I/O for the block layer and lower layers (ie drivers and * stacking drivers) */ struct bio { - sector_t bi_sector; /* device address in 512 byte - sectors */ struct bio *bi_next; /* request queue link */ struct block_device *bi_bdev; unsigned long bi_flags; /* status, command, etc */ @@ -42,16 +51,13 @@ struct bio { * top bits priority */ - unsigned short bi_vcnt; /* how many bio_vec's */ - unsigned short bi_idx; /* current index into bvl_vec */ + struct bvec_iter bi_iter; /* Number of segments in this BIO after * physical address coalescing is performed. */ unsigned int bi_phys_segments; - unsigned int bi_size; /* residual I/O count */ - /* * To keep track of the max segment size, we account for the * sizes of the first and last mergeable segments in this bio. @@ -59,6 +65,8 @@ struct bio { unsigned int bi_seg_front_size; unsigned int bi_seg_back_size; + atomic_t bi_remaining; + bio_end_io_t *bi_end_io; void *bi_private; @@ -74,11 +82,13 @@ struct bio { struct bio_integrity_payload *bi_integrity; /* data integrity */ #endif + unsigned short bi_vcnt; /* how many bio_vec's */ + /* * Everything starting with bi_max_vecs will be preserved by bio_reset() */ - unsigned int bi_max_vecs; /* max bvl_vecs we can hold */ + unsigned short bi_max_vecs; /* max bvl_vecs we can hold */ atomic_t bi_cnt; /* pin count */ diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 1b135d49b279..02cb6f0ea71d 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -735,7 +735,7 @@ struct rq_map_data { }; struct req_iterator { - int i; + struct bvec_iter iter; struct bio *bio; }; @@ -748,10 +748,11 @@ struct req_iterator { #define rq_for_each_segment(bvl, _rq, _iter) \ __rq_for_each_bio(_iter.bio, _rq) \ - bio_for_each_segment(bvl, _iter.bio, _iter.i) + bio_for_each_segment(bvl, _iter.bio, _iter.iter) -#define rq_iter_last(rq, _iter) \ - (_iter.bio->bi_next == NULL && _iter.i == _iter.bio->bi_vcnt-1) +#define rq_iter_last(bvec, _iter) \ + (_iter.bio->bi_next == NULL && \ + bio_iter_last(bvec, _iter.iter)) #ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE # error "You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform" diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index 20ee8b63a968..d21f2dba0731 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -1,6 +1,7 @@ #ifndef __FS_CEPH_MESSENGER_H #define __FS_CEPH_MESSENGER_H +#include <linux/blk_types.h> #include <linux/kref.h> #include <linux/mutex.h> #include <linux/net.h> @@ -119,8 +120,7 @@ struct ceph_msg_data_cursor { #ifdef CONFIG_BLOCK struct { /* bio */ struct bio *bio; /* bio from list */ - unsigned int vector_index; /* vector from bio */ - unsigned int vector_offset; /* bytes from vector */ + struct bvec_iter bvec_iter; }; #endif /* CONFIG_BLOCK */ struct { /* pages */ diff --git a/include/linux/cmdline-parser.h b/include/linux/cmdline-parser.h index a0f9280421ec..2e6dce6e5c2a 100644 --- a/include/linux/cmdline-parser.h +++ b/include/linux/cmdline-parser.h @@ -37,9 +37,9 @@ int cmdline_parts_parse(struct cmdline_parts **parts, const char *cmdline); struct cmdline_parts *cmdline_parts_find(struct cmdline_parts *parts, const char *bdev); -void cmdline_parts_set(struct cmdline_parts *parts, sector_t disk_size, - int slot, - int (*add_part)(int, struct cmdline_subpart *, void *), - void *param); +int cmdline_parts_set(struct cmdline_parts *parts, sector_t disk_size, + int slot, + int (*add_part)(int, struct cmdline_subpart *, void *), + void *param); #endif /* CMDLINEPARSEH */ diff --git a/include/linux/dm-io.h b/include/linux/dm-io.h index f4b0aa3126f5..a68cbe59e6ad 100644 --- a/include/linux/dm-io.h +++ b/include/linux/dm-io.h @@ -29,7 +29,7 @@ typedef void (*io_notify_fn)(unsigned long error, void *context); enum dm_io_mem_type { DM_IO_PAGE_LIST,/* Page list */ - DM_IO_BVEC, /* Bio vector */ + DM_IO_BIO, /* Bio vector */ DM_IO_VMA, /* Virtual memory area */ DM_IO_KMEM, /* Kernel memory */ }; @@ -41,7 +41,7 @@ struct dm_io_memory { union { struct page_list *pl; - struct bio_vec *bvec; + struct bio *bio; void *vma; void *addr; } ptr; |