From 4fa253f33c1d4d6402de9eaacefdb45a47ed116d Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 18 Jul 2007 13:13:10 +0200 Subject: block: ll_rw_blk.c: cosmetics Fix ?: construct, a typo, whitespace, and similar. Signed-off-by: Jens Axboe --- block/ll_rw_blk.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'block/ll_rw_blk.c') diff --git a/block/ll_rw_blk.c b/block/ll_rw_blk.c index d875673e76cd..0fa5d3d556d9 100644 --- a/block/ll_rw_blk.c +++ b/block/ll_rw_blk.c @@ -377,10 +377,12 @@ void blk_ordered_complete_seq(struct request_queue *q, unsigned seq, int error) /* * Okay, sequence complete. */ - rq = q->orig_bar_rq; - uptodate = q->orderr ? q->orderr : 1; + uptodate = 1; + if (q->orderr) + uptodate = q->orderr; q->ordseq = 0; + rq = q->orig_bar_rq; end_that_request_first(rq, uptodate, rq->hard_nr_sectors); end_that_request_last(rq, uptodate); @@ -445,7 +447,8 @@ static inline struct request *start_ordered(struct request_queue *q, rq_init(q, rq); if (bio_data_dir(q->orig_bar_rq->bio) == WRITE) rq->cmd_flags |= REQ_RW; - rq->cmd_flags |= q->ordered & QUEUE_ORDERED_FUA ? REQ_FUA : 0; + if (q->ordered & QUEUE_ORDERED_FUA) + rq->cmd_flags |= REQ_FUA; rq->elevator_private = NULL; rq->elevator_private2 = NULL; init_request_from_bio(rq, q->orig_bar_rq->bio); @@ -3191,7 +3194,7 @@ end_io: break; } - if (unlikely(bio_sectors(bio) > q->max_hw_sectors)) { + if (unlikely(nr_sectors > q->max_hw_sectors)) { printk("bio too big device %s (%u > %u)\n", bdevname(bio->bi_bdev, b), bio_sectors(bio), @@ -3212,7 +3215,7 @@ end_io: blk_partition_remap(bio); if (old_sector != -1) - blk_add_trace_remap(q, bio, old_dev, bio->bi_sector, + blk_add_trace_remap(q, bio, old_dev, bio->bi_sector, old_sector); blk_add_trace_bio(q, bio, BLK_TA_QUEUE); @@ -3564,7 +3567,7 @@ static struct notifier_block blk_cpu_notifier __cpuinitdata = { * Description: * Ends all I/O on a request. It does not handle partial completions, * unless the driver actually implements this in its completion callback - * through requeueing. Theh actual completion happens out-of-order, + * through requeueing. The actual completion happens out-of-order, * through a softirq handler. The user must have registered a completion * callback through blk_queue_softirq_done(). **/ -- cgit v1.2.3 From a0cd128542cd9c67f27458a08e989db486a293ce Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 21 Sep 2007 10:41:07 +0200 Subject: block: add end_queued_request() and end_dequeued_request() helpers We can use this helper in the elevator core for BLKPREP_KILL, and it'll also be useful for the empty barrier patch. Signed-off-by: Jens Axboe --- block/ll_rw_blk.c | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 73 insertions(+), 5 deletions(-) (limited to 'block/ll_rw_blk.c') diff --git a/block/ll_rw_blk.c b/block/ll_rw_blk.c index 0fa5d3d556d9..8904f8b1f417 100644 --- a/block/ll_rw_blk.c +++ b/block/ll_rw_blk.c @@ -3630,15 +3630,83 @@ void end_that_request_last(struct request *req, int uptodate) EXPORT_SYMBOL(end_that_request_last); -void end_request(struct request *req, int uptodate) +static inline void __end_request(struct request *rq, int uptodate, + unsigned int nr_bytes, int dequeue) { - if (!end_that_request_first(req, uptodate, req->hard_cur_sectors)) { - add_disk_randomness(req->rq_disk); - blkdev_dequeue_request(req); - end_that_request_last(req, uptodate); + if (!end_that_request_chunk(rq, uptodate, nr_bytes)) { + if (dequeue) + blkdev_dequeue_request(rq); + add_disk_randomness(rq->rq_disk); + end_that_request_last(rq, uptodate); } } +static unsigned int rq_byte_size(struct request *rq) +{ + if (blk_fs_request(rq)) + return rq->hard_nr_sectors << 9; + + return rq->data_len; +} + +/** + * end_queued_request - end all I/O on a queued request + * @rq: the request being processed + * @uptodate: error value or 0/1 uptodate flag + * + * Description: + * Ends all I/O on a request, and removes it from the block layer queues. + * Not suitable for normal IO completion, unless the driver still has + * the request attached to the block layer. + * + **/ +void end_queued_request(struct request *rq, int uptodate) +{ + __end_request(rq, uptodate, rq_byte_size(rq), 1); +} +EXPORT_SYMBOL(end_queued_request); + +/** + * end_dequeued_request - end all I/O on a dequeued request + * @rq: the request being processed + * @uptodate: error value or 0/1 uptodate flag + * + * Description: + * Ends all I/O on a request. The request must already have been + * dequeued using blkdev_dequeue_request(), as is normally the case + * for most drivers. + * + **/ +void end_dequeued_request(struct request *rq, int uptodate) +{ + __end_request(rq, uptodate, rq_byte_size(rq), 0); +} +EXPORT_SYMBOL(end_dequeued_request); + + +/** + * end_request - end I/O on the current segment of the request + * @rq: the request being processed + * @uptodate: error value or 0/1 uptodate flag + * + * Description: + * Ends I/O on the current segment of a request. If that is the only + * remaining segment, the request is also completed and freed. + * + * This is a remnant of how older block drivers handled IO completions. + * Modern drivers typically end IO on the full request in one go, unless + * they have a residual value to account for. For that case this function + * isn't really useful, unless the residual just happens to be the + * full current segment. In other words, don't use this function in new + * code. Either use end_request_completely(), or the + * end_that_request_chunk() (along with end_that_request_last()) for + * partial completions. + * + **/ +void end_request(struct request *req, int uptodate) +{ + __end_request(req, uptodate, req->hard_cur_sectors << 9, 1); +} EXPORT_SYMBOL(end_request); static void blk_rq_bio_prep(struct request_queue *q, struct request *rq, -- cgit v1.2.3 From c07e2b41291853b19fff11ceee3657df252a4e42 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 18 Jul 2007 13:27:58 +0200 Subject: block: factor our bio_check_eod() End of device check is done twice in __generic_make_request() and it's fully inlined each time. Factor out bio_check_eod(). Signed-off-by: Tejun Heo Signed-off-by: Jens Axboe --- block/ll_rw_blk.c | 63 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 30 deletions(-) (limited to 'block/ll_rw_blk.c') diff --git a/block/ll_rw_blk.c b/block/ll_rw_blk.c index 8904f8b1f417..548f0d826679 100644 --- a/block/ll_rw_blk.c +++ b/block/ll_rw_blk.c @@ -3120,6 +3120,35 @@ static inline int should_fail_request(struct bio *bio) #endif /* CONFIG_FAIL_MAKE_REQUEST */ +/* + * Check whether this bio extends beyond the end of the device. + */ +static inline int bio_check_eod(struct bio *bio, unsigned int nr_sectors) +{ + sector_t maxsector; + + if (!nr_sectors) + return 0; + + /* Test device or partition size, when known. */ + maxsector = bio->bi_bdev->bd_inode->i_size >> 9; + if (maxsector) { + sector_t sector = bio->bi_sector; + + if (maxsector < nr_sectors || maxsector - nr_sectors < sector) { + /* + * This may well happen - the kernel calls bread() + * without checking the size of the device, e.g., when + * mounting a device. + */ + handle_bad_sector(bio); + return 1; + } + } + + return 0; +} + /** * generic_make_request: hand a buffer to its device driver for I/O * @bio: The bio describing the location in memory and on the device. @@ -3147,27 +3176,14 @@ static inline int should_fail_request(struct bio *bio) static inline void __generic_make_request(struct bio *bio) { struct request_queue *q; - sector_t maxsector; sector_t old_sector; int ret, nr_sectors = bio_sectors(bio); dev_t old_dev; might_sleep(); - /* Test device or partition size, when known. */ - maxsector = bio->bi_bdev->bd_inode->i_size >> 9; - if (maxsector) { - sector_t sector = bio->bi_sector; - if (maxsector < nr_sectors || maxsector - nr_sectors < sector) { - /* - * This may well happen - the kernel calls bread() - * without checking the size of the device, e.g., when - * mounting a device. - */ - handle_bad_sector(bio); - goto end_io; - } - } + if (bio_check_eod(bio, nr_sectors)) + goto end_io; /* * Resolve the mapping until finished. (drivers are @@ -3223,21 +3239,8 @@ end_io: old_sector = bio->bi_sector; old_dev = bio->bi_bdev->bd_dev; - maxsector = bio->bi_bdev->bd_inode->i_size >> 9; - if (maxsector) { - sector_t sector = bio->bi_sector; - - if (maxsector < nr_sectors || - maxsector - nr_sectors < sector) { - /* - * This may well happen - partitions are not - * checked to make sure they are within the size - * of the whole device. - */ - handle_bad_sector(bio); - goto end_io; - } - } + if (bio_check_eod(bio, nr_sectors)) + goto end_io; ret = q->make_request_fn(q, bio); } while (ret); -- cgit v1.2.3 From bf2de6f5a4faf0197268f18d08969b003b87b6e8 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 27 Sep 2007 13:01:25 +0200 Subject: block: Initial support for data-less (or empty) barrier support This implements functionality to pass down or insert a barrier in a queue, without having data attached to it. The ->prepare_flush_fn() infrastructure from data barriers are reused to provide this functionality. Signed-off-by: Jens Axboe --- block/ll_rw_blk.c | 58 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 19 deletions(-) (limited to 'block/ll_rw_blk.c') diff --git a/block/ll_rw_blk.c b/block/ll_rw_blk.c index 548f0d826679..4fde3a3c92d3 100644 --- a/block/ll_rw_blk.c +++ b/block/ll_rw_blk.c @@ -458,9 +458,12 @@ static inline struct request *start_ordered(struct request_queue *q, * Queue ordered sequence. As we stack them at the head, we * need to queue in reverse order. Note that we rely on that * no fs request uses ELEVATOR_INSERT_FRONT and thus no fs - * request gets inbetween ordered sequence. + * request gets inbetween ordered sequence. If this request is + * an empty barrier, we don't need to do a postflush ever since + * there will be no data written between the pre and post flush. + * Hence a single flush will suffice. */ - if (q->ordered & QUEUE_ORDERED_POSTFLUSH) + if ((q->ordered & QUEUE_ORDERED_POSTFLUSH) && !blk_empty_barrier(rq)) queue_flush(q, QUEUE_ORDERED_POSTFLUSH); else q->ordseq |= QUEUE_ORDSEQ_POSTFLUSH; @@ -484,7 +487,7 @@ static inline struct request *start_ordered(struct request_queue *q, int blk_do_ordered(struct request_queue *q, struct request **rqp) { struct request *rq = *rqp; - int is_barrier = blk_fs_request(rq) && blk_barrier_rq(rq); + const int is_barrier = blk_fs_request(rq) && blk_barrier_rq(rq); if (!q->ordseq) { if (!is_barrier) @@ -3054,7 +3057,7 @@ static inline void blk_partition_remap(struct bio *bio) { struct block_device *bdev = bio->bi_bdev; - if (bdev != bdev->bd_contains) { + if (bio_sectors(bio) && bdev != bdev->bd_contains) { struct hd_struct *p = bdev->bd_part; const int rw = bio_data_dir(bio); @@ -3313,23 +3316,32 @@ void submit_bio(int rw, struct bio *bio) { int count = bio_sectors(bio); - BIO_BUG_ON(!bio->bi_size); - BIO_BUG_ON(!bio->bi_io_vec); bio->bi_rw |= rw; - if (rw & WRITE) { - count_vm_events(PGPGOUT, count); - } else { - task_io_account_read(bio->bi_size); - count_vm_events(PGPGIN, count); - } - if (unlikely(block_dump)) { - char b[BDEVNAME_SIZE]; - printk(KERN_DEBUG "%s(%d): %s block %Lu on %s\n", - current->comm, current->pid, - (rw & WRITE) ? "WRITE" : "READ", - (unsigned long long)bio->bi_sector, - bdevname(bio->bi_bdev,b)); + /* + * If it's a regular read/write or a barrier with data attached, + * go through the normal accounting stuff before submission. + */ + if (!bio_empty_barrier(bio)) { + + BIO_BUG_ON(!bio->bi_size); + BIO_BUG_ON(!bio->bi_io_vec); + + if (rw & WRITE) { + count_vm_events(PGPGOUT, count); + } else { + task_io_account_read(bio->bi_size); + count_vm_events(PGPGIN, count); + } + + if (unlikely(block_dump)) { + char b[BDEVNAME_SIZE]; + printk(KERN_DEBUG "%s(%d): %s block %Lu on %s\n", + current->comm, current->pid, + (rw & WRITE) ? "WRITE" : "READ", + (unsigned long long)bio->bi_sector, + bdevname(bio->bi_bdev,b)); + } } generic_make_request(bio); @@ -3405,6 +3417,14 @@ static int __end_that_request_first(struct request *req, int uptodate, while ((bio = req->bio) != NULL) { int nbytes; + /* + * For an empty barrier request, the low level driver must + * store a potential error location in ->sector. We pass + * that back up in ->bi_sector. + */ + if (blk_empty_barrier(req)) + bio->bi_sector = req->sector; + if (nr_bytes >= bio->bi_size) { req->bio = bio->bi_next; nbytes = bio->bi_size; -- cgit v1.2.3 From fd5d806266935179deda1502101624832eacd01f Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 16 Oct 2007 11:05:02 +0200 Subject: block: convert blkdev_issue_flush() to use empty barriers Then we can get rid of ->issue_flush_fn() and all the driver private implementations of that. Signed-off-by: Jens Axboe --- block/ll_rw_blk.c | 56 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 20 deletions(-) (limited to 'block/ll_rw_blk.c') diff --git a/block/ll_rw_blk.c b/block/ll_rw_blk.c index 4fde3a3c92d3..4df7d027eb06 100644 --- a/block/ll_rw_blk.c +++ b/block/ll_rw_blk.c @@ -304,23 +304,6 @@ int blk_queue_ordered(struct request_queue *q, unsigned ordered, EXPORT_SYMBOL(blk_queue_ordered); -/** - * blk_queue_issue_flush_fn - set function for issuing a flush - * @q: the request queue - * @iff: the function to be called issuing the flush - * - * Description: - * If a driver supports issuing a flush command, the support is notified - * to the block layer by defining it through this call. - * - **/ -void blk_queue_issue_flush_fn(struct request_queue *q, issue_flush_fn *iff) -{ - q->issue_flush_fn = iff; -} - -EXPORT_SYMBOL(blk_queue_issue_flush_fn); - /* * Cache flushing for ordered writes handling */ @@ -2666,6 +2649,14 @@ int blk_execute_rq(struct request_queue *q, struct gendisk *bd_disk, EXPORT_SYMBOL(blk_execute_rq); +static void bio_end_empty_barrier(struct bio *bio, int err) +{ + if (err) + clear_bit(BIO_UPTODATE, &bio->bi_flags); + + complete(bio->bi_private); +} + /** * blkdev_issue_flush - queue a flush * @bdev: blockdev to issue flush for @@ -2678,7 +2669,10 @@ EXPORT_SYMBOL(blk_execute_rq); */ int blkdev_issue_flush(struct block_device *bdev, sector_t *error_sector) { + DECLARE_COMPLETION_ONSTACK(wait); struct request_queue *q; + struct bio *bio; + int ret; if (bdev->bd_disk == NULL) return -ENXIO; @@ -2686,10 +2680,32 @@ int blkdev_issue_flush(struct block_device *bdev, sector_t *error_sector) q = bdev_get_queue(bdev); if (!q) return -ENXIO; - if (!q->issue_flush_fn) - return -EOPNOTSUPP; - return q->issue_flush_fn(q, bdev->bd_disk, error_sector); + bio = bio_alloc(GFP_KERNEL, 0); + if (!bio) + return -ENOMEM; + + bio->bi_end_io = bio_end_empty_barrier; + bio->bi_private = &wait; + bio->bi_bdev = bdev; + submit_bio(1 << BIO_RW_BARRIER, bio); + + wait_for_completion(&wait); + + /* + * The driver must store the error location in ->bi_sector, if + * it supports it. For non-stacked drivers, this should be copied + * from rq->sector. + */ + if (error_sector) + *error_sector = bio->bi_sector; + + ret = 0; + if (!bio_flagged(bio, BIO_UPTODATE)) + ret = -EIO; + + bio_put(bio); + return ret; } EXPORT_SYMBOL(blkdev_issue_flush); -- cgit v1.2.3 From f565913ef8a8d0cfa46a1faaf8340cc357a46f3a Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 21 Sep 2007 10:44:19 +0200 Subject: block: convert to using sg helpers Convert the main rq mapper (blk_rq_map_sg()) to the sg helper setup. Signed-off-by: Jens Axboe --- block/ll_rw_blk.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'block/ll_rw_blk.c') diff --git a/block/ll_rw_blk.c b/block/ll_rw_blk.c index 4df7d027eb06..36d205128f9a 100644 --- a/block/ll_rw_blk.c +++ b/block/ll_rw_blk.c @@ -30,6 +30,7 @@ #include #include #include +#include /* * for max sense size @@ -1318,9 +1319,10 @@ static int blk_hw_contig_segment(struct request_queue *q, struct bio *bio, * must make sure sg can hold rq->nr_phys_segments entries */ int blk_rq_map_sg(struct request_queue *q, struct request *rq, - struct scatterlist *sg) + struct scatterlist *sglist) { struct bio_vec *bvec, *bvprv; + struct scatterlist *next_sg, *sg; struct req_iterator iter; int nsegs, cluster; @@ -1331,11 +1333,12 @@ int blk_rq_map_sg(struct request_queue *q, struct request *rq, * for each bio in rq */ bvprv = NULL; + sg = next_sg = &sglist[0]; rq_for_each_segment(bvec, rq, iter) { int nbytes = bvec->bv_len; if (bvprv && cluster) { - if (sg[nsegs - 1].length + nbytes > q->max_segment_size) + if (sg->length + nbytes > q->max_segment_size) goto new_segment; if (!BIOVEC_PHYS_MERGEABLE(bvprv, bvec)) @@ -1343,14 +1346,15 @@ int blk_rq_map_sg(struct request_queue *q, struct request *rq, if (!BIOVEC_SEG_BOUNDARY(q, bvprv, bvec)) goto new_segment; - sg[nsegs - 1].length += nbytes; + sg->length += nbytes; } else { new_segment: - memset(&sg[nsegs],0,sizeof(struct scatterlist)); - sg[nsegs].page = bvec->bv_page; - sg[nsegs].length = nbytes; - sg[nsegs].offset = bvec->bv_offset; + sg = next_sg; + next_sg = sg_next(sg); + sg->page = bvec->bv_page; + sg->length = nbytes; + sg->offset = bvec->bv_offset; nsegs++; } bvprv = bvec; -- cgit v1.2.3 From 563063a808de6b2004d5b8a09ddcb6125481f4b2 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 10 May 2007 13:29:51 +0200 Subject: ll_rw_blk: temporarily enable max_segments tweaking Expose this setting for now, so that users can play with enabling large commands without defaulting it to on globally. This is a debug patch, it will be dropped for the final versions. Signed-off-by: Jens Axboe --- block/ll_rw_blk.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'block/ll_rw_blk.c') diff --git a/block/ll_rw_blk.c b/block/ll_rw_blk.c index 36d205128f9a..527bd8d4db50 100644 --- a/block/ll_rw_blk.c +++ b/block/ll_rw_blk.c @@ -4072,7 +4072,23 @@ static ssize_t queue_max_hw_sectors_show(struct request_queue *q, char *page) return queue_var_show(max_hw_sectors_kb, (page)); } +static ssize_t queue_max_segments_show(struct request_queue *q, char *page) +{ + return queue_var_show(q->max_phys_segments, page); +} + +static ssize_t queue_max_segments_store(struct request_queue *q, + const char *page, size_t count) +{ + unsigned long segments; + ssize_t ret = queue_var_store(&segments, page, count); + spin_lock_irq(q->queue_lock); + q->max_phys_segments = segments; + spin_unlock_irq(q->queue_lock); + + return ret; +} static struct queue_sysfs_entry queue_requests_entry = { .attr = {.name = "nr_requests", .mode = S_IRUGO | S_IWUSR }, .show = queue_requests_show, @@ -4096,6 +4112,12 @@ static struct queue_sysfs_entry queue_max_hw_sectors_entry = { .show = queue_max_hw_sectors_show, }; +static struct queue_sysfs_entry queue_max_segments_entry = { + .attr = {.name = "max_segments", .mode = S_IRUGO | S_IWUSR }, + .show = queue_max_segments_show, + .store = queue_max_segments_store, +}; + static struct queue_sysfs_entry queue_iosched_entry = { .attr = {.name = "scheduler", .mode = S_IRUGO | S_IWUSR }, .show = elv_iosched_show, @@ -4107,6 +4129,7 @@ static struct attribute *default_attrs[] = { &queue_ra_entry.attr, &queue_max_hw_sectors_entry.attr, &queue_max_sectors_entry.attr, + &queue_max_segments_entry.attr, &queue_iosched_entry.attr, NULL, }; -- cgit v1.2.3