diff options
Diffstat (limited to 'drivers/md/dm-thin.c')
-rw-r--r-- | drivers/md/dm-thin.c | 277 |
1 files changed, 180 insertions, 97 deletions
diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c index 5409607d4875..009339d62828 100644 --- a/drivers/md/dm-thin.c +++ b/drivers/md/dm-thin.c @@ -26,6 +26,9 @@ #define PRISON_CELLS 1024 #define COMMIT_PERIOD HZ +DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle, + "A percentage of time allocated for copy on write"); + /* * The block size of the device holding pool data must be * between 64KB and 1GB. @@ -227,6 +230,78 @@ struct thin_c { /*----------------------------------------------------------------*/ /* + * wake_worker() is used when new work is queued and when pool_resume is + * ready to continue deferred IO processing. + */ +static void wake_worker(struct pool *pool) +{ + queue_work(pool->wq, &pool->worker); +} + +/*----------------------------------------------------------------*/ + +static int bio_detain(struct pool *pool, struct dm_cell_key *key, struct bio *bio, + struct dm_bio_prison_cell **cell_result) +{ + int r; + struct dm_bio_prison_cell *cell_prealloc; + + /* + * Allocate a cell from the prison's mempool. + * This might block but it can't fail. + */ + cell_prealloc = dm_bio_prison_alloc_cell(pool->prison, GFP_NOIO); + + r = dm_bio_detain(pool->prison, key, bio, cell_prealloc, cell_result); + if (r) + /* + * We reused an old cell; we can get rid of + * the new one. + */ + dm_bio_prison_free_cell(pool->prison, cell_prealloc); + + return r; +} + +static void cell_release(struct pool *pool, + struct dm_bio_prison_cell *cell, + struct bio_list *bios) +{ + dm_cell_release(pool->prison, cell, bios); + dm_bio_prison_free_cell(pool->prison, cell); +} + +static void cell_release_no_holder(struct pool *pool, + struct dm_bio_prison_cell *cell, + struct bio_list *bios) +{ + dm_cell_release_no_holder(pool->prison, cell, bios); + dm_bio_prison_free_cell(pool->prison, cell); +} + +static void cell_defer_no_holder_no_free(struct thin_c *tc, + struct dm_bio_prison_cell *cell) +{ + struct pool *pool = tc->pool; + unsigned long flags; + + spin_lock_irqsave(&pool->lock, flags); + dm_cell_release_no_holder(pool->prison, cell, &pool->deferred_bios); + spin_unlock_irqrestore(&pool->lock, flags); + + wake_worker(pool); +} + +static void cell_error(struct pool *pool, + struct dm_bio_prison_cell *cell) +{ + dm_cell_error(pool->prison, cell); + dm_bio_prison_free_cell(pool->prison, cell); +} + +/*----------------------------------------------------------------*/ + +/* * A global list of pools that uses a struct mapped_device as a key. */ static struct dm_thin_pool_table { @@ -330,14 +405,20 @@ static void requeue_io(struct thin_c *tc) * target. */ +static bool block_size_is_power_of_two(struct pool *pool) +{ + return pool->sectors_per_block_shift >= 0; +} + static dm_block_t get_bio_block(struct thin_c *tc, struct bio *bio) { + struct pool *pool = tc->pool; sector_t block_nr = bio->bi_sector; - if (tc->pool->sectors_per_block_shift < 0) - (void) sector_div(block_nr, tc->pool->sectors_per_block); + if (block_size_is_power_of_two(pool)) + block_nr >>= pool->sectors_per_block_shift; else - block_nr >>= tc->pool->sectors_per_block_shift; + (void) sector_div(block_nr, pool->sectors_per_block); return block_nr; } @@ -348,12 +429,12 @@ static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block) sector_t bi_sector = bio->bi_sector; bio->bi_bdev = tc->pool_dev->bdev; - if (tc->pool->sectors_per_block_shift < 0) - bio->bi_sector = (block * pool->sectors_per_block) + - sector_div(bi_sector, pool->sectors_per_block); - else + if (block_size_is_power_of_two(pool)) bio->bi_sector = (block << pool->sectors_per_block_shift) | (bi_sector & (pool->sectors_per_block - 1)); + else + bio->bi_sector = (block * pool->sectors_per_block) + + sector_div(bi_sector, pool->sectors_per_block); } static void remap_to_origin(struct thin_c *tc, struct bio *bio) @@ -420,15 +501,6 @@ static void remap_and_issue(struct thin_c *tc, struct bio *bio, issue(tc, bio); } -/* - * wake_worker() is used when new work is queued and when pool_resume is - * ready to continue deferred IO processing. - */ -static void wake_worker(struct pool *pool) -{ - queue_work(pool->wq, &pool->worker); -} - /*----------------------------------------------------------------*/ /* @@ -515,14 +587,14 @@ static void cell_defer(struct thin_c *tc, struct dm_bio_prison_cell *cell) unsigned long flags; spin_lock_irqsave(&pool->lock, flags); - dm_cell_release(cell, &pool->deferred_bios); + cell_release(pool, cell, &pool->deferred_bios); spin_unlock_irqrestore(&tc->pool->lock, flags); wake_worker(pool); } /* - * Same as cell_defer except it omits the original holder of the cell. + * Same as cell_defer above, except it omits the original holder of the cell. */ static void cell_defer_no_holder(struct thin_c *tc, struct dm_bio_prison_cell *cell) { @@ -530,7 +602,7 @@ static void cell_defer_no_holder(struct thin_c *tc, struct dm_bio_prison_cell *c unsigned long flags; spin_lock_irqsave(&pool->lock, flags); - dm_cell_release_no_holder(cell, &pool->deferred_bios); + cell_release_no_holder(pool, cell, &pool->deferred_bios); spin_unlock_irqrestore(&pool->lock, flags); wake_worker(pool); @@ -540,13 +612,15 @@ static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m) { if (m->bio) m->bio->bi_end_io = m->saved_bi_end_io; - dm_cell_error(m->cell); + cell_error(m->tc->pool, m->cell); list_del(&m->list); mempool_free(m, m->tc->pool->mapping_pool); } + static void process_prepared_mapping(struct dm_thin_new_mapping *m) { struct thin_c *tc = m->tc; + struct pool *pool = tc->pool; struct bio *bio; int r; @@ -555,7 +629,7 @@ static void process_prepared_mapping(struct dm_thin_new_mapping *m) bio->bi_end_io = m->saved_bi_end_io; if (m->err) { - dm_cell_error(m->cell); + cell_error(pool, m->cell); goto out; } @@ -567,7 +641,7 @@ static void process_prepared_mapping(struct dm_thin_new_mapping *m) r = dm_thin_insert_block(tc->td, m->virt_block, m->data_block); if (r) { DMERR_LIMIT("dm_thin_insert_block() failed"); - dm_cell_error(m->cell); + cell_error(pool, m->cell); goto out; } @@ -585,7 +659,7 @@ static void process_prepared_mapping(struct dm_thin_new_mapping *m) out: list_del(&m->list); - mempool_free(m, tc->pool->mapping_pool); + mempool_free(m, pool->mapping_pool); } static void process_prepared_discard_fail(struct dm_thin_new_mapping *m) @@ -736,7 +810,7 @@ static void schedule_copy(struct thin_c *tc, dm_block_t virt_block, if (r < 0) { mempool_free(m, pool->mapping_pool); DMERR_LIMIT("dm_kcopyd_copy() failed"); - dm_cell_error(cell); + cell_error(pool, cell); } } } @@ -802,7 +876,7 @@ static void schedule_zero(struct thin_c *tc, dm_block_t virt_block, if (r < 0) { mempool_free(m, pool->mapping_pool); DMERR_LIMIT("dm_kcopyd_zero() failed"); - dm_cell_error(cell); + cell_error(pool, cell); } } } @@ -908,13 +982,13 @@ static void retry_on_resume(struct bio *bio) spin_unlock_irqrestore(&pool->lock, flags); } -static void no_space(struct dm_bio_prison_cell *cell) +static void no_space(struct pool *pool, struct dm_bio_prison_cell *cell) { struct bio *bio; struct bio_list bios; bio_list_init(&bios); - dm_cell_release(cell, &bios); + cell_release(pool, cell, &bios); while ((bio = bio_list_pop(&bios))) retry_on_resume(bio); @@ -932,7 +1006,7 @@ static void process_discard(struct thin_c *tc, struct bio *bio) struct dm_thin_new_mapping *m; build_virtual_key(tc->td, block, &key); - if (dm_bio_detain(tc->pool->prison, &key, bio, &cell)) + if (bio_detain(tc->pool, &key, bio, &cell)) return; r = dm_thin_find_block(tc->td, block, 1, &lookup_result); @@ -944,7 +1018,7 @@ static void process_discard(struct thin_c *tc, struct bio *bio) * on this block. */ build_data_key(tc->td, lookup_result.block, &key2); - if (dm_bio_detain(tc->pool->prison, &key2, bio, &cell2)) { + if (bio_detain(tc->pool, &key2, bio, &cell2)) { cell_defer_no_holder(tc, cell); break; } @@ -1020,13 +1094,13 @@ static void break_sharing(struct thin_c *tc, struct bio *bio, dm_block_t block, break; case -ENOSPC: - no_space(cell); + no_space(tc->pool, cell); break; default: DMERR_LIMIT("%s: alloc_data_block() failed: error = %d", __func__, r); - dm_cell_error(cell); + cell_error(tc->pool, cell); break; } } @@ -1044,7 +1118,7 @@ static void process_shared_bio(struct thin_c *tc, struct bio *bio, * of being broken so we have nothing further to do here. */ build_data_key(tc->td, lookup_result->block, &key); - if (dm_bio_detain(pool->prison, &key, bio, &cell)) + if (bio_detain(pool, &key, bio, &cell)) return; if (bio_data_dir(bio) == WRITE && bio->bi_size) @@ -1065,12 +1139,13 @@ static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block { int r; dm_block_t data_block; + struct pool *pool = tc->pool; /* * Remap empty bios (flushes) immediately, without provisioning. */ if (!bio->bi_size) { - inc_all_io_entry(tc->pool, bio); + inc_all_io_entry(pool, bio); cell_defer_no_holder(tc, cell); remap_and_issue(tc, bio, 0); @@ -1097,14 +1172,14 @@ static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block break; case -ENOSPC: - no_space(cell); + no_space(pool, cell); break; default: DMERR_LIMIT("%s: alloc_data_block() failed: error = %d", __func__, r); - set_pool_mode(tc->pool, PM_READ_ONLY); - dm_cell_error(cell); + set_pool_mode(pool, PM_READ_ONLY); + cell_error(pool, cell); break; } } @@ -1112,6 +1187,7 @@ static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block static void process_bio(struct thin_c *tc, struct bio *bio) { int r; + struct pool *pool = tc->pool; dm_block_t block = get_bio_block(tc, bio); struct dm_bio_prison_cell *cell; struct dm_cell_key key; @@ -1122,7 +1198,7 @@ static void process_bio(struct thin_c *tc, struct bio *bio) * being provisioned so we have nothing further to do here. */ build_virtual_key(tc->td, block, &key); - if (dm_bio_detain(tc->pool->prison, &key, bio, &cell)) + if (bio_detain(pool, &key, bio, &cell)) return; r = dm_thin_find_block(tc->td, block, 1, &lookup_result); @@ -1130,9 +1206,9 @@ static void process_bio(struct thin_c *tc, struct bio *bio) case 0: if (lookup_result.shared) { process_shared_bio(tc, bio, block, &lookup_result); - cell_defer_no_holder(tc, cell); + cell_defer_no_holder(tc, cell); /* FIXME: pass this cell into process_shared? */ } else { - inc_all_io_entry(tc->pool, bio); + inc_all_io_entry(pool, bio); cell_defer_no_holder(tc, cell); remap_and_issue(tc, bio, lookup_result.block); @@ -1141,7 +1217,7 @@ static void process_bio(struct thin_c *tc, struct bio *bio) case -ENODATA: if (bio_data_dir(bio) == READ && tc->origin_dev) { - inc_all_io_entry(tc->pool, bio); + inc_all_io_entry(pool, bio); cell_defer_no_holder(tc, cell); remap_to_origin_and_issue(tc, bio); @@ -1378,7 +1454,8 @@ static int thin_bio_map(struct dm_target *ti, struct bio *bio) dm_block_t block = get_bio_block(tc, bio); struct dm_thin_device *td = tc->td; struct dm_thin_lookup_result result; - struct dm_bio_prison_cell *cell1, *cell2; + struct dm_bio_prison_cell cell1, cell2; + struct dm_bio_prison_cell *cell_result; struct dm_cell_key key; thin_hook_bio(tc, bio); @@ -1420,18 +1497,18 @@ static int thin_bio_map(struct dm_target *ti, struct bio *bio) } build_virtual_key(tc->td, block, &key); - if (dm_bio_detain(tc->pool->prison, &key, bio, &cell1)) + if (dm_bio_detain(tc->pool->prison, &key, bio, &cell1, &cell_result)) return DM_MAPIO_SUBMITTED; build_data_key(tc->td, result.block, &key); - if (dm_bio_detain(tc->pool->prison, &key, bio, &cell2)) { - cell_defer_no_holder(tc, cell1); + if (dm_bio_detain(tc->pool->prison, &key, bio, &cell2, &cell_result)) { + cell_defer_no_holder_no_free(tc, &cell1); return DM_MAPIO_SUBMITTED; } inc_all_io_entry(tc->pool, bio); - cell_defer_no_holder(tc, cell2); - cell_defer_no_holder(tc, cell1); + cell_defer_no_holder_no_free(tc, &cell2); + cell_defer_no_holder_no_free(tc, &cell1); remap(tc, bio, result.block); return DM_MAPIO_REMAPPED; @@ -1636,7 +1713,7 @@ static struct pool *pool_create(struct mapped_device *pool_md, goto bad_prison; } - pool->copier = dm_kcopyd_client_create(); + pool->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle); if (IS_ERR(pool->copier)) { r = PTR_ERR(pool->copier); *error = "Error creating pool's kcopyd client"; @@ -1938,7 +2015,7 @@ static int pool_ctr(struct dm_target *ti, unsigned argc, char **argv) pt->data_dev = data_dev; pt->low_water_blocks = low_water_blocks; pt->adjusted_pf = pt->requested_pf = pf; - ti->num_flush_requests = 1; + ti->num_flush_bios = 1; /* * Only need to enable discards if the pool should pass @@ -1946,7 +2023,7 @@ static int pool_ctr(struct dm_target *ti, unsigned argc, char **argv) * processing will cause mappings to be removed from the btree. */ if (pf.discard_enabled && pf.discard_passdown) { - ti->num_discard_requests = 1; + ti->num_discard_bios = 1; /* * Setting 'discards_supported' circumvents the normal @@ -2299,8 +2376,8 @@ static void emit_flags(struct pool_features *pf, char *result, * <transaction id> <used metadata sectors>/<total metadata sectors> * <used data sectors>/<total data sectors> <held metadata root> */ -static int pool_status(struct dm_target *ti, status_type_t type, - unsigned status_flags, char *result, unsigned maxlen) +static void pool_status(struct dm_target *ti, status_type_t type, + unsigned status_flags, char *result, unsigned maxlen) { int r; unsigned sz = 0; @@ -2326,32 +2403,41 @@ static int pool_status(struct dm_target *ti, status_type_t type, if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti)) (void) commit_or_fallback(pool); - r = dm_pool_get_metadata_transaction_id(pool->pmd, - &transaction_id); - if (r) - return r; + r = dm_pool_get_metadata_transaction_id(pool->pmd, &transaction_id); + if (r) { + DMERR("dm_pool_get_metadata_transaction_id returned %d", r); + goto err; + } - r = dm_pool_get_free_metadata_block_count(pool->pmd, - &nr_free_blocks_metadata); - if (r) - return r; + r = dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free_blocks_metadata); + if (r) { + DMERR("dm_pool_get_free_metadata_block_count returned %d", r); + goto err; + } r = dm_pool_get_metadata_dev_size(pool->pmd, &nr_blocks_metadata); - if (r) - return r; + if (r) { + DMERR("dm_pool_get_metadata_dev_size returned %d", r); + goto err; + } - r = dm_pool_get_free_block_count(pool->pmd, - &nr_free_blocks_data); - if (r) - return r; + r = dm_pool_get_free_block_count(pool->pmd, &nr_free_blocks_data); + if (r) { + DMERR("dm_pool_get_free_block_count returned %d", r); + goto err; + } r = dm_pool_get_data_dev_size(pool->pmd, &nr_blocks_data); - if (r) - return r; + if (r) { + DMERR("dm_pool_get_data_dev_size returned %d", r); + goto err; + } r = dm_pool_get_metadata_snap(pool->pmd, &held_root); - if (r) - return r; + if (r) { + DMERR("dm_pool_get_metadata_snap returned %d", r); + goto err; + } DMEMIT("%llu %llu/%llu %llu/%llu ", (unsigned long long)transaction_id, @@ -2388,8 +2474,10 @@ static int pool_status(struct dm_target *ti, status_type_t type, emit_flags(&pt->requested_pf, result, sz, maxlen); break; } + return; - return 0; +err: + DMEMIT("Error"); } static int pool_iterate_devices(struct dm_target *ti, @@ -2414,11 +2502,6 @@ static int pool_merge(struct dm_target *ti, struct bvec_merge_data *bvm, return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); } -static bool block_size_is_power_of_two(struct pool *pool) -{ - return pool->sectors_per_block_shift >= 0; -} - static void set_discard_limits(struct pool_c *pt, struct queue_limits *limits) { struct pool *pool = pt->pool; @@ -2432,15 +2515,8 @@ static void set_discard_limits(struct pool_c *pt, struct queue_limits *limits) if (pt->adjusted_pf.discard_passdown) { data_limits = &bdev_get_queue(pt->data_dev->bdev)->limits; limits->discard_granularity = data_limits->discard_granularity; - } else if (block_size_is_power_of_two(pool)) + } else limits->discard_granularity = pool->sectors_per_block << SECTOR_SHIFT; - else - /* - * Use largest power of 2 that is a factor of sectors_per_block - * but at least DATA_DEV_BLOCK_SIZE_MIN_SECTORS. - */ - limits->discard_granularity = max(1 << (ffs(pool->sectors_per_block) - 1), - DATA_DEV_BLOCK_SIZE_MIN_SECTORS) << SECTOR_SHIFT; } static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits) @@ -2468,7 +2544,7 @@ static struct target_type pool_target = { .name = "thin-pool", .features = DM_TARGET_SINGLETON | DM_TARGET_ALWAYS_WRITEABLE | DM_TARGET_IMMUTABLE, - .version = {1, 6, 0}, + .version = {1, 6, 1}, .module = THIS_MODULE, .ctr = pool_ctr, .dtr = pool_dtr, @@ -2588,17 +2664,17 @@ static int thin_ctr(struct dm_target *ti, unsigned argc, char **argv) if (r) goto bad_thin_open; - ti->num_flush_requests = 1; + ti->num_flush_bios = 1; ti->flush_supported = true; ti->per_bio_data_size = sizeof(struct dm_thin_endio_hook); /* In case the pool supports discards, pass them on. */ if (tc->pool->pf.discard_enabled) { ti->discards_supported = true; - ti->num_discard_requests = 1; + ti->num_discard_bios = 1; ti->discard_zeroes_data_unsupported = true; - /* Discard requests must be split on a block boundary */ - ti->split_discard_requests = true; + /* Discard bios must be split on a block boundary */ + ti->split_discard_bios = true; } dm_put(pool_md); @@ -2676,8 +2752,8 @@ static void thin_postsuspend(struct dm_target *ti) /* * <nr mapped sectors> <highest mapped sector> */ -static int thin_status(struct dm_target *ti, status_type_t type, - unsigned status_flags, char *result, unsigned maxlen) +static void thin_status(struct dm_target *ti, status_type_t type, + unsigned status_flags, char *result, unsigned maxlen) { int r; ssize_t sz = 0; @@ -2687,7 +2763,7 @@ static int thin_status(struct dm_target *ti, status_type_t type, if (get_pool_mode(tc->pool) == PM_FAIL) { DMEMIT("Fail"); - return 0; + return; } if (!tc->td) @@ -2696,12 +2772,16 @@ static int thin_status(struct dm_target *ti, status_type_t type, switch (type) { case STATUSTYPE_INFO: r = dm_thin_get_mapped_count(tc->td, &mapped); - if (r) - return r; + if (r) { + DMERR("dm_thin_get_mapped_count returned %d", r); + goto err; + } r = dm_thin_get_highest_mapped_block(tc->td, &highest); - if (r < 0) - return r; + if (r < 0) { + DMERR("dm_thin_get_highest_mapped_block returned %d", r); + goto err; + } DMEMIT("%llu ", mapped * tc->pool->sectors_per_block); if (r) @@ -2721,7 +2801,10 @@ static int thin_status(struct dm_target *ti, status_type_t type, } } - return 0; + return; + +err: + DMEMIT("Error"); } static int thin_iterate_devices(struct dm_target *ti, @@ -2748,7 +2831,7 @@ static int thin_iterate_devices(struct dm_target *ti, static struct target_type thin_target = { .name = "thin", - .version = {1, 7, 0}, + .version = {1, 7, 1}, .module = THIS_MODULE, .ctr = thin_ctr, .dtr = thin_dtr, |