diff options
Diffstat (limited to 'fs/verity')
| -rw-r--r-- | fs/verity/enable.c | 32 | ||||
| -rw-r--r-- | fs/verity/fsverity_private.h | 16 | ||||
| -rw-r--r-- | fs/verity/open.c | 77 |
3 files changed, 78 insertions, 47 deletions
diff --git a/fs/verity/enable.c b/fs/verity/enable.c index c56c18e2605b..c9448074cce1 100644 --- a/fs/verity/enable.c +++ b/fs/verity/enable.c @@ -266,8 +266,25 @@ static int enable_verity(struct file *filp, } /* + * Add the fsverity_info into the hash table before finishing the + * initialization so that we don't have to undo the enabling when memory + * allocation for the hash table fails. This is safe because looking up + * the fsverity_info always first checks the S_VERITY flag on the inode, + * which will only be set at the very end of the ->end_enable_verity + * method. + */ + err = fsverity_set_info(vi); + if (err) { + fsverity_free_info(vi); + goto rollback; + } + + /* * Tell the filesystem to finish enabling verity on the file. - * Serialized with ->begin_enable_verity() by the inode lock. + * Serialized with ->begin_enable_verity() by the inode lock. The file + * system needs to set the S_VERITY flag on the inode at the very end of + * the method, at which point the fsverity information can be accessed + * by other threads. */ inode_lock(inode); err = vops->end_enable_verity(filp, desc, desc_size, params.tree_size); @@ -275,19 +292,10 @@ static int enable_verity(struct file *filp, if (err) { fsverity_err(inode, "%ps() failed with err %d", vops->end_enable_verity, err); - fsverity_free_info(vi); + fsverity_remove_info(vi); } else if (WARN_ON_ONCE(!IS_VERITY(inode))) { + fsverity_remove_info(vi); err = -EINVAL; - fsverity_free_info(vi); - } else { - /* Successfully enabled verity */ - - /* - * Readers can start using the inode's verity info immediately, - * so it can't be rolled back once set. So don't set it until - * just after the filesystem has successfully enabled verity. - */ - fsverity_set_info(inode, vi); } out: kfree(params.hashstate); diff --git a/fs/verity/fsverity_private.h b/fs/verity/fsverity_private.h index f9f3936b0a89..2887cb849cec 100644 --- a/fs/verity/fsverity_private.h +++ b/fs/verity/fsverity_private.h @@ -11,6 +11,7 @@ #define pr_fmt(fmt) "fs-verity: " fmt #include <linux/fsverity.h> +#include <linux/rhashtable.h> /* * Implementation limit: maximum depth of the Merkle tree. For now 8 is plenty; @@ -63,13 +64,14 @@ struct merkle_tree_params { * fsverity_info - cached verity metadata for an inode * * When a verity file is first opened, an instance of this struct is allocated - * and a pointer to it is stored in the file's in-memory inode. It remains - * until the inode is evicted. It caches information about the Merkle tree - * that's needed to efficiently verify data read from the file. It also caches - * the file digest. The Merkle tree pages themselves are not cached here, but - * the filesystem may cache them. + * and a pointer to it is stored in the global hash table, indexed by the inode + * pointer value. It remains alive until the inode is evicted. It caches + * information about the Merkle tree that's needed to efficiently verify data + * read from the file. It also caches the file digest. The Merkle tree pages + * themselves are not cached here, but the filesystem may cache them. */ struct fsverity_info { + struct rhash_head rhash_head; struct merkle_tree_params tree_params; u8 root_hash[FS_VERITY_MAX_DIGEST_SIZE]; u8 file_digest[FS_VERITY_MAX_DIGEST_SIZE]; @@ -127,9 +129,9 @@ int fsverity_init_merkle_tree_params(struct merkle_tree_params *params, struct fsverity_info *fsverity_create_info(struct inode *inode, struct fsverity_descriptor *desc); -void fsverity_set_info(struct inode *inode, struct fsverity_info *vi); - +int fsverity_set_info(struct fsverity_info *vi); void fsverity_free_info(struct fsverity_info *vi); +void fsverity_remove_info(struct fsverity_info *vi); int fsverity_get_descriptor(struct inode *inode, struct fsverity_descriptor **desc_ret); diff --git a/fs/verity/open.c b/fs/verity/open.c index 128502cf0a23..dfa0d1afe0fe 100644 --- a/fs/verity/open.c +++ b/fs/verity/open.c @@ -12,6 +12,14 @@ #include <linux/slab.h> static struct kmem_cache *fsverity_info_cachep; +static struct rhashtable fsverity_info_hash; + +static const struct rhashtable_params fsverity_info_hash_params = { + .key_len = sizeof_field(struct fsverity_info, inode), + .key_offset = offsetof(struct fsverity_info, inode), + .head_offset = offsetof(struct fsverity_info, rhash_head), + .automatic_shrinking = true, +}; /** * fsverity_init_merkle_tree_params() - initialize Merkle tree parameters @@ -241,33 +249,19 @@ fail: return ERR_PTR(err); } -void fsverity_set_info(struct inode *inode, struct fsverity_info *vi) +int fsverity_set_info(struct fsverity_info *vi) { - /* - * Multiple tasks may race to set the inode's verity info pointer, so - * use cmpxchg_release(). This pairs with the smp_load_acquire() in - * fsverity_get_info(). I.e., publish the pointer with a RELEASE - * barrier so that other tasks can ACQUIRE it. - */ - if (cmpxchg_release(fsverity_info_addr(inode), NULL, vi) != NULL) { - /* Lost the race, so free the verity info we allocated. */ - fsverity_free_info(vi); - /* - * Afterwards, the caller may access the inode's verity info - * directly, so make sure to ACQUIRE the winning verity info. - */ - (void)fsverity_get_info(inode); - } + return rhashtable_lookup_insert_fast(&fsverity_info_hash, + &vi->rhash_head, + fsverity_info_hash_params); } -void fsverity_free_info(struct fsverity_info *vi) +struct fsverity_info *__fsverity_get_info(const struct inode *inode) { - if (!vi) - return; - kfree(vi->tree_params.hashstate); - kvfree(vi->hash_block_verified); - kmem_cache_free(fsverity_info_cachep, vi); + return rhashtable_lookup_fast(&fsverity_info_hash, &inode, + fsverity_info_hash_params); } +EXPORT_SYMBOL_GPL(__fsverity_get_info); static bool validate_fsverity_descriptor(struct inode *inode, const struct fsverity_descriptor *desc, @@ -352,7 +346,7 @@ int fsverity_get_descriptor(struct inode *inode, static int ensure_verity_info(struct inode *inode) { - struct fsverity_info *vi = fsverity_get_info(inode); + struct fsverity_info *vi = fsverity_get_info(inode), *found; struct fsverity_descriptor *desc; int err; @@ -369,8 +363,19 @@ static int ensure_verity_info(struct inode *inode) goto out_free_desc; } - fsverity_set_info(inode, vi); - err = 0; + /* + * Multiple tasks may race to set the inode's verity info, in which case + * we might find an existing fsverity_info in the hash table. + */ + found = rhashtable_lookup_get_insert_fast(&fsverity_info_hash, + &vi->rhash_head, + fsverity_info_hash_params); + if (found) { + fsverity_free_info(vi); + if (IS_ERR(found)) + err = PTR_ERR(found); + } + out_free_desc: kfree(desc); return err; @@ -384,16 +389,32 @@ int __fsverity_file_open(struct inode *inode, struct file *filp) } EXPORT_SYMBOL_GPL(__fsverity_file_open); +void fsverity_free_info(struct fsverity_info *vi) +{ + kfree(vi->tree_params.hashstate); + kvfree(vi->hash_block_verified); + kmem_cache_free(fsverity_info_cachep, vi); +} + +void fsverity_remove_info(struct fsverity_info *vi) +{ + rhashtable_remove_fast(&fsverity_info_hash, &vi->rhash_head, + fsverity_info_hash_params); + fsverity_free_info(vi); +} + void fsverity_cleanup_inode(struct inode *inode) { - struct fsverity_info **vi_addr = fsverity_info_addr(inode); + struct fsverity_info *vi = fsverity_get_info(inode); - fsverity_free_info(*vi_addr); - *vi_addr = NULL; + if (vi) + fsverity_remove_info(vi); } void __init fsverity_init_info_cache(void) { + if (rhashtable_init(&fsverity_info_hash, &fsverity_info_hash_params)) + panic("failed to initialize fsverity hash\n"); fsverity_info_cachep = KMEM_CACHE_USERCOPY( fsverity_info, SLAB_RECLAIM_ACCOUNT | SLAB_PANIC, |