diff options
Diffstat (limited to 'fs/crypto/crypto.c')
| -rw-r--r-- | fs/crypto/crypto.c | 182 |
1 files changed, 55 insertions, 127 deletions
diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c index 07f9cbfe3ea4..5286a124b0d9 100644 --- a/fs/crypto/crypto.c +++ b/fs/crypto/crypto.c @@ -38,18 +38,11 @@ MODULE_PARM_DESC(num_prealloc_crypto_pages, static mempool_t *fscrypt_bounce_page_pool = NULL; -static struct workqueue_struct *fscrypt_read_workqueue; static DEFINE_MUTEX(fscrypt_init_mutex); struct kmem_cache *fscrypt_inode_info_cachep; -void fscrypt_enqueue_decrypt_work(struct work_struct *work) -{ - queue_work(fscrypt_read_workqueue, work); -} -EXPORT_SYMBOL(fscrypt_enqueue_decrypt_work); - -struct page *fscrypt_alloc_bounce_page(gfp_t gfp_flags) +static struct page *fscrypt_alloc_bounce_page(gfp_t gfp_flags) { if (WARN_ON_ONCE(!fscrypt_bounce_page_pool)) { /* @@ -65,8 +58,7 @@ struct page *fscrypt_alloc_bounce_page(gfp_t gfp_flags) * fscrypt_free_bounce_page() - free a ciphertext bounce page * @bounce_page: the bounce page to free, or NULL * - * Free a bounce page that was allocated by fscrypt_encrypt_pagecache_blocks(), - * or by fscrypt_alloc_bounce_page() directly. + * Free a bounce page that was allocated by fscrypt_encrypt_pagecache_blocks(). */ void fscrypt_free_bounce_page(struct page *bounce_page) { @@ -91,7 +83,7 @@ void fscrypt_generate_iv(union fscrypt_iv *iv, u64 index, { u8 flags = fscrypt_policy_flags(&ci->ci_policy); - memset(iv, 0, ci->ci_mode->ivsize); + memset(iv, 0, sizeof(*iv)); if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64) { WARN_ON_ONCE(index > U32_MAX); @@ -107,17 +99,23 @@ void fscrypt_generate_iv(union fscrypt_iv *iv, u64 index, } /* Encrypt or decrypt a single "data unit" of file contents. */ -int fscrypt_crypt_data_unit(const struct fscrypt_inode_info *ci, - fscrypt_direction_t rw, u64 index, - struct page *src_page, struct page *dest_page, - unsigned int len, unsigned int offs) +static int fscrypt_crypt_data_unit(const struct fscrypt_inode_info *ci, + fscrypt_direction_t rw, u64 index, + struct page *src_page, + struct page *dest_page, unsigned int len, + unsigned int offs) { - struct crypto_sync_skcipher *tfm = ci->ci_enc_key.tfm; - SYNC_SKCIPHER_REQUEST_ON_STACK(req, tfm); + struct crypto_sync_skcipher *tfm; union fscrypt_iv iv; struct scatterlist dst, src; int err; + if (WARN_ON_ONCE(ci == NULL)) /* File hasn't been opened yet? */ + return -ENOKEY; + tfm = ci->ci_enc_key.tfm; + if (WARN_ON_ONCE(tfm == NULL)) /* Called on block-based filesystem? */ + return -ENOKEY; + if (WARN_ON_ONCE(len <= 0)) return -EINVAL; if (WARN_ON_ONCE(len % FSCRYPT_CONTENTS_ALIGNMENT != 0)) @@ -125,18 +123,22 @@ int fscrypt_crypt_data_unit(const struct fscrypt_inode_info *ci, fscrypt_generate_iv(&iv, index, ci); - skcipher_request_set_callback( - req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, - NULL, NULL); - sg_init_table(&dst, 1); - sg_set_page(&dst, dest_page, len, offs); - sg_init_table(&src, 1); - sg_set_page(&src, src_page, len, offs); - skcipher_request_set_crypt(req, &src, &dst, len, &iv); - if (rw == FS_DECRYPT) - err = crypto_skcipher_decrypt(req); - else - err = crypto_skcipher_encrypt(req); + { + SYNC_SKCIPHER_REQUEST_ON_STACK(req, tfm); + skcipher_request_set_callback(req, + CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP, + NULL, NULL); + sg_init_table(&dst, 1); + sg_set_page(&dst, dest_page, len, offs); + sg_init_table(&src, 1); + sg_set_page(&src, src_page, len, offs); + skcipher_request_set_crypt(req, &src, &dst, len, &iv); + if (rw == FS_DECRYPT) + err = crypto_skcipher_decrypt(req); + else + err = crypto_skcipher_encrypt(req); + } if (err) fscrypt_err(ci->ci_inode, "%scryption failed for data unit %llu: %d", @@ -160,7 +162,7 @@ int fscrypt_crypt_data_unit(const struct fscrypt_inode_info *ci, * which the plaintext data was located in the source page. Any other parts of * the bounce page will be left uninitialized. * - * This is for use by the filesystem's ->writepages() method. + * This is for use by the ->writepages() method of non-block-based filesystems. * * The bounce page allocation is mempool-backed, so it will always succeed when * @gfp_flags includes __GFP_DIRECT_RECLAIM, e.g. when it's GFP_NOFS. However, @@ -174,14 +176,20 @@ struct page *fscrypt_encrypt_pagecache_blocks(struct folio *folio, { const struct inode *inode = folio->mapping->host; const struct fscrypt_inode_info *ci = fscrypt_get_inode_info_raw(inode); - const unsigned int du_bits = ci->ci_data_unit_bits; - const unsigned int du_size = 1U << du_bits; + unsigned int du_bits; + unsigned int du_size; struct page *ciphertext_page; - u64 index = ((u64)folio->index << (PAGE_SHIFT - du_bits)) + - (offs >> du_bits); + u64 index; unsigned int i; int err; + if (WARN_ON_ONCE(ci == NULL)) /* File hasn't been opened yet? */ + return ERR_PTR(-ENOKEY); + + du_bits = ci->ci_data_unit_bits; + du_size = 1U << du_bits; + index = (folio_pos(folio) + offs) >> du_bits; + VM_BUG_ON_FOLIO(folio_test_large(folio), folio); if (WARN_ON_ONCE(!folio_test_locked(folio))) return ERR_PTR(-EINVAL); @@ -222,7 +230,8 @@ EXPORT_SYMBOL(fscrypt_encrypt_pagecache_blocks); * arbitrary page, not necessarily in the original pagecache page. The @inode * and @lblk_num must be specified, as they can't be determined from @page. * - * This is not compatible with fscrypt_operations::supports_subblock_data_units. + * This function only supports non-block-based filesystems that don't support + * sub-block data units (as indicated by the fscrypt_operations fields). * * Return: 0 on success; -errno on failure */ @@ -239,50 +248,6 @@ int fscrypt_encrypt_block_inplace(const struct inode *inode, struct page *page, EXPORT_SYMBOL(fscrypt_encrypt_block_inplace); /** - * fscrypt_decrypt_pagecache_blocks() - Decrypt data from a pagecache folio - * @folio: the pagecache folio containing the data to decrypt - * @len: size of the data to decrypt, in bytes - * @offs: offset within @folio of the data to decrypt, in bytes - * - * Decrypt data that has just been read from an encrypted file. The data must - * be located in a pagecache folio that is still locked and not yet uptodate. - * The length and offset of the data must be aligned to the file's crypto data - * unit size. Alignment to the filesystem block size fulfills this requirement, - * as the filesystem block size is always a multiple of the data unit size. - * - * Return: 0 on success; -errno on failure - */ -int fscrypt_decrypt_pagecache_blocks(struct folio *folio, size_t len, - size_t offs) -{ - const struct inode *inode = folio->mapping->host; - const struct fscrypt_inode_info *ci = fscrypt_get_inode_info_raw(inode); - const unsigned int du_bits = ci->ci_data_unit_bits; - const unsigned int du_size = 1U << du_bits; - u64 index = ((u64)folio->index << (PAGE_SHIFT - du_bits)) + - (offs >> du_bits); - size_t i; - int err; - - if (WARN_ON_ONCE(!folio_test_locked(folio))) - return -EINVAL; - - if (WARN_ON_ONCE(len <= 0 || !IS_ALIGNED(len | offs, du_size))) - return -EINVAL; - - for (i = offs; i < offs + len; i += du_size, index++) { - struct page *page = folio_page(folio, i >> PAGE_SHIFT); - - err = fscrypt_crypt_data_unit(ci, FS_DECRYPT, index, page, - page, du_size, i & ~PAGE_MASK); - if (err) - return err; - } - return 0; -} -EXPORT_SYMBOL(fscrypt_decrypt_pagecache_blocks); - -/** * fscrypt_decrypt_block_inplace() - Decrypt a filesystem block in-place * @inode: The inode to which this block belongs * @page: The page containing the block to decrypt @@ -296,7 +261,8 @@ EXPORT_SYMBOL(fscrypt_decrypt_pagecache_blocks); * arbitrary page, not necessarily in the original pagecache page. The @inode * and @lblk_num must be specified, as they can't be determined from @page. * - * This is not compatible with fscrypt_operations::supports_subblock_data_units. + * This function only supports non-block-based filesystems that don't support + * sub-block data units (as indicated by the fscrypt_operations fields). * * Return: 0 on success; -errno on failure */ @@ -323,31 +289,26 @@ EXPORT_SYMBOL(fscrypt_decrypt_block_inplace); */ int fscrypt_initialize(struct super_block *sb) { - int err = 0; mempool_t *pool; /* pairs with smp_store_release() below */ - if (likely(smp_load_acquire(&fscrypt_bounce_page_pool))) + if (smp_load_acquire(&fscrypt_bounce_page_pool)) return 0; /* No need to allocate a bounce page pool if this FS won't use it. */ if (!sb->s_cop->needs_bounce_pages) return 0; - mutex_lock(&fscrypt_init_mutex); + guard(mutex)(&fscrypt_init_mutex); if (fscrypt_bounce_page_pool) - goto out_unlock; + return 0; - err = -ENOMEM; pool = mempool_create_page_pool(num_prealloc_crypto_pages, 0); if (!pool) - goto out_unlock; + return -ENOMEM; /* pairs with smp_load_acquire() above */ smp_store_release(&fscrypt_bounce_page_pool, pool); - err = 0; -out_unlock: - mutex_unlock(&fscrypt_init_mutex); - return err; + return 0; } void fscrypt_msg(const struct inode *inode, const char *level, @@ -365,7 +326,7 @@ void fscrypt_msg(const struct inode *inode, const char *level, vaf.fmt = fmt; vaf.va = &args; if (inode && inode->i_ino) - printk("%sfscrypt (%s, inode %lu): %pV\n", + printk("%sfscrypt (%s, inode %llu): %pV\n", level, inode->i_sb->s_id, inode->i_ino, &vaf); else if (inode) printk("%sfscrypt (%s): %pV\n", level, inode->i_sb->s_id, &vaf); @@ -374,45 +335,12 @@ void fscrypt_msg(const struct inode *inode, const char *level, va_end(args); } -/** - * fscrypt_init() - Set up for fs encryption. - * - * Return: 0 on success; -errno on failure - */ static int __init fscrypt_init(void) { - int err = -ENOMEM; - - /* - * Use an unbound workqueue to allow bios to be decrypted in parallel - * even when they happen to complete on the same CPU. This sacrifices - * locality, but it's worthwhile since decryption is CPU-intensive. - * - * Also use a high-priority workqueue to prioritize decryption work, - * which blocks reads from completing, over regular application tasks. - */ - fscrypt_read_workqueue = alloc_workqueue("fscrypt_read_queue", - WQ_UNBOUND | WQ_HIGHPRI, - num_online_cpus()); - if (!fscrypt_read_workqueue) - goto fail; - fscrypt_inode_info_cachep = KMEM_CACHE(fscrypt_inode_info, - SLAB_RECLAIM_ACCOUNT); - if (!fscrypt_inode_info_cachep) - goto fail_free_queue; - - err = fscrypt_init_keyring(); - if (err) - goto fail_free_inode_info; - + SLAB_RECLAIM_ACCOUNT | + SLAB_PANIC); + fscrypt_init_keyring(); return 0; - -fail_free_inode_info: - kmem_cache_destroy(fscrypt_inode_info_cachep); -fail_free_queue: - destroy_workqueue(fscrypt_read_workqueue); -fail: - return err; } late_initcall(fscrypt_init) |