diff options
27 files changed, 3073 insertions, 799 deletions
diff --git a/Documentation/crypto/libcrypto-auth-encryption.rst b/Documentation/crypto/libcrypto-auth-encryption.rst new file mode 100644 index 000000000000..1e527685a42f --- /dev/null +++ b/Documentation/crypto/libcrypto-auth-encryption.rst @@ -0,0 +1,20 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later + +Authenticated encryption +======================== + +These APIs provide support for authenticated encryption and decryption. + +AES-CCM +------- + +This API provides support for AES in the CCM mode of operation. + +.. kernel-doc:: include/crypto/aes-ccm.h + +AES-GCM +------- + +This API provides support for AES in the GCM mode of operation. + +.. kernel-doc:: include/crypto/aes-gcm.h diff --git a/Documentation/crypto/libcrypto-unauth-encryption.rst b/Documentation/crypto/libcrypto-unauth-encryption.rst new file mode 100644 index 000000000000..b4639b8927c6 --- /dev/null +++ b/Documentation/crypto/libcrypto-unauth-encryption.rst @@ -0,0 +1,49 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later + +Unauthenticated encryption +========================== + +These APIs provide support for unauthenticated encryption and decryption, +including bare stream ciphers and other length-preserving algorithms such as +block ciphers in XTS mode. The legitimate use cases for these algorithms are: + +- Support for legacy protocols that really should have chosen an authenticated + mode (or even another primitive entirely) but didn't. + +- Internal components of authenticated modes. For example, AES-CTR is used by + AES-GCM and AES-CCM internally. + +- Storage encryption that cannot accommodate ciphertext expansion. Usually + AES-XTS is used for this. + +- Stream ciphers for key derivation and random number generation. + +Besides the above, these shouldn't be used. + +AES-CBC and AES-CBC-CTS +----------------------- + +This API provides support for AES in the CBC and CBC-CTS modes of operation. + +.. kernel-doc:: include/crypto/aes-cbc.h + +AES-CTR and AES-XCTR +-------------------- + +This API provides support for AES in the CTR and XCTR modes of operation. + +.. kernel-doc:: include/crypto/aes-ctr.h + +AES-ECB +------- + +This API provides support for AES in the ECB mode of operation. + +.. kernel-doc:: include/crypto/aes-ecb.h + +AES-XTS +------- + +This API provides support for AES in the XTS mode of operation. + +.. kernel-doc:: include/crypto/aes-xts.h diff --git a/Documentation/crypto/libcrypto.rst b/Documentation/crypto/libcrypto.rst index 0733e603d229..e911e0521597 100644 --- a/Documentation/crypto/libcrypto.rst +++ b/Documentation/crypto/libcrypto.rst @@ -159,8 +159,10 @@ API documentation .. toctree:: :maxdepth: 2 + libcrypto-auth-encryption libcrypto-blockcipher libcrypto-hash libcrypto-signature + libcrypto-unauth-encryption libcrypto-utils sha3 diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index bdad90f210e4..0b89641cac16 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1495,7 +1495,7 @@ config AMD_MEM_ENCRYPT select ARCH_HAS_CC_PLATFORM select X86_MEM_ENCRYPT select UNACCEPTED_MEMORY - select CRYPTO_LIB_AESGCM + select CRYPTO_LIB_AES_GCM help Say yes to enable support for the encryption of system memory. This requires an AMD processor that supports Secure Memory diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c index ecd77d3217f3..cc292d7c6fd1 100644 --- a/arch/x86/coco/sev/core.c +++ b/arch/x86/coco/sev/core.c @@ -25,7 +25,7 @@ #include <linux/psp-sev.h> #include <linux/dmi.h> #include <uapi/linux/sev-guest.h> -#include <crypto/gcm.h> +#include <crypto/aes-gcm.h> #include <asm/init.h> #include <asm/cpu_entry_area.h> @@ -1535,21 +1535,21 @@ static u8 *get_vmpck(int id, struct snp_secrets_page *secrets, u32 **seqno) return key; } -static struct aesgcm_ctx *snp_init_crypto(u8 *key, size_t keylen) +static struct aes_gcm_key *snp_init_crypto(const u8 *key, size_t keylen) { - struct aesgcm_ctx *ctx; + struct aes_gcm_key *gcm_key; - ctx = kzalloc_obj(*ctx); - if (!ctx) + gcm_key = kzalloc_obj(*gcm_key); + if (!gcm_key) return NULL; - if (aesgcm_expandkey(ctx, key, keylen, AUTHTAG_LEN)) { - pr_err("Crypto context initialization failed\n"); - kfree(ctx); + if (aes_gcm_preparekey(gcm_key, key, keylen, AUTHTAG_LEN)) { + pr_err("AES-GCM key preparation failed\n"); + kfree_sensitive(gcm_key); return NULL; } - return ctx; + return gcm_key; } int snp_msg_init(struct snp_msg_desc *mdesc, int vmpck_id) @@ -1572,8 +1572,8 @@ int snp_msg_init(struct snp_msg_desc *mdesc, int vmpck_id) mdesc->vmpck_id = vmpck_id; - mdesc->ctx = snp_init_crypto(mdesc->vmpck, VMPCK_KEY_LEN); - if (!mdesc->ctx) + mdesc->gcm_key = snp_init_crypto(mdesc->vmpck, VMPCK_KEY_LEN); + if (!mdesc->gcm_key) return -ENOMEM; return 0; @@ -1624,7 +1624,7 @@ void snp_msg_free(struct snp_msg_desc *mdesc) if (!mdesc) return; - kfree(mdesc->ctx); + kfree_sensitive(mdesc->gcm_key); free_shared_pages(mdesc->response, sizeof(struct snp_guest_msg)); free_shared_pages(mdesc->request, sizeof(struct snp_guest_msg)); iounmap((__force void __iomem *)mdesc->secrets); @@ -1709,7 +1709,7 @@ static int verify_and_dec_payload(struct snp_msg_desc *mdesc, struct snp_guest_r struct snp_guest_msg *req_msg = &mdesc->secret_request; struct snp_guest_msg_hdr *req_msg_hdr = &req_msg->hdr; struct snp_guest_msg_hdr *resp_msg_hdr = &resp_msg->hdr; - struct aesgcm_ctx *ctx = mdesc->ctx; + struct aes_gcm_key *gcm_key = mdesc->gcm_key; u8 iv[GCM_AES_IV_SIZE] = {}; pr_debug("response [seqno %lld type %d version %d sz %d]\n", @@ -1732,23 +1732,21 @@ static int verify_and_dec_payload(struct snp_msg_desc *mdesc, struct snp_guest_r * If the message size is greater than our buffer length then return * an error. */ - if (unlikely((resp_msg_hdr->msg_sz + ctx->authsize) > req->resp_sz)) + if (unlikely(resp_msg_hdr->msg_sz + AUTHTAG_LEN > req->resp_sz)) return -EBADMSG; /* Decrypt the payload */ memcpy(iv, &resp_msg_hdr->msg_seqno, min(sizeof(iv), sizeof(resp_msg_hdr->msg_seqno))); - if (!aesgcm_decrypt(ctx, req->resp_buf, resp_msg->payload, resp_msg_hdr->msg_sz, - &resp_msg_hdr->algo, AAD_LEN, iv, resp_msg_hdr->authtag)) - return -EBADMSG; - - return 0; + return aes_gcm_decrypt(req->resp_buf, resp_msg->payload, + resp_msg_hdr->msg_sz, resp_msg_hdr->authtag, + &resp_msg_hdr->algo, AAD_LEN, iv, gcm_key); } static int enc_payload(struct snp_msg_desc *mdesc, u64 seqno, struct snp_guest_req *req) { struct snp_guest_msg *msg = &mdesc->secret_request; struct snp_guest_msg_hdr *hdr = &msg->hdr; - struct aesgcm_ctx *ctx = mdesc->ctx; + struct aes_gcm_key *gcm_key = mdesc->gcm_key; u8 iv[GCM_AES_IV_SIZE] = {}; memset(msg, 0, sizeof(*msg)); @@ -1769,12 +1767,12 @@ static int enc_payload(struct snp_msg_desc *mdesc, u64 seqno, struct snp_guest_r pr_debug("request [seqno %lld type %d version %d sz %d]\n", hdr->msg_seqno, hdr->msg_type, hdr->msg_version, hdr->msg_sz); - if (WARN_ON((req->req_sz + ctx->authsize) > sizeof(msg->payload))) + if (WARN_ON(req->req_sz + AUTHTAG_LEN > sizeof(msg->payload))) return -EBADMSG; memcpy(iv, &hdr->msg_seqno, min(sizeof(iv), sizeof(hdr->msg_seqno))); - aesgcm_encrypt(ctx, msg->payload, req->req_buf, req->req_sz, &hdr->algo, - AAD_LEN, iv, hdr->authtag); + aes_gcm_encrypt(msg->payload, req->req_buf, req->req_sz, hdr->authtag, + &hdr->algo, AAD_LEN, iv, gcm_key); return 0; } @@ -1869,15 +1867,6 @@ int snp_send_guest_request(struct snp_msg_desc *mdesc, struct snp_guest_req *req u64 seqno; int rc; - /* - * enc_payload() calls aesgcm_encrypt(), which can potentially offload to HW. - * The offload's DMA SG list of data to encrypt has to be in linear mapping. - */ - if (!virt_addr_valid(req->req_buf) || !virt_addr_valid(req->resp_buf)) { - pr_warn("AES-GSM buffers must be in linear mapping"); - return -EINVAL; - } - guard(mutex)(&snp_cmd_mutex); /* Check if the VMPCK is not empty */ diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h index 594cfa19cbd4..9e7a077c445d 100644 --- a/arch/x86/include/asm/sev.h +++ b/arch/x86/include/asm/sev.h @@ -314,7 +314,7 @@ struct snp_msg_desc { struct snp_secrets_page *secrets; - struct aesgcm_ctx *ctx; + struct aes_gcm_key *gcm_key; u32 *os_area_msg_seqno; u8 *vmpck; diff --git a/crypto/Kconfig b/crypto/Kconfig index b61401bd3ef6..981d5dc422d4 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -358,8 +358,19 @@ config CRYPTO_AES tristate "AES (Advanced Encryption Standard)" select CRYPTO_ALGAPI select CRYPTO_LIB_AES + select CRYPTO_LIB_AES_CBC if CRYPTO_CBC != n || CRYPTO_CTS != n select CRYPTO_LIB_AES_CBC_MACS if CRYPTO_CMAC != n || CRYPTO_XCBC != n || CRYPTO_CCM != n + select CRYPTO_LIB_AES_CCM if CRYPTO_CCM != n + select CRYPTO_LIB_AES_CTR if CRYPTO_CTR != n || CRYPTO_XCTR != n + select CRYPTO_LIB_AES_ECB if CRYPTO_ECB != n + select CRYPTO_LIB_AES_GCM if CRYPTO_GCM != n + select CRYPTO_LIB_AES_XTS if CRYPTO_XTS != n + select CRYPTO_AEAD if CRYPTO_GCM != n || CRYPTO_CCM != n select CRYPTO_HASH if CRYPTO_CMAC != n || CRYPTO_XCBC != n || CRYPTO_CCM != n + # CRYPTO_SKCIPHER should be selected only if a mode that needs it is + # enabled, but that doesn't work due to a recursive dependency caused by + # CRYPTO_SKCIPHER selecting CRYPTO_ECB. So just always select it. + select CRYPTO_SKCIPHER help AES cipher algorithms (Rijndael)(FIPS-197, ISO/IEC 18033-3) diff --git a/crypto/aes.c b/crypto/aes.c index 6bf23eb0503f..94791f481e98 100644 --- a/crypto/aes.c +++ b/crypto/aes.c @@ -6,12 +6,22 @@ */ #include <crypto/aes-cbc-macs.h> +#include <crypto/aes-cbc.h> +#include <crypto/aes-ccm.h> +#include <crypto/aes-ctr.h> +#include <crypto/aes-ecb.h> +#include <crypto/aes-gcm.h> +#include <crypto/aes-xts.h> #include <crypto/aes.h> #include <crypto/algapi.h> +#include <crypto/internal/aead.h> #include <crypto/internal/hash.h> +#include <crypto/internal/skcipher.h> +#include <crypto/scatterwalk.h> #include <linux/module.h> static_assert(__alignof__(struct aes_key) <= CRYPTO_MINALIGN); +static_assert(__alignof__(struct aes_enckey) <= CRYPTO_MINALIGN); static int crypto_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key, unsigned int key_len) @@ -85,7 +95,6 @@ static int __maybe_unused crypto_aes_cmac_digest(struct shash_desc *desc, return 0; } -static_assert(__alignof__(struct aes_enckey) <= CRYPTO_MINALIGN); #define AES_CBCMAC_KEY(tfm) ((struct aes_enckey *)crypto_shash_ctx(tfm)) #define AES_CBCMAC_CTX(desc) ((struct aes_cbcmac_ctx *)shash_desc_ctx(desc)) @@ -200,6 +209,828 @@ static struct shash_alg mac_algs[] = { #endif }; +static __maybe_unused int +crypto_aes_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct aes_key *key = crypto_skcipher_ctx(tfm); + + return aes_preparekey(key, in_key, key_len); +} + +static __maybe_unused int +crypto_aes_skcipher_setenckey(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct aes_enckey *key = crypto_skcipher_ctx(tfm); + + return aes_prepareenckey(key, in_key, key_len); +} + +/* + * Return true if the request uses only a single scatterlist element and high + * memory isn't enabled. This assumes that both scatterlists are non-NULL, i.e. + * the caller must have handled the cryptlen == 0 case already. + */ +static inline bool +skcipher_request_is_linear_lowmem(const struct skcipher_request *req) +{ + return !IS_ENABLED(CONFIG_HIGHMEM) && + req->dst->length >= req->cryptlen && + req->src->length >= req->cryptlen; +} + +/* + * Call crypt_func() (a function that operates on simple virtual addresses) zero + * or more times to en/decrypt 'cryptlen' bytes of data from the source + * scatterlist 'src' and write it into the destination scatterlist 'dst', + * starting at 'start_pos' bytes into both. + * + * This always calls crypt_func() with a length that's a multiple of + * AES_BLOCK_SIZE, except the last call which includes any remainder. This is + * implemented by using an on-stack bounce buffer when necessary. The current + * implementation also tries to prefer passing at least 4 blocks, so e.g. + * scatterlist entries [16,16,16,16] result in a single 64-byte call. + * + * The scatterlists must describe either entirely different memory + * (out-of-place) or entirely the same memory (in-place). In the latter case, + * crypt_func() is always called with the source and dest pointers the same. + */ +#define AES_CRYPT_SG(crypt_func, dst, src, cryptlen, start_pos, ...) \ + ({ \ + unsigned int remaining = (cryptlen); \ + unsigned int spos = (start_pos); \ + \ + if (remaining != 0) { \ + struct scatter_walk dst_walk, src_walk; \ + u8 tmp[4 * AES_BLOCK_SIZE] __aligned( \ + __alignof__(long)); \ + \ + scatterwalk_start_at_pos(&dst_walk, (dst), spos); \ + scatterwalk_start_at_pos(&src_walk, (src), spos); \ + do { \ + unsigned int dst_avail = scatterwalk_clamp( \ + &dst_walk, remaining); \ + unsigned int src_avail = scatterwalk_clamp( \ + &src_walk, remaining); \ + unsigned int n = min(dst_avail, src_avail); \ + u8 *dst_virt; \ + const u8 *src_virt; \ + \ + if (n < remaining) { \ + if (n < sizeof(tmp)) { \ + n = min(remaining, \ + sizeof(tmp)); \ + memcpy_from_scatterwalk( \ + tmp, &src_walk, n); \ + crypt_func(tmp, tmp, n, \ + ##__VA_ARGS__); \ + memcpy_to_scatterwalk( \ + &dst_walk, tmp, n); \ + remaining -= n; \ + continue; \ + } \ + n = round_down(n, AES_BLOCK_SIZE); \ + } \ + \ + scatterwalk_map(&dst_walk); \ + dst_virt = dst_walk.addr; \ + if (IS_ENABLED(CONFIG_HIGHMEM) && \ + offset_in_page(src_walk.offset) == \ + offset_in_page(dst_walk.offset) && \ + sg_page(src_walk.sg) + (src_walk.offset / \ + PAGE_SIZE) == \ + sg_page(dst_walk.sg) + \ + (dst_walk.offset / \ + PAGE_SIZE)) { \ + src_virt = dst_virt; \ + } else { \ + scatterwalk_map(&src_walk); \ + src_virt = src_walk.addr; \ + } \ + crypt_func(dst_virt, src_virt, n, \ + ##__VA_ARGS__); \ + if (src_virt != dst_virt) \ + scatterwalk_unmap(&src_walk); \ + scatterwalk_advance(&src_walk, n); \ + scatterwalk_done_dst(&dst_walk, n); \ + remaining -= n; \ + } while (remaining); \ + memzero_explicit(tmp, sizeof(tmp)); \ + } \ + }) + +/* + * Call ad_func() as needed to process the associated data in the first + * 'assoclen' bytes of the scatterlist 'src'. + */ +#define AES_PROCESS_ASSOC_DATA(ad_func, src, assoclen, ctx) \ + ({ \ + unsigned int remaining = (assoclen); \ + \ + if (remaining != 0) { \ + struct scatter_walk walk; \ + \ + scatterwalk_start(&walk, (src)); \ + do { \ + unsigned int n = \ + scatterwalk_next(&walk, remaining); \ + \ + ad_func((ctx), walk.addr, n); \ + scatterwalk_done_src(&walk, n); \ + remaining -= n; \ + } while (remaining); \ + } \ + }) + +/* AES-ECB */ + +static __maybe_unused int crypto_aes_ecb_encrypt(struct skcipher_request *req) +{ + const struct aes_key *key = + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + + if (unlikely(req->cryptlen % AES_BLOCK_SIZE)) + return -EINVAL; + AES_CRYPT_SG(aes_ecb_encrypt, req->dst, req->src, req->cryptlen, 0, + key); + return 0; +} + +static __maybe_unused int crypto_aes_ecb_decrypt(struct skcipher_request *req) +{ + const struct aes_key *key = + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + + if (unlikely(req->cryptlen % AES_BLOCK_SIZE)) + return -EINVAL; + AES_CRYPT_SG(aes_ecb_decrypt, req->dst, req->src, req->cryptlen, 0, + key); + return 0; +} + +/* AES-CBC */ + +static void crypto_aes_cbc_encrypt_sg(struct skcipher_request *req, + unsigned int cryptlen, + const struct aes_key *key) +{ + AES_CRYPT_SG(aes_cbc_encrypt, req->dst, req->src, cryptlen, 0, req->iv, + key); +} + +static void crypto_aes_cbc_decrypt_sg(struct skcipher_request *req, + unsigned int cryptlen, + const struct aes_key *key) +{ + AES_CRYPT_SG(aes_cbc_decrypt, req->dst, req->src, cryptlen, 0, req->iv, + key); +} + +static __maybe_unused int crypto_aes_cbc_encrypt(struct skcipher_request *req) +{ + const struct aes_key *key = + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + + if (unlikely(req->cryptlen % AES_BLOCK_SIZE)) + return -EINVAL; + crypto_aes_cbc_encrypt_sg(req, req->cryptlen, key); + return 0; +} + +static __maybe_unused int crypto_aes_cbc_decrypt(struct skcipher_request *req) +{ + const struct aes_key *key = + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + + if (unlikely(req->cryptlen % AES_BLOCK_SIZE)) + return -EINVAL; + crypto_aes_cbc_decrypt_sg(req, req->cryptlen, key); + return 0; +} + +/* AES-CBC-CTS */ + +/* + * This handles AES-CBC-CTS en/decryption requests that use a nonlinear + * scatterlist layout or where HIGHMEM is enabled. It is explicitly 'noinline' + * to keep the temporary buffer out of the stack frame of the fast path. + */ +static noinline int +crypto_aes_cbc_cts_crypt_nonlinear(struct skcipher_request *req, bool enc) +{ + const struct aes_key *key = + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + unsigned int main_len = req->cryptlen; + unsigned int tail_len; + u8 tmp[2 * AES_BLOCK_SIZE] __aligned(__alignof__(long)); + + if (main_len == AES_BLOCK_SIZE) { + /* Single block is a special case that just does CBC. */ + if (enc) + crypto_aes_cbc_encrypt_sg(req, main_len, key); + else + crypto_aes_cbc_decrypt_sg(req, main_len, key); + return 0; + } + /* Just do the last two blocks separately. */ + tail_len = AES_BLOCK_SIZE + ((main_len - 1) % AES_BLOCK_SIZE) + 1; + main_len -= tail_len; + if (enc) + crypto_aes_cbc_encrypt_sg(req, main_len, key); + else + crypto_aes_cbc_decrypt_sg(req, main_len, key); + memcpy_from_sglist(tmp, req->src, main_len, tail_len); + if (enc) + aes_cbc_cts_encrypt(tmp, tmp, tail_len, req->iv, key); + else + aes_cbc_cts_decrypt(tmp, tmp, tail_len, req->iv, key); + memcpy_to_sglist(req->dst, main_len, tmp, tail_len); + memzero_explicit(tmp, sizeof(tmp)); + return 0; +} + +static __maybe_unused int +crypto_aes_cbc_cts_encrypt(struct skcipher_request *req) +{ + const struct aes_key *key = + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + + if (unlikely(req->cryptlen < AES_BLOCK_SIZE)) + return -EINVAL; + if (likely(skcipher_request_is_linear_lowmem(req))) { + /* Fast path */ + aes_cbc_cts_encrypt(sg_virt(req->dst), sg_virt(req->src), + req->cryptlen, req->iv, key); + return 0; + } + return crypto_aes_cbc_cts_crypt_nonlinear(req, /* enc= */ true); +} + +static __maybe_unused int +crypto_aes_cbc_cts_decrypt(struct skcipher_request *req) +{ + const struct aes_key *key = + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + + if (unlikely(req->cryptlen < AES_BLOCK_SIZE)) + return -EINVAL; + if (likely(skcipher_request_is_linear_lowmem(req))) { + /* Fast path */ + aes_cbc_cts_decrypt(sg_virt(req->dst), sg_virt(req->src), + req->cryptlen, req->iv, key); + return 0; + } + return crypto_aes_cbc_cts_crypt_nonlinear(req, /* enc= */ false); +} + +/* AES-CTR */ + +static __maybe_unused int crypto_aes_ctr_crypt(struct skcipher_request *req) +{ + const struct aes_enckey *key = + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + + AES_CRYPT_SG(aes_ctr, req->dst, req->src, req->cryptlen, 0, req->iv, + key); + return 0; +} + +/* AES-XCTR */ + +static __maybe_unused int crypto_aes_xctr_crypt(struct skcipher_request *req) +{ + const struct aes_enckey *key = + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + u64 ctr = 1; + + AES_CRYPT_SG(aes_xctr, req->dst, req->src, req->cryptlen, 0, &ctr, + req->iv, key); + return 0; +} + +/* AES-XTS */ + +static __maybe_unused int crypto_aes_xts_setkey(struct crypto_skcipher *tfm, + const u8 *in_key, + unsigned int key_len) +{ + struct aes_xts_key *key = crypto_skcipher_ctx(tfm); + int flags = (crypto_skcipher_get_flags(tfm) & + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) ? + XTS_FORBID_WEAK_KEYS : + 0; + + return aes_xts_preparekey(key, in_key, key_len, flags); +} + +static void aes_xts_crypt_wrapper(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], + const struct aes_xts_key *key, bool enc, + bool *cont) +{ + if (enc) + aes_xts_encrypt(dst, src, len, iv, key, *cont); + else + aes_xts_decrypt(dst, src, len, iv, key, *cont); + *cont = true; +} + +/* + * This handles AES-XTS en/decryption requests that use a nonlinear scatterlist + * layout or where HIGHMEM is enabled. It is explicitly 'noinline' to keep the + * temporary buffer out of the stack frame of the fast path. + */ +static noinline int crypto_aes_xts_crypt_nonlinear(struct skcipher_request *req, + bool enc) +{ + const struct aes_xts_key *key = + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + u8 tmp[2 * AES_BLOCK_SIZE] __aligned(__alignof__(long)); + unsigned int main_len = req->cryptlen; + unsigned int tail_len = main_len % AES_BLOCK_SIZE; + bool cont = false; + + if (unlikely(tail_len)) { + /* + * Ciphertext stealing is needed. + * Just do the last two blocks separately. + */ + tail_len += AES_BLOCK_SIZE; + main_len -= tail_len; + } + + AES_CRYPT_SG(aes_xts_crypt_wrapper, req->dst, req->src, main_len, 0, + req->iv, key, enc, &cont); + + if (unlikely(tail_len)) { + memcpy_from_sglist(tmp, req->src, main_len, tail_len); + aes_xts_crypt_wrapper(tmp, tmp, tail_len, req->iv, key, enc, + &cont); + memcpy_to_sglist(req->dst, main_len, tmp, tail_len); + memzero_explicit(tmp, sizeof(tmp)); + } + return 0; +} + +static __maybe_unused int crypto_aes_xts_encrypt(struct skcipher_request *req) +{ + const struct aes_xts_key *key = + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + + if (unlikely(req->cryptlen < AES_BLOCK_SIZE)) + return -EINVAL; + if (likely(skcipher_request_is_linear_lowmem(req))) { + /* Fast path */ + aes_xts_encrypt(sg_virt(req->dst), sg_virt(req->src), + req->cryptlen, req->iv, key, /* cont= */ false); + return 0; + } + return crypto_aes_xts_crypt_nonlinear(req, /* enc= */ true); +} + +static __maybe_unused int crypto_aes_xts_decrypt(struct skcipher_request *req) +{ + const struct aes_xts_key *key = + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + + if (unlikely(req->cryptlen < AES_BLOCK_SIZE)) + return -EINVAL; + if (likely(skcipher_request_is_linear_lowmem(req))) { + /* Fast path */ + aes_xts_decrypt(sg_virt(req->dst), sg_virt(req->src), + req->cryptlen, req->iv, key, /* cont= */ false); + return 0; + } + return crypto_aes_xts_crypt_nonlinear(req, /* enc= */ false); +} + +static struct skcipher_alg skcipher_algs[] = { +#if IS_ENABLED(CONFIG_CRYPTO_ECB) + { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-lib", + .base.cra_priority = 110, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct aes_key), + .base.cra_module = THIS_MODULE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = crypto_aes_skcipher_setkey, + .encrypt = crypto_aes_ecb_encrypt, + .decrypt = crypto_aes_ecb_decrypt, + }, +#endif +#if IS_ENABLED(CONFIG_CRYPTO_CBC) + { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-lib", + .base.cra_priority = 110, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct aes_key), + .base.cra_module = THIS_MODULE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = crypto_aes_skcipher_setkey, + .encrypt = crypto_aes_cbc_encrypt, + .decrypt = crypto_aes_cbc_decrypt, + }, +#endif +#if IS_ENABLED(CONFIG_CRYPTO_CTS) + { + .base.cra_name = "cts(cbc(aes))", + .base.cra_driver_name = "cts-cbc-aes-lib", + .base.cra_priority = 110, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct aes_key), + .base.cra_module = THIS_MODULE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = crypto_aes_skcipher_setkey, + .encrypt = crypto_aes_cbc_cts_encrypt, + .decrypt = crypto_aes_cbc_cts_decrypt, + }, +#endif +#if IS_ENABLED(CONFIG_CRYPTO_CTR) + { + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-lib", + .base.cra_priority = 110, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct aes_enckey), + .base.cra_module = THIS_MODULE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .chunksize = AES_BLOCK_SIZE, + .setkey = crypto_aes_skcipher_setenckey, + .encrypt = crypto_aes_ctr_crypt, + .decrypt = crypto_aes_ctr_crypt, + }, +#endif +#if IS_ENABLED(CONFIG_CRYPTO_XCTR) + { + .base.cra_name = "xctr(aes)", + .base.cra_driver_name = "xctr-aes-lib", + .base.cra_priority = 110, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct aes_enckey), + .base.cra_module = THIS_MODULE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .chunksize = AES_BLOCK_SIZE, + .setkey = crypto_aes_skcipher_setenckey, + .encrypt = crypto_aes_xctr_crypt, + .decrypt = crypto_aes_xctr_crypt, + }, +#endif +#if IS_ENABLED(CONFIG_CRYPTO_XTS) + { + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "xts-aes-lib", + .base.cra_priority = 110, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct aes_xts_key), + .base.cra_module = THIS_MODULE, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = crypto_aes_xts_setkey, + .encrypt = crypto_aes_xts_encrypt, + .decrypt = crypto_aes_xts_decrypt, + }, +#endif +}; + +/* AES-GCM */ + +static __maybe_unused int crypto_aes_gcm_setkey(struct crypto_aead *tfm, + const u8 *in_key, + unsigned int key_len) +{ + struct aes_gcm_key *key = crypto_aead_ctx(tfm); + + return aes_gcm_preparekey(key, in_key, key_len, + crypto_aead_authsize(tfm)); +} + +static __maybe_unused int crypto_aes_gcm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + struct aes_gcm_key *key = crypto_aead_ctx(tfm); + + if (crypto_gcm_check_authsize(authsize) != 0) + return -EINVAL; + /* Synchronize the tag length to the struct aes_gcm_key. */ + key->authtag_len = authsize; + return 0; +} + +static void crypto_aes_gcm_auth_update(struct aes_gcm_ctx *ctx, + struct scatterlist *src, + unsigned int assoclen) +{ + AES_PROCESS_ASSOC_DATA(aes_gcm_auth_update, src, assoclen, ctx); +} + +static void aes_gcm_encrypt_update_helper(u8 *dst, const u8 *src, + unsigned int len, + struct aes_gcm_ctx *ctx) +{ + aes_gcm_encrypt_update(ctx, dst, src, len); +} + +static void aes_gcm_decrypt_update_helper(u8 *dst, const u8 *src, + unsigned int len, + struct aes_gcm_ctx *ctx) +{ + aes_gcm_decrypt_update(ctx, dst, src, len); +} + +static int crypto_aes_gcm_encrypt_common(struct aead_request *req, + const struct aes_gcm_key *key, + u8 iv[12], unsigned int assoclen) +{ + struct aes_gcm_ctx ctx; + u8 authtag[16]; + + aes_gcm_init(&ctx, iv, key); + crypto_aes_gcm_auth_update(&ctx, req->src, assoclen); + AES_CRYPT_SG(aes_gcm_encrypt_update_helper, req->dst, req->src, + req->cryptlen, req->assoclen, &ctx); + aes_gcm_encrypt_final(&ctx, authtag); + memcpy_to_sglist(req->dst, req->assoclen + req->cryptlen, authtag, + key->authtag_len); + memzero_explicit(authtag, sizeof(authtag)); + return 0; +} + +static int crypto_aes_gcm_decrypt_common(struct aead_request *req, + const struct aes_gcm_key *key, + u8 iv[12], unsigned int assoclen) +{ + struct aes_gcm_ctx ctx; + unsigned int data_len; + u8 authtag[16]; + int err; + + aes_gcm_init(&ctx, iv, key); + crypto_aes_gcm_auth_update(&ctx, req->src, assoclen); + + /* crypto_aead_decrypt() already checked cryptlen >= authtag_len. */ + data_len = req->cryptlen - key->authtag_len; + AES_CRYPT_SG(aes_gcm_decrypt_update_helper, req->dst, req->src, + data_len, req->assoclen, &ctx); + + memcpy_from_sglist(authtag, req->src, req->assoclen + data_len, + key->authtag_len); + err = aes_gcm_decrypt_final(&ctx, authtag); + memzero_explicit(authtag, sizeof(authtag)); + return err; +} + +static __maybe_unused int crypto_aes_gcm_encrypt(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + const struct aes_gcm_key *key = crypto_aead_ctx(tfm); + + return crypto_aes_gcm_encrypt_common(req, key, req->iv, req->assoclen); +} + +static __maybe_unused int crypto_aes_gcm_decrypt(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + const struct aes_gcm_key *key = crypto_aead_ctx(tfm); + + return crypto_aes_gcm_decrypt_common(req, key, req->iv, req->assoclen); +} + +struct aes_rfc4106_key { + struct aes_gcm_key gcm; + u8 nonce[4]; +}; + +static __maybe_unused int crypto_aes_rfc4106_setkey(struct crypto_aead *tfm, + const u8 *in_key, + unsigned int key_len) +{ + struct aes_rfc4106_key *key = crypto_aead_ctx(tfm); + + if (key_len < 4) + return -EINVAL; + + key_len -= 4; + memcpy(key->nonce, in_key + key_len, 4); + + return aes_gcm_preparekey(&key->gcm, in_key, key_len, + crypto_aead_authsize(tfm)); +} + +static __maybe_unused int +crypto_aes_rfc4106_setauthsize(struct crypto_aead *tfm, unsigned int authsize) +{ + struct aes_rfc4106_key *key = crypto_aead_ctx(tfm); + + if (crypto_rfc4106_check_authsize(authsize) != 0) + return -EINVAL; + + /* Synchronize the tag length to the struct aes_gcm_key. */ + key->gcm.authtag_len = authsize; + return 0; +} + +static __maybe_unused int crypto_aes_rfc4106_encrypt(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + const struct aes_rfc4106_key *key = crypto_aead_ctx(tfm); + u8 iv[12]; + + if (crypto_ipsec_check_assoclen(req->assoclen) != 0) + return -EINVAL; + memcpy(iv, key->nonce, 4); + memcpy(&iv[4], req->iv, 8); + + return crypto_aes_gcm_encrypt_common(req, &key->gcm, iv, + req->assoclen - 8); +} + +static __maybe_unused int crypto_aes_rfc4106_decrypt(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + const struct aes_rfc4106_key *key = crypto_aead_ctx(tfm); + u8 iv[12]; + + if (crypto_ipsec_check_assoclen(req->assoclen) != 0) + return -EINVAL; + memcpy(iv, key->nonce, 4); + memcpy(&iv[4], req->iv, 8); + + return crypto_aes_gcm_decrypt_common(req, &key->gcm, iv, + req->assoclen - 8); +} + +/* AES-CCM */ + +static __maybe_unused int crypto_aes_ccm_setkey(struct crypto_aead *tfm, + const u8 *in_key, + unsigned int key_len) +{ + struct aes_ccm_key *key = crypto_aead_ctx(tfm); + + return aes_ccm_preparekey(key, in_key, key_len, + crypto_aead_authsize(tfm)); +} + +static __maybe_unused int crypto_aes_ccm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + struct aes_ccm_key *key = crypto_aead_ctx(tfm); + + if (authsize < 4 || authsize > 16 || authsize % 2) + return -EINVAL; + /* Synchronize the tag length to the struct aes_ccm_key. */ + key->authtag_len = authsize; + return 0; +} + +static int crypto_aes_ccm_init(struct aes_ccm_ctx *ctx, + struct aead_request *req, unsigned int data_len, + const struct aes_ccm_key *key) +{ + int nonce_len; + const u8 *nonce; + int err; + + /* + * CCM accepts a variable-length nonce between 7 and 13 bytes + * inclusively, while crypto_aead assumes a fixed-length IV. This is + * worked around by requiring that iv[0] contain '14 - nonce_len' and + * iv[1..] contain the actual nonce. Extra bytes at the end are unused. + */ + nonce_len = 14 - (int)req->iv[0]; + if (unlikely(nonce_len < 7 || nonce_len > 13)) + return -EINVAL; + nonce = &req->iv[1]; + err = aes_ccm_init(ctx, data_len, req->assoclen, nonce, nonce_len, key); + if (unlikely(err)) + return err; + AES_PROCESS_ASSOC_DATA(aes_ccm_auth_update, req->src, req->assoclen, + ctx); + return 0; +} + +static void aes_ccm_encrypt_update_helper(u8 *dst, const u8 *src, + unsigned int len, + struct aes_ccm_ctx *ctx) +{ + aes_ccm_encrypt_update(ctx, dst, src, len); +} + +static void aes_ccm_decrypt_update_helper(u8 *dst, const u8 *src, + unsigned int len, + struct aes_ccm_ctx *ctx) +{ + aes_ccm_decrypt_update(ctx, dst, src, len); +} + +static __maybe_unused int crypto_aes_ccm_encrypt(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + const struct aes_ccm_key *key = crypto_aead_ctx(tfm); + struct aes_ccm_ctx ctx; + u8 authtag[16]; + int err; + + err = crypto_aes_ccm_init(&ctx, req, req->cryptlen, key); + if (unlikely(err)) + return err; + AES_CRYPT_SG(aes_ccm_encrypt_update_helper, req->dst, req->src, + req->cryptlen, req->assoclen, &ctx); + aes_ccm_encrypt_final(&ctx, authtag); + memcpy_to_sglist(req->dst, req->assoclen + req->cryptlen, authtag, + key->authtag_len); + memzero_explicit(authtag, sizeof(authtag)); + return 0; +} + +static __maybe_unused int crypto_aes_ccm_decrypt(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + const struct aes_ccm_key *key = crypto_aead_ctx(tfm); + unsigned int data_len; + struct aes_ccm_ctx ctx; + u8 authtag[16]; + int err; + + /* crypto_aead_decrypt() already checked cryptlen >= authtag_len. */ + data_len = req->cryptlen - key->authtag_len; + err = crypto_aes_ccm_init(&ctx, req, data_len, key); + if (unlikely(err)) + return err; + AES_CRYPT_SG(aes_ccm_decrypt_update_helper, req->dst, req->src, + data_len, req->assoclen, &ctx); + memcpy_from_sglist(authtag, req->src, req->assoclen + data_len, + key->authtag_len); + err = aes_ccm_decrypt_final(&ctx, authtag); + memzero_explicit(authtag, sizeof(authtag)); + return err; +} + +static struct aead_alg aead_algs[] = { +#if IS_ENABLED(CONFIG_CRYPTO_GCM) + { + .base.cra_name = "gcm(aes)", + .base.cra_driver_name = "gcm-aes-lib", + .base.cra_priority = 110, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct aes_gcm_key), + .base.cra_module = THIS_MODULE, + .setkey = crypto_aes_gcm_setkey, + .setauthsize = crypto_aes_gcm_setauthsize, + .encrypt = crypto_aes_gcm_encrypt, + .decrypt = crypto_aes_gcm_decrypt, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .chunksize = AES_BLOCK_SIZE, + }, + { + .base.cra_name = "rfc4106(gcm(aes))", + .base.cra_driver_name = "rfc4106-gcm-aes-lib", + .base.cra_priority = 110, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct aes_rfc4106_key), + .base.cra_module = THIS_MODULE, + .setkey = crypto_aes_rfc4106_setkey, + .setauthsize = crypto_aes_rfc4106_setauthsize, + .encrypt = crypto_aes_rfc4106_encrypt, + .decrypt = crypto_aes_rfc4106_decrypt, + .ivsize = GCM_RFC4106_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .chunksize = AES_BLOCK_SIZE, + }, +#endif /* CONFIG_CRYPTO_GCM */ +#if IS_ENABLED(CONFIG_CRYPTO_CCM) + { + .base.cra_name = "ccm(aes)", + .base.cra_driver_name = "ccm-aes-lib", + .base.cra_priority = 110, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct aes_ccm_key), + .base.cra_module = THIS_MODULE, + .setkey = crypto_aes_ccm_setkey, + .setauthsize = crypto_aes_ccm_setauthsize, + .encrypt = crypto_aes_ccm_encrypt, + .decrypt = crypto_aes_ccm_decrypt, + .ivsize = 16, + .maxauthsize = 16, + .chunksize = AES_BLOCK_SIZE, + }, +#endif /* CONFIG_CRYPTO_CCM */ +}; + static int __init crypto_aes_mod_init(void) { int err = crypto_register_alg(&alg); @@ -212,8 +1043,28 @@ static int __init crypto_aes_mod_init(void) if (err) goto err_unregister_alg; } /* Else, CONFIG_CRYPTO_HASH might not be enabled. */ + + if (ARRAY_SIZE(skcipher_algs) > 0) { + err = crypto_register_skciphers(skcipher_algs, + ARRAY_SIZE(skcipher_algs)); + if (err) + goto err_unregister_macs; + } + + if (ARRAY_SIZE(aead_algs) > 0) { + err = crypto_register_aeads(aead_algs, ARRAY_SIZE(aead_algs)); + if (err) + goto err_unregister_skciphers; + } /* Else, CONFIG_CRYPTO_AEAD might not be enabled. */ return 0; +err_unregister_skciphers: + if (ARRAY_SIZE(skcipher_algs) > 0) + crypto_unregister_skciphers(skcipher_algs, + ARRAY_SIZE(skcipher_algs)); +err_unregister_macs: + if (ARRAY_SIZE(mac_algs) > 0) + crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs)); err_unregister_alg: crypto_unregister_alg(&alg); return err; @@ -222,6 +1073,11 @@ module_init(crypto_aes_mod_init); static void __exit crypto_aes_mod_exit(void) { + if (ARRAY_SIZE(aead_algs) > 0) + crypto_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs)); + if (ARRAY_SIZE(skcipher_algs) > 0) + crypto_unregister_skciphers(skcipher_algs, + ARRAY_SIZE(skcipher_algs)); if (ARRAY_SIZE(mac_algs) > 0) crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs)); crypto_unregister_alg(&alg); @@ -245,3 +1101,37 @@ MODULE_ALIAS_CRYPTO("xcbc-aes-lib"); MODULE_ALIAS_CRYPTO("cbcmac(aes)"); MODULE_ALIAS_CRYPTO("cbcmac-aes-lib"); #endif +#if IS_ENABLED(CONFIG_CRYPTO_ECB) +MODULE_ALIAS_CRYPTO("ecb(aes)"); +MODULE_ALIAS_CRYPTO("ecb-aes-lib"); +#endif +#if IS_ENABLED(CONFIG_CRYPTO_CBC) +MODULE_ALIAS_CRYPTO("cbc(aes)"); +MODULE_ALIAS_CRYPTO("cbc-aes-lib"); +#endif +#if IS_ENABLED(CONFIG_CRYPTO_CTS) +MODULE_ALIAS_CRYPTO("cts(cbc(aes))"); +MODULE_ALIAS_CRYPTO("cts-cbc-aes-lib"); +#endif +#if IS_ENABLED(CONFIG_CRYPTO_CTR) +MODULE_ALIAS_CRYPTO("ctr(aes)"); +MODULE_ALIAS_CRYPTO("ctr-aes-lib"); +#endif +#if IS_ENABLED(CONFIG_CRYPTO_XCTR) +MODULE_ALIAS_CRYPTO("xctr(aes)"); +MODULE_ALIAS_CRYPTO("xctr-aes-lib"); +#endif +#if IS_ENABLED(CONFIG_CRYPTO_XTS) +MODULE_ALIAS_CRYPTO("xts(aes)"); +MODULE_ALIAS_CRYPTO("xts-aes-lib"); +#endif +#if IS_ENABLED(CONFIG_CRYPTO_GCM) +MODULE_ALIAS_CRYPTO("gcm(aes)"); +MODULE_ALIAS_CRYPTO("gcm-aes-lib"); +MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))"); +MODULE_ALIAS_CRYPTO("rfc4106-gcm-aes-lib"); +#endif +#if IS_ENABLED(CONFIG_CRYPTO_CCM) +MODULE_ALIAS_CRYPTO("ccm(aes)"); +MODULE_ALIAS_CRYPTO("ccm-aes-lib"); +#endif diff --git a/drivers/virt/coco/sev-guest/sev-guest.c b/drivers/virt/coco/sev-guest/sev-guest.c index d186ae55cf63..935537a41469 100644 --- a/drivers/virt/coco/sev-guest/sev-guest.c +++ b/drivers/virt/coco/sev-guest/sev-guest.c @@ -17,7 +17,6 @@ #include <linux/set_memory.h> #include <linux/fs.h> #include <linux/tsm.h> -#include <crypto/gcm.h> #include <linux/psp-sev.h> #include <linux/sockptr.h> #include <linux/cleanup.h> @@ -87,7 +86,7 @@ static int get_report(struct snp_guest_dev *snp_dev, struct snp_guest_request_io * response payload. Make sure that it has enough space to cover the * authtag. */ - resp_len = sizeof(report_resp->data) + mdesc->ctx->authsize; + resp_len = sizeof(report_resp->data) + AUTHTAG_LEN; report_resp = kzalloc(resp_len, GFP_KERNEL_ACCOUNT); if (!report_resp) return -ENOMEM; @@ -130,7 +129,7 @@ static int get_derived_key(struct snp_guest_dev *snp_dev, struct snp_guest_reque * response payload. Make sure that it has enough space to cover the * authtag. */ - resp_len = sizeof(derived_key_resp->data) + mdesc->ctx->authsize; + resp_len = sizeof(derived_key_resp->data) + AUTHTAG_LEN; derived_key_resp = kzalloc(resp_len, GFP_KERNEL_ACCOUNT); if (!derived_key_resp) return -ENOMEM; @@ -230,7 +229,7 @@ cmd: * response payload. Make sure that it has enough space to cover the * authtag. */ - resp_len = sizeof(report_resp->data) + mdesc->ctx->authsize; + resp_len = sizeof(report_resp->data) + AUTHTAG_LEN; report_resp = kzalloc(resp_len, GFP_KERNEL_ACCOUNT); if (!report_resp) { ret = -ENOMEM; diff --git a/fs/smb/client/smb2transport.c b/fs/smb/client/smb2transport.c index 1143ee52470a..d23566da2ac8 100644 --- a/fs/smb/client/smb2transport.c +++ b/fs/smb/client/smb2transport.c @@ -464,8 +464,8 @@ smb3_calc_signature(struct smb_rqst *rqst, struct TCP_Server_Info *server) unsigned char smb3_signature[SMB2_CMACAES_SIZE]; struct kvec *iov = rqst->rq_iov; struct smb2_hdr *shdr = (struct smb2_hdr *)iov[0].iov_base; - struct aes_cmac_key cmac_key; - struct aes_cmac_ctx cmac_ctx; + struct aes_cmac_key cmac_key __cleanup(aes_cmac_zeroize_key); + struct aes_cmac_ctx cmac_ctx __cleanup(aes_cmac_zeroize_ctx); struct smb_rqst drqst; u8 key[SMB3_SIGN_KEY_SIZE]; diff --git a/fs/smb/server/auth.c b/fs/smb/server/auth.c index 4e7b6f0e6b8c..e8d1c068a43e 100644 --- a/fs/smb/server/auth.c +++ b/fs/smb/server/auth.c @@ -509,7 +509,7 @@ void ksmbd_sign_smb2_pdu(struct ksmbd_conn *conn, char *key, struct kvec *iov, void ksmbd_sign_smb3_pdu(struct ksmbd_conn *conn, char *key, struct kvec *iov, int n_vec, char *sig) { - struct aes_cmac_key cmac_key; + struct aes_cmac_key cmac_key __cleanup(aes_cmac_zeroize_key); struct aes_cmac_ctx cmac_ctx; int i; diff --git a/include/crypto/aes-cbc-macs.h b/include/crypto/aes-cbc-macs.h index e61df108b926..06e8a22f8a0a 100644 --- a/include/crypto/aes-cbc-macs.h +++ b/include/crypto/aes-cbc-macs.h @@ -8,6 +8,7 @@ #define _CRYPTO_AES_CBC_MACS_H #include <crypto/aes.h> +#include <linux/string.h> /** * struct aes_cmac_key - Prepared key for AES-CMAC or AES-XCBC-MAC @@ -25,6 +26,19 @@ struct aes_cmac_key { }; /** + * aes_cmac_zeroize_key() - Zeroize an aes_cmac_key structure + * @key: The location of the key structure that should be zeroized + * + * Explicitly fills the aes_cmac_key with zeroes. This should be done once + * the key is not required anymore to avoid that its contents are leaked + * on the stack or heap (if not using kfree_sensitive()). + */ +static inline void aes_cmac_zeroize_key(struct aes_cmac_key *key) +{ + memzero_explicit(key, sizeof(*key)); +} + +/** * struct aes_cmac_ctx - Context for computing an AES-CMAC or AES-XCBC-MAC value * @key: Pointer to the key struct. A pointer is used rather than a copy of the * struct, since the key struct size may be large. It is assumed that the @@ -41,12 +55,29 @@ struct aes_cmac_ctx { }; /** + * aes_cmac_zeroize_ctx() - Zeroize an aes_cmac_ctx structure + * @ctx: The location of the context that should be zeroized + * + * Explicitly fills the aes_cmac_ctx with zeroes. This should be done once + * the context is not required anymore to avoid that its contents are + * leaked on the stack or heap. Only required if not using aes_cmac_final(). + */ +static inline void aes_cmac_zeroize_ctx(struct aes_cmac_ctx *ctx) +{ + memzero_explicit(ctx, sizeof(*ctx)); +} + +/** * aes_cmac_preparekey() - Prepare a key for AES-CMAC * @key: (output) The key struct to initialize * @in_key: The raw AES key * @key_len: Length of the raw key in bytes. The supported values are * AES_KEYSIZE_128, AES_KEYSIZE_192, and AES_KEYSIZE_256. * + * On success, the caller should ensure that the prepared key is zeroized + * at the end of its lifetime, e.g. by calling aes_cmac_zeroize_key() or + * kfree_sensitive(). + * * Context: Any context. * Return: 0 on success or -EINVAL if the given key length is invalid. No other * errors are possible, so callers that always pass a valid key length @@ -79,6 +110,9 @@ void aes_xcbcmac_preparekey(struct aes_cmac_key *key, * * This supports both AES-CMAC and AES-XCBC-MAC. Which one is done depends on * whether aes_cmac_preparekey() or aes_xcbcmac_preparekey() was called. + * + * The caller should ensure that the context is zeroized at the end of its + * lifetime, e.g. by calling aes_cmac_final() or aes_cmac_zeroize_ctx(). */ static inline void aes_cmac_init(struct aes_cmac_ctx *ctx, const struct aes_cmac_key *key) diff --git a/include/crypto/aes-cbc.h b/include/crypto/aes-cbc.h new file mode 100644 index 000000000000..7bae340935f9 --- /dev/null +++ b/include/crypto/aes-cbc.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AES-CBC and AES-CBC-CTS unauthenticated encryption and decryption + * + * Copyright 2026 Google LLC + */ +#ifndef _CRYPTO_AES_CBC_H +#define _CRYPTO_AES_CBC_H + +#include <crypto/aes.h> + +/** + * aes_cbc_encrypt() - Encrypt data using AES-CBC + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to encrypt. Must be a multiple of AES_BLOCK_SIZE. + * @iv: The initialization vector. It is updated with the next value, i.e. the + * last ciphertext block (or left unchanged if @len == 0). + * @key: The key, already prepared using aes_preparekey() or aes_prepareenckey() + * + * This supports incremental encryption. The length of each chunk must be a + * multiple of AES_BLOCK_SIZE, and the updated @iv must be passed in each time. + * + * Context: Any context. + */ +void aes_cbc_encrypt(u8 *dst, const u8 *src, size_t len, + u8 iv[at_least AES_BLOCK_SIZE], aes_encrypt_arg key); + +/** + * aes_cbc_decrypt() - Decrypt data using AES-CBC + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to decrypt. Must be a multiple of AES_BLOCK_SIZE. + * @iv: The initialization vector. It is updated with the next value, i.e. the + * last ciphertext block (or left unchanged if @len == 0). + * @key: The key, already prepared using aes_preparekey() + * + * This supports incremental decryption. The length of each chunk must be a + * multiple of AES_BLOCK_SIZE, and the updated @iv must be passed in each time. + * + * Context: Any context. + */ +void aes_cbc_decrypt(u8 *dst, const u8 *src, size_t len, + u8 iv[at_least AES_BLOCK_SIZE], const struct aes_key *key); + +/** + * aes_cbc_cts_encrypt() - Encrypt data using AES-CBC-CTS (CS3 variant) + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to encrypt, at least AES_BLOCK_SIZE + * @iv: The initialization vector, clobbered by this function + * @key: The key, already prepared using aes_preparekey() or aes_prepareenckey() + * + * Context: Any context. + */ +void aes_cbc_cts_encrypt(u8 *dst, const u8 *src, size_t len, + u8 iv[at_least AES_BLOCK_SIZE], aes_encrypt_arg key); + +/** + * aes_cbc_cts_decrypt() - Decrypt data using AES-CBC-CTS (CS3 variant) + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to decrypt, at least AES_BLOCK_SIZE + * @iv: The initialization vector, clobbered by this function + * @key: The key, already prepared using aes_preparekey() + * + * Context: Any context. + */ +void aes_cbc_cts_decrypt(u8 *dst, const u8 *src, size_t len, + u8 iv[at_least AES_BLOCK_SIZE], + const struct aes_key *key); + +#endif /* _CRYPTO_AES_CBC_H */ diff --git a/include/crypto/aes-ccm.h b/include/crypto/aes-ccm.h new file mode 100644 index 000000000000..8b00859ac4d6 --- /dev/null +++ b/include/crypto/aes-ccm.h @@ -0,0 +1,266 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AES-CCM authenticated encryption and decryption + * + * Copyright 2026 Google LLC + */ +#ifndef _CRYPTO_AES_CCM_H +#define _CRYPTO_AES_CCM_H + +#include <crypto/aes.h> + +/** + * struct aes_ccm_key - A key prepared for AES-CCM encryption and decryption + */ +struct aes_ccm_key { + /* private: */ + struct aes_enckey aes; + size_t authtag_len; /* Length of authentication tags in bytes */ +}; + +/** + * struct aes_ccm_ctx - Context for incrementally en/decrypting a message + */ +struct aes_ccm_ctx { + /* private: */ + /* + * Pointer to the key, which is assumed to live at least as long as this + * struct. + */ + const struct aes_ccm_key *key; + /* + * The current CBC-MAC chaining value. When not on a block boundary, + * the partial block has been XOR'ed into this. The number of partial + * bytes is 'partial_len'. + */ + u8 mac[AES_BLOCK_SIZE] __aligned(__alignof__(__be64)); + /* The current counter, a 128-bit big endian value */ + u8 ctr[AES_BLOCK_SIZE] __aligned(__alignof__(__be64)); + /* Buffered keystream for partial block updates */ + u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(__be64)); + /* Encrypted counter of 0. This gets XOR'ed with the tag at the end. */ + u8 s0[AES_BLOCK_SIZE] __aligned(__alignof__(__be64)); + /* Number of associated data bytes remaining to be provided */ + u64 ad_remaining; + /* Number of en/decrypted data bytes remaining to be provided */ + u64 data_remaining; + /* Current partial block length, 0 <= partial_len < AES_BLOCK_SIZE */ + u32 partial_len; + /* True if associated data padding has been done */ + bool ad_padded; +}; + +/** + * aes_ccm_preparekey() - Prepare a key for AES-CCM encryption and decryption + * @key: (output) The key structure to initialize + * @in_key: The raw AES-CCM key + * @key_len: Length of the raw key in bytes: 16, 24, or 32 + * @authtag_len: Length of the authentication tag in bytes: + * 4, 6, 8, 10, 12, 14, or 16. 16 is recommended. + * + * Users should use memzero_explicit() to zeroize the key struct at the end of + * its lifetime. (But if this function fails, zeroization is unnecessary.) + * + * Context: Any context. + * Return: + * * 0 on success + * * -EINVAL if either of the lengths is invalid + */ +int __must_check aes_ccm_preparekey(struct aes_ccm_key *key, const u8 *in_key, + size_t key_len, size_t authtag_len); + +/** + * aes_ccm_encrypt() - Encrypt a message with AES-CCM + * @dst: The destination ciphertext data. Can be in-place or out-of-place. + * For other overlaps the behavior is unspecified. + * @src: The source plaintext data + * @data_len: Length of plaintext in bytes (and ciphertext excluding the tag): + * at most 2^(120 - (8 * @nonce_len)) - 1 + * @authtag: The output authentication tag. Length is the authtag_len that was + * passed to aes_ccm_preparekey(). Usually protocols using AES-CCM + * put the tag at the end of the ciphertext, in which case this should + * be set to @dst + @data_len and @dst must have room for the tag. + * @ad: The associated data + * @ad_len: Length of associated data in bytes + * @nonce: The nonce. All (key, nonce) pairs used MUST be distinct. + * @nonce_len: Length of the nonce in bytes: between 7 and 13 inclusive + * @key: The key, already prepared using aes_ccm_preparekey() + * + * Context: Any context. + * Return: + * * 0 on success + * * -EINVAL if @nonce_len is invalid + * * -EOVERFLOW if @data_len is too large for the selected @nonce_len + */ +int __must_check aes_ccm_encrypt(u8 *dst, const u8 *src, size_t data_len, + u8 *authtag, const u8 *ad, size_t ad_len, + const u8 *nonce, size_t nonce_len, + const struct aes_ccm_key *key); + +/** + * aes_ccm_decrypt() - Decrypt a message with AES-CCM + * @dst: The destination plaintext data. Can be in-place or out-of-place. + * For other overlaps the behavior is unspecified. + * @src: The source ciphertext data + * @data_len: Length of plaintext in bytes (and ciphertext excluding the tag): + * at most 2^(120 - (8 * @nonce_len)) - 1 + * @authtag: The stored authentication tag. Length is the authtag_len that was + * passed to aes_ccm_preparekey(). Usually protocols using AES-CCM + * put the tag at the end of the ciphertext, in which case this should + * be set to @src + @data_len and @src must have room for the tag. + * @ad: The associated data + * @ad_len: Length of associated data in bytes + * @nonce: The nonce + * @nonce_len: Length of the nonce in bytes: between 7 and 13 inclusive + * @key: The key, already prepared using aes_ccm_preparekey() + * + * Context: Any context. + * Return: + * * 0 on success. This is the only case where any decrypted or associated data + * can be used. + * * -EBADMSG if the message is inauthentic + * * -EINVAL if @nonce_len is invalid + * * -EOVERFLOW if @data_len is too large for the selected @nonce_len + */ +int __must_check aes_ccm_decrypt(u8 *dst, const u8 *src, size_t data_len, + const u8 *authtag, const u8 *ad, size_t ad_len, + const u8 *nonce, size_t nonce_len, + const struct aes_ccm_key *key); + +/** + * aes_ccm_init() - Initialize context for incremental AES-CCM encryption or + * decryption + * @ctx: The context to initialize + * @data_len: Length of the en/decrypted data that will be provided in bytes: + * at most 2^(120 - (8 * @nonce_len)) - 1 + * @ad_len: Length of the associated data that will be provided in bytes + * @nonce: The nonce. All (key, nonce) pairs used for encryption MUST be + * distinct. + * @nonce_len: Length of the nonce in bytes: between 7 and 13 inclusive + * @key: The key, already prepared using aes_ccm_preparekey(). Note that a + * pointer to the key is saved in the context, so the key must live at + * least as long as the context. + * + * Unlike AES-GCM, AES-CCM requires the total lengths of the associated data and + * the en/decrypted data to be known during initialization. Callers MUST ensure + * that these lengths are correct. + * + * If this function returns success, the context should be zeroized at the end + * of its lifetime. Normally that happens in aes_ccm_encrypt_final() or + * aes_ccm_decrypt_final(), but callers that abandon a context without + * finalizing it should explicitly zeroize it. + * + * IMPORTANT: Callers that are decrypting MUST NOT assume that any decrypted or + * associated data is authentic until the authentication tag has been verified. + * This incremental API is provided solely to support callers that can't + * efficiently use the one-shot functions due to using a nonlinear data layout. + * + * For incremental AES-CCM encryption, use: + * + * 1. aes_ccm_init() + * 2. aes_ccm_auth_update() (any number of times) + * 3. aes_ccm_encrypt_update() (any number of times) + * 4. aes_ccm_encrypt_final() + * + * For incremental AES-CCM decryption, use: + * + * 1. aes_ccm_init() + * 2. aes_ccm_auth_update() (any number of times) + * 3. aes_ccm_decrypt_update() (any number of times) + * 4. aes_ccm_decrypt_final() + * + * Context: Any context. + * Return: + * * 0 on success + * * -EINVAL if @nonce_len is invalid + * * -EOVERFLOW if @data_len is too large for the selected @nonce_len + */ +int __must_check aes_ccm_init(struct aes_ccm_ctx *ctx, u64 data_len, u64 ad_len, + const u8 *nonce, size_t nonce_len, + const struct aes_ccm_key *key); + +/** + * aes_ccm_auth_update() - Incrementally process AES-CCM associated data + * @ctx: An AES-CCM context + * @ad: The associated data + * @len: Length of the associated data in bytes + * + * IMPORTANT: Callers MUST NOT assume that any decrypted or associated data is + * authentic until the authentication tag has been verified. + * + * The total length of the associated data (over all calls to this function) + * MUST match the ad_len that was passed to aes_ccm_init(). + * + * Context: Any context. + */ +void aes_ccm_auth_update(struct aes_ccm_ctx *ctx, const u8 *ad, size_t len); + +/** + * aes_ccm_encrypt_update() - Incrementally encrypt data with AES-CCM + * @ctx: An AES-CCM context + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source plaintext data + * @len: Number of bytes to encrypt + * + * This can be called only after all associated data has been processed. + * + * The total length of the encrypted data (over all calls to this function) MUST + * match the data_len that was passed to aes_ccm_init(). + * + * Context: Any context. + */ +void aes_ccm_encrypt_update(struct aes_ccm_ctx *ctx, u8 *dst, const u8 *src, + size_t len); + +/** + * aes_ccm_decrypt_update() - Incrementally decrypt data with AES-CCM + * @ctx: An AES-CCM context + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source ciphertext data (not including auth tag) + * @len: Number of bytes to decrypt + * + * This can be called only after all associated data has been processed. + * + * The total length of the decrypted data (over all calls to this function) MUST + * match the data_len that was passed to aes_ccm_init(). + * + * IMPORTANT: Callers MUST NOT assume that any decrypted or associated data is + * authentic until the authentication tag has been verified. + * + * Context: Any context. + */ +void aes_ccm_decrypt_update(struct aes_ccm_ctx *ctx, u8 *dst, const u8 *src, + size_t len); + +/** + * aes_ccm_encrypt_final() - Finish encrypting a message with AES-CCM + * @ctx: An AES-CCM context + * @authtag: The output authentication tag. Length is the authtag_len that was + * passed to aes_ccm_preparekey(). + * + * This also zeroizes @ctx, so the caller doesn't need to do it. + * + * Context: Any context. + */ +void aes_ccm_encrypt_final(struct aes_ccm_ctx *ctx, u8 *authtag); + +/** + * aes_ccm_decrypt_final() - Finish decrypting a message with AES-CCM + * @ctx: An AES-CCM context + * @authtag: The stored authentication tag. Length is the authtag_len that was + * passed to aes_ccm_preparekey(). + * + * This also zeroizes @ctx, so the caller doesn't need to do it. + * + * Context: Any context. + * Return: + * * 0 on success. This is the only case where any decrypted or associated data + * can be used. + * * -EBADMSG if the message is inauthentic + */ +int __must_check aes_ccm_decrypt_final(struct aes_ccm_ctx *ctx, + const u8 *authtag); + +#endif /* _CRYPTO_AES_CCM_H */ diff --git a/include/crypto/aes-ctr.h b/include/crypto/aes-ctr.h new file mode 100644 index 000000000000..8e214a85716b --- /dev/null +++ b/include/crypto/aes-ctr.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AES-CTR and AES-XCTR stream ciphers + * + * Copyright 2026 Google LLC + */ +#ifndef _CRYPTO_AES_CTR_H +#define _CRYPTO_AES_CTR_H + +#include <crypto/aes.h> + +/** + * aes_ctr() - AES-CTR en/decryption + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to en/decrypt + * @ctr: The counter. It will be incremented by ceil(@len / AES_BLOCK_SIZE). + * @key: The key, already prepared using aes_preparekey() or aes_prepareenckey() + * + * This implements AES in counter mode with a 128-bit big endian counter. + * + * This exists only for use by the implementation of modes built on top of CTR + * (e.g., GCM and CCM) and some legacy protocols that use CTR mode directly. + * Callers are expected to know how to use CTR mode appropriately, including + * choosing (key, counter) pairs appropriately to avoid keystream reuse. + * + * This supports incremental en/decryption. The length of each non-final chunk + * must be a multiple of AES_BLOCK_SIZE, and the updated @ctr must be passed in + * each time. + * + * Context: Any context. + */ +void aes_ctr(u8 *dst, const u8 *src, size_t len, + u8 ctr[at_least AES_BLOCK_SIZE], aes_encrypt_arg key); + +/** + * aes_xctr() - AES-XCTR en/decryption + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to en/decrypt + * @ctr: The block counter (in host endianness). For the first call, set it to + * 1. It will be incremented by ceil(@len / AES_BLOCK_SIZE). + * @iv: The initialization vector + * @key: The key, already prepared using aes_preparekey() or aes_prepareenckey() + * + * This implements AES in XOR Counter mode, as specified in the paper + * "Length-preserving encryption with HCTR2" + * (https://eprint.iacr.org/2021/1441.pdf). + * + * This exists only for use by the implementation of modes built on top of XCTR. + * Callers are expected to know how to use XCTR mode appropriately, including + * choosing (key, IV) pairs appropriately to avoid keystream reuse. + * + * This supports incremental en/decryption. The length of each non-final chunk + * must be a multiple of AES_BLOCK_SIZE, and the updated @ctr must be passed in + * each time. + * + * Context: Any context. + */ +void aes_xctr(u8 *dst, const u8 *src, size_t len, u64 *ctr, + const u8 iv[at_least AES_BLOCK_SIZE], aes_encrypt_arg key); + +#endif /* _CRYPTO_AES_CTR_H */ diff --git a/include/crypto/aes-ecb.h b/include/crypto/aes-ecb.h new file mode 100644 index 000000000000..8cac1f8794c7 --- /dev/null +++ b/include/crypto/aes-ecb.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AES-ECB unauthenticated encryption and decryption + * + * Copyright 2026 Google LLC + */ +#ifndef _CRYPTO_AES_ECB_H +#define _CRYPTO_AES_ECB_H + +#include <crypto/aes.h> + +/** + * aes_ecb_encrypt() - Encrypt data using AES-ECB + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to encrypt. Must be a multiple of AES_BLOCK_SIZE. + * @key: The key, already prepared using aes_preparekey() or aes_prepareenckey() + * + * ECB mode is insecure by itself. This function exists only for compatibility + * with legacy protocols and for internal use by other modes. + * + * This supports incremental encryption, but the length of each chunk must be a + * multiple of AES_BLOCK_SIZE. + * + * Context: Any context. + */ +void aes_ecb_encrypt(u8 *dst, const u8 *src, size_t len, aes_encrypt_arg key); + +/** + * aes_ecb_decrypt() - Decrypt data using AES-ECB + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to decrypt. Must be a multiple of AES_BLOCK_SIZE. + * @key: The key, already prepared using aes_preparekey() + * + * ECB mode is insecure by itself. This function exists only for compatibility + * with legacy protocols and for internal use by other modes. + * + * This supports incremental decryption, but the length of each chunk must be a + * multiple of AES_BLOCK_SIZE. + * + * Context: Any context. + */ +void aes_ecb_decrypt(u8 *dst, const u8 *src, size_t len, + const struct aes_key *key); + +#endif /* _CRYPTO_AES_ECB_H */ diff --git a/include/crypto/aes-gcm.h b/include/crypto/aes-gcm.h new file mode 100644 index 000000000000..2aee62f01989 --- /dev/null +++ b/include/crypto/aes-gcm.h @@ -0,0 +1,260 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AES-GCM authenticated encryption and decryption + * + * Copyright 2026 Google LLC + */ +#ifndef _CRYPTO_AES_GCM_H +#define _CRYPTO_AES_GCM_H + +#include <crypto/aes.h> +#include <crypto/gcm.h> +#include <crypto/gf128hash.h> + +/** + * struct aes_gcm_key - A key prepared for AES-GCM encryption and decryption + */ +struct aes_gcm_key { + /* private: */ + struct aes_enckey aes; + struct ghash_key ghash; + size_t authtag_len; /* Length of authentication tags in bytes */ +}; + +/** + * struct aes_gcm_ctx - Context for incrementally en/decrypting a message + */ +struct aes_gcm_ctx { + /* private: */ + /* + * Pointer to the key, which is assumed to live at least as long as this + * struct. + */ + const struct aes_gcm_key *key; + /* The current GHASH context */ + struct ghash_ctx ghash; + /* + * The current counter. This can be viewed as either a 128-bit big + * endian counter, or as a 96-bit nonce followed by a 32-bit big endian + * counter; it doesn't matter, since the last 32-bit word starts at 1, + * and AES-GCM is undefined for messages that would overflow that part. + * In practice this means that code optimized for AES-GCM can just + * increment the last 32-bit word (wrapping at 2^32), but when needed it + * can still call AES-CTR code that does a 128-bit increment. + * + * 'long' alignment is for crypto_xor() to work more efficiently. + */ + union { + u8 ctr[AES_BLOCK_SIZE]; + __be32 ctr32[AES_BLOCK_SIZE / 4]; + } __aligned(__alignof__(long)); + /* Buffered keystream for partial block updates */ + u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + /* Encrypted counter of 1. This gets XOR'ed with the tag at the end. */ + u8 j0_enc[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + /* Number of associated data bytes processed so far */ + u64 ad_len; + /* Number of en/decrypted bytes processed so far */ + u64 data_len; +}; + +/** + * aes_gcm_preparekey() - Prepare a key for AES-GCM encryption and decryption + * @key: (output) The key structure to initialize + * @in_key: The raw AES-GCM key + * @key_len: Length of the raw key in bytes: 16, 24, or 32 + * @authtag_len: Length of the authentication tag in bytes: + * 4, 8, 12, 13, 14, 15, or 16. 16 is recommended. + * + * Users should use memzero_explicit() to zeroize the key struct at the end of + * its lifetime. (But if this function fails, zeroization is unnecessary.) + * + * Context: Any context. + * Return: + * * 0 on success + * * -EINVAL if either of the lengths is invalid + */ +int __must_check aes_gcm_preparekey(struct aes_gcm_key *key, const u8 *in_key, + size_t key_len, size_t authtag_len); + +/** + * aes_gcm_encrypt() - Encrypt a message with AES-GCM + * @dst: The destination ciphertext data. Can be in-place or out-of-place. + * For other overlaps the behavior is unspecified. + * @src: The source plaintext data + * @data_len: Length of plaintext in bytes (and ciphertext excluding the tag): + * at most 2^36 - 32 + * @authtag: The output authentication tag. Length is the authtag_len that was + * passed to aes_gcm_preparekey(). Usually protocols using AES-GCM + * put the tag at the end of the ciphertext, in which case this should + * be set to @dst + @data_len and @dst must have room for the tag. + * @ad: The associated data + * @ad_len: Length of associated data in bytes: at most 2^61 - 1 + * @nonce: The 12-byte nonce. All (key, nonce) pairs used MUST be distinct. + * @key: The key, already prepared using aes_gcm_preparekey() + * + * For AES-GMAC (i.e., AES-GCM without any data en/decrypted), use dst=NULL, + * src=NULL, and data_len=0 to generate the AES-GMAC value. + * + * Context: Any context. + */ +void aes_gcm_encrypt(u8 *dst, const u8 *src, size_t data_len, u8 *authtag, + const u8 *ad, size_t ad_len, const u8 nonce[at_least 12], + const struct aes_gcm_key *key); + +/** + * aes_gcm_decrypt() - Decrypt a message with AES-GCM + * @dst: The destination plaintext data. Can be in-place or out-of-place. + * For other overlaps the behavior is unspecified. + * @src: The source ciphertext data + * @data_len: Length of plaintext in bytes (and ciphertext excluding the tag): + * at most 2^36 - 32 + * @authtag: The stored authentication tag. Length is the authtag_len that was + * passed to aes_gcm_preparekey(). Usually protocols using AES-GCM + * put the tag at the end of the ciphertext, in which case this should + * be set to @src + @data_len and @src must have room for the tag. + * @ad: The associated data + * @ad_len: Length of associated data in bytes: at most 2^61 - 1 + * @nonce: The 12-byte nonce + * @key: The key, already prepared using aes_gcm_preparekey() + * + * For AES-GMAC (i.e., AES-GCM without any data en/decrypted), use dst=NULL, + * src=NULL, and data_len=0 to verify the AES-GMAC value. + * + * Context: Any context. + * Return: + * * 0 on success. This is the only case where any decrypted or associated data + * can be used. + * * -EBADMSG if the message is inauthentic + */ +int __must_check aes_gcm_decrypt(u8 *dst, const u8 *src, size_t data_len, + const u8 *authtag, const u8 *ad, size_t ad_len, + const u8 nonce[at_least 12], + const struct aes_gcm_key *key); + +/** + * aes_gcm_init() - Initialize context for incremental AES-GCM encryption or + * decryption, or for AES-GMAC computation + * @ctx: The context to initialize + * @nonce: The 12-byte nonce. All (key, nonce) pairs used for encryption or MAC + * generation MUST be distinct. + * @key: The key, already prepared using aes_gcm_preparekey(). Note that a + * pointer to the key is saved in the context, so the key must live at + * least as long as the context. + * + * The context should be zeroized at the end of its lifetime. Normally that + * happens in aes_gcm_encrypt_final() or aes_gcm_decrypt_final(), but callers + * that abandon a context without finalizing it should explicitly zeroize it. + * + * IMPORTANT: Callers that are decrypting data or computing a GMAC value for + * verification MUST NOT assume that any decrypted or associated data is + * authentic until the authentication tag has been verified. This incremental + * API is provided solely to support callers that can't efficiently use the + * one-shot functions due to using a nonlinear data layout. + * + * For incremental AES-GCM encryption, use: + * + * 1. aes_gcm_init() + * 2. aes_gcm_auth_update() (any number of times) + * 3. aes_gcm_encrypt_update() (any number of times) + * 4. aes_gcm_encrypt_final() + * + * For incremental AES-GCM decryption, use: + * + * 1. aes_gcm_init() + * 2. aes_gcm_auth_update() (any number of times) + * 3. aes_gcm_decrypt_update() (any number of times) + * 4. aes_gcm_decrypt_final() + * + * AES-GMAC is just AES-GCM with zero bytes en/decrypted. For incremental + * AES-GMAC computation, use: + * + * 1. aes_gcm_init() + * 2. aes_gcm_auth_update() (any number of times) + * 3. aes_gcm_encrypt_final() to return the computed tag to the caller, or + * aes_gcm_decrypt_final() to directly verify the computed tag + * + * Context: Any context. + */ +void aes_gcm_init(struct aes_gcm_ctx *ctx, const u8 nonce[at_least 12], + const struct aes_gcm_key *key); + +/** + * aes_gcm_auth_update() - Incrementally process AES-GCM associated data + * @ctx: An AES-GCM context + * @ad: The associated data + * @len: Number of bytes provided. The caller must ensure that the total + * associated data length doesn't exceed GCM's limit of 2^61 - 1. + * + * IMPORTANT: Callers MUST NOT assume that any decrypted or associated data is + * authentic until the authentication tag has been verified. + * + * Context: Any context. + */ +void aes_gcm_auth_update(struct aes_gcm_ctx *ctx, const u8 *ad, size_t len); + +/** + * aes_gcm_encrypt_update() - Incrementally encrypt data with AES-GCM + * @ctx: An AES-GCM context + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source plaintext data + * @len: Number of bytes to encrypt. The caller must ensure that the total + * number of bytes encrypted doesn't exceed GCM's limit of 2^36 - 32. + * + * This can be called only after all associated data has been processed. + * + * Context: Any context. + */ +void aes_gcm_encrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *src, + size_t len); + +/** + * aes_gcm_decrypt_update() - Incrementally decrypt data with AES-GCM + * @ctx: An AES-GCM context + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source ciphertext data (not including auth tag) + * @len: Number of bytes to decrypt. The caller must ensure that the total + * number of bytes decrypted doesn't exceed GCM's limit of 2^36 - 32. + * + * This can be called only after all associated data has been processed. + * + * IMPORTANT: Callers MUST NOT assume that any decrypted or associated data is + * authentic until the authentication tag has been verified. + * + * Context: Any context. + */ +void aes_gcm_decrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *src, + size_t len); + +/** + * aes_gcm_encrypt_final() - Finish encrypting a message with AES-GCM + * @ctx: An AES-GCM context + * @authtag: The output authentication tag. Length is the authtag_len that was + * passed to aes_gcm_preparekey(). + * + * This also zeroizes @ctx, so the caller doesn't need to do it. + * + * Context: Any context. + */ +void aes_gcm_encrypt_final(struct aes_gcm_ctx *ctx, u8 *authtag); + +/** + * aes_gcm_decrypt_final() - Finish decrypting a message with AES-GCM + * @ctx: An AES-GCM context + * @authtag: The stored authentication tag. Length is the authtag_len that was + * passed to aes_gcm_preparekey(). + * + * This also zeroizes @ctx, so the caller doesn't need to do it. + * + * Context: Any context. + * Return: + * * 0 on success. This is the only case where any decrypted or associated data + * can be used. + * * -EBADMSG if the message is inauthentic + */ +int __must_check aes_gcm_decrypt_final(struct aes_gcm_ctx *ctx, + const u8 *authtag); + +#endif /* _CRYPTO_AES_GCM_H */ diff --git a/include/crypto/aes-xts.h b/include/crypto/aes-xts.h new file mode 100644 index 000000000000..b9e828265e58 --- /dev/null +++ b/include/crypto/aes-xts.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AES-XTS unauthenticated encryption and decryption + * + * Copyright 2026 Google LLC + */ +#ifndef _CRYPTO_AES_XTS_H +#define _CRYPTO_AES_XTS_H + +#include <crypto/aes.h> +#include <crypto/xts.h> + +/** + * struct aes_xts_key - A key prepared for AES-XTS encryption and decryption + * + * Note that (depending on the architecture) this typically is around 768 bytes, + * which makes it a bit too large to allocate on the stack in most cases. + */ +struct aes_xts_key { + /* private: */ + struct aes_key main_key; + struct aes_enckey tweak_key; +}; + +/** + * aes_xts_preparekey() - Prepare a key for AES-XTS encryption and decryption + * @key: (output) The key structure to initialize + * @in_key: The raw AES-XTS key + * @key_len: Length of the raw key in bytes + * @flags: Optional flag XTS_FORBID_WEAK_KEYS to forbid keys whose two halves + * are the same. + * + * Users should use memzero_explicit() to zeroize the key struct at the end of + * its lifetime. (But if this function fails, zeroization is unnecessary.) + * + * Context: Any context. + * Return: + * * 0 on success + * * -EINVAL if the key is rejected because its length isn't 32, 64, or (when + * FIPS mode isn't enabled) 48; or because its two halves are the same and + * either XTS_FORBID_WEAK_KEYS is given or FIPS mode is enabled. + */ +int __must_check aes_xts_preparekey(struct aes_xts_key *key, const u8 *in_key, + size_t key_len, int flags); + +/** + * aes_xts_encrypt() - Encrypt data using AES-XTS + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to encrypt. On non-final calls it must be a nonzero + * multiple of AES_BLOCK_SIZE. On the final call it can be any value >= + * AES_BLOCK_SIZE, i.e. ciphertext stealing is supported. + * @tweak: The tweak. It is updated with the next value, unless @len isn't a + * multiple of AES_BLOCK_SIZE in which case the value is unspecified. + * @key: The key, already prepared using aes_xts_preparekey() + * @cont: %false to begin encrypting a new message (do the tweak encryption); + * %true to continue encrypting a message (skip tweak encryption) + * + * This supports both one-shot and incremental encryption. On the first call, + * pass @cont = %false. On any later calls, pass @cont = %true and the updated + * @tweak; all earlier @len must have been multiples of AES_BLOCK_SIZE. + * + * Context: Any context. + */ +void aes_xts_encrypt(u8 *dst, const u8 *src, size_t len, + u8 tweak[at_least AES_BLOCK_SIZE], + const struct aes_xts_key *key, bool cont); + +/** + * aes_xts_decrypt() - Decrypt data using AES-XTS + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to decrypt. On non-final calls it must be a nonzero + * multiple of AES_BLOCK_SIZE. On the final call it can be any value >= + * AES_BLOCK_SIZE, i.e. ciphertext stealing is supported. + * @tweak: The tweak. It is updated with the next value, unless @len isn't a + * multiple of AES_BLOCK_SIZE in which case the value is unspecified. + * @key: The key, already prepared using aes_xts_preparekey() + * @cont: %false to begin decrypting a new message (do the tweak encryption); + * %true to continue decrypting a message (skip tweak encryption) + * + * This supports both one-shot and incremental decryption. On the first call, + * pass @cont = %false. On any later calls, pass @cont = %true and the updated + * @tweak; all earlier @len must have been multiples of AES_BLOCK_SIZE. + * + * Context: Any context. + */ +void aes_xts_decrypt(u8 *dst, const u8 *src, size_t len, + u8 tweak[at_least AES_BLOCK_SIZE], + const struct aes_xts_key *key, bool cont); + +#endif /* _CRYPTO_AES_XTS_H */ diff --git a/include/crypto/gcm.h b/include/crypto/gcm.h index 1d5f39ff1dc4..154038ca7343 100644 --- a/include/crypto/gcm.h +++ b/include/crypto/gcm.h @@ -3,9 +3,6 @@ #include <linux/errno.h> -#include <crypto/aes.h> -#include <crypto/gf128hash.h> - #define GCM_AES_IV_SIZE 12 #define GCM_RFC4106_IV_SIZE 8 #define GCM_RFC4543_IV_SIZE 8 @@ -13,7 +10,7 @@ /* * validate authentication tag for GCM */ -static inline int crypto_gcm_check_authsize(unsigned int authsize) +static inline int crypto_gcm_check_authsize(size_t authsize) { switch (authsize) { case 4: @@ -34,7 +31,7 @@ static inline int crypto_gcm_check_authsize(unsigned int authsize) /* * validate authentication tag for RFC4106 */ -static inline int crypto_rfc4106_check_authsize(unsigned int authsize) +static inline int crypto_rfc4106_check_authsize(size_t authsize) { switch (authsize) { case 8: @@ -64,22 +61,4 @@ static inline int crypto_ipsec_check_assoclen(unsigned int assoclen) return 0; } -struct aesgcm_ctx { - struct ghash_key ghash_key; - struct aes_enckey aes_key; - unsigned int authsize; -}; - -int aesgcm_expandkey(struct aesgcm_ctx *ctx, const u8 *key, - unsigned int keysize, unsigned int authsize); - -void aesgcm_encrypt(const struct aesgcm_ctx *ctx, u8 *dst, const u8 *src, - int crypt_len, const u8 *assoc, int assoc_len, - const u8 iv[GCM_AES_IV_SIZE], u8 *authtag); - -bool __must_check aesgcm_decrypt(const struct aesgcm_ctx *ctx, u8 *dst, - const u8 *src, int crypt_len, const u8 *assoc, - int assoc_len, const u8 iv[GCM_AES_IV_SIZE], - const u8 *authtag); - #endif diff --git a/include/crypto/xts.h b/include/crypto/xts.h index 15b16c4853d8..16aef89f021f 100644 --- a/include/crypto/xts.h +++ b/include/crypto/xts.h @@ -7,9 +7,9 @@ #include <linux/fips.h> #define XTS_BLOCK_SIZE 16 +#define XTS_FORBID_WEAK_KEYS (1 << 0) -static inline int xts_verify_key(struct crypto_skcipher *tfm, - const u8 *key, unsigned int keylen) +static inline int __xts_verify_key(const u8 *key, size_t keylen, int flags) { /* * key consists of keys of equal size concatenated, therefore @@ -29,12 +29,22 @@ static inline int xts_verify_key(struct crypto_skcipher *tfm, * Ensure that the AES and tweak key are not identical when * in FIPS mode or the FORBID_WEAK_KEYS flag is set. */ - if ((fips_enabled || (crypto_skcipher_get_flags(tfm) & - CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) && + if ((fips_enabled || (flags & XTS_FORBID_WEAK_KEYS)) && !crypto_memneq(key, key + (keylen / 2), keylen / 2)) return -EINVAL; return 0; } +static inline int xts_verify_key(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + int flags = (crypto_skcipher_get_flags(tfm) & + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) ? + XTS_FORBID_WEAK_KEYS : + 0; + + return __xts_verify_key(key, keylen, flags); +} + #endif /* _CRYPTO_XTS_H */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 83d4c95e079e..56f308dd0273 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -8,6 +8,8 @@ config CRYPTO_LIB_UTILS config CRYPTO_LIB_AES tristate + # Select dependencies of modes that are part of libaes. + select CRYPTO_LIB_GF128HASH if CRYPTO_LIB_AES_GCM != n select CRYPTO_LIB_UTILS config CRYPTO_LIB_AES_ARCH @@ -27,6 +29,12 @@ config CRYPTO_LIB_AESCFB select CRYPTO_LIB_AES select CRYPTO_LIB_UTILS +config CRYPTO_LIB_AES_CBC + tristate + select CRYPTO_LIB_AES + help + The AES-CBC and AES-CBC-CTS library functions. + config CRYPTO_LIB_AES_CBC_MACS tristate select CRYPTO_LIB_AES @@ -35,11 +43,38 @@ config CRYPTO_LIB_AES_CBC_MACS this if your module uses any of the functions from <crypto/aes-cbc-macs.h>. -config CRYPTO_LIB_AESGCM +config CRYPTO_LIB_AES_CCM tristate select CRYPTO_LIB_AES - select CRYPTO_LIB_GF128HASH - select CRYPTO_LIB_UTILS + select CRYPTO_LIB_AES_CBC_MACS + select CRYPTO_LIB_AES_CTR + help + The AES-CCM library functions. + +config CRYPTO_LIB_AES_CTR + tristate + select CRYPTO_LIB_AES + help + The AES-CTR and AES-XCTR library functions. + +config CRYPTO_LIB_AES_ECB + tristate + select CRYPTO_LIB_AES + help + The AES-ECB library functions. + +config CRYPTO_LIB_AES_GCM + tristate + select CRYPTO_LIB_AES + select CRYPTO_LIB_AES_CTR + help + The AES-GCM library functions. + +config CRYPTO_LIB_AES_XTS + tristate + select CRYPTO_LIB_AES + help + The AES-XTS library functions. config CRYPTO_LIB_ARC4 tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index f1e9bf89785f..ca068df1f71f 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -63,9 +63,6 @@ clean-files += powerpc/aesp8-ppc.S obj-$(CONFIG_CRYPTO_LIB_AESCFB) += libaescfb.o libaescfb-y := aescfb.o -obj-$(CONFIG_CRYPTO_LIB_AESGCM) += libaesgcm.o -libaesgcm-y := aesgcm.o - obj-$(CONFIG_CRYPTO_LIB_ARC4) += libarc4.o libarc4-y := arc4.o diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c index ca733f15b2a8..4222a4cec2f2 100644 --- a/lib/crypto/aes.c +++ b/lib/crypto/aes.c @@ -5,7 +5,14 @@ */ #include <crypto/aes-cbc-macs.h> +#include <crypto/aes-cbc.h> +#include <crypto/aes-ccm.h> +#include <crypto/aes-ctr.h> +#include <crypto/aes-ecb.h> +#include <crypto/aes-gcm.h> +#include <crypto/aes-xts.h> #include <crypto/aes.h> +#include <crypto/gf128mul.h> #include <crypto/utils.h> #include <linux/cache.h> #include <linux/crypto.h> @@ -737,6 +744,1162 @@ static inline void aes_cmac_fips_test(void) } #endif /* !CONFIG_CRYPTO_LIB_AES_CBC_MACS */ +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB) +/* + * Hooks for optimized AES-ECB implementations, overridable by the architecture. + * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. Returning false + * causes the fallback implementation to be used instead. + */ +#ifndef aes_ecb_encrypt_arch +static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len, + const struct aes_enckey *key) +{ + return false; +} +#endif +#ifndef aes_ecb_decrypt_arch +static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len, + const struct aes_key *key) +{ + return false; +} +#endif + +void aes_ecb_encrypt(u8 *dst, const u8 *src, size_t len, aes_encrypt_arg key) +{ + if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) + len = round_down(len, AES_BLOCK_SIZE); + + if (unlikely(len == 0)) + return; + + if (likely(aes_ecb_encrypt_arch(dst, src, len, key.enc_key))) + return; + + for (size_t i = 0; i < len; i += AES_BLOCK_SIZE) + aes_encrypt(key, &dst[i], &src[i]); +} +EXPORT_SYMBOL_GPL(aes_ecb_encrypt); + +void aes_ecb_decrypt(u8 *dst, const u8 *src, size_t len, + const struct aes_key *key) +{ + if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) + len = round_down(len, AES_BLOCK_SIZE); + + if (unlikely(len == 0)) + return; + + if (likely(aes_ecb_decrypt_arch(dst, src, len, key))) + return; + + for (size_t i = 0; i < len; i += AES_BLOCK_SIZE) + aes_decrypt(key, &dst[i], &src[i]); +} +EXPORT_SYMBOL_GPL(aes_ecb_decrypt); +#endif /* CONFIG_CRYPTO_LIB_AES_ECB */ + +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC) +/* + * Hooks for optimized AES-CBC implementations, overridable by the architecture. + * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. Returning false + * causes the fallback implementation to be used instead. + */ +#ifndef aes_cbc_encrypt_arch +static bool aes_cbc_encrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], + const struct aes_enckey *key) +{ + return false; +} +#endif +#ifndef aes_cbc_decrypt_arch +static bool aes_cbc_decrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], + const struct aes_key *key) +{ + return false; +} +#endif + +void aes_cbc_encrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_SIZE], + aes_encrypt_arg key) +{ + const u8 *prev = iv; + + if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) + len = round_down(len, AES_BLOCK_SIZE); + + if (unlikely(len == 0)) + return; + + if (likely(aes_cbc_encrypt_arch(dst, src, len, iv, key.enc_key))) + return; + + do { + crypto_xor_cpy(dst, src, prev, AES_BLOCK_SIZE); + aes_encrypt(key, dst, dst); + prev = dst; + dst += AES_BLOCK_SIZE; + src += AES_BLOCK_SIZE; + len -= AES_BLOCK_SIZE; + } while (len); + memcpy(iv, prev, AES_BLOCK_SIZE); +} +EXPORT_SYMBOL_GPL(aes_cbc_encrypt); + +void aes_cbc_decrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_SIZE], + const struct aes_key *key) +{ + u8 next_iv[AES_BLOCK_SIZE]; + + if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) + len = round_down(len, AES_BLOCK_SIZE); + + if (unlikely(len == 0)) + return; + + if (likely(aes_cbc_decrypt_arch(dst, src, len, iv, key))) + return; + + len -= AES_BLOCK_SIZE; + dst += len; + src += len; + memcpy(next_iv, src, AES_BLOCK_SIZE); + for (;;) { + aes_decrypt(key, dst, src); + if (len == 0) + break; + src -= AES_BLOCK_SIZE; + crypto_xor(dst, src, AES_BLOCK_SIZE); + dst -= AES_BLOCK_SIZE; + len -= AES_BLOCK_SIZE; + } + crypto_xor(dst, iv, AES_BLOCK_SIZE); + memcpy(iv, next_iv, AES_BLOCK_SIZE); +} +EXPORT_SYMBOL_GPL(aes_cbc_decrypt); + +/* + * Hooks for optimized AES-CBC-CTS implementations, overridable by the + * architecture. They are called with len > AES_BLOCK_SIZE. Returning false + * causes the fallback implementation to be used instead. The fallback + * implementation still uses the arch-optimized AES-CBC code if available, but + * direct implementation of AES-CBC-CTS is helpful on short messages. + */ +#ifndef aes_cbc_cts_encrypt_arch +static bool aes_cbc_cts_encrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], + const struct aes_enckey *key) +{ + return false; +} +#endif +#ifndef aes_cbc_cts_decrypt_arch +static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], + const struct aes_key *key) +{ + return false; +} +#endif + +void aes_cbc_cts_encrypt(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], aes_encrypt_arg key) +{ + /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ + size_t pn_offset = round_down(len - 1, AES_BLOCK_SIZE); + /* Length of P[n] and C[n], 1 <= pn_len <= AES_BLOCK_SIZE */ + size_t pn_len = len - pn_offset; + u8 tmp[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + u8 *pad; + + if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) + return; + + if (len == AES_BLOCK_SIZE) { + aes_cbc_encrypt(dst, src, len, iv, key); + return; + } + if (likely(aes_cbc_cts_encrypt_arch(dst, src, len, iv, key.enc_key))) + return; + + /* CBC-encrypt all blocks except the last. */ + aes_cbc_encrypt(dst, src, pn_offset, iv, key); + + /* + * Compute C[n] and C[n - 1]. + * + * Careful: src may equal dst (i.e., the encryption can be in-place), so + * src[pn_offset..] can't be read after dst[pn_offset..] is written. + */ + pad = &dst[pn_offset - AES_BLOCK_SIZE]; + memcpy(tmp, pad, AES_BLOCK_SIZE); + crypto_xor(tmp, &src[pn_offset], pn_len); + memcpy(&dst[pn_offset], pad, pn_len); /* C[n] */ + aes_encrypt(key, pad, tmp); /* C[n - 1] */ + + memzero_explicit(tmp, sizeof(tmp)); +} +EXPORT_SYMBOL_GPL(aes_cbc_cts_encrypt); + +void aes_cbc_cts_decrypt(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], const struct aes_key *key) +{ + /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ + size_t pn_offset = round_down(len - 1, AES_BLOCK_SIZE); + /* Length of P[n] and C[n], 1 <= pn_len <= AES_BLOCK_SIZE */ + size_t pn_len = len - pn_offset; + u8 *pad; + + if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) + return; + + if (len == AES_BLOCK_SIZE) { + aes_cbc_decrypt(dst, src, len, iv, key); + return; + } + if (likely(aes_cbc_cts_decrypt_arch(dst, src, len, iv, key))) + return; + + /* Compute P[0]..P[n - 2]. */ + aes_cbc_decrypt(dst, src, pn_offset - AES_BLOCK_SIZE, iv, key); + + /* + * Compute P[n] and P[n - 1]. + * + * Careful: src may equal dst (i.e., the decryption can be in-place), so + * src[pn_offset..] can't be read after dst[pn_offset..] is written. + * + * To avoid needing a temporary buffer, do a "redundant" XOR to recover + * src[pn_offset..] from dst[pn_offset..] after the latter is written. + */ + pad = &dst[pn_offset - AES_BLOCK_SIZE]; + aes_decrypt(key, pad, &src[pn_offset - AES_BLOCK_SIZE]); + crypto_xor_cpy(&dst[pn_offset], &src[pn_offset], pad, + pn_len); /* P[n] */ + crypto_xor(pad, &dst[pn_offset], pn_len); + aes_decrypt(key, pad, pad); + crypto_xor(pad, iv, AES_BLOCK_SIZE); /* P[n - 1] */ +} +EXPORT_SYMBOL_GPL(aes_cbc_cts_decrypt); +#endif /* CONFIG_CRYPTO_LIB_AES_CBC */ + +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR) +/* + * Hooks for optimized AES-CTR and AES-XCTR implementations, overridable by the + * architecture. They are called with any len >= 0. Returning false causes the + * fallback implementation to be used instead. + */ +#ifndef aes_ctr_arch +static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len, + u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key) +{ + return false; +} +#endif +#ifndef aes_xctr_arch +static bool aes_xctr_arch(u8 *dst, const u8 *src, size_t len, u64 *ctr, + const u8 iv[AES_BLOCK_SIZE], + const struct aes_enckey *key) +{ + return false; +} +#endif + +static __always_inline void inc_be128_ctr(u8 ctr[AES_BLOCK_SIZE]) +{ + /* + * 255 times out of 256 the first iteration is enough, so unroll the + * first iteration as a micro-optimization. + */ + if ((++ctr[AES_BLOCK_SIZE - 1]) != 0) + return; + for (int i = AES_BLOCK_SIZE - 2; i >= 0; i--) { + if (++ctr[i] != 0) + break; + } +} + +void aes_ctr(u8 *dst, const u8 *src, size_t len, u8 ctr[AES_BLOCK_SIZE], + aes_encrypt_arg key) +{ + u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + + if (likely(aes_ctr_arch(dst, src, len, ctr, key.enc_key))) + return; + + /* Handle the full blocks. */ + for (; len >= AES_BLOCK_SIZE; len -= AES_BLOCK_SIZE) { + aes_encrypt(key, keystream, ctr); + crypto_xor_cpy(dst, src, keystream, AES_BLOCK_SIZE); + inc_be128_ctr(ctr); + dst += AES_BLOCK_SIZE; + src += AES_BLOCK_SIZE; + } + /* Handle any partial block at the end. */ + if (len) { + aes_encrypt(key, keystream, ctr); + crypto_xor_cpy(dst, src, keystream, len); + /* Counter is incremented even with just a partial block. */ + inc_be128_ctr(ctr); + } + memzero_explicit(keystream, sizeof(keystream)); +} +EXPORT_SYMBOL_GPL(aes_ctr); + +void aes_xctr(u8 *dst, const u8 *src, size_t len, u64 *ctr, + const u8 iv[AES_BLOCK_SIZE], aes_encrypt_arg key) +{ + const __le64 iv0 = get_unaligned((const __le64 *)&iv[0]); + __le64 aes_input[2]; + u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + + if (likely(aes_xctr_arch(dst, src, len, ctr, iv, key.enc_key))) + return; + + aes_input[1] = get_unaligned((const __le64 *)&iv[8]); + /* Handle the full blocks. */ + for (; len >= AES_BLOCK_SIZE; len -= AES_BLOCK_SIZE) { + aes_input[0] = iv0 ^ cpu_to_le64((*ctr)++); + aes_encrypt(key, keystream, (const u8 *)aes_input); + crypto_xor_cpy(dst, src, keystream, AES_BLOCK_SIZE); + dst += AES_BLOCK_SIZE; + src += AES_BLOCK_SIZE; + } + /* Handle any partial block at the end. */ + if (len) { + /* Counter is incremented even with just a partial block. */ + aes_input[0] = iv0 ^ cpu_to_le64((*ctr)++); + aes_encrypt(key, keystream, (const u8 *)aes_input); + crypto_xor_cpy(dst, src, keystream, len); + } + memzero_explicit(keystream, sizeof(keystream)); + memzero_explicit(aes_input, sizeof(aes_input)); +} +EXPORT_SYMBOL_GPL(aes_xctr); +#endif /* CONFIG_CRYPTO_LIB_AES_CTR */ + +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS) +int aes_xts_preparekey(struct aes_xts_key *key, const u8 *in_key, + size_t key_len, int flags) +{ + int err; + + err = __xts_verify_key(in_key, key_len, flags); + if (unlikely(err)) + goto out_zeroize; + /* First half of XTS key is the main key */ + err = aes_preparekey(&key->main_key, in_key, key_len / 2); + if (unlikely(err)) + goto out_zeroize; + /* Second half of XTS key is the tweak key */ + err = aes_prepareenckey(&key->tweak_key, &in_key[key_len / 2], + key_len / 2); + if (unlikely(err)) + goto out_zeroize; + return 0; + +out_zeroize: + memzero_explicit(key, sizeof(*key)); + return err; +} +EXPORT_SYMBOL_GPL(aes_xts_preparekey); + +/* + * Hooks for optimized AES-XTS implementations, overridable by the architecture. + * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. In other words, + * they aren't expected to handle ciphertext stealing or empty inputs. + * Returning false causes the fallback implementation to be used instead. + * + * (Currently, all users of AES-XTS in the kernel seem to en/decrypt whole + * numbers of blocks anyway, with len >= 512. So there's no need to heavily + * optimize ciphertext stealing for short messages.) + */ +#ifndef aes_xts_encrypt_arch +static bool aes_xts_encrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, bool cont) +{ + return false; +} +#endif +#ifndef aes_xts_decrypt_arch +static bool aes_xts_decrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, bool cont) +{ + return false; +} +#endif + +static noinline void aes_xts_crypt_nocts_blockbyblock( + u8 *dst, const u8 *src, size_t len, u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, bool cont, bool enc) +{ + le128 t; + + if (cont) + memcpy(&t, tweak, sizeof(t)); + else + aes_encrypt(&key->tweak_key, (u8 *)&t, tweak); + do { + crypto_xor_cpy(dst, src, (const u8 *)&t, AES_BLOCK_SIZE); + if (enc) + aes_encrypt(&key->main_key, dst, dst); + else + aes_decrypt(&key->main_key, dst, dst); + crypto_xor(dst, (const u8 *)&t, AES_BLOCK_SIZE); + gf128mul_x_ble(&t, &t); + dst += AES_BLOCK_SIZE; + src += AES_BLOCK_SIZE; + len -= AES_BLOCK_SIZE; + } while (len); + memcpy(tweak, &t, sizeof(t)); + memzero_explicit(&t, sizeof(t)); +} + +/* Requires len > 0 && len % AES_BLOCK_SIZE == 0 */ +static __always_inline void aes_xts_encrypt_nocts(u8 *dst, const u8 *src, + size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, + bool cont) +{ + if (likely(aes_xts_encrypt_arch(dst, src, len, tweak, key, cont))) + return; + + /* + * For the fallback, just go block-by-block. It could be implemented on + * top of AES-ECB, which could be significantly faster than this if the + * arch has optimized AES-ECB code but not AES-XTS. However, AES-XTS + * performance is important enough that it needs to be (and has been) + * implemented directly by every non-obsolete arch anyway. + */ + aes_xts_crypt_nocts_blockbyblock(dst, src, len, tweak, key, cont, + /* enc= */ true); +} + +/* Requires len > 0 && len % AES_BLOCK_SIZE == 0 */ +static __always_inline void aes_xts_decrypt_nocts(u8 *dst, const u8 *src, + size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, + bool cont) +{ + if (likely(aes_xts_decrypt_arch(dst, src, len, tweak, key, cont))) + return; + + /* Just go block-by-block. See comment in aes_xts_encrypt_nocts(). */ + aes_xts_crypt_nocts_blockbyblock(dst, src, len, tweak, key, cont, + /* enc= */ false); +} + +static noinline void aes_xts_encrypt_cts(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, + bool cont) +{ + size_t partial_len = len % AES_BLOCK_SIZE; /* Length of partial block */ + size_t nocts_len = round_down(len, AES_BLOCK_SIZE); + u8 tmp_block[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + + /* Encrypt all full blocks. */ + aes_xts_encrypt_nocts(dst, src, nocts_len, tweak, key, cont); + dst += nocts_len - AES_BLOCK_SIZE; + src += nocts_len - AES_BLOCK_SIZE; + + /* + * Swap the partial block with the first 'partial_len' bytes of the + * encrypted last full block. Note that a temporary buffer is needed to + * support in-place encryption. + */ + memcpy(tmp_block, src + AES_BLOCK_SIZE, partial_len); + memcpy(dst + AES_BLOCK_SIZE, dst, partial_len); + memcpy(dst, tmp_block, partial_len); + + /* Encrypt the last full block again. */ + crypto_xor(dst, tweak, AES_BLOCK_SIZE); + aes_encrypt(&key->main_key, dst, dst); + crypto_xor(dst, tweak, AES_BLOCK_SIZE); + memzero_explicit(tmp_block, sizeof(tmp_block)); +} + +static noinline void aes_xts_decrypt_cts(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, + bool cont) +{ + size_t partial_len = len % AES_BLOCK_SIZE; /* Length of partial block */ + size_t nocts_len = round_down(len, AES_BLOCK_SIZE) - AES_BLOCK_SIZE; + union { + u8 block[AES_BLOCK_SIZE]; + le128 tweak; + } tmp __aligned(__alignof__(long)); + + /* + * Decrypt all blocks except the last full block and the partial block. + * The last full block has to be handled specially because decryption + * ciphertext stealing uses the last two tweaks in reverse order. + * + * nocts_len == 0 is possible here, which aes_xts_decrypt_nocts() + * doesn't handle (so that the length doesn't get checked redundantly in + * the fast path). So handle that case specially as well. + */ + if (nocts_len) + aes_xts_decrypt_nocts(dst, src, nocts_len, tweak, key, cont); + else if (!cont) + aes_encrypt(&key->tweak_key, tweak, tweak); + dst += nocts_len; + src += nocts_len; + + /* Copy the tweak, advance it again, then decrypt last full block. */ + memcpy(&tmp.tweak, tweak, AES_BLOCK_SIZE); + gf128mul_x_ble(&tmp.tweak, &tmp.tweak); + crypto_xor_cpy(dst, src, tmp.block, AES_BLOCK_SIZE); + aes_decrypt(&key->main_key, dst, dst); + crypto_xor(dst, tmp.block, AES_BLOCK_SIZE); + + /* + * Swap the partial block with the first 'partial_len' bytes of the + * decrypted last full block. Note that a temporary buffer is needed to + * support in-place decryption. + */ + memcpy(tmp.block, src + AES_BLOCK_SIZE, partial_len); + memcpy(dst + AES_BLOCK_SIZE, dst, partial_len); + memcpy(dst, tmp.block, partial_len); + + /* Decrypt the last full block again. */ + crypto_xor(dst, tweak, AES_BLOCK_SIZE); + aes_decrypt(&key->main_key, dst, dst); + crypto_xor(dst, tweak, AES_BLOCK_SIZE); + memzero_explicit(&tmp, sizeof(tmp)); +} + +void aes_xts_encrypt(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], const struct aes_xts_key *key, + bool cont) +{ + if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) + return; + + if (unlikely(len % AES_BLOCK_SIZE)) { + aes_xts_encrypt_cts(dst, src, len, tweak, key, cont); + return; + } + + aes_xts_encrypt_nocts(dst, src, len, tweak, key, cont); +} +EXPORT_SYMBOL_GPL(aes_xts_encrypt); + +void aes_xts_decrypt(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], const struct aes_xts_key *key, + bool cont) +{ + if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) + return; + + if (unlikely(len % AES_BLOCK_SIZE)) { + aes_xts_decrypt_cts(dst, src, len, tweak, key, cont); + return; + } + + aes_xts_decrypt_nocts(dst, src, len, tweak, key, cont); +} +EXPORT_SYMBOL_GPL(aes_xts_decrypt); +#endif /* CONFIG_CRYPTO_LIB_AES_XTS */ + +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_GCM) +/* + * Hooks for optimized AES-GCM implementations, overridable by the architecture. + * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. I.e. they aren't + * expected to handle empty inputs or partial blocks, as those cases are handled + * by non-arch-specific code instead. + * + * The GHASH accumulator is provided in POLYVAL format. The counter is provided + * in big endian format, and it's read-only, as the caller handles updating it. + * + * Returning false causes the fallback implementation to be used instead. + * + * These hooks are used only for en/decrypted data. For the associated data the + * GHASH functions are called instead, so those should be implemented too. + */ +#ifndef aes_gcm_encrypt_update_arch +static bool aes_gcm_encrypt_update_arch(u8 *dst, const u8 *src, size_t len, + struct polyval_elem *ghash_acc, + const __be32 ctr32[4], + const struct aes_enckey *aes_key, + const struct ghash_key *ghash_key) +{ + return false; +} +#endif +#ifndef aes_gcm_decrypt_update_arch +static bool aes_gcm_decrypt_update_arch(u8 *dst, const u8 *src, size_t len, + struct polyval_elem *ghash_acc, + const __be32 ctr32[4], + const struct aes_enckey *aes_key, + const struct ghash_key *ghash_key) +{ + return false; +} +#endif + +int aes_gcm_preparekey(struct aes_gcm_key *key, const u8 *in_key, + size_t key_len, size_t authtag_len) +{ + u8 h[AES_BLOCK_SIZE] = { 0 }; + int err; + + err = crypto_gcm_check_authsize(authtag_len); + if (unlikely(err)) + return err; + + err = aes_prepareenckey(&key->aes, in_key, key_len); + if (unlikely(err)) + return err; + + aes_encrypt(&key->aes, h, h); + ghash_preparekey(&key->ghash, h); + + key->authtag_len = authtag_len; + + memzero_explicit(h, sizeof(h)); + return 0; +} +EXPORT_SYMBOL_GPL(aes_gcm_preparekey); + +void aes_gcm_init(struct aes_gcm_ctx *ctx, const u8 nonce[12], + const struct aes_gcm_key *key) +{ + ctx->key = key; + ctx->ad_len = 0; + ctx->data_len = 0; + ghash_init(&ctx->ghash, &key->ghash); + memset(ctx->keystream, 0, sizeof(ctx->keystream)); + + memcpy(ctx->ctr32, nonce, 12); + ctx->ctr32[3] = cpu_to_be32(1); + + aes_encrypt(&key->aes, ctx->j0_enc, ctx->ctr); + ctx->ctr32[3] = cpu_to_be32(2); +} +EXPORT_SYMBOL_GPL(aes_gcm_init); + +void aes_gcm_auth_update(struct aes_gcm_ctx *ctx, const u8 *ad, size_t len) +{ + WARN_ON_ONCE(ctx->data_len != 0); + if (len) { + ghash_update(&ctx->ghash, ad, len); + ctx->ad_len += len; + } +} +EXPORT_SYMBOL_GPL(aes_gcm_auth_update); + +static const u8 gcm_zeroes[AES_BLOCK_SIZE]; + +static __always_inline void ghash_pad(struct ghash_ctx *ghash, u64 len) +{ + if (len % AES_BLOCK_SIZE) + ghash_update(ghash, gcm_zeroes, -len % AES_BLOCK_SIZE); +} + +static __always_inline void aes_gcm_crypt_update(struct aes_gcm_ctx *ctx, + u8 *dst, const u8 *src, + size_t len, bool enc) +{ + size_t partial_len, n; + + if (unlikely(len == 0)) + return; + + partial_len = ctx->data_len % AES_BLOCK_SIZE; + if (ctx->data_len == 0) + ghash_pad(&ctx->ghash, ctx->ad_len); + ctx->data_len += len; + + if (unlikely(partial_len != 0)) { + /* + * The previous call ended on a non-block-aligned data_len, so + * continue using a previously-generated keystream block. + */ + n = min(len, AES_BLOCK_SIZE - partial_len); + if (enc) { + crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], + n); + ghash_update(&ctx->ghash, dst, n); + } else { + ghash_update(&ctx->ghash, src, n); + crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], + n); + } + dst += n; + src += n; + len -= n; + } + + if (len >= AES_BLOCK_SIZE) { + n = round_down(len, AES_BLOCK_SIZE); + if (enc) { + if (likely(aes_gcm_encrypt_update_arch( + dst, src, n, &ctx->ghash.acc, ctx->ctr32, + &ctx->key->aes, &ctx->key->ghash))) { + be32_add_cpu(&ctx->ctr32[3], + n / AES_BLOCK_SIZE); + } else { + aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); + ghash_update(&ctx->ghash, dst, n); + } + } else { + if (likely(aes_gcm_decrypt_update_arch( + dst, src, n, &ctx->ghash.acc, ctx->ctr32, + &ctx->key->aes, &ctx->key->ghash))) { + be32_add_cpu(&ctx->ctr32[3], + n / AES_BLOCK_SIZE); + } else { + ghash_update(&ctx->ghash, src, n); + aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); + } + } + dst += n; + src += n; + len -= n; + } + + if (len != 0) { + /* + * Ending on a non-block aligned data_len. Generate the next + * keystream block, use the needed portion of it, and leave it + * cached in ctx->keystream in case this isn't the final call. + */ + aes_encrypt(&ctx->key->aes, ctx->keystream, ctx->ctr); + be32_add_cpu(&ctx->ctr32[3], 1); + if (enc) { + crypto_xor_cpy(dst, src, ctx->keystream, len); + ghash_update(&ctx->ghash, dst, len); + } else { + ghash_update(&ctx->ghash, src, len); + crypto_xor_cpy(dst, src, ctx->keystream, len); + } + } +} + +void aes_gcm_encrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *src, + size_t len) +{ + aes_gcm_crypt_update(ctx, dst, src, len, /* enc= */ true); +} +EXPORT_SYMBOL_GPL(aes_gcm_encrypt_update); + +void aes_gcm_decrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *src, + size_t len) +{ + aes_gcm_crypt_update(ctx, dst, src, len, /* enc= */ false); +} +EXPORT_SYMBOL_GPL(aes_gcm_decrypt_update); + +/* Maximum AES-GCM associated data length in bytes */ +#define AES_GCM_MAX_AD_LEN ((1ULL << 61) - 1) +/* Maximum AES-GCM en/decrypted data length in bytes */ +#define AES_GCM_MAX_DATA_LEN ((1ULL << 36) - 32) + +void aes_gcm_encrypt_final(struct aes_gcm_ctx *ctx, u8 *authtag) +{ + __be64 tail[2]; + + WARN_ON_ONCE(ctx->ad_len > AES_GCM_MAX_AD_LEN); + WARN_ON_ONCE(ctx->data_len > AES_GCM_MAX_DATA_LEN); + + ghash_pad(&ctx->ghash, + ctx->data_len == 0 ? ctx->ad_len : ctx->data_len); + + tail[0] = cpu_to_be64(ctx->ad_len * 8); + tail[1] = cpu_to_be64(ctx->data_len * 8); + ghash_update(&ctx->ghash, (const u8 *)tail, 16); + ghash_final(&ctx->ghash, ctx->ctr); /* Use ctr as temp buffer */ + + crypto_xor_cpy(authtag, ctx->ctr, ctx->j0_enc, ctx->key->authtag_len); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(aes_gcm_encrypt_final); + +int aes_gcm_decrypt_final(struct aes_gcm_ctx *ctx, const u8 *authtag) +{ + __be64 tail[2]; + int err; + + if (WARN_ON_ONCE(ctx->ad_len > AES_GCM_MAX_AD_LEN) || + WARN_ON_ONCE(ctx->data_len > AES_GCM_MAX_DATA_LEN)) { + err = -EBADMSG; + goto out; + } + + ghash_pad(&ctx->ghash, + ctx->data_len == 0 ? ctx->ad_len : ctx->data_len); + + tail[0] = cpu_to_be64(ctx->ad_len * 8); + tail[1] = cpu_to_be64(ctx->data_len * 8); + ghash_update(&ctx->ghash, (const u8 *)tail, 16); + ghash_final(&ctx->ghash, ctx->ctr); /* Use ctr as temp buffer */ + crypto_xor(ctx->ctr, ctx->j0_enc, ctx->key->authtag_len); + err = crypto_memneq(ctx->ctr, authtag, ctx->key->authtag_len) ? + -EBADMSG : + 0; +out: + memzero_explicit(ctx, sizeof(*ctx)); + return err; +} +EXPORT_SYMBOL_GPL(aes_gcm_decrypt_final); + +void aes_gcm_encrypt(u8 *dst, const u8 *src, size_t data_len, u8 *authtag, + const u8 *ad, size_t ad_len, const u8 nonce[12], + const struct aes_gcm_key *key) +{ + struct aes_gcm_ctx ctx; + + aes_gcm_init(&ctx, nonce, key); + aes_gcm_auth_update(&ctx, ad, ad_len); + aes_gcm_encrypt_update(&ctx, dst, src, data_len); + aes_gcm_encrypt_final(&ctx, authtag); +} +EXPORT_SYMBOL_GPL(aes_gcm_encrypt); + +int aes_gcm_decrypt(u8 *dst, const u8 *src, size_t data_len, const u8 *authtag, + const u8 *ad, size_t ad_len, const u8 nonce[12], + const struct aes_gcm_key *key) +{ + struct aes_gcm_ctx ctx; + int err; + + aes_gcm_init(&ctx, nonce, key); + aes_gcm_auth_update(&ctx, ad, ad_len); + aes_gcm_decrypt_update(&ctx, dst, src, data_len); + err = aes_gcm_decrypt_final(&ctx, authtag); + if (unlikely(err) && data_len) { + /* + * Clear the inauthentic decrypted data so that callers won't + * receive it even if they fail to correctly handle errors. + */ + memset(dst, 0, data_len); + } + return err; +} +EXPORT_SYMBOL_GPL(aes_gcm_decrypt); + +#endif /* CONFIG_CRYPTO_LIB_AES_GCM */ + +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CCM) +int aes_ccm_preparekey(struct aes_ccm_key *key, const u8 *in_key, + size_t key_len, size_t authtag_len) +{ + int err; + + if (unlikely(authtag_len < 4 || authtag_len > 16 || authtag_len % 2)) + return -EINVAL; + + err = aes_prepareenckey(&key->aes, in_key, key_len); + if (unlikely(err)) + return err; + + key->authtag_len = authtag_len; + return 0; +} +EXPORT_SYMBOL_GPL(aes_ccm_preparekey); + +int aes_ccm_init(struct aes_ccm_ctx *ctx, u64 data_len, u64 ad_len, + const u8 *nonce, size_t nonce_len, + const struct aes_ccm_key *key) +{ + /* + * This is the value L defined in the CCM specification. It determines + * the maximum allowed message length, and it is itself determined by + * the nonce length. They are inversely related, i.e. the longer the + * nonce the smaller the maximum message length is. + */ + unsigned int l = 15 - nonce_len; + + if (unlikely(nonce_len < 7 || nonce_len > 13)) + return -EINVAL; + /* Thus 2 <= l <= 8. */ + + /* Check whether data_len can be represented in 'l' bytes. */ + if (unlikely(data_len > U64_MAX >> (64 - 8 * l))) + return -EOVERFLOW; + + ctx->key = key; + ctx->ad_remaining = ad_len; + ctx->data_remaining = data_len; + ctx->ad_padded = false; + + /* + * Initialize the zero-th counter block to: + * + * L - 1 || nonce || 0 + * + * ... and the zero-th CBC-MAC block to: + * + * Flags || nonce || data_len + */ + *(__be64 *)&ctx->ctr[8] = 0; + *(__be64 *)&ctx->mac[8] = cpu_to_be64(data_len); + ctx->ctr[0] = l - 1; + ctx->mac[0] = (ad_len ? 0x40 : 0) | + (((key->authtag_len - 2) / 2) << 3) | (l - 1); + memcpy(&ctx->ctr[1], nonce, nonce_len); /* Overlapping store */ + memcpy(&ctx->mac[1], nonce, nonce_len); /* Overlapping store */ + + /* + * Generate S_0 by encrypting the counter (this is used to encrypt the + * auth tag later), and encrypt the zero-th CBC-MAC block. + */ + aes_encrypt(&key->aes, ctx->s0, ctx->ctr); + aes_encrypt(&key->aes, ctx->mac, ctx->mac); + + /* Increment the counter from 0 to 1. */ + ctx->ctr[15] = 1; + + if (ad_len) { + /* + * Update CBC-MAC with the associated data length, represented + * using either 2, 6, or 10 bytes depending on the length. + */ + if (likely(ad_len < 0xff00)) { + *(__be16 *)&ctx->mac[0] ^= cpu_to_be16(ad_len); + ctx->partial_len = 2; + } else if (ad_len <= U32_MAX) { + __be32 *p = (__be32 *)&ctx->mac[2]; + + *(__be16 *)&ctx->mac[0] ^= cpu_to_be16(0xfffe); + put_unaligned(get_unaligned(p) ^ cpu_to_be32(ad_len), + p); + ctx->partial_len = 6; + } else { + __be64 *p = (__be64 *)&ctx->mac[2]; + + *(__be16 *)&ctx->mac[0] ^= cpu_to_be16(0xffff); + put_unaligned(get_unaligned(p) ^ cpu_to_be64(ad_len), + p); + ctx->partial_len = 10; + } + } else { + ctx->partial_len = 0; + } + return 0; +} +EXPORT_SYMBOL_GPL(aes_ccm_init); + +void aes_ccm_auth_update(struct aes_ccm_ctx *ctx, const u8 *ad, size_t len) +{ + size_t partial_len = ctx->partial_len; + bool enc_before = false; + size_t nblocks; + + WARN_ON_ONCE(ctx->ad_padded); + + /* + * We could warn on len > ad_remaining here, but underflow will be + * caught by the != 0 check at the end anyway. (It's a u64, so it isn't + * going to underflow all the way back to 0.) + */ + ctx->ad_remaining -= len; + + if (partial_len) { + size_t n = min(len, AES_BLOCK_SIZE - partial_len); + + crypto_xor(&ctx->mac[partial_len], ad, n); + ad += n; + len -= n; + partial_len += n; + if (partial_len < AES_BLOCK_SIZE) { + ctx->partial_len = partial_len; + return; + } + enc_before = true; + } + + nblocks = len / AES_BLOCK_SIZE; + len %= AES_BLOCK_SIZE; + if (nblocks == 0) { + if (enc_before) + aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); + } else { + aes_cbcmac_blocks(ctx->mac, &ctx->key->aes, ad, nblocks, + enc_before, /* enc_after= */ true); + ad += nblocks * AES_BLOCK_SIZE; + } + crypto_xor(ctx->mac, ad, len); + ctx->partial_len = len; +} +EXPORT_SYMBOL_GPL(aes_ccm_auth_update); + +static __always_inline void aes_ccm_crypt_update(struct aes_ccm_ctx *ctx, + u8 *dst, const u8 *src, + size_t len, bool enc) +{ + size_t partial_len = ctx->partial_len; + size_t n, nblocks; + + if (unlikely(len == 0)) + return; + + WARN_ON_ONCE(ctx->ad_remaining != 0); + + /* + * We could warn on len > data_remaining here, but underflow will be + * caught by the != 0 check at the end anyway. (It's a u64, so it isn't + * going to underflow all the way back to 0.) + */ + ctx->data_remaining -= len; + + if (!ctx->ad_padded) { + ctx->ad_padded = true; + if (partial_len) + aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); + } else if (partial_len) { + /* + * The previous call ended on a non-block-aligned data_len, so + * continue using a previously-generated keystream block. + */ + n = min(len, AES_BLOCK_SIZE - partial_len); + if (enc) + crypto_xor(&ctx->mac[partial_len], src, n); + crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], n); + if (!enc) + crypto_xor(&ctx->mac[partial_len], dst, n); + dst += n; + src += n; + len -= n; + partial_len += n; + if (partial_len < AES_BLOCK_SIZE) { + ctx->partial_len = partial_len; + return; + } + aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); + } + + if (len >= AES_BLOCK_SIZE) { + n = round_down(len, AES_BLOCK_SIZE); + nblocks = len / AES_BLOCK_SIZE; + if (enc) + aes_cbcmac_blocks(ctx->mac, &ctx->key->aes, src, + nblocks, /* enc_before= */ false, + /* enc_after= */ true); + aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); + if (!enc) + aes_cbcmac_blocks(ctx->mac, &ctx->key->aes, dst, + nblocks, /* enc_before= */ false, + /* enc_after= */ true); + dst += n; + src += n; + len -= n; + } + + if (len) { + /* + * Ending on a non-block aligned data_len. Generate the next + * keystream block, use the needed portion of it, and leave it + * cached in ctx->keystream in case this isn't the final call. + */ + aes_encrypt(&ctx->key->aes, ctx->keystream, ctx->ctr); + inc_be128_ctr(ctx->ctr); + if (enc) + crypto_xor(ctx->mac, src, len); + crypto_xor_cpy(dst, src, ctx->keystream, len); + if (!enc) + crypto_xor(ctx->mac, dst, len); + } + ctx->partial_len = len; +} + +void aes_ccm_encrypt_update(struct aes_ccm_ctx *ctx, u8 *dst, const u8 *src, + size_t len) +{ + aes_ccm_crypt_update(ctx, dst, src, len, /* enc= */ true); +} +EXPORT_SYMBOL_GPL(aes_ccm_encrypt_update); + +void aes_ccm_decrypt_update(struct aes_ccm_ctx *ctx, u8 *dst, const u8 *src, + size_t len) +{ + aes_ccm_crypt_update(ctx, dst, src, len, /* enc= */ false); +} +EXPORT_SYMBOL_GPL(aes_ccm_decrypt_update); + +void aes_ccm_encrypt_final(struct aes_ccm_ctx *ctx, u8 *authtag) +{ + WARN_ON_ONCE(ctx->ad_remaining != 0); + WARN_ON_ONCE(ctx->data_remaining != 0); + if (ctx->partial_len) + aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); + crypto_xor_cpy(authtag, ctx->mac, ctx->s0, ctx->key->authtag_len); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(aes_ccm_encrypt_final); + +int aes_ccm_decrypt_final(struct aes_ccm_ctx *ctx, const u8 *authtag) +{ + int err; + + if (WARN_ON_ONCE(ctx->ad_remaining != 0) || + WARN_ON_ONCE(ctx->data_remaining != 0)) { + err = -EBADMSG; + goto out; + } + + if (ctx->partial_len) + aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); + crypto_xor(ctx->mac, ctx->s0, ctx->key->authtag_len); + err = crypto_memneq(ctx->mac, authtag, ctx->key->authtag_len) ? + -EBADMSG : + 0; +out: + memzero_explicit(ctx, sizeof(*ctx)); + return err; +} +EXPORT_SYMBOL_GPL(aes_ccm_decrypt_final); + +int aes_ccm_encrypt(u8 *dst, const u8 *src, size_t data_len, u8 *authtag, + const u8 *ad, size_t ad_len, const u8 *nonce, + size_t nonce_len, const struct aes_ccm_key *key) +{ + struct aes_ccm_ctx ctx; + int err; + + err = aes_ccm_init(&ctx, data_len, ad_len, nonce, nonce_len, key); + if (unlikely(err)) + return err; + aes_ccm_auth_update(&ctx, ad, ad_len); + aes_ccm_encrypt_update(&ctx, dst, src, data_len); + aes_ccm_encrypt_final(&ctx, authtag); + return 0; +} +EXPORT_SYMBOL_GPL(aes_ccm_encrypt); + +int aes_ccm_decrypt(u8 *dst, const u8 *src, size_t data_len, const u8 *authtag, + const u8 *ad, size_t ad_len, const u8 *nonce, + size_t nonce_len, const struct aes_ccm_key *key) +{ + struct aes_ccm_ctx ctx; + int err; + + err = aes_ccm_init(&ctx, data_len, ad_len, nonce, nonce_len, key); + if (unlikely(err)) + return err; + aes_ccm_auth_update(&ctx, ad, ad_len); + aes_ccm_decrypt_update(&ctx, dst, src, data_len); + err = aes_ccm_decrypt_final(&ctx, authtag); + if (unlikely(err) && data_len) { + /* + * Clear the inauthentic decrypted data so that callers won't + * receive it even if they fail to correctly handle errors. + */ + memset(dst, 0, data_len); + } + return err; +} +EXPORT_SYMBOL_GPL(aes_ccm_decrypt); +#endif /* CONFIG_CRYPTO_LIB_AES_CCM */ + static int __init aes_mod_init(void) { #ifdef aes_mod_init_arch diff --git a/lib/crypto/aesgcm.c b/lib/crypto/aesgcm.c deleted file mode 100644 index 1da31e1f747d..000000000000 --- a/lib/crypto/aesgcm.c +++ /dev/null @@ -1,721 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Minimal library implementation of GCM - * - * Copyright 2022 Google LLC - */ - -#include <crypto/gcm.h> -#include <crypto/utils.h> -#include <linux/export.h> -#include <linux/module.h> - -/** - * aesgcm_expandkey - Expands the AES and GHASH keys for the AES-GCM key - * schedule - * - * @ctx: The data structure that will hold the AES-GCM key schedule - * @key: The AES encryption input key - * @keysize: The length in bytes of the input key - * @authsize: The size in bytes of the GCM authentication tag - * - * Returns: 0 on success, or -EINVAL if @keysize or @authsize contain values - * that are not permitted by the GCM specification. - */ -int aesgcm_expandkey(struct aesgcm_ctx *ctx, const u8 *key, - unsigned int keysize, unsigned int authsize) -{ - u8 h[AES_BLOCK_SIZE] = {}; - int ret; - - ret = crypto_gcm_check_authsize(authsize) ?: - aes_prepareenckey(&ctx->aes_key, key, keysize); - if (ret) - return ret; - - ctx->authsize = authsize; - aes_encrypt(&ctx->aes_key, h, h); - ghash_preparekey(&ctx->ghash_key, h); - memzero_explicit(h, sizeof(h)); - return 0; -} -EXPORT_SYMBOL(aesgcm_expandkey); - -/** - * aesgcm_mac - Generates the authentication tag using AES-GCM algorithm. - * @ctx: The data structure that will hold the AES-GCM key schedule - * @src: The input source data. - * @src_len: Length of the source data. - * @assoc: Points to the associated data. - * @assoc_len: Length of the associated data values. - * @ctr: Points to the counter value. - * @authtag: The output buffer for the authentication tag. - * - * It takes in the AES-GCM context, source data, associated data, counter value, - * and an output buffer for the authentication tag. - */ -static void aesgcm_mac(const struct aesgcm_ctx *ctx, const u8 *src, int src_len, - const u8 *assoc, int assoc_len, __be32 *ctr, u8 *authtag) -{ - static const u8 zeroes[GHASH_BLOCK_SIZE]; - __be64 tail[2] = { - cpu_to_be64((u64)assoc_len * 8), - cpu_to_be64((u64)src_len * 8), - }; - struct ghash_ctx ghash; - u8 ghash_out[AES_BLOCK_SIZE]; - u8 enc_ctr[AES_BLOCK_SIZE]; - - ghash_init(&ghash, &ctx->ghash_key); - - ghash_update(&ghash, assoc, assoc_len); - ghash_update(&ghash, zeroes, -assoc_len & (GHASH_BLOCK_SIZE - 1)); - - ghash_update(&ghash, src, src_len); - ghash_update(&ghash, zeroes, -src_len & (GHASH_BLOCK_SIZE - 1)); - - ghash_update(&ghash, (const u8 *)&tail, sizeof(tail)); - - ghash_final(&ghash, ghash_out); - - ctr[3] = cpu_to_be32(1); - aes_encrypt(&ctx->aes_key, enc_ctr, (const u8 *)ctr); - crypto_xor_cpy(authtag, ghash_out, enc_ctr, ctx->authsize); - - memzero_explicit(ghash_out, sizeof(ghash_out)); - memzero_explicit(enc_ctr, sizeof(enc_ctr)); -} - -static void aesgcm_crypt(const struct aesgcm_ctx *ctx, u8 *dst, const u8 *src, - int len, __be32 *ctr) -{ - u8 buf[AES_BLOCK_SIZE]; - unsigned int n = 2; - - while (len > 0) { - /* - * The counter increment below must not result in overflow or - * carry into the next 32-bit word, as this could result in - * inadvertent IV reuse, which must be avoided at all cost for - * stream ciphers such as AES-CTR. Given the range of 'int - * len', this cannot happen, so no explicit test is necessary. - */ - ctr[3] = cpu_to_be32(n++); - aes_encrypt(&ctx->aes_key, buf, (const u8 *)ctr); - crypto_xor_cpy(dst, src, buf, min(len, AES_BLOCK_SIZE)); - - dst += AES_BLOCK_SIZE; - src += AES_BLOCK_SIZE; - len -= AES_BLOCK_SIZE; - } - memzero_explicit(buf, sizeof(buf)); -} - -/** - * aesgcm_encrypt - Perform AES-GCM encryption on a block of data - * - * @ctx: The AES-GCM key schedule - * @dst: Pointer to the ciphertext output buffer - * @src: Pointer the plaintext (may equal @dst for encryption in place) - * @crypt_len: The size in bytes of the plaintext and ciphertext. - * @assoc: Pointer to the associated data, - * @assoc_len: The size in bytes of the associated data - * @iv: The initialization vector (IV) to use for this block of data - * (must be 12 bytes in size as per the GCM spec recommendation) - * @authtag: The address of the buffer in memory where the authentication - * tag should be stored. The buffer is assumed to have space for - * @ctx->authsize bytes. - */ -void aesgcm_encrypt(const struct aesgcm_ctx *ctx, u8 *dst, const u8 *src, - int crypt_len, const u8 *assoc, int assoc_len, - const u8 iv[GCM_AES_IV_SIZE], u8 *authtag) -{ - __be32 ctr[4]; - - memcpy(ctr, iv, GCM_AES_IV_SIZE); - - aesgcm_crypt(ctx, dst, src, crypt_len, ctr); - aesgcm_mac(ctx, dst, crypt_len, assoc, assoc_len, ctr, authtag); -} -EXPORT_SYMBOL(aesgcm_encrypt); - -/** - * aesgcm_decrypt - Perform AES-GCM decryption on a block of data - * - * @ctx: The AES-GCM key schedule - * @dst: Pointer to the plaintext output buffer - * @src: Pointer the ciphertext (may equal @dst for decryption in place) - * @crypt_len: The size in bytes of the plaintext and ciphertext. - * @assoc: Pointer to the associated data, - * @assoc_len: The size in bytes of the associated data - * @iv: The initialization vector (IV) to use for this block of data - * (must be 12 bytes in size as per the GCM spec recommendation) - * @authtag: The address of the buffer in memory where the authentication - * tag is stored. - * - * Returns: true on success, or false if the ciphertext failed authentication. - * On failure, no plaintext will be returned. - */ -bool __must_check aesgcm_decrypt(const struct aesgcm_ctx *ctx, u8 *dst, - const u8 *src, int crypt_len, const u8 *assoc, - int assoc_len, const u8 iv[GCM_AES_IV_SIZE], - const u8 *authtag) -{ - u8 tagbuf[AES_BLOCK_SIZE]; - __be32 ctr[4]; - - memcpy(ctr, iv, GCM_AES_IV_SIZE); - - aesgcm_mac(ctx, src, crypt_len, assoc, assoc_len, ctr, tagbuf); - if (crypto_memneq(authtag, tagbuf, ctx->authsize)) { - memzero_explicit(tagbuf, sizeof(tagbuf)); - return false; - } - aesgcm_crypt(ctx, dst, src, crypt_len, ctr); - return true; -} -EXPORT_SYMBOL(aesgcm_decrypt); - -MODULE_DESCRIPTION("Generic AES-GCM library"); -MODULE_AUTHOR("Ard Biesheuvel <ardb@kernel.org>"); -MODULE_LICENSE("GPL"); - -#ifdef CONFIG_CRYPTO_SELFTESTS - -/* - * Test code below. Vectors taken from crypto/testmgr.h - */ - -static const u8 __initconst ctext0[16] __nonstring = - "\x58\xe2\xfc\xce\xfa\x7e\x30\x61" - "\x36\x7f\x1d\x57\xa4\xe7\x45\x5a"; - -static const u8 __initconst ptext1[16]; - -static const u8 __initconst ctext1[32] __nonstring = - "\x03\x88\xda\xce\x60\xb6\xa3\x92" - "\xf3\x28\xc2\xb9\x71\xb2\xfe\x78" - "\xab\x6e\x47\xd4\x2c\xec\x13\xbd" - "\xf5\x3a\x67\xb2\x12\x57\xbd\xdf"; - -static const u8 __initconst ptext2[64] __nonstring = - "\xd9\x31\x32\x25\xf8\x84\x06\xe5" - "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" - "\x86\xa7\xa9\x53\x15\x34\xf7\xda" - "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" - "\x1c\x3c\x0c\x95\x95\x68\x09\x53" - "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" - "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" - "\xba\x63\x7b\x39\x1a\xaf\xd2\x55"; - -static const u8 __initconst ctext2[80] __nonstring = - "\x42\x83\x1e\xc2\x21\x77\x74\x24" - "\x4b\x72\x21\xb7\x84\xd0\xd4\x9c" - "\xe3\xaa\x21\x2f\x2c\x02\xa4\xe0" - "\x35\xc1\x7e\x23\x29\xac\xa1\x2e" - "\x21\xd5\x14\xb2\x54\x66\x93\x1c" - "\x7d\x8f\x6a\x5a\xac\x84\xaa\x05" - "\x1b\xa3\x0b\x39\x6a\x0a\xac\x97" - "\x3d\x58\xe0\x91\x47\x3f\x59\x85" - "\x4d\x5c\x2a\xf3\x27\xcd\x64\xa6" - "\x2c\xf3\x5a\xbd\x2b\xa6\xfa\xb4"; - -static const u8 __initconst ptext3[60] __nonstring = - "\xd9\x31\x32\x25\xf8\x84\x06\xe5" - "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" - "\x86\xa7\xa9\x53\x15\x34\xf7\xda" - "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" - "\x1c\x3c\x0c\x95\x95\x68\x09\x53" - "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" - "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" - "\xba\x63\x7b\x39"; - -static const u8 __initconst ctext3[76] __nonstring = - "\x42\x83\x1e\xc2\x21\x77\x74\x24" - "\x4b\x72\x21\xb7\x84\xd0\xd4\x9c" - "\xe3\xaa\x21\x2f\x2c\x02\xa4\xe0" - "\x35\xc1\x7e\x23\x29\xac\xa1\x2e" - "\x21\xd5\x14\xb2\x54\x66\x93\x1c" - "\x7d\x8f\x6a\x5a\xac\x84\xaa\x05" - "\x1b\xa3\x0b\x39\x6a\x0a\xac\x97" - "\x3d\x58\xe0\x91" - "\x5b\xc9\x4f\xbc\x32\x21\xa5\xdb" - "\x94\xfa\xe9\x5a\xe7\x12\x1a\x47"; - -static const u8 __initconst ctext4[16] __nonstring = - "\xcd\x33\xb2\x8a\xc7\x73\xf7\x4b" - "\xa0\x0e\xd1\xf3\x12\x57\x24\x35"; - -static const u8 __initconst ctext5[32] __nonstring = - "\x98\xe7\x24\x7c\x07\xf0\xfe\x41" - "\x1c\x26\x7e\x43\x84\xb0\xf6\x00" - "\x2f\xf5\x8d\x80\x03\x39\x27\xab" - "\x8e\xf4\xd4\x58\x75\x14\xf0\xfb"; - -static const u8 __initconst ptext6[64] __nonstring = - "\xd9\x31\x32\x25\xf8\x84\x06\xe5" - "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" - "\x86\xa7\xa9\x53\x15\x34\xf7\xda" - "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" - "\x1c\x3c\x0c\x95\x95\x68\x09\x53" - "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" - "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" - "\xba\x63\x7b\x39\x1a\xaf\xd2\x55"; - -static const u8 __initconst ctext6[80] __nonstring = - "\x39\x80\xca\x0b\x3c\x00\xe8\x41" - "\xeb\x06\xfa\xc4\x87\x2a\x27\x57" - "\x85\x9e\x1c\xea\xa6\xef\xd9\x84" - "\x62\x85\x93\xb4\x0c\xa1\xe1\x9c" - "\x7d\x77\x3d\x00\xc1\x44\xc5\x25" - "\xac\x61\x9d\x18\xc8\x4a\x3f\x47" - "\x18\xe2\x44\x8b\x2f\xe3\x24\xd9" - "\xcc\xda\x27\x10\xac\xad\xe2\x56" - "\x99\x24\xa7\xc8\x58\x73\x36\xbf" - "\xb1\x18\x02\x4d\xb8\x67\x4a\x14"; - -static const u8 __initconst ctext7[16] __nonstring = - "\x53\x0f\x8a\xfb\xc7\x45\x36\xb9" - "\xa9\x63\xb4\xf1\xc4\xcb\x73\x8b"; - -static const u8 __initconst ctext8[32] __nonstring = - "\xce\xa7\x40\x3d\x4d\x60\x6b\x6e" - "\x07\x4e\xc5\xd3\xba\xf3\x9d\x18" - "\xd0\xd1\xc8\xa7\x99\x99\x6b\xf0" - "\x26\x5b\x98\xb5\xd4\x8a\xb9\x19"; - -static const u8 __initconst ptext9[64] __nonstring = - "\xd9\x31\x32\x25\xf8\x84\x06\xe5" - "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" - "\x86\xa7\xa9\x53\x15\x34\xf7\xda" - "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" - "\x1c\x3c\x0c\x95\x95\x68\x09\x53" - "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" - "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" - "\xba\x63\x7b\x39\x1a\xaf\xd2\x55"; - -static const u8 __initconst ctext9[80] __nonstring = - "\x52\x2d\xc1\xf0\x99\x56\x7d\x07" - "\xf4\x7f\x37\xa3\x2a\x84\x42\x7d" - "\x64\x3a\x8c\xdc\xbf\xe5\xc0\xc9" - "\x75\x98\xa2\xbd\x25\x55\xd1\xaa" - "\x8c\xb0\x8e\x48\x59\x0d\xbb\x3d" - "\xa7\xb0\x8b\x10\x56\x82\x88\x38" - "\xc5\xf6\x1e\x63\x93\xba\x7a\x0a" - "\xbc\xc9\xf6\x62\x89\x80\x15\xad" - "\xb0\x94\xda\xc5\xd9\x34\x71\xbd" - "\xec\x1a\x50\x22\x70\xe3\xcc\x6c"; - -static const u8 __initconst ptext10[60] __nonstring = - "\xd9\x31\x32\x25\xf8\x84\x06\xe5" - "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" - "\x86\xa7\xa9\x53\x15\x34\xf7\xda" - "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" - "\x1c\x3c\x0c\x95\x95\x68\x09\x53" - "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" - "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" - "\xba\x63\x7b\x39"; - -static const u8 __initconst ctext10[76] __nonstring = - "\x52\x2d\xc1\xf0\x99\x56\x7d\x07" - "\xf4\x7f\x37\xa3\x2a\x84\x42\x7d" - "\x64\x3a\x8c\xdc\xbf\xe5\xc0\xc9" - "\x75\x98\xa2\xbd\x25\x55\xd1\xaa" - "\x8c\xb0\x8e\x48\x59\x0d\xbb\x3d" - "\xa7\xb0\x8b\x10\x56\x82\x88\x38" - "\xc5\xf6\x1e\x63\x93\xba\x7a\x0a" - "\xbc\xc9\xf6\x62" - "\x76\xfc\x6e\xce\x0f\x4e\x17\x68" - "\xcd\xdf\x88\x53\xbb\x2d\x55\x1b"; - -static const u8 __initconst ptext11[60] __nonstring = - "\xd9\x31\x32\x25\xf8\x84\x06\xe5" - "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" - "\x86\xa7\xa9\x53\x15\x34\xf7\xda" - "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" - "\x1c\x3c\x0c\x95\x95\x68\x09\x53" - "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" - "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" - "\xba\x63\x7b\x39"; - -static const u8 __initconst ctext11[76] __nonstring = - "\x39\x80\xca\x0b\x3c\x00\xe8\x41" - "\xeb\x06\xfa\xc4\x87\x2a\x27\x57" - "\x85\x9e\x1c\xea\xa6\xef\xd9\x84" - "\x62\x85\x93\xb4\x0c\xa1\xe1\x9c" - "\x7d\x77\x3d\x00\xc1\x44\xc5\x25" - "\xac\x61\x9d\x18\xc8\x4a\x3f\x47" - "\x18\xe2\x44\x8b\x2f\xe3\x24\xd9" - "\xcc\xda\x27\x10" - "\x25\x19\x49\x8e\x80\xf1\x47\x8f" - "\x37\xba\x55\xbd\x6d\x27\x61\x8c"; - -static const u8 __initconst ptext12[719] __nonstring = - "\x42\xc1\xcc\x08\x48\x6f\x41\x3f" - "\x2f\x11\x66\x8b\x2a\x16\xf0\xe0" - "\x58\x83\xf0\xc3\x70\x14\xc0\x5b" - "\x3f\xec\x1d\x25\x3c\x51\xd2\x03" - "\xcf\x59\x74\x1f\xb2\x85\xb4\x07" - "\xc6\x6a\x63\x39\x8a\x5b\xde\xcb" - "\xaf\x08\x44\xbd\x6f\x91\x15\xe1" - "\xf5\x7a\x6e\x18\xbd\xdd\x61\x50" - "\x59\xa9\x97\xab\xbb\x0e\x74\x5c" - "\x00\xa4\x43\x54\x04\x54\x9b\x3b" - "\x77\xec\xfd\x5c\xa6\xe8\x7b\x08" - "\xae\xe6\x10\x3f\x32\x65\xd1\xfc" - "\xa4\x1d\x2c\x31\xfb\x33\x7a\xb3" - "\x35\x23\xf4\x20\x41\xd4\xad\x82" - "\x8b\xa4\xad\x96\x1c\x20\x53\xbe" - "\x0e\xa6\xf4\xdc\x78\x49\x3e\x72" - "\xb1\xa9\xb5\x83\xcb\x08\x54\xb7" - "\xad\x49\x3a\xae\x98\xce\xa6\x66" - "\x10\x30\x90\x8c\x55\x83\xd7\x7c" - "\x8b\xe6\x53\xde\xd2\x6e\x18\x21" - "\x01\x52\xd1\x9f\x9d\xbb\x9c\x73" - "\x57\xcc\x89\x09\x75\x9b\x78\x70" - "\xed\x26\x97\x4d\xb4\xe4\x0c\xa5" - "\xfa\x70\x04\x70\xc6\x96\x1c\x7d" - "\x54\x41\x77\xa8\xe3\xb0\x7e\x96" - "\x82\xd9\xec\xa2\x87\x68\x55\xf9" - "\x8f\x9e\x73\x43\x47\x6a\x08\x36" - "\x93\x67\xa8\x2d\xde\xac\x41\xa9" - "\x5c\x4d\x73\x97\x0f\x70\x68\xfa" - "\x56\x4d\x00\xc2\x3b\x1f\xc8\xb9" - "\x78\x1f\x51\x07\xe3\x9a\x13\x4e" - "\xed\x2b\x2e\xa3\xf7\x44\xb2\xe7" - "\xab\x19\x37\xd9\xba\x76\x5e\xd2" - "\xf2\x53\x15\x17\x4c\x6b\x16\x9f" - "\x02\x66\x49\xca\x7c\x91\x05\xf2" - "\x45\x36\x1e\xf5\x77\xad\x1f\x46" - "\xa8\x13\xfb\x63\xb6\x08\x99\x63" - "\x82\xa2\xed\xb3\xac\xdf\x43\x19" - "\x45\xea\x78\x73\xd9\xb7\x39\x11" - "\xa3\x13\x7c\xf8\x3f\xf7\xad\x81" - "\x48\x2f\xa9\x5c\x5f\xa0\xf0\x79" - "\xa4\x47\x7d\x80\x20\x26\xfd\x63" - "\x0a\xc7\x7e\x6d\x75\x47\xff\x76" - "\x66\x2e\x8a\x6c\x81\x35\xaf\x0b" - "\x2e\x6a\x49\x60\xc1\x10\xe1\xe1" - "\x54\x03\xa4\x09\x0c\x37\x7a\x15" - "\x23\x27\x5b\x8b\x4b\xa5\x64\x97" - "\xae\x4a\x50\x73\x1f\x66\x1c\x5c" - "\x03\x25\x3c\x8d\x48\x58\x71\x34" - "\x0e\xec\x4e\x55\x1a\x03\x6a\xe5" - "\xb6\x19\x2b\x84\x2a\x20\xd1\xea" - "\x80\x6f\x96\x0e\x05\x62\xc7\x78" - "\x87\x79\x60\x38\x46\xb4\x25\x57" - "\x6e\x16\x63\xf8\xad\x6e\xd7\x42" - "\x69\xe1\x88\xef\x6e\xd5\xb4\x9a" - "\x3c\x78\x6c\x3b\xe5\xa0\x1d\x22" - "\x86\x5c\x74\x3a\xeb\x24\x26\xc7" - "\x09\xfc\x91\x96\x47\x87\x4f\x1a" - "\xd6\x6b\x2c\x18\x47\xc0\xb8\x24" - "\xa8\x5a\x4a\x9e\xcb\x03\xe7\x2a" - "\x09\xe6\x4d\x9c\x6d\x86\x60\xf5" - "\x2f\x48\x69\x37\x9f\xf2\xd2\xcb" - "\x0e\x5a\xdd\x6e\x8a\xfb\x6a\xfe" - "\x0b\x63\xde\x87\x42\x79\x8a\x68" - "\x51\x28\x9b\x7a\xeb\xaf\xb8\x2f" - "\x9d\xd1\xc7\x45\x90\x08\xc9\x83" - "\xe9\x83\x84\xcb\x28\x69\x09\x69" - "\xce\x99\x46\x00\x54\xcb\xd8\x38" - "\xf9\x53\x4a\xbf\x31\xce\x57\x15" - "\x33\xfa\x96\x04\x33\x42\xe3\xc0" - "\xb7\x54\x4a\x65\x7a\x7c\x02\xe6" - "\x19\x95\xd0\x0e\x82\x07\x63\xf9" - "\xe1\x2b\x2a\xfc\x55\x92\x52\xc9" - "\xb5\x9f\x23\x28\x60\xe7\x20\x51" - "\x10\xd3\xed\x6d\x9b\xab\xb8\xe2" - "\x5d\x9a\x34\xb3\xbe\x9c\x64\xcb" - "\x78\xc6\x91\x22\x40\x91\x80\xbe" - "\xd7\x78\x5c\x0e\x0a\xdc\x08\xe9" - "\x67\x10\xa4\x83\x98\x79\x23\xe7" - "\x92\xda\xa9\x22\x16\xb1\xe7\x78" - "\xa3\x1c\x6c\x8f\x35\x7c\x4d\x37" - "\x2f\x6e\x0b\x50\x5c\x34\xb9\xf9" - "\xe6\x3d\x91\x0d\x32\x95\xaa\x3d" - "\x48\x11\x06\xbb\x2d\xf2\x63\x88" - "\x3f\x73\x09\xe2\x45\x56\x31\x51" - "\xfa\x5e\x4e\x62\xf7\x90\xf9\xa9" - "\x7d\x7b\x1b\xb1\xc8\x26\x6e\x66" - "\xf6\x90\x9a\x7f\xf2\x57\xcc\x23" - "\x59\xfa\xfa\xaa\x44\x04\x01\xa7" - "\xa4\x78\xdb\x74\x3d\x8b\xb5"; - -static const u8 __initconst ctext12[735] __nonstring = - "\x84\x0b\xdb\xd5\xb7\xa8\xfe\x20" - "\xbb\xb1\x12\x7f\x41\xea\xb3\xc0" - "\xa2\xb4\x37\x19\x11\x58\xb6\x0b" - "\x4c\x1d\x38\x05\x54\xd1\x16\x73" - "\x8e\x1c\x20\x90\xa2\x9a\xb7\x74" - "\x47\xe6\xd8\xfc\x18\x3a\xb4\xea" - "\xd5\x16\x5a\x2c\x53\x01\x46\xb3" - "\x18\x33\x74\x6c\x50\xf2\xe8\xc0" - "\x73\xda\x60\x22\xeb\xe3\xe5\x9b" - "\x20\x93\x6c\x4b\x37\x99\xb8\x23" - "\x3b\x4e\xac\xe8\x5b\xe8\x0f\xb7" - "\xc3\x8f\xfb\x4a\x37\xd9\x39\x95" - "\x34\xf1\xdb\x8f\x71\xd9\xc7\x0b" - "\x02\xf1\x63\xfc\x9b\xfc\xc5\xab" - "\xb9\x14\x13\x21\xdf\xce\xaa\x88" - "\x44\x30\x1e\xce\x26\x01\x92\xf8" - "\x9f\x00\x4b\x0c\x4b\xf7\x5f\xe0" - "\x89\xca\x94\x66\x11\x21\x97\xca" - "\x3e\x83\x74\x2d\xdb\x4d\x11\xeb" - "\x97\xc2\x14\xff\x9e\x1e\xa0\x6b" - "\x08\xb4\x31\x2b\x85\xc6\x85\x6c" - "\x90\xec\x39\xc0\xec\xb3\xb5\x4e" - "\xf3\x9c\xe7\x83\x3a\x77\x0a\xf4" - "\x56\xfe\xce\x18\x33\x6d\x0b\x2d" - "\x33\xda\xc8\x05\x5c\xb4\x09\x2a" - "\xde\x6b\x52\x98\x01\xef\x36\x3d" - "\xbd\xf9\x8f\xa8\x3e\xaa\xcd\xd1" - "\x01\x2d\x42\x49\xc3\xb6\x84\xbb" - "\x48\x96\xe0\x90\x93\x6c\x48\x64" - "\xd4\xfa\x7f\x93\x2c\xa6\x21\xc8" - "\x7a\x23\x7b\xaa\x20\x56\x12\xae" - "\x16\x9d\x94\x0f\x54\xa1\xec\xca" - "\x51\x4e\xf2\x39\xf4\xf8\x5f\x04" - "\x5a\x0d\xbf\xf5\x83\xa1\x15\xe1" - "\xf5\x3c\xd8\x62\xa3\xed\x47\x89" - "\x85\x4c\xe5\xdb\xac\x9e\x17\x1d" - "\x0c\x09\xe3\x3e\x39\x5b\x4d\x74" - "\x0e\xf5\x34\xee\x70\x11\x4c\xfd" - "\xdb\x34\xb1\xb5\x10\x3f\x73\xb7" - "\xf5\xfa\xed\xb0\x1f\xa5\xcd\x3c" - "\x8d\x35\x83\xd4\x11\x44\x6e\x6c" - "\x5b\xe0\x0e\x69\xa5\x39\xe5\xbb" - "\xa9\x57\x24\x37\xe6\x1f\xdd\xcf" - "\x16\x2a\x13\xf9\x6a\x2d\x90\xa0" - "\x03\x60\x7a\xed\x69\xd5\x00\x8b" - "\x7e\x4f\xcb\xb9\xfa\x91\xb9\x37" - "\xc1\x26\xce\x90\x97\x22\x64\x64" - "\xc1\x72\x43\x1b\xf6\xac\xc1\x54" - "\x8a\x10\x9c\xdd\x8d\xd5\x8e\xb2" - "\xe4\x85\xda\xe0\x20\x5f\xf4\xb4" - "\x15\xb5\xa0\x8d\x12\x74\x49\x23" - "\x3a\xdf\x4a\xd3\xf0\x3b\x89\xeb" - "\xf8\xcc\x62\x7b\xfb\x93\x07\x41" - "\x61\x26\x94\x58\x70\xa6\x3c\xe4" - "\xff\x58\xc4\x13\x3d\xcb\x36\x6b" - "\x32\xe5\xb2\x6d\x03\x74\x6f\x76" - "\x93\x77\xde\x48\xc4\xfa\x30\x4a" - "\xda\x49\x80\x77\x0f\x1c\xbe\x11" - "\xc8\x48\xb1\xe5\xbb\xf2\x8a\xe1" - "\x96\x2f\x9f\xd1\x8e\x8a\x5c\xe2" - "\xf7\xd7\xd8\x54\xf3\x3f\xc4\x91" - "\xb8\xfb\x86\xdc\x46\x24\x91\x60" - "\x6c\x2f\xc9\x41\x37\x51\x49\x54" - "\x09\x81\x21\xf3\x03\x9f\x2b\xe3" - "\x1f\x39\x63\xaf\xf4\xd7\x53\x60" - "\xa7\xc7\x54\xf9\xee\xb1\xb1\x7d" - "\x75\x54\x65\x93\xfe\xb1\x68\x6b" - "\x57\x02\xf9\xbb\x0e\xf9\xf8\xbf" - "\x01\x12\x27\xb4\xfe\xe4\x79\x7a" - "\x40\x5b\x51\x4b\xdf\x38\xec\xb1" - "\x6a\x56\xff\x35\x4d\x42\x33\xaa" - "\x6f\x1b\xe4\xdc\xe0\xdb\x85\x35" - "\x62\x10\xd4\xec\xeb\xc5\x7e\x45" - "\x1c\x6f\x17\xca\x3b\x8e\x2d\x66" - "\x4f\x4b\x36\x56\xcd\x1b\x59\xaa" - "\xd2\x9b\x17\xb9\x58\xdf\x7b\x64" - "\x8a\xff\x3b\x9c\xa6\xb5\x48\x9e" - "\xaa\xe2\x5d\x09\x71\x32\x5f\xb6" - "\x29\xbe\xe7\xc7\x52\x7e\x91\x82" - "\x6b\x6d\x33\xe1\x34\x06\x36\x21" - "\x5e\xbe\x1e\x2f\x3e\xc1\xfb\xea" - "\x49\x2c\xb5\xca\xf7\xb0\x37\xea" - "\x1f\xed\x10\x04\xd9\x48\x0d\x1a" - "\x1c\xfb\xe7\x84\x0e\x83\x53\x74" - "\xc7\x65\xe2\x5c\xe5\xba\x73\x4c" - "\x0e\xe1\xb5\x11\x45\x61\x43\x46" - "\xaa\x25\x8f\xbd\x85\x08\xfa\x4c" - "\x15\xc1\xc0\xd8\xf5\xdc\x16\xbb" - "\x7b\x1d\xe3\x87\x57\xa7\x2a\x1d" - "\x38\x58\x9e\x8a\x43\xdc\x57" - "\xd1\x81\x7d\x2b\xe9\xff\x99\x3a" - "\x4b\x24\x52\x58\x55\xe1\x49\x14"; - -static struct { - const u8 *ptext; - const u8 *ctext; - - u8 key[AES_MAX_KEY_SIZE] __nonstring; - u8 iv[GCM_AES_IV_SIZE] __nonstring; - u8 assoc[20] __nonstring; - - int klen; - int clen; - int plen; - int alen; -} const aesgcm_tv[] __initconst = { - { /* From McGrew & Viega - http://citeseer.ist.psu.edu/656989.html */ - .klen = 16, - .ctext = ctext0, - .clen = sizeof(ctext0), - }, { - .klen = 16, - .ptext = ptext1, - .plen = sizeof(ptext1), - .ctext = ctext1, - .clen = sizeof(ctext1), - }, { - .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08", - .klen = 16, - .iv = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" - "\xde\xca\xf8\x88", - .ptext = ptext2, - .plen = sizeof(ptext2), - .ctext = ctext2, - .clen = sizeof(ctext2), - }, { - .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08", - .klen = 16, - .iv = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" - "\xde\xca\xf8\x88", - .ptext = ptext3, - .plen = sizeof(ptext3), - .assoc = "\xfe\xed\xfa\xce\xde\xad\xbe\xef" - "\xfe\xed\xfa\xce\xde\xad\xbe\xef" - "\xab\xad\xda\xd2", - .alen = 20, - .ctext = ctext3, - .clen = sizeof(ctext3), - }, { - .klen = 24, - .ctext = ctext4, - .clen = sizeof(ctext4), - }, { - .klen = 24, - .ptext = ptext1, - .plen = sizeof(ptext1), - .ctext = ctext5, - .clen = sizeof(ctext5), - }, { - .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\xfe\xff\xe9\x92\x86\x65\x73\x1c", - .klen = 24, - .iv = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" - "\xde\xca\xf8\x88", - .ptext = ptext6, - .plen = sizeof(ptext6), - .ctext = ctext6, - .clen = sizeof(ctext6), - }, { - .klen = 32, - .ctext = ctext7, - .clen = sizeof(ctext7), - }, { - .klen = 32, - .ptext = ptext1, - .plen = sizeof(ptext1), - .ctext = ctext8, - .clen = sizeof(ctext8), - }, { - .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08", - .klen = 32, - .iv = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" - "\xde\xca\xf8\x88", - .ptext = ptext9, - .plen = sizeof(ptext9), - .ctext = ctext9, - .clen = sizeof(ctext9), - }, { - .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08", - .klen = 32, - .iv = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" - "\xde\xca\xf8\x88", - .ptext = ptext10, - .plen = sizeof(ptext10), - .assoc = "\xfe\xed\xfa\xce\xde\xad\xbe\xef" - "\xfe\xed\xfa\xce\xde\xad\xbe\xef" - "\xab\xad\xda\xd2", - .alen = 20, - .ctext = ctext10, - .clen = sizeof(ctext10), - }, { - .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\xfe\xff\xe9\x92\x86\x65\x73\x1c", - .klen = 24, - .iv = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" - "\xde\xca\xf8\x88", - .ptext = ptext11, - .plen = sizeof(ptext11), - .assoc = "\xfe\xed\xfa\xce\xde\xad\xbe\xef" - "\xfe\xed\xfa\xce\xde\xad\xbe\xef" - "\xab\xad\xda\xd2", - .alen = 20, - .ctext = ctext11, - .clen = sizeof(ctext11), - }, { - .key = "\x62\x35\xf8\x95\xfc\xa5\xeb\xf6" - "\x0e\x92\x12\x04\xd3\xa1\x3f\x2e" - "\x8b\x32\xcf\xe7\x44\xed\x13\x59" - "\x04\x38\x77\xb0\xb9\xad\xb4\x38", - .klen = 32, - .iv = "\x00\xff\xff\xff\xff\x00\x00\xff" - "\xff\xff\x00\xff", - .ptext = ptext12, - .plen = sizeof(ptext12), - .ctext = ctext12, - .clen = sizeof(ctext12), - } -}; - -static int __init libaesgcm_init(void) -{ - for (int i = 0; i < ARRAY_SIZE(aesgcm_tv); i++) { - u8 tagbuf[AES_BLOCK_SIZE]; - int plen = aesgcm_tv[i].plen; - struct aesgcm_ctx ctx; - static u8 buf[sizeof(ptext12)]; - - if (aesgcm_expandkey(&ctx, aesgcm_tv[i].key, aesgcm_tv[i].klen, - aesgcm_tv[i].clen - plen)) { - pr_err("aesgcm_expandkey() failed on vector %d\n", i); - return -ENODEV; - } - - if (!aesgcm_decrypt(&ctx, buf, aesgcm_tv[i].ctext, plen, - aesgcm_tv[i].assoc, aesgcm_tv[i].alen, - aesgcm_tv[i].iv, aesgcm_tv[i].ctext + plen) - || memcmp(buf, aesgcm_tv[i].ptext, plen)) { - pr_err("aesgcm_decrypt() #1 failed on vector %d\n", i); - return -ENODEV; - } - - /* encrypt in place */ - aesgcm_encrypt(&ctx, buf, buf, plen, aesgcm_tv[i].assoc, - aesgcm_tv[i].alen, aesgcm_tv[i].iv, tagbuf); - if (memcmp(buf, aesgcm_tv[i].ctext, plen)) { - pr_err("aesgcm_encrypt() failed on vector %d\n", i); - return -ENODEV; - } - - /* decrypt in place */ - if (!aesgcm_decrypt(&ctx, buf, buf, plen, aesgcm_tv[i].assoc, - aesgcm_tv[i].alen, aesgcm_tv[i].iv, tagbuf) - || memcmp(buf, aesgcm_tv[i].ptext, plen)) { - pr_err("aesgcm_decrypt() #2 failed on vector %d\n", i); - return -ENODEV; - } - } - return 0; -} -module_init(libaesgcm_init); - -static void __exit libaesgcm_exit(void) -{ -} -module_exit(libaesgcm_exit); -#endif diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig index 9409c1a935c3..bc084dde424f 100644 --- a/lib/crypto/tests/Kconfig +++ b/lib/crypto/tests/Kconfig @@ -144,7 +144,13 @@ config CRYPTO_LIB_SM3_KUNIT_TEST config CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT tristate "Enable all crypto library code for KUnit tests" depends on KUNIT + select CRYPTO_LIB_AES_CBC select CRYPTO_LIB_AES_CBC_MACS + select CRYPTO_LIB_AES_CCM + select CRYPTO_LIB_AES_CTR + select CRYPTO_LIB_AES_ECB + select CRYPTO_LIB_AES_GCM + select CRYPTO_LIB_AES_XTS select CRYPTO_LIB_BLAKE2B select CRYPTO_LIB_CHACHA20POLY1305 select CRYPTO_LIB_CURVE25519 diff --git a/net/bluetooth/smp.c b/net/bluetooth/smp.c index c4470958b0d5..4a32e4f80b48 100644 --- a/net/bluetooth/smp.c +++ b/net/bluetooth/smp.c @@ -164,7 +164,7 @@ static inline void swap_buf(const u8 *src, u8 *dst, size_t len) static int smp_aes_cmac(const u8 k[16], const u8 *m, size_t len, u8 mac[16]) { uint8_t tmp[16], mac_msb[16], msg_msb[CMAC_MSG_MAX]; - struct aes_cmac_key key; + struct aes_cmac_key key __cleanup(aes_cmac_zeroize_key); int err; if (len > CMAC_MSG_MAX) @@ -178,6 +178,7 @@ static int smp_aes_cmac(const u8 k[16], const u8 *m, size_t len, u8 mac[16]) SMP_DBG("key %16phN", k); err = aes_cmac_preparekey(&key, tmp, 16); + memzero_explicit(tmp, sizeof(tmp)); if (WARN_ON_ONCE(err)) /* Should never happen, as 16 is valid keylen */ return err; aes_cmac(&key, msg_msb, len, mac_msb); diff --git a/net/mac80211/fils_aead.c b/net/mac80211/fils_aead.c index d2f4a17eab99..293590976489 100644 --- a/net/mac80211/fils_aead.c +++ b/net/mac80211/fils_aead.c @@ -24,7 +24,7 @@ static int aes_s2v(const u8 *in_key, size_t key_len, size_t num_elem, const u8 *addr[], size_t len[], u8 *v) { u8 d[AES_BLOCK_SIZE], tmp[AES_BLOCK_SIZE] = {}; - struct aes_cmac_key key; + struct aes_cmac_key key __cleanup(aes_cmac_zeroize_key); struct aes_cmac_ctx ctx; size_t i; int res; @@ -62,7 +62,6 @@ static int aes_s2v(const u8 *in_key, size_t key_len, aes_cmac_update(&ctx, d, AES_BLOCK_SIZE); aes_cmac_final(&ctx, v); - memzero_explicit(&key, sizeof(key)); return 0; } |