From 7b930def8ef6224f19e96f100993ac7ba3c44223 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 31 May 2019 10:13:06 +0200 Subject: crypto: caam - limit output IV to CBC to work around CTR mode DMA issue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit ed527b13d800dd515a9e6c582f0a73eca65b2e1b upstream. The CAAM driver currently violates an undocumented and slightly controversial requirement imposed by the crypto stack that a buffer referred to by the request structure via its virtual address may not be modified while any scatterlists passed via the same request structure are mapped for inbound DMA. This may result in errors like alg: aead: decryption failed on test 1 for gcm_base(ctr-aes-caam,ghash-generic): ret=74 alg: aead: Failed to load transform for gcm(aes): -2 on non-cache coherent systems, due to the fact that the GCM driver passes an IV buffer by virtual address which shares a cacheline with the auth_tag buffer passed via a scatterlist, resulting in corruption of the auth_tag when the IV is updated while the DMA mapping is live. Since the IV that is returned to the caller is only valid for CBC mode, and given that the in-kernel users of CBC (such as CTS) don't trigger the same issue as the GCM driver, let's just disable the output IV generation for all modes except CBC for the time being. Fixes: 854b06f76879 ("crypto: caam - properly set IV after {en,de}crypt") Cc: Horia Geanta Cc: Iuliana Prodan Reported-by: Sascha Hauer Cc: Signed-off-by: Ard Biesheuvel Reviewed-by: Horia Geanta Signed-off-by: Herbert Xu [ Horia: backported to 4.9 ] Signed-off-by: Horia Geantă Signed-off-by: Greg Kroah-Hartman --- drivers/crypto/caam/caamalg.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'drivers/crypto/caam/caamalg.c') diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 88caca3370f2..f8ac768ed5d7 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -2015,6 +2015,7 @@ static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, struct ablkcipher_request *req = context; struct ablkcipher_edesc *edesc; struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); int ivsize = crypto_ablkcipher_ivsize(ablkcipher); #ifdef DEBUG @@ -2040,10 +2041,11 @@ static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, /* * The crypto API expects us to set the IV (req->info) to the last - * ciphertext block. This is used e.g. by the CTS mode. + * ciphertext block when running in CBC mode. */ - scatterwalk_map_and_copy(req->info, req->dst, req->nbytes - ivsize, - ivsize, 0); + if ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_CBC) + scatterwalk_map_and_copy(req->info, req->dst, req->nbytes - + ivsize, ivsize, 0); kfree(edesc); @@ -2056,6 +2058,7 @@ static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, struct ablkcipher_request *req = context; struct ablkcipher_edesc *edesc; struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); int ivsize = crypto_ablkcipher_ivsize(ablkcipher); #ifdef DEBUG @@ -2080,10 +2083,11 @@ static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, /* * The crypto API expects us to set the IV (req->info) to the last - * ciphertext block. + * ciphertext block when running in CBC mode. */ - scatterwalk_map_and_copy(req->info, req->src, req->nbytes - ivsize, - ivsize, 0); + if ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_CBC) + scatterwalk_map_and_copy(req->info, req->src, req->nbytes - + ivsize, ivsize, 0); kfree(edesc); -- cgit v1.2.3 From 99f1f6b3090eab0ea3831cf41b873fd634a1cab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Tue, 30 Jul 2019 08:48:33 +0300 Subject: crypto: caam - fix concurrency issue in givencrypt descriptor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit 48f89d2a2920166c35b1c0b69917dbb0390ebec7 upstream. IV transfer from ofifo to class2 (set up at [29][30]) is not guaranteed to be scheduled before the data transfer from ofifo to external memory (set up at [38]: [29] 10FA0004 ld: ind-nfifo (len=4) imm [30] 81F00010 class2 type=msg len=16> [31] 14820004 ld: ccb2-datasz len=4 offs=0 imm [32] 00000010 data:0x00000010 [33] 8210010D operation: cls1-op aes cbc init-final enc [34] A8080B04 math: (seqin + math0)->vseqout len=4 [35] 28000010 seqfifold: skip len=16 [36] A8080A04 math: (seqin + math0)->vseqin len=4 [37] 2F1E0000 seqfifold: both msg1->2-last2-last1 len=vseqinsz [38] 69300000 seqfifostr: msg len=vseqoutsz [39] 5C20000C seqstr: ccb2 ctx len=12 offs=0 If ofifo -> external memory transfer happens first, DECO will hang (issuing a Watchdog Timeout error, if WDOG is enabled) waiting for data availability in ofifo for the ofifo -> c2 ififo transfer. Make sure IV transfer happens first by waiting for all CAAM internal transfers to end before starting payload transfer. New descriptor with jump command inserted at [37]: [..] [36] A8080A04 math: (seqin + math0)->vseqin len=4 [37] A1000401 jump: jsl1 all-match[!nfifopend] offset=[01] local->[38] [38] 2F1E0000 seqfifold: both msg1->2-last2-last1 len=vseqinsz [39] 69300000 seqfifostr: msg len=vseqoutsz [40] 5C20000C seqstr: ccb2 ctx len=12 offs=0 [Note: the issue is present in the descriptor from the very beginning (cf. Fixes tag). However I've marked it v4.19+ since it's the oldest maintained kernel that the patch applies clean against.] Cc: # v4.19+ Fixes: 1acebad3d8db8 ("crypto: caam - faster aead implementation") Signed-off-by: Herbert Xu [Horia: backport to v4.4, v4.9] Signed-off-by: Horia Geantă Signed-off-by: Greg Kroah-Hartman --- drivers/crypto/caam/caamalg.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'drivers/crypto/caam/caamalg.c') diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index f8ac768ed5d7..413e1f35773f 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -75,7 +75,7 @@ #define DESC_AEAD_BASE (4 * CAAM_CMD_SZ) #define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 11 * CAAM_CMD_SZ) #define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ) -#define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 9 * CAAM_CMD_SZ) +#define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 10 * CAAM_CMD_SZ) /* Note: Nonce is counted in enckeylen */ #define DESC_AEAD_CTR_RFC3686_LEN (4 * CAAM_CMD_SZ) @@ -474,6 +474,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead) u32 geniv, moveiv; u32 ctx1_iv_off = 0; u32 *desc; + u32 *wait_cmd; const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_CTR_MOD128); const bool is_rfc3686 = alg->caam.rfc3686; @@ -736,6 +737,14 @@ copy_iv: /* Will read cryptlen */ append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + + /* + * Wait for IV transfer (ofifo -> class2) to finish before starting + * ciphertext transfer (ofifo -> external memory). + */ + wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NIFP); + set_jump_tgt_here(desc, wait_cmd); + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF | FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH); append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); -- cgit v1.2.3