lib/crypto: arm64/aes: Migrate optimized CBC-based MACs into library

Instead of exposing the arm64-optimized CMAC, XCBC-MAC, and CBC-MAC code
via arm64-specific crypto_shash algorithms, instead just implement the
aes_cbcmac_blocks_arch() library function.  This is much simpler, it
makes the corresponding library functions be arm64-optimized, and it
fixes the longstanding issue where this optimized code was disabled by
default.  The corresponding algorithms still remain available through
crypto_shash, but individual architectures no longer need to handle it.

Note that to be compatible with the library using 'size_t' lengths, the
type of the return value and 'blocks' parameter to the assembly
functions had to be changed to 'size_t', and the assembly code had to be
updated accordingly to use the corresponding 64-bit registers.

Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260218213501.136844-6-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>

2 files changed, 56 insertions, 11 deletions

diff --git a/lib/crypto/arm64/aes-modes.S b/lib/crypto/arm64/aes-modes.S
index f4df6f84a3c7..fc89cd02b642 100644
--- a/lib/crypto/arm64/aes-modes.S
+++ b/lib/crypto/arm64/aes-modes.S
@@ -815,9 +815,11 @@ AES_FUNC_START(aes_xts_decrypt)
 	b		.Lxtsdecctsout
 AES_FUNC_END(aes_xts_decrypt)
 
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS)
 	/*
-	 * aes_mac_update(u8 const in[], u32 const rk[], int rounds,
-	 *		  int blocks, u8 dg[], int enc_before, int enc_after)
+	 * size_t aes_mac_update(u8 const in[], u32 const rk[], int rounds,
+	 *			 size_t blocks, u8 dg[], int enc_before,
+	 *			 int enc_after);
 	 */
 AES_FUNC_START(aes_mac_update)
 	ld1		{v0.16b}, [x4]			/* get dg */
@@ -827,7 +829,7 @@ AES_FUNC_START(aes_mac_update)
 	encrypt_block	v0, w2, x1, x7, w8
 
 .Lmacloop4x:
-	subs		w3, w3, #4
+	subs		x3, x3, #4
 	bmi		.Lmac1x
 	ld1		{v1.16b-v4.16b}, [x0], #64	/* get next pt block */
 	eor		v0.16b, v0.16b, v1.16b		/* ..and xor with dg */
@@ -837,7 +839,7 @@ AES_FUNC_START(aes_mac_update)
 	eor		v0.16b, v0.16b, v3.16b
 	encrypt_block	v0, w2, x1, x7, w8
 	eor		v0.16b, v0.16b, v4.16b
-	cmp		w3, wzr
+	cmp		x3, xzr
 	csinv		w5, w6, wzr, eq
 	cbz		w5, .Lmacout
 	encrypt_block	v0, w2, x1, x7, w8
@@ -845,13 +847,13 @@ AES_FUNC_START(aes_mac_update)
 	cond_yield	.Lmacout, x7, x8
 	b		.Lmacloop4x
 .Lmac1x:
-	add		w3, w3, #4
+	add		x3, x3, #4
 .Lmacloop:
-	cbz		w3, .Lmacout
+	cbz		x3, .Lmacout
 	ld1		{v1.16b}, [x0], #16		/* get next pt block */
 	eor		v0.16b, v0.16b, v1.16b		/* ..and xor with dg */
 
-	subs		w3, w3, #1
+	subs		x3, x3, #1
 	csinv		w5, w6, wzr, eq
 	cbz		w5, .Lmacout
 
@@ -861,6 +863,7 @@ AES_FUNC_START(aes_mac_update)
 
 .Lmacout:
 	st1		{v0.16b}, [x4]			/* return dg */
-	mov		w0, w3
+	mov		x0, x3
 	ret
 AES_FUNC_END(aes_mac_update)
+#endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */
diff --git a/lib/crypto/arm64/aes.h b/lib/crypto/arm64/aes.h
index 69f465c668f0..78e7b4e5f120 100644
--- a/lib/crypto/arm64/aes.h
+++ b/lib/crypto/arm64/aes.h
@@ -11,6 +11,7 @@
 #include <linux/unaligned.h>
 #include <linux/cpufeature.h>
 
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aes);
 
 struct aes_block {
@@ -28,6 +29,9 @@ asmlinkage void __aes_ce_decrypt(const u32 inv_rk[], u8 out[AES_BLOCK_SIZE],
 asmlinkage u32 __aes_ce_sub(u32 l);
 asmlinkage void __aes_ce_invert(struct aes_block *out,
 				const struct aes_block *in);
+asmlinkage size_t neon_aes_mac_update(u8 const in[], u32 const rk[], int rounds,
+				      size_t blocks, u8 dg[], int enc_before,
+				      int enc_after);
 
 /*
  * Expand an AES key using the crypto extensions if supported and usable or
@@ -139,7 +143,6 @@ EXPORT_SYMBOL_NS_GPL(neon_aes_xts_encrypt, "CRYPTO_INTERNAL");
 EXPORT_SYMBOL_NS_GPL(neon_aes_xts_decrypt, "CRYPTO_INTERNAL");
 EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL");
 EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL");
-EXPORT_SYMBOL_NS_GPL(neon_aes_mac_update, "CRYPTO_INTERNAL");
 
 EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_encrypt, "CRYPTO_INTERNAL");
 EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_decrypt, "CRYPTO_INTERNAL");
@@ -153,6 +156,8 @@ EXPORT_SYMBOL_NS_GPL(ce_aes_xts_encrypt, "CRYPTO_INTERNAL");
 EXPORT_SYMBOL_NS_GPL(ce_aes_xts_decrypt, "CRYPTO_INTERNAL");
 EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL");
 EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL");
+#endif
+#if IS_MODULE(CONFIG_CRYPTO_AES_ARM64_CE_CCM)
 EXPORT_SYMBOL_NS_GPL(ce_aes_mac_update, "CRYPTO_INTERNAL");
 #endif
 
@@ -184,11 +189,48 @@ static void aes_decrypt_arch(const struct aes_key *key,
 	}
 }
 
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS)
+#define aes_cbcmac_blocks_arch aes_cbcmac_blocks_arch
+static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE],
+				   const struct aes_enckey *key, const u8 *data,
+				   size_t nblocks, bool enc_before,
+				   bool enc_after)
+{
+	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
+	    static_branch_likely(&have_neon) && likely(may_use_simd())) {
+		do {
+			size_t rem;
+
+			scoped_ksimd() {
+				if (static_branch_likely(&have_aes))
+					rem = ce_aes_mac_update(
+						data, key->k.rndkeys,
+						key->nrounds, nblocks, h,
+						enc_before, enc_after);
+				else
+					rem = neon_aes_mac_update(
+						data, key->k.rndkeys,
+						key->nrounds, nblocks, h,
+						enc_before, enc_after);
+			}
+			data += (nblocks - rem) * AES_BLOCK_SIZE;
+			nblocks = rem;
+			enc_before = false;
+		} while (nblocks);
+		return true;
+	}
+	return false;
+}
+#endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */
+
 #ifdef CONFIG_KERNEL_MODE_NEON
 #define aes_mod_init_arch aes_mod_init_arch
 static void aes_mod_init_arch(void)
 {
-	if (cpu_have_named_feature(AES))
-		static_branch_enable(&have_aes);
+	if (cpu_have_named_feature(ASIMD)) {
+		static_branch_enable(&have_neon);
+		if (cpu_have_named_feature(AES))
+			static_branch_enable(&have_aes);
+	}
 }
 #endif /* CONFIG_KERNEL_MODE_NEON */