Merge tag 'libcrypto-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux

Pull crypto library updates from Eric Biggers:

 - Migrate more hash algorithms from the traditional crypto subsystem to
   lib/crypto/

   Like the algorithms migrated earlier (e.g. SHA-*), this simplifies
   the implementations, improves performance, enables further
   simplifications in calling code, and solves various other issues:

     - AES CBC-based MACs (AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC)

         - Support these algorithms in lib/crypto/ using the AES library
           and the existing arm64 assembly code

         - Reimplement the traditional crypto API's "cmac(aes)",
           "xcbc(aes)", and "cbcmac(aes)" on top of the library

         - Convert mac80211 to use the AES-CMAC library. Note: several
           other subsystems can use it too and will be converted later

         - Drop the broken, nonstandard, and likely unused support for
           "xcbc(aes)" with key lengths other than 128 bits

         - Enable optimizations by default

     - GHASH

         - Migrate the standalone GHASH code into lib/crypto/

         - Integrate the GHASH code more closely with the very similar
           POLYVAL code, and improve the generic GHASH implementation to
           resist cache-timing attacks and use much less memory

         - Reimplement the AES-GCM library and the "gcm" crypto_aead
           template on top of the GHASH library. Remove "ghash" from the
           crypto_shash API, as it's no longer needed

         - Enable optimizations by default

     - SM3

         - Migrate the kernel's existing SM3 code into lib/crypto/, and
           reimplement the traditional crypto API's "sm3" on top of it

         - I don't recommend using SM3, but this cleanup is worthwhile
           to organize the code the same way as other algorithms

 - Testing improvements:

     - Add a KUnit test suite for each of the new library APIs

     - Migrate the existing ChaCha20Poly1305 test to KUnit

     - Make the KUnit all_tests.config enable all crypto library tests

     - Move the test kconfig options to the Runtime Testing menu

 - Other updates to arch-optimized crypto code:

     - Optimize SHA-256 for Zhaoxin CPUs using the Padlock Hash Engine

     - Remove some MD5 implementations that are no longer worth keeping

     - Drop big endian and voluntary preemption support from the arm64
       code, as those configurations are no longer supported on arm64

 - Make jitterentropy and samples/tsm-mr use the crypto library APIs

* tag 'libcrypto-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux: (66 commits)
  lib/crypto: arm64: Assume a little-endian kernel
  arm64: fpsimd: Remove obsolete cond_yield macro
  lib/crypto: arm64/sha3: Remove obsolete chunking logic
  lib/crypto: arm64/sha512: Remove obsolete chunking logic
  lib/crypto: arm64/sha256: Remove obsolete chunking logic
  lib/crypto: arm64/sha1: Remove obsolete chunking logic
  lib/crypto: arm64/poly1305: Remove obsolete chunking logic
  lib/crypto: arm64/gf128hash: Remove obsolete chunking logic
  lib/crypto: arm64/chacha: Remove obsolete chunking logic
  lib/crypto: arm64/aes: Remove obsolete chunking logic
  lib/crypto: Include <crypto/utils.h> instead of <crypto/algapi.h>
  lib/crypto: aesgcm: Don't disable IRQs during AES block encryption
  lib/crypto: aescfb: Don't disable IRQs during AES block encryption
  lib/crypto: tests: Migrate ChaCha20Poly1305 self-test to KUnit
  lib/crypto: sparc: Drop optimized MD5 code
  lib/crypto: mips: Drop optimized MD5 code
  lib: Move crypto library tests to Runtime Testing menu
  crypto: sm3 - Remove 'struct sm3_state'
  crypto: sm3 - Remove the original "sm3_block_generic()"
  crypto: sm3 - Remove sm3_base.h
  ...

7 files changed, 43 insertions, 111 deletions

diff --git a/net/mac80211/Kconfig b/net/mac80211/Kconfig
index cf0f7780fb10..d6bc295e23a1 100644
--- a/net/mac80211/Kconfig
+++ b/net/mac80211/Kconfig
@@ -3,11 +3,11 @@ config MAC80211
 	tristate "Generic IEEE 802.11 Networking Stack (mac80211)"
 	depends on CFG80211
 	select CRYPTO
+	select CRYPTO_LIB_AES_CBC_MACS
 	select CRYPTO_LIB_ARC4
 	select CRYPTO_AES
 	select CRYPTO_CCM
 	select CRYPTO_GCM
-	select CRYPTO_CMAC
 	select CRC32
 	help
 	  This option enables the hardware independent IEEE 802.11
diff --git a/net/mac80211/aes_cmac.c b/net/mac80211/aes_cmac.c
index 0827965455dc..55b674ad7d7a 100644
--- a/net/mac80211/aes_cmac.c
+++ b/net/mac80211/aes_cmac.c
@@ -7,10 +7,9 @@
 
 #include <linux/kernel.h>
 #include <linux/types.h>
-#include <linux/crypto.h>
 #include <linux/export.h>
 #include <linux/err.h>
-#include <crypto/aes.h>
+#include <crypto/aes-cbc-macs.h>
 
 #include <net/mac80211.h>
 #include "key.h"
@@ -20,65 +19,25 @@
 
 static const u8 zero[IEEE80211_CMAC_256_MIC_LEN];
 
-int ieee80211_aes_cmac(struct crypto_shash *tfm, const u8 *aad,
-		       const u8 *data, size_t data_len, u8 *mic,
-		       unsigned int mic_len)
+void ieee80211_aes_cmac(const struct aes_cmac_key *key, const u8 *aad,
+			const u8 *data, size_t data_len, u8 *mic,
+			unsigned int mic_len)
 {
-	int err;
-	SHASH_DESC_ON_STACK(desc, tfm);
+	struct aes_cmac_ctx ctx;
 	u8 out[AES_BLOCK_SIZE];
 	const __le16 *fc;
 
-	desc->tfm = tfm;
-
-	err = crypto_shash_init(desc);
-	if (err)
-		return err;
-	err = crypto_shash_update(desc, aad, AAD_LEN);
-	if (err)
-		return err;
+	aes_cmac_init(&ctx, key);
+	aes_cmac_update(&ctx, aad, AAD_LEN);
 	fc = (const __le16 *)aad;
 	if (ieee80211_is_beacon(*fc)) {
 		/* mask Timestamp field to zero */
-		err = crypto_shash_update(desc, zero, 8);
-		if (err)
-			return err;
-		err = crypto_shash_update(desc, data + 8,
-					  data_len - 8 - mic_len);
-		if (err)
-			return err;
+		aes_cmac_update(&ctx, zero, 8);
+		aes_cmac_update(&ctx, data + 8, data_len - 8 - mic_len);
 	} else {
-		err = crypto_shash_update(desc, data, data_len - mic_len);
-		if (err)
-			return err;
+		aes_cmac_update(&ctx, data, data_len - mic_len);
 	}
-	err = crypto_shash_finup(desc, zero, mic_len, out);
-	if (err)
-		return err;
+	aes_cmac_update(&ctx, zero, mic_len);
+	aes_cmac_final(&ctx, out);
 	memcpy(mic, out, mic_len);
-
-	return 0;
-}
-
-struct crypto_shash *ieee80211_aes_cmac_key_setup(const u8 key[],
-						  size_t key_len)
-{
-	struct crypto_shash *tfm;
-
-	tfm = crypto_alloc_shash("cmac(aes)", 0, 0);
-	if (!IS_ERR(tfm)) {
-		int err = crypto_shash_setkey(tfm, key, key_len);
-
-		if (err) {
-			crypto_free_shash(tfm);
-			return ERR_PTR(err);
-		}
-	}
-
-	return tfm;
-}
-
-void ieee80211_aes_cmac_key_free(struct crypto_shash *tfm)
-{
-	crypto_free_shash(tfm);
 }
diff --git a/net/mac80211/aes_cmac.h b/net/mac80211/aes_cmac.h
index 5f971a8298cb..c7a6df47b327 100644
--- a/net/mac80211/aes_cmac.h
+++ b/net/mac80211/aes_cmac.h
@@ -6,14 +6,10 @@
 #ifndef AES_CMAC_H
 #define AES_CMAC_H
 
-#include <linux/crypto.h>
-#include <crypto/hash.h>
+#include <crypto/aes-cbc-macs.h>
 
-struct crypto_shash *ieee80211_aes_cmac_key_setup(const u8 key[],
-						  size_t key_len);
-int ieee80211_aes_cmac(struct crypto_shash *tfm, const u8 *aad,
-		       const u8 *data, size_t data_len, u8 *mic,
-		       unsigned int mic_len);
-void ieee80211_aes_cmac_key_free(struct crypto_shash *tfm);
+void ieee80211_aes_cmac(const struct aes_cmac_key *key, const u8 *aad,
+			const u8 *data, size_t data_len, u8 *mic,
+			unsigned int mic_len);
 
 #endif /* AES_CMAC_H */
diff --git a/net/mac80211/fils_aead.c b/net/mac80211/fils_aead.c
index 912c46f74d24..d2f4a17eab99 100644
--- a/net/mac80211/fils_aead.c
+++ b/net/mac80211/fils_aead.c
@@ -4,13 +4,11 @@
  * Copyright 2016, Qualcomm Atheros, Inc.
  */
 
-#include <crypto/aes.h>
-#include <crypto/hash.h>
+#include <crypto/aes-cbc-macs.h>
 #include <crypto/skcipher.h>
 #include <crypto/utils.h>
 
 #include "ieee80211_i.h"
-#include "aes_cmac.h"
 #include "fils_aead.h"
 
 static void gf_mulx(u8 *pad)
@@ -22,31 +20,35 @@ static void gf_mulx(u8 *pad)
 	put_unaligned_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0), pad + 8);
 }
 
-static int aes_s2v(struct crypto_shash *tfm,
+static int aes_s2v(const u8 *in_key, size_t key_len,
 		   size_t num_elem, const u8 *addr[], size_t len[], u8 *v)
 {
 	u8 d[AES_BLOCK_SIZE], tmp[AES_BLOCK_SIZE] = {};
-	SHASH_DESC_ON_STACK(desc, tfm);
+	struct aes_cmac_key key;
+	struct aes_cmac_ctx ctx;
 	size_t i;
+	int res;
 
-	desc->tfm = tfm;
+	res = aes_cmac_preparekey(&key, in_key, key_len);
+	if (res)
+		return res;
 
 	/* D = AES-CMAC(K, <zero>) */
-	crypto_shash_digest(desc, tmp, AES_BLOCK_SIZE, d);
+	aes_cmac(&key, tmp, AES_BLOCK_SIZE, d);
 
 	for (i = 0; i < num_elem - 1; i++) {
 		/* D = dbl(D) xor AES_CMAC(K, Si) */
 		gf_mulx(d); /* dbl */
-		crypto_shash_digest(desc, addr[i], len[i], tmp);
+		aes_cmac(&key, addr[i], len[i], tmp);
 		crypto_xor(d, tmp, AES_BLOCK_SIZE);
 	}
 
-	crypto_shash_init(desc);
+	aes_cmac_init(&ctx, &key);
 
 	if (len[i] >= AES_BLOCK_SIZE) {
 		/* len(Sn) >= 128 */
 		/* T = Sn xorend D */
-		crypto_shash_update(desc, addr[i], len[i] - AES_BLOCK_SIZE);
+		aes_cmac_update(&ctx, addr[i], len[i] - AES_BLOCK_SIZE);
 		crypto_xor(d, addr[i] + len[i] - AES_BLOCK_SIZE,
 			   AES_BLOCK_SIZE);
 	} else {
@@ -57,8 +59,10 @@ static int aes_s2v(struct crypto_shash *tfm,
 		d[len[i]] ^= 0x80;
 	}
 	/* V = AES-CMAC(K, T) */
-	crypto_shash_finup(desc, d, AES_BLOCK_SIZE, v);
+	aes_cmac_update(&ctx, d, AES_BLOCK_SIZE);
+	aes_cmac_final(&ctx, v);
 
+	memzero_explicit(&key, sizeof(key));
 	return 0;
 }
 
@@ -69,7 +73,6 @@ static int aes_siv_encrypt(const u8 *key, size_t key_len,
 			   size_t len[], u8 *out)
 {
 	u8 v[AES_BLOCK_SIZE];
-	struct crypto_shash *tfm;
 	struct crypto_skcipher *tfm2;
 	struct skcipher_request *req;
 	int res;
@@ -83,15 +86,7 @@ static int aes_siv_encrypt(const u8 *key, size_t key_len,
 	num_elem++;
 
 	/* S2V */
-
-	tfm = crypto_alloc_shash("cmac(aes)", 0, 0);
-	if (IS_ERR(tfm))
-		return PTR_ERR(tfm);
-	/* K1 for S2V */
-	res = crypto_shash_setkey(tfm, key, key_len);
-	if (!res)
-		res = aes_s2v(tfm, num_elem, addr, len, v);
-	crypto_free_shash(tfm);
+	res = aes_s2v(key /* K1 */, key_len, num_elem, addr, len, v);
 	if (res)
 		return res;
 
@@ -146,7 +141,6 @@ static int aes_siv_decrypt(const u8 *key, size_t key_len,
 			   size_t num_elem, const u8 *addr[], size_t len[],
 			   u8 *out)
 {
-	struct crypto_shash *tfm;
 	struct crypto_skcipher *tfm2;
 	struct skcipher_request *req;
 	struct scatterlist src[1], dst[1];
@@ -198,15 +192,7 @@ static int aes_siv_decrypt(const u8 *key, size_t key_len,
 		return res;
 
 	/* S2V */
-
-	tfm = crypto_alloc_shash("cmac(aes)", 0, 0);
-	if (IS_ERR(tfm))
-		return PTR_ERR(tfm);
-	/* K1 for S2V */
-	res = crypto_shash_setkey(tfm, key, key_len);
-	if (!res)
-		res = aes_s2v(tfm, num_elem, addr, len, check);
-	crypto_free_shash(tfm);
+	res = aes_s2v(key /* K1 */, key_len, num_elem, addr, len, check);
 	if (res)
 		return res;
 	if (memcmp(check, frame_iv, AES_BLOCK_SIZE) != 0)
diff --git a/net/mac80211/key.c b/net/mac80211/key.c
index 04c8809173d7..4b8965633df3 100644
--- a/net/mac80211/key.c
+++ b/net/mac80211/key.c
@@ -690,10 +690,9 @@ ieee80211_key_alloc(u32 cipher, int idx, size_t key_len,
 		 * Initialize AES key state here as an optimization so that
 		 * it does not need to be initialized for every packet.
 		 */
-		key->u.aes_cmac.tfm =
-			ieee80211_aes_cmac_key_setup(key_data, key_len);
-		if (IS_ERR(key->u.aes_cmac.tfm)) {
-			err = PTR_ERR(key->u.aes_cmac.tfm);
+		err = aes_cmac_preparekey(&key->u.aes_cmac.key, key_data,
+					  key_len);
+		if (err) {
 			kfree(key);
 			return ERR_PTR(err);
 		}
@@ -750,10 +749,6 @@ static void ieee80211_key_free_common(struct ieee80211_key *key)
 	case WLAN_CIPHER_SUITE_CCMP_256:
 		ieee80211_aes_key_free(key->u.ccmp.tfm);
 		break;
-	case WLAN_CIPHER_SUITE_AES_CMAC:
-	case WLAN_CIPHER_SUITE_BIP_CMAC_256:
-		ieee80211_aes_cmac_key_free(key->u.aes_cmac.tfm);
-		break;
 	case WLAN_CIPHER_SUITE_BIP_GMAC_128:
 	case WLAN_CIPHER_SUITE_BIP_GMAC_256:
 		ieee80211_aes_gmac_key_free(key->u.aes_gmac.tfm);
diff --git a/net/mac80211/key.h b/net/mac80211/key.h
index 1fa0f4f78962..826e4e9387c5 100644
--- a/net/mac80211/key.h
+++ b/net/mac80211/key.h
@@ -12,6 +12,7 @@
 #include <linux/list.h>
 #include <linux/crypto.h>
 #include <linux/rcupdate.h>
+#include <crypto/aes-cbc-macs.h>
 #include <crypto/arc4.h>
 #include <net/mac80211.h>
 
@@ -93,7 +94,7 @@ struct ieee80211_key {
 		} ccmp;
 		struct {
 			u8 rx_pn[IEEE80211_CMAC_PN_LEN];
-			struct crypto_shash *tfm;
+			struct aes_cmac_key key;
 			u32 replays; /* dot11RSNAStatsCMACReplays */
 			u32 icverrors; /* dot11RSNAStatsCMACICVErrors */
 		} aes_cmac;
diff --git a/net/mac80211/wpa.c b/net/mac80211/wpa.c
index fdf98c21d32c..59324b367bdd 100644
--- a/net/mac80211/wpa.c
+++ b/net/mac80211/wpa.c
@@ -872,11 +872,8 @@ ieee80211_crypto_aes_cmac_encrypt(struct ieee80211_tx_data *tx,
 
 	bip_aad(skb, aad);
 
-	if (ieee80211_aes_cmac(key->u.aes_cmac.tfm, aad,
-			       skb->data + 24, skb->len - 24,
-			       mmie->mic, mic_len))
-		return TX_DROP;
-
+	ieee80211_aes_cmac(&key->u.aes_cmac.key, aad, skb->data + 24,
+			   skb->len - 24, mmie->mic, mic_len);
 	return TX_CONTINUE;
 }
 
@@ -918,10 +915,8 @@ ieee80211_crypto_aes_cmac_decrypt(struct ieee80211_rx_data *rx,
 	if (!(status->flag & RX_FLAG_DECRYPTED)) {
 		/* hardware didn't decrypt/verify MIC */
 		bip_aad(skb, aad);
-		if (ieee80211_aes_cmac(key->u.aes_cmac.tfm, aad,
-				       skb->data + 24, skb->len - 24,
-				       mic, mic_len))
-			return RX_DROP_U_DECRYPT_FAIL;
+		ieee80211_aes_cmac(&key->u.aes_cmac.key, aad, skb->data + 24,
+				   skb->len - 24, mic, mic_len);
 		if (crypto_memneq(mic, mmie->mic, mic_len)) {
 			key->u.aes_cmac.icverrors++;
 			return RX_DROP_U_MIC_FAIL;