Merge tag 'libcrypto-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux

Pull crypto library updates from Eric Biggers: - Migrate more hash algorithms from the traditional crypto subsystem to lib/crypto/ Like the algorithms migrated earlier (e.g. SHA-*), this simplifies the implementations, improves performance, enables further simplifications in calling code, and solves various other issues: - AES CBC-based MACs (AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC) - Support these algorithms in lib/crypto/ using the AES library and the existing arm64 assembly code - Reimplement the traditional crypto API's "cmac(aes)", "xcbc(aes)", and "cbcmac(aes)" on top of the library - Convert mac80211 to use the AES-CMAC library. Note: several other subsystems can use it too and will be converted later - Drop the broken, nonstandard, and likely unused support for "xcbc(aes)" with key lengths other than 128 bits - Enable optimizations by default - GHASH - Migrate the standalone GHASH code into lib/crypto/ - Integrate the GHASH code more closely with the very similar POLYVAL code, and improve the generic GHASH implementation to resist cache-timing attacks and use much less memory - Reimplement the AES-GCM library and the "gcm" crypto_aead template on top of the GHASH library. Remove "ghash" from the crypto_shash API, as it's no longer needed - Enable optimizations by default - SM3 - Migrate the kernel's existing SM3 code into lib/crypto/, and reimplement the traditional crypto API's "sm3" on top of it - I don't recommend using SM3, but this cleanup is worthwhile to organize the code the same way as other algorithms - Testing improvements: - Add a KUnit test suite for each of the new library APIs - Migrate the existing ChaCha20Poly1305 test to KUnit - Make the KUnit all_tests.config enable all crypto library tests - Move the test kconfig options to the Runtime Testing menu - Other updates to arch-optimized crypto code: - Optimize SHA-256 for Zhaoxin CPUs using the Padlock Hash Engine - Remove some MD5 implementations that are no longer worth keeping - Drop big endian and voluntary preemption support from the arm64 code, as those configurations are no longer supported on arm64 - Make jitterentropy and samples/tsm-mr use the crypto library APIs * tag 'libcrypto-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux: (66 commits) lib/crypto: arm64: Assume a little-endian kernel arm64: fpsimd: Remove obsolete cond_yield macro lib/crypto: arm64/sha3: Remove obsolete chunking logic lib/crypto: arm64/sha512: Remove obsolete chunking logic lib/crypto: arm64/sha256: Remove obsolete chunking logic lib/crypto: arm64/sha1: Remove obsolete chunking logic lib/crypto: arm64/poly1305: Remove obsolete chunking logic lib/crypto: arm64/gf128hash: Remove obsolete chunking logic lib/crypto: arm64/chacha: Remove obsolete chunking logic lib/crypto: arm64/aes: Remove obsolete chunking logic lib/crypto: Include <crypto/utils.h> instead of <crypto/algapi.h> lib/crypto: aesgcm: Don't disable IRQs during AES block encryption lib/crypto: aescfb: Don't disable IRQs during AES block encryption lib/crypto: tests: Migrate ChaCha20Poly1305 self-test to KUnit lib/crypto: sparc: Drop optimized MD5 code lib/crypto: mips: Drop optimized MD5 code lib: Move crypto library tests to Runtime Testing menu crypto: sm3 - Remove 'struct sm3_state' crypto: sm3 - Remove the original "sm3_block_generic()" crypto: sm3 - Remove sm3_base.h ...
author: Linus Torvalds <torvalds@linux-foundation.org> 2026-04-13 17:31:39 -0700
committer: Linus Torvalds <torvalds@linux-foundation.org> 2026-04-13 17:31:39 -0700
commit: 370c3883195566ee3e7d79e0146c3d735a406573 (patch)
tree: 5ef1663d7822a3094ce80d1379e7c0630200e5d9 /lib
parent: 9932f00bf40d281151de5694bc0f097cb9b5616c (diff)
parent: 12b11e47f126d097839fd2f077636e2139b0151b (diff)
64 files changed, 6778 insertions, 1502 deletions
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 30d069676309..94eb2667b7e1 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -3058,6 +3058,8 @@ config HW_BREAKPOINT_KUNIT_TEST
 
 	  If unsure, say N.
 
+source "lib/crypto/tests/Kconfig"
+
 config SIPHASH_KUNIT_TEST
 	tristate "Perform selftest on siphash functions" if !KUNIT_ALL_TESTS
 	depends on KUNIT
diff --git a/lib/crypto/.kunitconfig b/lib/crypto/.kunitconfig
index 6b2ce28ae509..3efc854a2c08 100644
--- a/lib/crypto/.kunitconfig
+++ b/lib/crypto/.kunitconfig
@@ -1,28 +1,13 @@
 CONFIG_KUNIT=y
 
-# These kconfig options select all the CONFIG_CRYPTO_LIB_* symbols that have a
-# corresponding KUnit test.  Those symbols cannot be directly enabled here,
-# since they are hidden symbols.
-CONFIG_CRYPTO=y
-CONFIG_CRYPTO_ADIANTUM=y
-CONFIG_CRYPTO_BLAKE2B=y
-CONFIG_CRYPTO_CHACHA20POLY1305=y
-CONFIG_CRYPTO_HCTR2=y
-CONFIG_CRYPTO_MD5=y
-CONFIG_CRYPTO_MLDSA=y
-CONFIG_CRYPTO_SHA1=y
-CONFIG_CRYPTO_SHA256=y
-CONFIG_CRYPTO_SHA512=y
-CONFIG_CRYPTO_SHA3=y
-CONFIG_INET=y
-CONFIG_IPV6=y
-CONFIG_NET=y
-CONFIG_NETDEVICES=y
-CONFIG_WIREGUARD=y
+CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=y
 
+CONFIG_CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST=y
 CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST=y
 CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST=y
+CONFIG_CRYPTO_LIB_CHACHA20POLY1305_KUNIT_TEST=y
 CONFIG_CRYPTO_LIB_CURVE25519_KUNIT_TEST=y
+CONFIG_CRYPTO_LIB_GHASH_KUNIT_TEST=y
 CONFIG_CRYPTO_LIB_MD5_KUNIT_TEST=y
 CONFIG_CRYPTO_LIB_MLDSA_KUNIT_TEST=y
 CONFIG_CRYPTO_LIB_NH_KUNIT_TEST=y
@@ -32,3 +17,4 @@ CONFIG_CRYPTO_LIB_SHA1_KUNIT_TEST=y
 CONFIG_CRYPTO_LIB_SHA256_KUNIT_TEST=y
 CONFIG_CRYPTO_LIB_SHA512_KUNIT_TEST=y
 CONFIG_CRYPTO_LIB_SHA3_KUNIT_TEST=y
+CONFIG_CRYPTO_LIB_SM3_KUNIT_TEST=y
diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig
index 032f9755f999..d3904b72dae7 100644
--- a/lib/crypto/Kconfig
+++ b/lib/crypto/Kconfig
@@ -1,7 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
 
-menu "Crypto library routines"
-
 config CRYPTO_HASH_INFO
 	bool
 
@@ -10,6 +8,8 @@ config CRYPTO_LIB_UTILS
 
 config CRYPTO_LIB_AES
 	tristate
+	# Select dependencies of modes that are part of libaes.
+	select CRYPTO_LIB_UTILS if CRYPTO_LIB_AES_CBC_MACS
 
 config CRYPTO_LIB_AES_ARCH
 	bool
@@ -28,10 +28,18 @@ config CRYPTO_LIB_AESCFB
 	select CRYPTO_LIB_AES
 	select CRYPTO_LIB_UTILS
 
+config CRYPTO_LIB_AES_CBC_MACS
+	tristate
+	select CRYPTO_LIB_AES
+	help
+	  The AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC library functions.  Select
+	  this if your module uses any of the functions from
+	  <crypto/aes-cbc-macs.h>.
+
 config CRYPTO_LIB_AESGCM
 	tristate
 	select CRYPTO_LIB_AES
-	select CRYPTO_LIB_GF128MUL
+	select CRYPTO_LIB_GF128HASH
 	select CRYPTO_LIB_UTILS
 
 config CRYPTO_LIB_ARC4
@@ -70,7 +78,7 @@ config CRYPTO_LIB_CHACHA_ARCH
 	bool
 	depends on CRYPTO_LIB_CHACHA && !UML && !KMSAN
 	default y if ARM
-	default y if ARM64 && KERNEL_MODE_NEON
+	default y if ARM64
 	default y if MIPS && CPU_MIPS32_R2
 	default y if PPC64 && CPU_LITTLE_ENDIAN && VSX
 	default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
@@ -100,6 +108,23 @@ config CRYPTO_LIB_CURVE25519_GENERIC
 config CRYPTO_LIB_DES
 	tristate
 
+config CRYPTO_LIB_GF128HASH
+	tristate
+	help
+	  The GHASH and POLYVAL library functions.  Select this if your module
+	  uses any of the functions from <crypto/gf128hash.h>.
+
+config CRYPTO_LIB_GF128HASH_ARCH
+	bool
+	depends on CRYPTO_LIB_GF128HASH && !UML
+	default y if ARM && KERNEL_MODE_NEON
+	default y if ARM64
+	default y if PPC64 && VSX
+	default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
+		     RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
+	default y if S390
+	default y if X86_64
+
 config CRYPTO_LIB_MD5
 	tristate
 	help
@@ -109,9 +134,7 @@ config CRYPTO_LIB_MD5
 config CRYPTO_LIB_MD5_ARCH
 	bool
 	depends on CRYPTO_LIB_MD5 && !UML
-	default y if MIPS && CPU_CAVIUM_OCTEON
 	default y if PPC
-	default y if SPARC64
 
 config CRYPTO_LIB_MLDSA
 	tristate
@@ -130,7 +153,7 @@ config CRYPTO_LIB_NH_ARCH
 	bool
 	depends on CRYPTO_LIB_NH && !UML && !KMSAN
 	default y if ARM && KERNEL_MODE_NEON
-	default y if ARM64 && KERNEL_MODE_NEON
+	default y if ARM64
 	default y if X86_64
 
 config CRYPTO_LIB_POLY1305
@@ -143,7 +166,7 @@ config CRYPTO_LIB_POLY1305_ARCH
 	bool
 	depends on CRYPTO_LIB_POLY1305 && !UML && !KMSAN
 	default y if ARM
-	default y if ARM64 && KERNEL_MODE_NEON
+	default y if ARM64
 	default y if MIPS
 	# The PPC64 code needs to be fixed to work in softirq context.
 	default y if PPC64 && CPU_LITTLE_ENDIAN && VSX && BROKEN
@@ -168,18 +191,6 @@ config CRYPTO_LIB_POLY1305_RSIZE
 	default 9 if ARM || ARM64
 	default 1
 
-config CRYPTO_LIB_POLYVAL
-	tristate
-	help
-	  The POLYVAL library functions.  Select this if your module uses any of
-	  the functions from <crypto/polyval.h>.
-
-config CRYPTO_LIB_POLYVAL_ARCH
-	bool
-	depends on CRYPTO_LIB_POLYVAL && !UML
-	default y if ARM64 && KERNEL_MODE_NEON
-	default y if X86_64
-
 config CRYPTO_LIB_CHACHA20POLY1305
 	tristate
 	select CRYPTO_LIB_CHACHA
@@ -196,7 +207,7 @@ config CRYPTO_LIB_SHA1_ARCH
 	bool
 	depends on CRYPTO_LIB_SHA1 && !UML
 	default y if ARM
-	default y if ARM64 && KERNEL_MODE_NEON
+	default y if ARM64
 	default y if MIPS && CPU_CAVIUM_OCTEON
 	default y if PPC
 	default y if S390
@@ -252,12 +263,19 @@ config CRYPTO_LIB_SHA3
 config CRYPTO_LIB_SHA3_ARCH
 	bool
 	depends on CRYPTO_LIB_SHA3 && !UML
-	default y if ARM64 && KERNEL_MODE_NEON
+	default y if ARM64
 	default y if S390
 
 config CRYPTO_LIB_SM3
 	tristate
+	help
+	  The SM3 library functions.  Select this if your module uses any of the
+	  functions from <crypto/sm3.h>.
 
-source "lib/crypto/tests/Kconfig"
-
-endmenu
+config CRYPTO_LIB_SM3_ARCH
+	bool
+	depends on CRYPTO_LIB_SM3 && !UML
+	default y if ARM64
+	default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
+		     RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
+	default y if X86_64
diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
index dc7a56f7287d..4ad91f390038 100644
--- a/lib/crypto/Makefile
+++ b/lib/crypto/Makefile
@@ -8,6 +8,10 @@ quiet_cmd_perlasm = PERLASM $@
 quiet_cmd_perlasm_with_args = PERLASM $@
       cmd_perlasm_with_args = $(PERL) $(<) void $(@)
 
+ppc64-perlasm-flavour-y := linux-ppc64
+ppc64-perlasm-flavour-$(CONFIG_PPC64_ELF_ABI_V2) := linux-ppc64-elfv2
+ppc64-perlasm-flavour-$(CONFIG_CPU_LITTLE_ENDIAN) := linux-ppc64le
+
 obj-$(CONFIG_KUNIT)				+= tests/
 
 obj-$(CONFIG_CRYPTO_HASH_INFO)			+= hash_info.o
@@ -23,11 +27,10 @@ ifeq ($(CONFIG_CRYPTO_LIB_AES_ARCH),y)
 CFLAGS_aes.o += -I$(src)/$(SRCARCH)
 
 libaes-$(CONFIG_ARM) += arm/aes-cipher-core.o
-
-ifeq ($(CONFIG_ARM64),y)
-libaes-y += arm64/aes-cipher-core.o
-libaes-$(CONFIG_KERNEL_MODE_NEON) += arm64/aes-ce-core.o
-endif
+libaes-$(CONFIG_ARM64) += arm64/aes-cipher-core.o \
+			  arm64/aes-ce-core.o \
+			  arm64/aes-ce.o \
+			  arm64/aes-neon.o
 
 ifeq ($(CONFIG_PPC),y)
 ifeq ($(CONFIG_SPE),y)
@@ -37,11 +40,8 @@ libaes-y += powerpc/aes-spe-core.o \
 	    powerpc/aes-tab-4k.o
 else
 libaes-y += powerpc/aesp8-ppc.o
-aes-perlasm-flavour-y := linux-ppc64
-aes-perlasm-flavour-$(CONFIG_PPC64_ELF_ABI_V2) := linux-ppc64-elfv2
-aes-perlasm-flavour-$(CONFIG_CPU_LITTLE_ENDIAN) := linux-ppc64le
 quiet_cmd_perlasm_aes = PERLASM $@
-      cmd_perlasm_aes = $(PERL) $< $(aes-perlasm-flavour-y) $@
+      cmd_perlasm_aes = $(PERL) $< $(ppc64-perlasm-flavour-y) $@
 # Use if_changed instead of cmd, in case the flavour changed.
 $(obj)/powerpc/aesp8-ppc.S: $(src)/powerpc/aesp8-ppc.pl FORCE
 	$(call if_changed,perlasm_aes)
@@ -125,7 +125,6 @@ endif # CONFIG_CRYPTO_LIB_CHACHA_ARCH
 
 obj-$(CONFIG_CRYPTO_LIB_CHACHA20POLY1305)	+= libchacha20poly1305.o
 libchacha20poly1305-y				+= chacha20poly1305.o
-libchacha20poly1305-$(CONFIG_CRYPTO_SELFTESTS)	+= chacha20poly1305-selftest.o
 
 ################################################################################
 
@@ -158,12 +157,39 @@ libdes-y					:= des.o
 
 ################################################################################
 
+obj-$(CONFIG_CRYPTO_LIB_GF128HASH) += libgf128hash.o
+libgf128hash-y := gf128hash.o
+ifeq ($(CONFIG_CRYPTO_LIB_GF128HASH_ARCH),y)
+CFLAGS_gf128hash.o += -I$(src)/$(SRCARCH)
+libgf128hash-$(CONFIG_ARM) += arm/ghash-neon-core.o
+libgf128hash-$(CONFIG_ARM64) += arm64/ghash-neon-core.o \
+				arm64/polyval-ce-core.o
+
+ifeq ($(CONFIG_PPC),y)
+libgf128hash-y += powerpc/ghashp8-ppc.o
+quiet_cmd_perlasm_ghash = PERLASM $@
+      cmd_perlasm_ghash = $(PERL) $< $(ppc64-perlasm-flavour-y) $@
+$(obj)/powerpc/ghashp8-ppc.S: $(src)/powerpc/ghashp8-ppc.pl FORCE
+	$(call if_changed,perlasm_ghash)
+targets += powerpc/ghashp8-ppc.S
+OBJECT_FILES_NON_STANDARD_powerpc/ghashp8-ppc.o := y
+endif
+
+libgf128hash-$(CONFIG_RISCV) += riscv/ghash-riscv64-zvkg.o
+libgf128hash-$(CONFIG_X86) += x86/ghash-pclmul.o \
+			      x86/polyval-pclmul-avx.o
+endif # CONFIG_CRYPTO_LIB_GF128HASH_ARCH
+
+# clean-files must be defined unconditionally
+clean-files += powerpc/ghashp8-ppc.S
+
+################################################################################
+
 obj-$(CONFIG_CRYPTO_LIB_MD5) += libmd5.o
 libmd5-y := md5.o
 ifeq ($(CONFIG_CRYPTO_LIB_MD5_ARCH),y)
 CFLAGS_md5.o += -I$(src)/$(SRCARCH)
 libmd5-$(CONFIG_PPC) += powerpc/md5-asm.o
-libmd5-$(CONFIG_SPARC) += sparc/md5_asm.o
 endif # CONFIG_CRYPTO_LIB_MD5_ARCH
 
 ################################################################################
@@ -255,16 +281,6 @@ clean-files += arm/poly1305-core.S \
 
 ################################################################################
 
-obj-$(CONFIG_CRYPTO_LIB_POLYVAL) += libpolyval.o
-libpolyval-y := polyval.o
-ifeq ($(CONFIG_CRYPTO_LIB_POLYVAL_ARCH),y)
-CFLAGS_polyval.o += -I$(src)/$(SRCARCH)
-libpolyval-$(CONFIG_ARM64) += arm64/polyval-ce-core.o
-libpolyval-$(CONFIG_X86) += x86/polyval-pclmul-avx.o
-endif
-
-################################################################################
-
 obj-$(CONFIG_CRYPTO_LIB_SHA1) += libsha1.o
 libsha1-y := sha1.o
 ifeq ($(CONFIG_CRYPTO_LIB_SHA1_ARCH),y)
@@ -300,10 +316,9 @@ AFLAGS_arm/sha256-core.o += $(aflags-thumb2-y)
 endif
 
 ifeq ($(CONFIG_ARM64),y)
-libsha256-y += arm64/sha256-core.o
+libsha256-y += arm64/sha256-ce.o arm64/sha256-core.o
 $(obj)/arm64/sha256-core.S: $(src)/arm64/sha2-armv8.pl
 	$(call cmd,perlasm_with_args)
-libsha256-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha256-ce.o
 endif
 
 libsha256-$(CONFIG_PPC) += powerpc/sha256-spe-asm.o
@@ -330,10 +345,9 @@ AFLAGS_arm/sha512-core.o += $(aflags-thumb2-y)
 endif
 
 ifeq ($(CONFIG_ARM64),y)
-libsha512-y += arm64/sha512-core.o
+libsha512-y += arm64/sha512-ce-core.o arm64/sha512-core.o
 $(obj)/arm64/sha512-core.S: $(src)/arm64/sha2-armv8.pl
 	$(call cmd,perlasm_with_args)
-libsha512-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha512-ce-core.o
 endif
 
 libsha512-$(CONFIG_RISCV) += riscv/sha512-riscv64-zvknhb-zvkb.o
@@ -355,13 +369,22 @@ endif # CONFIG_CRYPTO_LIB_SHA3_ARCH
 
 ################################################################################
 
+obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o
+libsm3-y := sm3.o
+ifeq ($(CONFIG_CRYPTO_LIB_SM3_ARCH),y)
+CFLAGS_sm3.o += -I$(src)/$(SRCARCH)
+libsm3-$(CONFIG_ARM64) += arm64/sm3-ce-core.o \
+			  arm64/sm3-neon-core.o
+libsm3-$(CONFIG_RISCV) += riscv/sm3-riscv64-zvksh-zvkb.o
+libsm3-$(CONFIG_X86) += x86/sm3-avx-asm_64.o
+endif # CONFIG_CRYPTO_LIB_SM3_ARCH
+
+################################################################################
+
 obj-$(CONFIG_MPILIB) += mpi/
 
 obj-$(CONFIG_CRYPTO_SELFTESTS_FULL)		+= simd.o
 
-obj-$(CONFIG_CRYPTO_LIB_SM3)			+= libsm3.o
-libsm3-y					:= sm3.o
-
 # clean-files must be defined unconditionally
 clean-files += arm/sha256-core.S arm/sha512-core.S
 clean-files += arm64/sha256-core.S arm64/sha512-core.S
diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c
index b73e19f1bb95..ca733f15b2a8 100644
--- a/lib/crypto/aes.c
+++ b/lib/crypto/aes.c
@@ -4,12 +4,15 @@
  * Copyright 2026 Google LLC
  */
 
+#include <crypto/aes-cbc-macs.h>
 #include <crypto/aes.h>
+#include <crypto/utils.h>
 #include <linux/cache.h>
 #include <linux/crypto.h>
 #include <linux/export.h>
 #include <linux/module.h>
 #include <linux/unaligned.h>
+#include "fips.h"
 
 static const u8 ____cacheline_aligned aes_sbox[] = {
 	0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5,
@@ -512,10 +515,235 @@ void aes_decrypt(const struct aes_key *key, u8 out[AES_BLOCK_SIZE],
 }
 EXPORT_SYMBOL(aes_decrypt);
 
-#ifdef aes_mod_init_arch
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS)
+
+#ifndef aes_cbcmac_blocks_arch
+static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE],
+				   const struct aes_enckey *key, const u8 *data,
+				   size_t nblocks, bool enc_before,
+				   bool enc_after)
+{
+	return false;
+}
+#endif
+
+/* This assumes nblocks >= 1. */
+static void aes_cbcmac_blocks(u8 h[AES_BLOCK_SIZE],
+			      const struct aes_enckey *key, const u8 *data,
+			      size_t nblocks, bool enc_before, bool enc_after)
+{
+	if (aes_cbcmac_blocks_arch(h, key, data, nblocks, enc_before,
+				   enc_after))
+		return;
+
+	if (enc_before)
+		aes_encrypt(key, h, h);
+	for (; nblocks > 1; nblocks--) {
+		crypto_xor(h, data, AES_BLOCK_SIZE);
+		data += AES_BLOCK_SIZE;
+		aes_encrypt(key, h, h);
+	}
+	crypto_xor(h, data, AES_BLOCK_SIZE);
+	if (enc_after)
+		aes_encrypt(key, h, h);
+}
+
+int aes_cmac_preparekey(struct aes_cmac_key *key, const u8 *in_key,
+			size_t key_len)
+{
+	u64 hi, lo, mask;
+	int err;
+
+	/* Prepare the AES key. */
+	err = aes_prepareenckey(&key->aes, in_key, key_len);
+	if (err)
+		return err;
+
+	/*
+	 * Prepare the subkeys K1 and K2 by encrypting the all-zeroes block,
+	 * then multiplying by 'x' and 'x^2' (respectively) in GF(2^128).
+	 * Reference: NIST SP 800-38B, Section 6.1 "Subkey Generation".
+	 */
+	memset(key->k_final[0].b, 0, AES_BLOCK_SIZE);
+	aes_encrypt(&key->aes, key->k_final[0].b, key->k_final[0].b);
+	hi = be64_to_cpu(key->k_final[0].w[0]);
+	lo = be64_to_cpu(key->k_final[0].w[1]);
+	for (int i = 0; i < 2; i++) {
+		mask = ((s64)hi >> 63) & 0x87;
+		hi = (hi << 1) ^ (lo >> 63);
+		lo = (lo << 1) ^ mask;
+		key->k_final[i].w[0] = cpu_to_be64(hi);
+		key->k_final[i].w[1] = cpu_to_be64(lo);
+	}
+	return 0;
+}
+EXPORT_SYMBOL_GPL(aes_cmac_preparekey);
+
+void aes_xcbcmac_preparekey(struct aes_cmac_key *key,
+			    const u8 in_key[AES_KEYSIZE_128])
+{
+	static const u8 constants[3][AES_BLOCK_SIZE] = {
+		{ [0 ... AES_BLOCK_SIZE - 1] = 0x1 },
+		{ [0 ... AES_BLOCK_SIZE - 1] = 0x2 },
+		{ [0 ... AES_BLOCK_SIZE - 1] = 0x3 },
+	};
+	u8 new_aes_key[AES_BLOCK_SIZE];
+
+	static_assert(AES_BLOCK_SIZE == AES_KEYSIZE_128);
+	aes_prepareenckey(&key->aes, in_key, AES_BLOCK_SIZE);
+	aes_encrypt(&key->aes, new_aes_key, constants[0]);
+	aes_encrypt(&key->aes, key->k_final[0].b, constants[1]);
+	aes_encrypt(&key->aes, key->k_final[1].b, constants[2]);
+	aes_prepareenckey(&key->aes, new_aes_key, AES_BLOCK_SIZE);
+	memzero_explicit(new_aes_key, AES_BLOCK_SIZE);
+}
+EXPORT_SYMBOL_GPL(aes_xcbcmac_preparekey);
+
+void aes_cmac_update(struct aes_cmac_ctx *ctx, const u8 *data, size_t data_len)
+{
+	bool enc_before = false;
+	size_t nblocks;
+
+	if (ctx->partial_len) {
+		/* XOR data into a pending block. */
+		size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len);
+
+		crypto_xor(&ctx->h[ctx->partial_len], data, l);
+		data += l;
+		data_len -= l;
+		ctx->partial_len += l;
+		if (data_len == 0) {
+			/*
+			 * Either the pending block hasn't been filled yet, or
+			 * no more data was given so it's not yet known whether
+			 * the block is the final block.
+			 */
+			return;
+		}
+		/* Pending block has been filled and isn't the final block. */
+		enc_before = true;
+	}
+
+	nblocks = data_len / AES_BLOCK_SIZE;
+	data_len %= AES_BLOCK_SIZE;
+	if (nblocks == 0) {
+		/* 0 additional full blocks, then optionally a partial block */
+		if (enc_before)
+			aes_encrypt(&ctx->key->aes, ctx->h, ctx->h);
+		crypto_xor(ctx->h, data, data_len);
+		ctx->partial_len = data_len;
+	} else if (data_len != 0) {
+		/* 1 or more additional full blocks, then a partial block */
+		aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks,
+				  enc_before, /* enc_after= */ true);
+		data += nblocks * AES_BLOCK_SIZE;
+		crypto_xor(ctx->h, data, data_len);
+		ctx->partial_len = data_len;
+	} else {
+		/*
+		 * 1 or more additional full blocks only.  Encryption of the
+		 * last block is delayed until it's known whether it's the final
+		 * block in the message or not.
+		 */
+		aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks,
+				  enc_before, /* enc_after= */ false);
+		ctx->partial_len = AES_BLOCK_SIZE;
+	}
+}
+EXPORT_SYMBOL_GPL(aes_cmac_update);
+
+void aes_cmac_final(struct aes_cmac_ctx *ctx, u8 out[AES_BLOCK_SIZE])
+{
+	if (ctx->partial_len == AES_BLOCK_SIZE) {
+		/* Final block is a full block.  Use k_final[0]. */
+		crypto_xor(ctx->h, ctx->key->k_final[0].b, AES_BLOCK_SIZE);
+	} else {
+		/* Final block is a partial block.  Pad, and use k_final[1]. */
+		ctx->h[ctx->partial_len] ^= 0x80;
+		crypto_xor(ctx->h, ctx->key->k_final[1].b, AES_BLOCK_SIZE);
+	}
+	aes_encrypt(&ctx->key->aes, out, ctx->h);
+	memzero_explicit(ctx, sizeof(*ctx));
+}
+EXPORT_SYMBOL_GPL(aes_cmac_final);
+
+void aes_cbcmac_update(struct aes_cbcmac_ctx *ctx, const u8 *data,
+		       size_t data_len)
+{
+	bool enc_before = false;
+	size_t nblocks;
+
+	if (ctx->partial_len) {
+		size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len);
+
+		crypto_xor(&ctx->h[ctx->partial_len], data, l);
+		data += l;
+		data_len -= l;
+		ctx->partial_len += l;
+		if (ctx->partial_len < AES_BLOCK_SIZE)
+			return;
+		enc_before = true;
+	}
+
+	nblocks = data_len / AES_BLOCK_SIZE;
+	data_len %= AES_BLOCK_SIZE;
+	if (nblocks == 0) {
+		if (enc_before)
+			aes_encrypt(ctx->key, ctx->h, ctx->h);
+	} else {
+		aes_cbcmac_blocks(ctx->h, ctx->key, data, nblocks, enc_before,
+				  /* enc_after= */ true);
+		data += nblocks * AES_BLOCK_SIZE;
+	}
+	crypto_xor(ctx->h, data, data_len);
+	ctx->partial_len = data_len;
+}
+EXPORT_SYMBOL_NS_GPL(aes_cbcmac_update, "CRYPTO_INTERNAL");
+
+void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx, u8 out[AES_BLOCK_SIZE])
+{
+	if (ctx->partial_len)
+		aes_encrypt(ctx->key, out, ctx->h);
+	else
+		memcpy(out, ctx->h, AES_BLOCK_SIZE);
+	memzero_explicit(ctx, sizeof(*ctx));
+}
+EXPORT_SYMBOL_NS_GPL(aes_cbcmac_final, "CRYPTO_INTERNAL");
+
+/*
+ * FIPS cryptographic algorithm self-test for AES-CMAC.  As per the FIPS 140-3
+ * Implementation Guidance, a cryptographic algorithm self-test for at least one
+ * of AES-GCM, AES-CCM, AES-CMAC, or AES-GMAC is required if any of those modes
+ * is implemented.  This fulfills that requirement via AES-CMAC.
+ *
+ * This is just for FIPS.  The full tests are in the KUnit test suite.
+ */
+static void __init aes_cmac_fips_test(void)
+{
+	struct aes_cmac_key key;
+	u8 mac[AES_BLOCK_SIZE];
+
+	if (aes_cmac_preparekey(&key, fips_test_key, sizeof(fips_test_key)) !=
+	    0)
+		panic("aes: CMAC FIPS self-test failed (preparekey)\n");
+	aes_cmac(&key, fips_test_data, sizeof(fips_test_data), mac);
+	if (memcmp(fips_test_aes_cmac_value, mac, sizeof(mac)) != 0)
+		panic("aes: CMAC FIPS self-test failed (wrong MAC)\n");
+	memzero_explicit(&key, sizeof(key));
+}
+#else /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */
+static inline void aes_cmac_fips_test(void)
+{
+}
+#endif /* !CONFIG_CRYPTO_LIB_AES_CBC_MACS */
+
 static int __init aes_mod_init(void)
 {
+#ifdef aes_mod_init_arch
 	aes_mod_init_arch();
+#endif
+	if (fips_enabled)
+		aes_cmac_fips_test();
 	return 0;
 }
 subsys_initcall(aes_mod_init);
@@ -524,7 +752,6 @@ static void __exit aes_mod_exit(void)
 {
 }
 module_exit(aes_mod_exit);
-#endif
 
 MODULE_DESCRIPTION("AES block cipher");
 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
diff --git a/lib/crypto/aescfb.c b/lib/crypto/aescfb.c
index 147e5211728f..82cd55436055 100644
--- a/lib/crypto/aescfb.c
+++ b/lib/crypto/aescfb.c
@@ -6,28 +6,9 @@
  */
 
 #include <crypto/aes.h>
-#include <crypto/algapi.h>
+#include <crypto/utils.h>
 #include <linux/export.h>
 #include <linux/module.h>
-#include <asm/irqflags.h>
-
-static void aescfb_encrypt_block(const struct aes_enckey *key, void *dst,
-				 const void *src)
-{
-	unsigned long flags;
-
-	/*
-	 * In AES-CFB, the AES encryption operates on known 'plaintext' (the IV
-	 * and ciphertext), making it susceptible to timing attacks on the
-	 * encryption key. The AES library already mitigates this risk to some
-	 * extent by pulling the entire S-box into the caches before doing any
-	 * substitutions, but this strategy is more effective when running with
-	 * interrupts disabled.
-	 */
-	local_irq_save(flags);
-	aes_encrypt(key, dst, src);
-	local_irq_restore(flags);
-}
 
 /**
  * aescfb_encrypt - Perform AES-CFB encryption on a block of data
@@ -45,7 +26,7 @@ void aescfb_encrypt(const struct aes_enckey *key, u8 *dst, const u8 *src,
 	const u8 *v = iv;
 
 	while (len > 0) {
-		aescfb_encrypt_block(key, ks, v);
+		aes_encrypt(key, ks, v);
 		crypto_xor_cpy(dst, src, ks, min(len, AES_BLOCK_SIZE));
 		v = dst;
 
@@ -72,7 +53,7 @@ void aescfb_decrypt(const struct aes_enckey *key, u8 *dst, const u8 *src,
 {
 	u8 ks[2][AES_BLOCK_SIZE];
 
-	aescfb_encrypt_block(key, ks[0], iv);
+	aes_encrypt(key, ks[0], iv);
 
 	for (int i = 0; len > 0; i ^= 1) {
 		if (len > AES_BLOCK_SIZE)
@@ -81,7 +62,7 @@ void aescfb_decrypt(const struct aes_enckey *key, u8 *dst, const u8 *src,
 			 * performing the XOR, as that may update in place and
 			 * overwrite the ciphertext.
 			 */
-			aescfb_encrypt_block(key, ks[!i], src);
+			aes_encrypt(key, ks[!i], src);
 
 		crypto_xor_cpy(dst, src, ks[i], min(len, AES_BLOCK_SIZE));
 
diff --git a/lib/crypto/aesgcm.c b/lib/crypto/aesgcm.c
index 02f5b5f32c76..1da31e1f747d 100644
--- a/lib/crypto/aesgcm.c
+++ b/lib/crypto/aesgcm.c
@@ -5,30 +5,10 @@
  * Copyright 2022 Google LLC
  */
 
-#include <crypto/algapi.h>
 #include <crypto/gcm.h>
-#include <crypto/ghash.h>
+#include <crypto/utils.h>
 #include <linux/export.h>
 #include <linux/module.h>
-#include <asm/irqflags.h>
-
-static void aesgcm_encrypt_block(const struct aes_enckey *key, void *dst,
-				 const void *src)
-{
-	unsigned long flags;
-
-	/*
-	 * In AES-GCM, both the GHASH key derivation and the CTR mode
-	 * encryption operate on known plaintext, making them susceptible to
-	 * timing attacks on the encryption key. The AES library already
-	 * mitigates this risk to some extent by pulling the entire S-box into
-	 * the caches before doing any substitutions, but this strategy is more
-	 * effective when running with interrupts disabled.
-	 */
-	local_irq_save(flags);
-	aes_encrypt(key, dst, src);
-	local_irq_restore(flags);
-}
 
 /**
  * aesgcm_expandkey - Expands the AES and GHASH keys for the AES-GCM key
@@ -45,7 +25,7 @@ static void aesgcm_encrypt_block(const struct aes_enckey *key, void *dst,
 int aesgcm_expandkey(struct aesgcm_ctx *ctx, const u8 *key,
 		     unsigned int keysize, unsigned int authsize)
 {
-	u8 kin[AES_BLOCK_SIZE] = {};
+	u8 h[AES_BLOCK_SIZE] = {};
 	int ret;
 
 	ret = crypto_gcm_check_authsize(authsize) ?:
@@ -54,24 +34,13 @@ int aesgcm_expandkey(struct aesgcm_ctx *ctx, const u8 *key,
 		return ret;
 
 	ctx->authsize = authsize;
-	aesgcm_encrypt_block(&ctx->aes_key, &ctx->ghash_key, kin);
-
+	aes_encrypt(&ctx->aes_key, h, h);
+	ghash_preparekey(&ctx->ghash_key, h);
+	memzero_explicit(h, sizeof(h));
 	return 0;
 }
 EXPORT_SYMBOL(aesgcm_expandkey);
 
-static void aesgcm_ghash(be128 *ghash, const be128 *key, const void *src,
-			 int len)
-{
-	while (len > 0) {
-		crypto_xor((u8 *)ghash, src, min(len, GHASH_BLOCK_SIZE));
-		gf128mul_lle(ghash, key);
-
-		src += GHASH_BLOCK_SIZE;
-		len -= GHASH_BLOCK_SIZE;
-	}
-}
-
 /**
  * aesgcm_mac - Generates the authentication tag using AES-GCM algorithm.
  * @ctx: The data structure that will hold the AES-GCM key schedule
@@ -88,20 +57,33 @@ static void aesgcm_ghash(be128 *ghash, const be128 *key, const void *src,
 static void aesgcm_mac(const struct aesgcm_ctx *ctx, const u8 *src, int src_len,
 		       const u8 *assoc, int assoc_len, __be32 *ctr, u8 *authtag)
 {
-	be128 tail = { cpu_to_be64(assoc_len * 8), cpu_to_be64(src_len * 8) };
-	u8 buf[AES_BLOCK_SIZE];
-	be128 ghash = {};
+	static const u8 zeroes[GHASH_BLOCK_SIZE];
+	__be64 tail[2] = {
+		cpu_to_be64((u64)assoc_len * 8),
+		cpu_to_be64((u64)src_len * 8),
+	};
+	struct ghash_ctx ghash;
+	u8 ghash_out[AES_BLOCK_SIZE];
+	u8 enc_ctr[AES_BLOCK_SIZE];
+
+	ghash_init(&ghash, &ctx->ghash_key);
 
-	aesgcm_ghash(&ghash, &ctx->ghash_key, assoc, assoc_len);
-	aesgcm_ghash(&ghash, &ctx->ghash_key, src, src_len);
-	aesgcm_ghash(&ghash, &ctx->ghash_key, &tail, sizeof(tail));
+	ghash_update(&ghash, assoc, assoc_len);
+	ghash_update(&ghash, zeroes, -assoc_len & (GHASH_BLOCK_SIZE - 1));
+
+	ghash_update(&ghash, src, src_len);
+	ghash_update(&ghash, zeroes, -src_len & (GHASH_BLOCK_SIZE - 1));
+
+	ghash_update(&ghash, (const u8 *)&tail, sizeof(tail));
+
+	ghash_final(&ghash, ghash_out);
 
 	ctr[3] = cpu_to_be32(1);
-	aesgcm_encrypt_block(&ctx->aes_key, buf, ctr);
-	crypto_xor_cpy(authtag, buf, (u8 *)&ghash, ctx->authsize);
+	aes_encrypt(&ctx->aes_key, enc_ctr, (const u8 *)ctr);
+	crypto_xor_cpy(authtag, ghash_out, enc_ctr, ctx->authsize);
 
-	memzero_explicit(&ghash, sizeof(ghash));
-	memzero_explicit(buf, sizeof(buf));
+	memzero_explicit(ghash_out, sizeof(ghash_out));
+	memzero_explicit(enc_ctr, sizeof(enc_ctr));
 }
 
 static void aesgcm_crypt(const struct aesgcm_ctx *ctx, u8 *dst, const u8 *src,
@@ -119,7 +101,7 @@ static void aesgcm_crypt(const struct aesgcm_ctx *ctx, u8 *dst, const u8 *src,
 		 * len', this cannot happen, so no explicit test is necessary.
 		 */
 		ctr[3] = cpu_to_be32(n++);
-		aesgcm_encrypt_block(&ctx->aes_key, buf, ctr);
+		aes_encrypt(&ctx->aes_key, buf, (const u8 *)ctr);
 		crypto_xor_cpy(dst, src, buf, min(len, AES_BLOCK_SIZE));
 
 		dst += AES_BLOCK_SIZE;
diff --git a/lib/crypto/arm/gf128hash.h b/lib/crypto/arm/gf128hash.h
new file mode 100644
index 000000000000..c33c8cbe51fe
--- /dev/null
+++ b/lib/crypto/arm/gf128hash.h
@@ -0,0 +1,43 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * GHASH, arm optimized
+ *
+ * Copyright 2026 Google LLC
+ */
+
+#include <asm/hwcap.h>
+#include <asm/neon.h>
+#include <asm/simd.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
+
+void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg,
+			   const u8 *src, const struct polyval_elem *h);
+
+#define ghash_blocks_arch ghash_blocks_arch
+static void ghash_blocks_arch(struct polyval_elem *acc,
+			      const struct ghash_key *key,
+			      const u8 *data, size_t nblocks)
+{
+	if (static_branch_likely(&have_neon) && may_use_simd()) {
+		do {
+			/* Allow rescheduling every 4 KiB. */
+			size_t n =
+				min_t(size_t, nblocks, 4096 / GHASH_BLOCK_SIZE);
+
+			scoped_ksimd()
+				pmull_ghash_update_p8(n, acc, data, &key->h);
+			data += n * GHASH_BLOCK_SIZE;
+			nblocks -= n;
+		} while (nblocks);
+	} else {
+		ghash_blocks_generic(acc, &key->h, data, nblocks);
+	}
+}
+
+#define gf128hash_mod_init_arch gf128hash_mod_init_arch
+static void gf128hash_mod_init_arch(void)
+{
+	if (elf_hwcap & HWCAP_NEON)
+		static_branch_enable(&have_neon);
+}
diff --git a/lib/crypto/arm/ghash-neon-core.S b/lib/crypto/arm/ghash-neon-core.S
new file mode 100644
index 000000000000..eeffd12504a9
--- /dev/null
+++ b/lib/crypto/arm/ghash-neon-core.S
@@ -0,0 +1,209 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Accelerated GHASH implementation with NEON vmull.p8 instructions.
+ *
+ * Copyright (C) 2015 - 2017 Linaro Ltd.
+ * Copyright (C) 2023 Google LLC. <ardb@google.com>
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+	.fpu		neon
+
+	SHASH		.req	q0
+	T1		.req	q1
+	XL		.req	q2
+	XM		.req	q3
+	XH		.req	q4
+	IN1		.req	q4
+
+	SHASH_L		.req	d0
+	SHASH_H		.req	d1
+	T1_L		.req	d2
+	T1_H		.req	d3
+	XL_L		.req	d4
+	XL_H		.req	d5
+	XM_L		.req	d6
+	XM_H		.req	d7
+	XH_L		.req	d8
+
+	t0l		.req	d10
+	t0h		.req	d11
+	t1l		.req	d12
+	t1h		.req	d13
+	t2l		.req	d14
+	t2h		.req	d15
+	t3l		.req	d16
+	t3h		.req	d17
+	t4l		.req	d18
+	t4h		.req	d19
+
+	t0q		.req	q5
+	t1q		.req	q6
+	t2q		.req	q7
+	t3q		.req	q8
+	t4q		.req	q9
+
+	s1l		.req	d20
+	s1h		.req	d21
+	s2l		.req	d22
+	s2h		.req	d23
+	s3l		.req	d24
+	s3h		.req	d25
+	s4l		.req	d26
+	s4h		.req	d27
+
+	SHASH2_p8	.req	d28
+
+	k16		.req	d29
+	k32		.req	d30
+	k48		.req	d31
+
+	T2		.req	q7
+
+	.text
+
+	/*
+	 * This implementation of 64x64 -> 128 bit polynomial multiplication
+	 * using vmull.p8 instructions (8x8 -> 16) is taken from the paper
+	 * "Fast Software Polynomial Multiplication on ARM Processors Using
+	 * the NEON Engine" by Danilo Camara, Conrado Gouvea, Julio Lopez and
+	 * Ricardo Dahab (https://hal.inria.fr/hal-01506572)
+	 *
+	 * It has been slightly tweaked for in-order performance, and to allow
+	 * 'rq' to overlap with 'ad' or 'bd'.
+	 */
+	.macro		__pmull_p8, rq, ad, bd, b1=t4l, b2=t3l, b3=t4l, b4=t3l
+	vext.8		t0l, \ad, \ad, #1	@ A1
+	.ifc		\b1, t4l
+	vext.8		t4l, \bd, \bd, #1	@ B1
+	.endif
+	vmull.p8	t0q, t0l, \bd		@ F = A1*B
+	vext.8		t1l, \ad, \ad, #2	@ A2
+	vmull.p8	t4q, \ad, \b1		@ E = A*B1
+	.ifc		\b2, t3l
+	vext.8		t3l, \bd, \bd, #2	@ B2
+	.endif
+	vmull.p8	t1q, t1l, \bd		@ H = A2*B
+	vext.8		t2l, \ad, \ad, #3	@ A3
+	vmull.p8	t3q, \ad, \b2		@ G = A*B2
+	veor		t0q, t0q, t4q		@ L = E + F
+	.ifc		\b3, t4l
+	vext.8		t4l, \bd, \bd, #3	@ B3
+	.endif
+	vmull.p8	t2q, t2l, \bd		@ J = A3*B
+	veor		t0l, t0l, t0h		@ t0 = (L) (P0 + P1) << 8
+	veor		t1q, t1q, t3q		@ M = G + H
+	.ifc		\b4, t3l
+	vext.8		t3l, \bd, \bd, #4	@ B4
+	.endif
+	vmull.p8	t4q, \ad, \b3		@ I = A*B3
+	veor		t1l, t1l, t1h		@ t1 = (M) (P2 + P3) << 16
+	vmull.p8	t3q, \ad, \b4		@ K = A*B4
+	vand		t0h, t0h, k48
+	vand		t1h, t1h, k32
+	veor		t2q, t2q, t4q		@ N = I + J
+	veor		t0l, t0l, t0h
+	veor		t1l, t1l, t1h
+	veor		t2l, t2l, t2h		@ t2 = (N) (P4 + P5) << 24
+	vand		t2h, t2h, k16
+	veor		t3l, t3l, t3h		@ t3 = (K) (P6 + P7) << 32
+	vmov.i64	t3h, #0
+	vext.8		t0q, t0q, t0q, #15
+	veor		t2l, t2l, t2h
+	vext.8		t1q, t1q, t1q, #14
+	vmull.p8	\rq, \ad, \bd		@ D = A*B
+	vext.8		t2q, t2q, t2q, #13
+	vext.8		t3q, t3q, t3q, #12
+	veor		t0q, t0q, t1q
+	veor		t2q, t2q, t3q
+	veor		\rq, \rq, t0q
+	veor		\rq, \rq, t2q
+	.endm
+
+	.macro		__pmull_reduce_p8
+	veor		XL_H, XL_H, XM_L
+	veor		XH_L, XH_L, XM_H
+
+	vshl.i64	T1, XL, #57
+	vshl.i64	T2, XL, #62
+	veor		T1, T1, T2
+	vshl.i64	T2, XL, #63
+	veor		T1, T1, T2
+	veor		XL_H, XL_H, T1_L
+	veor		XH_L, XH_L, T1_H
+
+	vshr.u64	T1, XL, #1
+	veor		XH, XH, XL
+	veor		XL, XL, T1
+	vshr.u64	T1, T1, #6
+	vshr.u64	XL, XL, #1
+	.endm
+
+	.macro		vrev64_if_be	a
+#ifdef CONFIG_CPU_BIG_ENDIAN
+	vrev64.8	\a, \a
+#endif
+	.endm
+
+	.macro		ghash_update
+	vld1.64		{XL}, [r1]
+	vrev64_if_be	XL
+
+0:
+	vld1.8		{T1}, [r2]!
+	subs		r0, r0, #1
+
+	/* multiply XL by SHASH in GF(2^128) */
+	vrev64.8	T1, T1
+
+	vext.8		IN1, T1, T1, #8
+	veor		T1_L, T1_L, XL_H
+	veor		XL, XL, IN1
+
+	__pmull_p8	XH, XL_H, SHASH_H, s1h, s2h, s3h, s4h	@ a1 * b1
+	veor		T1, T1, XL
+	__pmull_p8	XL, XL_L, SHASH_L, s1l, s2l, s3l, s4l	@ a0 * b0
+	__pmull_p8	XM, T1_L, SHASH2_p8			@ (a1+a0)(b1+b0)
+
+	veor		T1, XL, XH
+	veor		XM, XM, T1
+
+	__pmull_reduce_p8
+
+	veor		T1, T1, XH
+	veor		XL, XL, T1
+
+	bne		0b
+	.endm
+
+	/*
+	 * void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg,
+	 *			      const u8 *src,
+	 *			      const struct polyval_elem *h)
+	 */
+ENTRY(pmull_ghash_update_p8)
+	vld1.64		{SHASH}, [r3]
+	vrev64_if_be	SHASH
+	veor		SHASH2_p8, SHASH_L, SHASH_H
+
+	vext.8		s1l, SHASH_L, SHASH_L, #1
+	vext.8		s2l, SHASH_L, SHASH_L, #2
+	vext.8		s3l, SHASH_L, SHASH_L, #3
+	vext.8		s4l, SHASH_L, SHASH_L, #4
+	vext.8		s1h, SHASH_H, SHASH_H, #1
+	vext.8		s2h, SHASH_H, SHASH_H, #2
+	vext.8		s3h, SHASH_H, SHASH_H, #3
+	vext.8		s4h, SHASH_H, SHASH_H, #4
+
+	vmov.i64	k16, #0xffff
+	vmov.i64	k32, #0xffffffff
+	vmov.i64	k48, #0xffffffffffff
+
+	ghash_update
+	vrev64_if_be	XL
+	vst1.64		{XL}, [r1]
+
+	bx		lr
+ENDPROC(pmull_ghash_update_p8)
diff --git a/lib/crypto/arm64/aes-ce.S b/lib/crypto/arm64/aes-ce.S
new file mode 100644
index 000000000000..b853e02f7b1e
--- /dev/null
+++ b/lib/crypto/arm64/aes-ce.S
@@ -0,0 +1,145 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * AES cipher for ARMv8 with Crypto Extensions
+ *
+ * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+#define AES_FUNC_START(func)		SYM_FUNC_START(ce_ ## func)
+#define AES_FUNC_END(func)		SYM_FUNC_END(ce_ ## func)
+
+	.arch		armv8-a+crypto
+
+	xtsmask		.req	v16
+	cbciv		.req	v16
+	vctr		.req	v16
+
+	.macro		xts_reload_mask, tmp
+	.endm
+
+	.macro		xts_cts_skip_tw, reg, lbl
+	.endm
+
+	/* preload all round keys */
+	.macro		load_round_keys, rk, nr, tmp
+	add		\tmp, \rk, \nr, sxtw #4
+	sub		\tmp, \tmp, #160
+	ld1		{v17.4s-v20.4s}, [\rk]
+	ld1		{v21.4s-v24.4s}, [\tmp], #64
+	ld1		{v25.4s-v28.4s}, [\tmp], #64
+	ld1		{v29.4s-v31.4s}, [\tmp]
+	.endm
+
+	/* prepare for encryption with key in rk[] */
+	.macro		enc_prepare, rounds, rk, temp
+	load_round_keys	\rk, \rounds, \temp
+	.endm
+
+	/* prepare for encryption (again) but with new key in rk[] */
+	.macro		enc_switch_key, rounds, rk, temp
+	load_round_keys	\rk, \rounds, \temp
+	.endm
+
+	/* prepare for decryption with key in rk[] */
+	.macro		dec_prepare, rounds, rk, temp
+	load_round_keys	\rk, \rounds, \temp
+	.endm
+
+	.macro		do_enc_Nx, de, mc, k, i0, i1, i2, i3, i4
+	aes\de		\i0\().16b, \k\().16b
+	aes\mc		\i0\().16b, \i0\().16b
+	.ifnb		\i1
+	aes\de		\i1\().16b, \k\().16b
+	aes\mc		\i1\().16b, \i1\().16b
+	.ifnb		\i3
+	aes\de		\i2\().16b, \k\().16b
+	aes\mc		\i2\().16b, \i2\().16b
+	aes\de		\i3\().16b, \k\().16b
+	aes\mc		\i3\().16b, \i3\().16b
+	.ifnb		\i4
+	aes\de		\i4\().16b, \k\().16b
+	aes\mc		\i4\().16b, \i4\().16b
+	.endif
+	.endif
+	.endif
+	.endm
+
+	/* up to 5 interleaved encryption rounds with the same round key */
+	.macro		round_Nx, enc, k, i0, i1, i2, i3, i4
+	.ifc		\enc, e
+	do_enc_Nx	e, mc, \k, \i0, \i1, \i2, \i3, \i4
+	.else
+	do_enc_Nx	d, imc, \k, \i0, \i1, \i2, \i3, \i4
+	.endif
+	.endm
+
+	/* up to 5 interleaved final rounds */
+	.macro		fin_round_Nx, de, k, k2, i0, i1, i2, i3, i4
+	aes\de		\i0\().16b, \k\().16b
+	.ifnb		\i1
+	aes\de		\i1\().16b, \k\().16b
+	.ifnb		\i3
+	aes\de		\i2\().16b, \k\().16b
+	aes\de		\i3\().16b, \k\().16b
+	.ifnb		\i4
+	aes\de		\i4\().16b, \k\().16b
+	.endif
+	.endif
+	.endif
+	eor		\i0\().16b, \i0\().16b, \k2\().16b
+	.ifnb		\i1
+	eor		\i1\().16b, \i1\().16b, \k2\().16b
+	.ifnb		\i3
+	eor		\i2\().16b, \i2\().16b, \k2\().16b
+	eor		\i3\().16b, \i3\().16b, \k2\().16b
+	.ifnb		\i4
+	eor		\i4\().16b, \i4\().16b, \k2\().16b
+	.endif
+	.endif
+	.endif
+	.endm
+
+	/* up to 5 interleaved blocks */
+	.macro		do_block_Nx, enc, rounds, i0, i1, i2, i3, i4
+	tbz		\rounds, #2, .L\@	/* 128 bits */
+	round_Nx	\enc, v17, \i0, \i1, \i2, \i3, \i4
+	round_Nx	\enc, v18, \i0, \i1, \i2, \i3, \i4
+	tbz		\rounds, #1, .L\@	/* 192 bits */
+	round_Nx	\enc, v19, \i0, \i1, \i2, \i3, \i4
+	round_Nx	\enc, v20, \i0, \i1, \i2, \i3, \i4
+.L\@:	.irp		key, v21, v22, v23, v24, v25, v26, v27, v28, v29
+	round_Nx	\enc, \key, \i0, \i1, \i2, \i3, \i4
+	.endr
+	fin_round_Nx	\enc, v30, v31, \i0, \i1, \i2, \i3, \i4
+	.endm
+
+	.macro		encrypt_block, in, rounds, t0, t1, t2
+	do_block_Nx	e, \rounds, \in
+	.endm
+
+	.macro		encrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
+	do_block_Nx	e, \rounds, \i0, \i1, \i2, \i3
+	.endm
+
+	.macro		encrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2
+	do_block_Nx	e, \rounds, \i0, \i1, \i2, \i3, \i4
+	.endm
+
+	.macro		decrypt_block, in, rounds, t0, t1, t2
+	do_block_Nx	d, \rounds, \in
+	.endm
+
+	.macro		decrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
+	do_block_Nx	d, \rounds, \i0, \i1, \i2, \i3
+	.endm
+
+	.macro		decrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2
+	do_block_Nx	d, \rounds, \i0, \i1, \i2, \i3, \i4
+	.endm
+
+#define MAX_STRIDE	5
+
+#include "aes-modes.S"
diff --git a/lib/crypto/arm64/aes-cipher-core.S b/lib/crypto/arm64/aes-cipher-core.S
index 651f701c56a8..0b05ec4be65f 100644
--- a/lib/crypto/arm64/aes-cipher-core.S
+++ b/lib/crypto/arm64/aes-cipher-core.S
@@ -87,11 +87,6 @@
 	ldp		w8, w9, [rk], #16
 	ldp		w10, w11, [rk, #-8]
 
-CPU_BE(	rev		w4, w4		)
-CPU_BE(	rev		w5, w5		)
-CPU_BE(	rev		w6, w6		)
-CPU_BE(	rev		w7, w7		)
-
 	eor		w4, w4, w8
 	eor		w5, w5, w9
 	eor		w6, w6, w10
@@ -112,11 +107,6 @@ CPU_BE(	rev		w7, w7		)
 3:	adr_l		tt, \ltab
 	\round		w4, w5, w6, w7, w8, w9, w10, w11, \bsz, b
 
-CPU_BE(	rev		w4, w4		)
-CPU_BE(	rev		w5, w5		)
-CPU_BE(	rev		w6, w6		)
-CPU_BE(	rev		w7, w7		)
-
 	stp		w4, w5, [out]
 	stp		w6, w7, [out, #8]
 	ret
diff --git a/lib/crypto/arm64/aes-modes.S b/lib/crypto/arm64/aes-modes.S
new file mode 100644
index 000000000000..10e537317eaf
--- /dev/null
+++ b/lib/crypto/arm64/aes-modes.S
@@ -0,0 +1,867 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Chaining mode wrappers for AES
+ *
+ * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
+ */
+
+/* included by aes-ce.S and aes-neon.S */
+
+	.text
+	.align		4
+
+#ifndef MAX_STRIDE
+#define MAX_STRIDE	4
+#endif
+
+#if MAX_STRIDE == 4
+#define ST4(x...) x
+#define ST5(x...)
+#else
+#define ST4(x...)
+#define ST5(x...) x
+#endif
+
+SYM_FUNC_START_LOCAL(aes_encrypt_block4x)
+	encrypt_block4x	v0, v1, v2, v3, w3, x2, x8, w7
+	ret
+SYM_FUNC_END(aes_encrypt_block4x)
+
+SYM_FUNC_START_LOCAL(aes_decrypt_block4x)
+	decrypt_block4x	v0, v1, v2, v3, w3, x2, x8, w7
+	ret
+SYM_FUNC_END(aes_decrypt_block4x)
+
+#if MAX_STRIDE == 5
+SYM_FUNC_START_LOCAL(aes_encrypt_block5x)
+	encrypt_block5x	v0, v1, v2, v3, v4, w3, x2, x8, w7
+	ret
+SYM_FUNC_END(aes_encrypt_block5x)
+
+SYM_FUNC_START_LOCAL(aes_decrypt_block5x)
+	decrypt_block5x	v0, v1, v2, v3, v4, w3, x2, x8, w7
+	ret
+SYM_FUNC_END(aes_decrypt_block5x)
+#endif
+
+	/*
+	 * aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+	 *		   int blocks)
+	 * aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+	 *		   int blocks)
+	 */
+
+AES_FUNC_START(aes_ecb_encrypt)
+	frame_push	0
+
+	enc_prepare	w3, x2, x5
+
+.LecbencloopNx:
+	subs		w4, w4, #MAX_STRIDE
+	bmi		.Lecbenc1x
+	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 pt blocks */
+ST4(	bl		aes_encrypt_block4x		)
+ST5(	ld1		{v4.16b}, [x1], #16		)
+ST5(	bl		aes_encrypt_block5x		)
+	st1		{v0.16b-v3.16b}, [x0], #64
+ST5(	st1		{v4.16b}, [x0], #16		)
+	b		.LecbencloopNx
+.Lecbenc1x:
+	adds		w4, w4, #MAX_STRIDE
+	beq		.Lecbencout
+.Lecbencloop:
+	ld1		{v0.16b}, [x1], #16		/* get next pt block */
+	encrypt_block	v0, w3, x2, x5, w6
+	st1		{v0.16b}, [x0], #16
+	subs		w4, w4, #1
+	bne		.Lecbencloop
+.Lecbencout:
+	frame_pop
+	ret
+AES_FUNC_END(aes_ecb_encrypt)
+
+
+AES_FUNC_START(aes_ecb_decrypt)
+	frame_push	0
+
+	dec_prepare	w3, x2, x5
+
+.LecbdecloopNx:
+	subs		w4, w4, #MAX_STRIDE
+	bmi		.Lecbdec1x
+	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 ct blocks */
+ST4(	bl		aes_decrypt_block4x		)
+ST5(	ld1		{v4.16b}, [x1], #16		)
+ST5(	bl		aes_decrypt_block5x		)
+	st1		{v0.16b-v3.16b}, [x0], #64
+ST5(	st1		{v4.16b}, [x0], #16		)
+	b		.LecbdecloopNx
+.Lecbdec1x:
+	adds		w4, w4, #MAX_STRIDE
+	beq		.Lecbdecout
+.Lecbdecloop:
+	ld1		{v0.16b}, [x1], #16		/* get next ct block */
+	decrypt_block	v0, w3, x2, x5, w6
+	st1		{v0.16b}, [x0], #16
+	subs		w4, w4, #1
+	bne		.Lecbdecloop
+.Lecbdecout:
+	frame_pop
+	ret
+AES_FUNC_END(aes_ecb_decrypt)
+
+
+	/*
+	 * aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+	 *		   int blocks, u8 iv[])
+	 * aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+	 *		   int blocks, u8 iv[])
+	 * aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[],
+	 *			 int rounds, int blocks, u8 iv[],
+	 *			 u32 const rk2[]);
+	 * aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[],
+	 *			 int rounds, int blocks, u8 iv[],
+	 *			 u32 const rk2[]);
+	 */
+
+AES_FUNC_START(aes_essiv_cbc_encrypt)
+	ld1		{v4.16b}, [x5]			/* get iv */
+
+	mov		w8, #14				/* AES-256: 14 rounds */
+	enc_prepare	w8, x6, x7
+	encrypt_block	v4, w8, x6, x7, w9
+	enc_switch_key	w3, x2, x6
+	b		.Lcbcencloop4x
+
+AES_FUNC_START(aes_cbc_encrypt)
+	ld1		{v4.16b}, [x5]			/* get iv */
+	enc_prepare	w3, x2, x6
+
+.Lcbcencloop4x:
+	subs		w4, w4, #4
+	bmi		.Lcbcenc1x
+	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 pt blocks */
+	eor		v0.16b, v0.16b, v4.16b		/* ..and xor with iv */
+	encrypt_block	v0, w3, x2, x6, w7
+	eor		v1.16b, v1.16b, v0.16b
+	encrypt_block	v1, w3, x2, x6, w7
+	eor		v2.16b, v2.16b, v1.16b
+	encrypt_block	v2, w3, x2, x6, w7
+	eor		v3.16b, v3.16b, v2.16b
+	encrypt_block	v3, w3, x2, x6, w7
+	st1		{v0.16b-v3.16b}, [x0], #64
+	mov		v4.16b, v3.16b
+	b		.Lcbcencloop4x
+.Lcbcenc1x:
+	adds		w4, w4, #4
+	beq		.Lcbcencout
+.Lcbcencloop:
+	ld1		{v0.16b}, [x1], #16		/* get next pt block */
+	eor		v4.16b, v4.16b, v0.16b		/* ..and xor with iv */
+	encrypt_block	v4, w3, x2, x6, w7
+	st1		{v4.16b}, [x0], #16
+	subs		w4, w4, #1
+	bne		.Lcbcencloop
+.Lcbcencout:
+	st1		{v4.16b}, [x5]			/* return iv */
+	ret
+AES_FUNC_END(aes_cbc_encrypt)
+AES_FUNC_END(aes_essiv_cbc_encrypt)
+
+AES_FUNC_START(aes_essiv_cbc_decrypt)
+	ld1		{cbciv.16b}, [x5]		/* get iv */
+
+	mov		w8, #14				/* AES-256: 14 rounds */
+	enc_prepare	w8, x6, x7
+	encrypt_block	cbciv, w8, x6, x7, w9
+	b		.Lessivcbcdecstart
+
+AES_FUNC_START(aes_cbc_decrypt)
+	ld1		{cbciv.16b}, [x5]		/* get iv */
+.Lessivcbcdecstart:
+	frame_push	0
+	dec_prepare	w3, x2, x6
+
+.LcbcdecloopNx:
+	subs		w4, w4, #MAX_STRIDE
+	bmi		.Lcbcdec1x
+	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 ct blocks */
+#if MAX_STRIDE == 5
+	ld1		{v4.16b}, [x1], #16		/* get 1 ct block */
+	mov		v5.16b, v0.16b
+	mov		v6.16b, v1.16b
+	mov		v7.16b, v2.16b
+	bl		aes_decrypt_block5x
+	sub		x1, x1, #32
+	eor		v0.16b, v0.16b, cbciv.16b
+	eor		v1.16b, v1.16b, v5.16b
+	ld1		{v5.16b}, [x1], #16		/* reload 1 ct block */
+	ld1		{cbciv.16b}, [x1], #16		/* reload 1 ct block */
+	eor		v2.16b, v2.16b, v6.16b
+	eor		v3.16b, v3.16b, v7.16b
+	eor		v4.16b, v4.16b, v5.16b
+#else
+	mov		v4.16b, v0.16b
+	mov		v5.16b, v1.16b
+	mov		v6.16b, v2.16b
+	bl		aes_decrypt_block4x
+	sub		x1, x1, #16
+	eor		v0.16b, v0.16b, cbciv.16b
+	eor		v1.16b, v1.16b, v4.16b
+	ld1		{cbciv.16b}, [x1], #16		/* reload 1 ct block */
+	eor		v2.16b, v2.16b, v5.16b
+	eor		v3.16b, v3.16b, v6.16b
+#endif
+	st1		{v0.16b-v3.16b}, [x0], #64
+ST5(	st1		{v4.16b}, [x0], #16		)
+	b		.LcbcdecloopNx
+.Lcbcdec1x:
+	adds		w4, w4, #MAX_STRIDE
+	beq		.Lcbcdecout
+.Lcbcdecloop:
+	ld1		{v1.16b}, [x1], #16		/* get next ct block */
+	mov		v0.16b, v1.16b			/* ...and copy to v0 */
+	decrypt_block	v0, w3, x2, x6, w7
+	eor		v0.16b, v0.16b, cbciv.16b	/* xor with iv => pt */
+	mov		cbciv.16b, v1.16b		/* ct is next iv */
+	st1		{v0.16b}, [x0], #16
+	subs		w4, w4, #1
+	bne		.Lcbcdecloop
+.Lcbcdecout:
+	st1		{cbciv.16b}, [x5]		/* return iv */
+	frame_pop
+	ret
+AES_FUNC_END(aes_cbc_decrypt)
+AES_FUNC_END(aes_essiv_cbc_decrypt)
+
+
+	/*
+	 * aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
+	 *		       int rounds, int bytes, u8 const iv[])
+	 * aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
+	 *		       int rounds, int bytes, u8 const iv[])
+	 */
+
+AES_FUNC_START(aes_cbc_cts_encrypt)
+	adr_l		x8, .Lcts_permute_table
+	sub		x4, x4, #16
+	add		x9, x8, #32
+	add		x8, x8, x4
+	sub		x9, x9, x4
+	ld1		{v3.16b}, [x8]
+	ld1		{v4.16b}, [x9]
+
+	ld1		{v0.16b}, [x1], x4		/* overlapping loads */
+	ld1		{v1.16b}, [x1]
+
+	ld1		{v5.16b}, [x5]			/* get iv */
+	enc_prepare	w3, x2, x6
+
+	eor		v0.16b, v0.16b, v5.16b		/* xor with iv */
+	tbl		v1.16b, {v1.16b}, v4.16b
+	encrypt_block	v0, w3, x2, x6, w7
+
+	eor		v1.16b, v1.16b, v0.16b
+	tbl		v0.16b, {v0.16b}, v3.16b
+	encrypt_block	v1, w3, x2, x6, w7
+
+	add		x4, x0, x4
+	st1		{v0.16b}, [x4]			/* overlapping stores */
+	st1		{v1.16b}, [x0]
+	ret
+AES_FUNC_END(aes_cbc_cts_encrypt)
+
+AES_FUNC_START(aes_cbc_cts_decrypt)
+	adr_l		x8, .Lcts_permute_table
+	sub		x4, x4, #16
+	add		x9, x8, #32
+	add		x8, x8, x4
+	sub		x9, x9, x4
+	ld1		{v3.16b}, [x8]
+	ld1		{v4.16b}, [x9]
+
+	ld1		{v0.16b}, [x1], x4		/* overlapping loads */
+	ld1		{v1.16b}, [x1]
+
+	ld1		{v5.16b}, [x5]			/* get iv */
+	dec_prepare	w3, x2, x6
+
+	decrypt_block	v0, w3, x2, x6, w7
+	tbl		v2.16b, {v0.16b}, v3.16b
+	eor		v2.16b, v2.16b, v1.16b
+
+	tbx		v0.16b, {v1.16b}, v4.16b
+	decrypt_block	v0, w3, x2, x6, w7
+	eor		v0.16b, v0.16b, v5.16b		/* xor with iv */
+
+	add		x4, x0, x4
+	st1		{v2.16b}, [x4]			/* overlapping stores */
+	st1		{v0.16b}, [x0]
+	ret
+AES_FUNC_END(aes_cbc_cts_decrypt)
+
+	.section	".rodata", "a"
+	.align		6
+.Lcts_permute_table:
+	.byte		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+	.byte		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+	.byte		 0x0,  0x1,  0x2,  0x3,  0x4,  0x5,  0x6,  0x7
+	.byte		 0x8,  0x9,  0xa,  0xb,  0xc,  0xd,  0xe,  0xf
+	.byte		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+	.byte		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+	.previous
+
+	/*
+	 * This macro generates the code for CTR and XCTR mode.
+	 */
+.macro ctr_encrypt xctr
+	// Arguments
+	OUT		.req x0
+	IN		.req x1
+	KEY		.req x2
+	ROUNDS_W	.req w3
+	BYTES_W		.req w4
+	IV		.req x5
+	BYTE_CTR_W 	.req w6		// XCTR only
+	// Intermediate values
+	CTR_W		.req w11	// XCTR only
+	CTR		.req x11	// XCTR only
+	IV_PART		.req x12
+	BLOCKS		.req x13
+	BLOCKS_W	.req w13
+
+	frame_push	0
+
+	enc_prepare	ROUNDS_W, KEY, IV_PART
+	ld1		{vctr.16b}, [IV]
+
+	/*
+	 * Keep 64 bits of the IV in a register.  For CTR mode this lets us
+	 * easily increment the IV.  For XCTR mode this lets us efficiently XOR
+	 * the 64-bit counter with the IV.
+	 */
+	.if \xctr
+		umov		IV_PART, vctr.d[0]
+		lsr		CTR_W, BYTE_CTR_W, #4
+	.else
+		umov		IV_PART, vctr.d[1]
+		rev		IV_PART, IV_PART
+	.endif
+
+.LctrloopNx\xctr:
+	add		BLOCKS_W, BYTES_W, #15
+	sub		BYTES_W, BYTES_W, #MAX_STRIDE << 4
+	lsr		BLOCKS_W, BLOCKS_W, #4
+	mov		w8, #MAX_STRIDE
+	cmp		BLOCKS_W, w8
+	csel		BLOCKS_W, BLOCKS_W, w8, lt
+
+	/*
+	 * Set up the counter values in v0-v{MAX_STRIDE-1}.
+	 *
+	 * If we are encrypting less than MAX_STRIDE blocks, the tail block
+	 * handling code expects the last keystream block to be in
+	 * v{MAX_STRIDE-1}.  For example: if encrypting two blocks with
+	 * MAX_STRIDE=5, then v3 and v4 should have the next two counter blocks.
+	 */
+	.if \xctr
+		add		CTR, CTR, BLOCKS
+	.else
+		adds		IV_PART, IV_PART, BLOCKS
+	.endif
+	mov		v0.16b, vctr.16b
+	mov		v1.16b, vctr.16b
+	mov		v2.16b, vctr.16b
+	mov		v3.16b, vctr.16b
+ST5(	mov		v4.16b, vctr.16b		)
+	.if \xctr
+		sub		x6, CTR, #MAX_STRIDE - 1
+		sub		x7, CTR, #MAX_STRIDE - 2
+		sub		x8, CTR, #MAX_STRIDE - 3
+		sub		x9, CTR, #MAX_STRIDE - 4
+ST5(		sub		x10, CTR, #MAX_STRIDE - 5	)
+		eor		x6, x6, IV_PART
+		eor		x7, x7, IV_PART
+		eor		x8, x8, IV_PART
+		eor		x9, x9, IV_PART
+ST5(		eor		x10, x10, IV_PART		)
+		mov		v0.d[0], x6
+		mov		v1.d[0], x7
+		mov		v2.d[0], x8
+		mov		v3.d[0], x9
+ST5(		mov		v4.d[0], x10			)
+	.else
+		bcs		0f
+		.subsection	1
+		/*
+		 * This subsection handles carries.
+		 *
+		 * Conditional branching here is allowed with respect to time
+		 * invariance since the branches are dependent on the IV instead
+		 * of the plaintext or key.  This code is rarely executed in
+		 * practice anyway.
+		 */
+
+		/* Apply carry to outgoing counter. */
+0:		umov		x8, vctr.d[0]
+		rev		x8, x8
+		add		x8, x8, #1
+		rev		x8, x8
+		ins		vctr.d[0], x8
+
+		/*
+		 * Apply carry to counter blocks if needed.
+		 *
+		 * Since the carry flag was set, we know 0 <= IV_PART <
+		 * MAX_STRIDE.  Using the value of IV_PART we can determine how
+		 * many counter blocks need to be updated.
+		 */
+		cbz		IV_PART, 2f
+		adr		x16, 1f
+		sub		x16, x16, IV_PART, lsl #3
+		br		x16
+		bti		c
+		mov		v0.d[0], vctr.d[0]
+		bti		c
+		mov		v1.d[0], vctr.d[0]
+		bti		c
+		mov		v2.d[0], vctr.d[0]
+		bti		c
+		mov		v3.d[0], vctr.d[0]
+ST5(		bti		c				)
+ST5(		mov		v4.d[0], vctr.d[0]		)
+1:		b		2f
+		.previous
+
+2:		rev		x7, IV_PART
+		ins		vctr.d[1], x7
+		sub		x7, IV_PART, #MAX_STRIDE - 1
+		sub		x8, IV_PART, #MAX_STRIDE - 2
+		sub		x9, IV_PART, #MAX_STRIDE - 3
+		rev		x7, x7
+		rev		x8, x8
+		mov		v1.d[1], x7
+		rev		x9, x9
+ST5(		sub		x10, IV_PART, #MAX_STRIDE - 4	)
+		mov		v2.d[1], x8
+ST5(		rev		x10, x10			)
+		mov		v3.d[1], x9
+ST5(		mov		v4.d[1], x10			)
+	.endif
+
+	/*
+	 * If there are at least MAX_STRIDE blocks left, XOR the data with
+	 * keystream and store.  Otherwise jump to tail handling.
+	 */
+	tbnz		BYTES_W, #31, .Lctrtail\xctr
+	ld1		{v5.16b-v7.16b}, [IN], #48
+ST4(	bl		aes_encrypt_block4x		)
+ST5(	bl		aes_encrypt_block5x		)
+	eor		v0.16b, v5.16b, v0.16b
+ST4(	ld1		{v5.16b}, [IN], #16		)
+	eor		v1.16b, v6.16b, v1.16b
+ST5(	ld1		{v5.16b-v6.16b}, [IN], #32	)
+	eor		v2.16b, v7.16b, v2.16b
+	eor		v3.16b, v5.16b, v3.16b
+ST5(	eor		v4.16b, v6.16b, v4.16b		)
+	st1		{v0.16b-v3.16b}, [OUT], #64
+ST5(	st1		{v4.16b}, [OUT], #16		)
+	cbz		BYTES_W, .Lctrout\xctr
+	b		.LctrloopNx\xctr
+
+.Lctrout\xctr:
+	.if !\xctr
+		st1		{vctr.16b}, [IV] /* return next CTR value */
+	.endif
+	frame_pop
+	ret
+
+.Lctrtail\xctr:
+	/*
+	 * Handle up to MAX_STRIDE * 16 - 1 bytes of plaintext
+	 *
+	 * This code expects the last keystream block to be in v{MAX_STRIDE-1}.
+	 * For example: if encrypting two blocks with MAX_STRIDE=5, then v3 and
+	 * v4 should have the next two counter blocks.
+	 *
+	 * This allows us to store the ciphertext by writing to overlapping
+	 * regions of memory.  Any invalid ciphertext blocks get overwritten by
+	 * correctly computed blocks.  This approach greatly simplifies the
+	 * logic for storing the ciphertext.
+	 */
+	mov		x16, #16
+	ands		w7, BYTES_W, #0xf
+	csel		x13, x7, x16, ne
+
+ST5(	cmp		BYTES_W, #64 - (MAX_STRIDE << 4))
+ST5(	csel		x14, x16, xzr, gt		)
+	cmp		BYTES_W, #48 - (MAX_STRIDE << 4)
+	csel		x15, x16, xzr, gt
+	cmp		BYTES_W, #32 - (MAX_STRIDE << 4)
+	csel		x16, x16, xzr, gt
+	cmp		BYTES_W, #16 - (MAX_STRIDE << 4)
+
+	adr_l		x9, .Lcts_permute_table
+	add		x9, x9, x13
+	ble		.Lctrtail1x\xctr
+
+ST5(	ld1		{v5.16b}, [IN], x14		)
+	ld1		{v6.16b}, [IN], x15
+	ld1		{v7.16b}, [IN], x16
+
+ST4(	bl		aes_encrypt_block4x		)
+ST5(	bl		aes_encrypt_block5x		)
+
+	ld1		{v8.16b}, [IN], x13
+	ld1		{v9.16b}, [IN]
+	ld1		{v10.16b}, [x9]
+
+ST4(	eor		v6.16b, v6.16b, v0.16b		)
+ST4(	eor		v7.16b, v7.16b, v1.16b		)
+ST4(	tbl		v3.16b, {v3.16b}, v10.16b	)
+ST4(	eor		v8.16b, v8.16b, v2.16b		)
+ST4(	eor		v9.16b, v9.16b, v3.16b		)
+
+ST5(	eor		v5.16b, v5.16b, v0.16b		)
+ST5(	eor		v6.16b, v6.16b, v1.16b		)
+ST5(	tbl		v4.16b, {v4.16b}, v10.16b	)
+ST5(	eor		v7.16b, v7.16b, v2.16b		)
+ST5(	eor		v8.16b, v8.16b, v3.16b		)
+ST5(	eor		v9.16b, v9.16b, v4.16b		)
+
+ST5(	st1		{v5.16b}, [OUT], x14		)
+	st1		{v6.16b}, [OUT], x15
+	st1		{v7.16b}, [OUT], x16
+	add		x13, x13, OUT
+	st1		{v9.16b}, [x13]		// overlapping stores
+	st1		{v8.16b}, [OUT]
+	b		.Lctrout\xctr
+
+.Lctrtail1x\xctr:
+	/*
+	 * Handle <= 16 bytes of plaintext
+	 *
+	 * This code always reads and writes 16 bytes.  To avoid out of bounds
+	 * accesses, XCTR and CTR modes must use a temporary buffer when
+	 * encrypting/decrypting less than 16 bytes.
+	 *
+	 * This code is unusual in that it loads the input and stores the output
+	 * relative to the end of the buffers rather than relative to the start.
+	 * This causes unusual behaviour when encrypting/decrypting less than 16
+	 * bytes; the end of the data is expected to be at the end of the
+	 * temporary buffer rather than the start of the data being at the start
+	 * of the temporary buffer.
+	 */
+	sub		x8, x7, #16
+	csel		x7, x7, x8, eq
+	add		IN, IN, x7
+	add		OUT, OUT, x7
+	ld1		{v5.16b}, [IN]
+	ld1		{v6.16b}, [OUT]
+ST5(	mov		v3.16b, v4.16b			)
+	encrypt_block	v3, ROUNDS_W, KEY, x8, w7
+	ld1		{v10.16b-v11.16b}, [x9]
+	tbl		v3.16b, {v3.16b}, v10.16b
+	sshr		v11.16b, v11.16b, #7
+	eor		v5.16b, v5.16b, v3.16b
+	bif		v5.16b, v6.16b, v11.16b
+	st1		{v5.16b}, [OUT]
+	b		.Lctrout\xctr
+
+	// Arguments
+	.unreq OUT
+	.unreq IN
+	.unreq KEY
+	.unreq ROUNDS_W
+	.unreq BYTES_W
+	.unreq IV
+	.unreq BYTE_CTR_W	// XCTR only
+	// Intermediate values
+	.unreq CTR_W		// XCTR only
+	.unreq CTR		// XCTR only
+	.unreq IV_PART
+	.unreq BLOCKS
+	.unreq BLOCKS_W
+.endm
+
+	/*
+	 * aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+	 *		   int bytes, u8 ctr[])
+	 *
+	 * The input and output buffers must always be at least 16 bytes even if
+	 * encrypting/decrypting less than 16 bytes.  Otherwise out of bounds
+	 * accesses will occur.  The data to be encrypted/decrypted is expected
+	 * to be at the end of this 16-byte temporary buffer rather than the
+	 * start.
+	 */
+
+AES_FUNC_START(aes_ctr_encrypt)
+	ctr_encrypt 0
+AES_FUNC_END(aes_ctr_encrypt)
+
+	/*
+	 * aes_xctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+	 *		   int bytes, u8 const iv[], int byte_ctr)
+	 *
+	 * The input and output buffers must always be at least 16 bytes even if
+	 * encrypting/decrypting less than 16 bytes.  Otherwise out of bounds
+	 * accesses will occur.  The data to be encrypted/decrypted is expected
+	 * to be at the end of this 16-byte temporary buffer rather than the
+	 * start.
+	 */
+
+AES_FUNC_START(aes_xctr_encrypt)
+	ctr_encrypt 1
+AES_FUNC_END(aes_xctr_encrypt)
+
+
+	/*
+	 * aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds,
+	 *		   int bytes, u8 const rk2[], u8 iv[], int first)
+	 * aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds,
+	 *		   int bytes, u8 const rk2[], u8 iv[], int first)
+	 */
+
+	.macro		next_tweak, out, in, tmp
+	sshr		\tmp\().2d,  \in\().2d,   #63
+	and		\tmp\().16b, \tmp\().16b, xtsmask.16b
+	add		\out\().2d,  \in\().2d,   \in\().2d
+	ext		\tmp\().16b, \tmp\().16b, \tmp\().16b, #8
+	eor		\out\().16b, \out\().16b, \tmp\().16b
+	.endm
+
+	.macro		xts_load_mask, tmp
+	movi		xtsmask.2s, #0x1
+	movi		\tmp\().2s, #0x87
+	uzp1		xtsmask.4s, xtsmask.4s, \tmp\().4s
+	.endm
+
+AES_FUNC_START(aes_xts_encrypt)
+	frame_push	0
+
+	ld1		{v4.16b}, [x6]
+	xts_load_mask	v8
+	cbz		w7, .Lxtsencnotfirst
+
+	enc_prepare	w3, x5, x8
+	xts_cts_skip_tw	w7, .LxtsencNx
+	encrypt_block	v4, w3, x5, x8, w7		/* first tweak */
+	enc_switch_key	w3, x2, x8
+	b		.LxtsencNx
+
+.Lxtsencnotfirst:
+	enc_prepare	w3, x2, x8
+.LxtsencloopNx:
+	next_tweak	v4, v4, v8
+.LxtsencNx:
+	subs		w4, w4, #64
+	bmi		.Lxtsenc1x
+	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 pt blocks */
+	next_tweak	v5, v4, v8
+	eor		v0.16b, v0.16b, v4.16b
+	next_tweak	v6, v5, v8
+	eor		v1.16b, v1.16b, v5.16b
+	eor		v2.16b, v2.16b, v6.16b
+	next_tweak	v7, v6, v8
+	eor		v3.16b, v3.16b, v7.16b
+	bl		aes_encrypt_block4x
+	eor		v3.16b, v3.16b, v7.16b
+	eor		v0.16b, v0.16b, v4.16b
+	eor		v1.16b, v1.16b, v5.16b
+	eor		v2.16b, v2.16b, v6.16b
+	st1		{v0.16b-v3.16b}, [x0], #64
+	mov		v4.16b, v7.16b
+	cbz		w4, .Lxtsencret
+	xts_reload_mask	v8
+	b		.LxtsencloopNx
+.Lxtsenc1x:
+	adds		w4, w4, #64
+	beq		.Lxtsencout
+	subs		w4, w4, #16
+	bmi		.LxtsencctsNx
+.Lxtsencloop:
+	ld1		{v0.16b}, [x1], #16
+.Lxtsencctsout:
+	eor		v0.16b, v0.16b, v4.16b
+	encrypt_block	v0, w3, x2, x8, w7
+	eor		v0.16b, v0.16b, v4.16b
+	cbz		w4, .Lxtsencout
+	subs		w4, w4, #16
+	next_tweak	v4, v4, v8
+	bmi		.Lxtsenccts
+	st1		{v0.16b}, [x0], #16
+	b		.Lxtsencloop
+.Lxtsencout:
+	st1		{v0.16b}, [x0]
+.Lxtsencret:
+	st1		{v4.16b}, [x6]
+	frame_pop
+	ret
+
+.LxtsencctsNx:
+	mov		v0.16b, v3.16b
+	sub		x0, x0, #16
+.Lxtsenccts:
+	adr_l		x8, .Lcts_permute_table
+
+	add		x1, x1, w4, sxtw	/* rewind input pointer */
+	add		w4, w4, #16		/* # bytes in final block */
+	add		x9, x8, #32
+	add		x8, x8, x4
+	sub		x9, x9, x4
+	add		x4, x0, x4		/* output address of final block */
+
+	ld1		{v1.16b}, [x1]		/* load final block */
+	ld1		{v2.16b}, [x8]
+	ld1		{v3.16b}, [x9]
+
+	tbl		v2.16b, {v0.16b}, v2.16b
+	tbx		v0.16b, {v1.16b}, v3.16b
+	st1		{v2.16b}, [x4]			/* overlapping stores */
+	mov		w4, wzr
+	b		.Lxtsencctsout
+AES_FUNC_END(aes_xts_encrypt)
+
+AES_FUNC_START(aes_xts_decrypt)
+	frame_push	0
+
+	/* subtract 16 bytes if we are doing CTS */
+	sub		w8, w4, #0x10
+	tst		w4, #0xf
+	csel		w4, w4, w8, eq
+
+	ld1		{v4.16b}, [x6]
+	xts_load_mask	v8
+	xts_cts_skip_tw	w7, .Lxtsdecskiptw
+	cbz		w7, .Lxtsdecnotfirst
+
+	enc_prepare	w3, x5, x8
+	encrypt_block	v4, w3, x5, x8, w7		/* first tweak */
+.Lxtsdecskiptw:
+	dec_prepare	w3, x2, x8
+	b		.LxtsdecNx
+
+.Lxtsdecnotfirst:
+	dec_prepare	w3, x2, x8
+.LxtsdecloopNx:
+	next_tweak	v4, v4, v8
+.LxtsdecNx:
+	subs		w4, w4, #64
+	bmi		.Lxtsdec1x
+	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 ct blocks */
+	next_tweak	v5, v4, v8
+	eor		v0.16b, v0.16b, v4.16b
+	next_tweak	v6, v5, v8
+	eor		v1.16b, v1.16b, v5.16b
+	eor		v2.16b, v2.16b, v6.16b
+	next_tweak	v7, v6, v8
+	eor		v3.16b, v3.16b, v7.16b
+	bl		aes_decrypt_block4x
+	eor		v3.16b, v3.16b, v7.16b
+	eor		v0.16b, v0.16b, v4.16b
+	eor		v1.16b, v1.16b, v5.16b
+	eor		v2.16b, v2.16b, v6.16b
+	st1		{v0.16b-v3.16b}, [x0], #64
+	mov		v4.16b, v7.16b
+	cbz		w4, .Lxtsdecout
+	xts_reload_mask	v8
+	b		.LxtsdecloopNx
+.Lxtsdec1x:
+	adds		w4, w4, #64
+	beq		.Lxtsdecout
+	subs		w4, w4, #16
+.Lxtsdecloop:
+	ld1		{v0.16b}, [x1], #16
+	bmi		.Lxtsdeccts
+.Lxtsdecctsout:
+	eor		v0.16b, v0.16b, v4.16b
+	decrypt_block	v0, w3, x2, x8, w7
+	eor		v0.16b, v0.16b, v4.16b
+	st1		{v0.16b}, [x0], #16
+	cbz		w4, .Lxtsdecout
+	subs		w4, w4, #16
+	next_tweak	v4, v4, v8
+	b		.Lxtsdecloop
+.Lxtsdecout:
+	st1		{v4.16b}, [x6]
+	frame_pop
+	ret
+
+.Lxtsdeccts:
+	adr_l		x8, .Lcts_permute_table
+
+	add		x1, x1, w4, sxtw	/* rewind input pointer */
+	add		w4, w4, #16		/* # bytes in final block */
+	add		x9, x8, #32
+	add		x8, x8, x4
+	sub		x9, x9, x4
+	add		x4, x0, x4		/* output address of final block */
+
+	next_tweak	v5, v4, v8
+
+	ld1		{v1.16b}, [x1]		/* load final block */
+	ld1		{v2.16b}, [x8]
+	ld1		{v3.16b}, [x9]
+
+	eor		v0.16b, v0.16b, v5.16b
+	decrypt_block	v0, w3, x2, x8, w7
+	eor		v0.16b, v0.16b, v5.16b
+
+	tbl		v2.16b, {v0.16b}, v2.16b
+	tbx		v0.16b, {v1.16b}, v3.16b
+
+	st1		{v2.16b}, [x4]			/* overlapping stores */
+	mov		w4, wzr
+	b		.Lxtsdecctsout
+AES_FUNC_END(aes_xts_decrypt)
+
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS)
+	/*
+	 * void aes_mac_update(u8 const in[], u32 const rk[], int rounds,
+	 *		       size_t blocks, u8 dg[], int enc_before,
+	 *		       int enc_after);
+	 */
+AES_FUNC_START(aes_mac_update)
+	ld1		{v0.16b}, [x4]			/* get dg */
+	enc_prepare	w2, x1, x7
+	cbz		w5, .Lmacloop4x
+
+	encrypt_block	v0, w2, x1, x7, w8
+
+.Lmacloop4x:
+	subs		x3, x3, #4
+	bmi		.Lmac1x
+	ld1		{v1.16b-v4.16b}, [x0], #64	/* get next pt block */
+	eor		v0.16b, v0.16b, v1.16b		/* ..and xor with dg */
+	encrypt_block	v0, w2, x1, x7, w8
+	eor		v0.16b, v0.16b, v2.16b
+	encrypt_block	v0, w2, x1, x7, w8
+	eor		v0.16b, v0.16b, v3.16b
+	encrypt_block	v0, w2, x1, x7, w8
+	eor		v0.16b, v0.16b, v4.16b
+	cmp		x3, xzr
+	csinv		w5, w6, wzr, eq
+	cbz		w5, .Lmacout
+	encrypt_block	v0, w2, x1, x7, w8
+	st1		{v0.16b}, [x4]			/* return dg */
+	b		.Lmacloop4x
+.Lmac1x:
+	add		x3, x3, #4
+.Lmacloop:
+	cbz		x3, .Lmacout
+	ld1		{v1.16b}, [x0], #16		/* get next pt block */
+	eor		v0.16b, v0.16b, v1.16b		/* ..and xor with dg */
+
+	subs		x3, x3, #1
+	csinv		w5, w6, wzr, eq
+	cbz		w5, .Lmacout
+
+.Lmacenc:
+	encrypt_block	v0, w2, x1, x7, w8
+	b		.Lmacloop
+
+.Lmacout:
+	st1		{v0.16b}, [x4]			/* return dg */
+	ret
+AES_FUNC_END(aes_mac_update)
+#endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */
diff --git a/lib/crypto/arm64/aes-neon.S b/lib/crypto/arm64/aes-neon.S
new file mode 100644
index 000000000000..f37b1dbd887f
--- /dev/null
+++ b/lib/crypto/arm64/aes-neon.S
@@ -0,0 +1,250 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * AES cipher for ARMv8 NEON
+ *
+ * Copyright (C) 2013 - 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+#define AES_FUNC_START(func)		SYM_FUNC_START(neon_ ## func)
+#define AES_FUNC_END(func)		SYM_FUNC_END(neon_ ## func)
+
+	xtsmask		.req	v7
+	cbciv		.req	v7
+	vctr		.req	v4
+
+	.macro		xts_reload_mask, tmp
+	xts_load_mask	\tmp
+	.endm
+
+	/* special case for the neon-bs driver calling into this one for CTS */
+	.macro		xts_cts_skip_tw, reg, lbl
+	tbnz		\reg, #1, \lbl
+	.endm
+
+	/* multiply by polynomial 'x' in GF(2^8) */
+	.macro		mul_by_x, out, in, temp, const
+	sshr		\temp, \in, #7
+	shl		\out, \in, #1
+	and		\temp, \temp, \const
+	eor		\out, \out, \temp
+	.endm
+
+	/* multiply by polynomial 'x^2' in GF(2^8) */
+	.macro		mul_by_x2, out, in, temp, const
+	ushr		\temp, \in, #6
+	shl		\out, \in, #2
+	pmul		\temp, \temp, \const
+	eor		\out, \out, \temp
+	.endm
+
+	/* preload the entire Sbox */
+	.macro		prepare, sbox, shiftrows, temp
+	movi		v12.16b, #0x1b
+	ldr_l		q13, \shiftrows, \temp
+	ldr_l		q14, .Lror32by8, \temp
+	adr_l		\temp, \sbox
+	ld1		{v16.16b-v19.16b}, [\temp], #64
+	ld1		{v20.16b-v23.16b}, [\temp], #64
+	ld1		{v24.16b-v27.16b}, [\temp], #64
+	ld1		{v28.16b-v31.16b}, [\temp]
+	.endm
+
+	/* do preload for encryption */
+	.macro		enc_prepare, ignore0, ignore1, temp
+	prepare		crypto_aes_sbox, .LForward_ShiftRows, \temp
+	.endm
+
+	.macro		enc_switch_key, ignore0, ignore1, temp
+	/* do nothing */
+	.endm
+
+	/* do preload for decryption */
+	.macro		dec_prepare, ignore0, ignore1, temp
+	prepare		crypto_aes_inv_sbox, .LReverse_ShiftRows, \temp
+	.endm
+
+	/* apply SubBytes transformation using the preloaded Sbox */
+	.macro		sub_bytes, in
+	sub		v9.16b, \in\().16b, v15.16b
+	tbl		\in\().16b, {v16.16b-v19.16b}, \in\().16b
+	sub		v10.16b, v9.16b, v15.16b
+	tbx		\in\().16b, {v20.16b-v23.16b}, v9.16b
+	sub		v11.16b, v10.16b, v15.16b
+	tbx		\in\().16b, {v24.16b-v27.16b}, v10.16b
+	tbx		\in\().16b, {v28.16b-v31.16b}, v11.16b
+	.endm
+
+	/* apply MixColumns transformation */
+	.macro		mix_columns, in, enc
+	.if		\enc == 0
+	/* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */
+	mul_by_x2	v8.16b, \in\().16b, v9.16b, v12.16b
+	eor		\in\().16b, \in\().16b, v8.16b
+	rev32		v8.8h, v8.8h
+	eor		\in\().16b, \in\().16b, v8.16b
+	.endif
+
+	mul_by_x	v9.16b, \in\().16b, v8.16b, v12.16b
+	rev32		v8.8h, \in\().8h
+	eor		v8.16b, v8.16b, v9.16b
+	eor		\in\().16b, \in\().16b, v8.16b
+	tbl		\in\().16b, {\in\().16b}, v14.16b
+	eor		\in\().16b, \in\().16b, v8.16b
+	.endm
+
+	.macro		do_block, enc, in, rounds, rk, rkp, i
+	ld1		{v15.4s}, [\rk]
+	add		\rkp, \rk, #16
+	mov		\i, \rounds
+.La\@:	eor		\in\().16b, \in\().16b, v15.16b		/* ^round key */
+	movi		v15.16b, #0x40
+	tbl		\in\().16b, {\in\().16b}, v13.16b	/* ShiftRows */
+	sub_bytes	\in
+	sub		\i, \i, #1
+	ld1		{v15.4s}, [\rkp], #16
+	cbz		\i, .Lb\@
+	mix_columns	\in, \enc
+	b		.La\@
+.Lb\@:	eor		\in\().16b, \in\().16b, v15.16b		/* ^round key */
+	.endm
+
+	.macro		encrypt_block, in, rounds, rk, rkp, i
+	do_block	1, \in, \rounds, \rk, \rkp, \i
+	.endm
+
+	.macro		decrypt_block, in, rounds, rk, rkp, i
+	do_block	0, \in, \rounds, \rk, \rkp, \i
+	.endm
+
+	/*
+	 * Interleaved versions: functionally equivalent to the
+	 * ones above, but applied to AES states in parallel.
+	 */
+
+	.macro		sub_bytes_4x, in0, in1, in2, in3
+	sub		v8.16b, \in0\().16b, v15.16b
+	tbl		\in0\().16b, {v16.16b-v19.16b}, \in0\().16b
+	sub		v9.16b, \in1\().16b, v15.16b
+	tbl		\in1\().16b, {v16.16b-v19.16b}, \in1\().16b
+	sub		v10.16b, \in2\().16b, v15.16b
+	tbl		\in2\().16b, {v16.16b-v19.16b}, \in2\().16b
+	sub		v11.16b, \in3\().16b, v15.16b
+	tbl		\in3\().16b, {v16.16b-v19.16b}, \in3\().16b
+	tbx		\in0\().16b, {v20.16b-v23.16b}, v8.16b
+	tbx		\in1\().16b, {v20.16b-v23.16b}, v9.16b
+	sub		v8.16b, v8.16b, v15.16b
+	tbx		\in2\().16b, {v20.16b-v23.16b}, v10.16b
+	sub		v9.16b, v9.16b, v15.16b
+	tbx		\in3\().16b, {v20.16b-v23.16b}, v11.16b
+	sub		v10.16b, v10.16b, v15.16b
+	tbx		\in0\().16b, {v24.16b-v27.16b}, v8.16b
+	sub		v11.16b, v11.16b, v15.16b
+	tbx		\in1\().16b, {v24.16b-v27.16b}, v9.16b
+	sub		v8.16b, v8.16b, v15.16b
+	tbx		\in2\().16b, {v24.16b-v27.16b}, v10.16b
+	sub		v9.16b, v9.16b, v15.16b
+	tbx		\in3\().16b, {v24.16b-v27.16b}, v11.16b
+	sub		v10.16b, v10.16b, v15.16b
+	tbx		\in0\().16b, {v28.16b-v31.16b}, v8.16b
+	sub		v11.16b, v11.16b, v15.16b
+	tbx		\in1\().16b, {v28.16b-v31.16b}, v9.16b
+	tbx		\in2\().16b, {v28.16b-v31.16b}, v10.16b
+	tbx		\in3\().16b, {v28.16b-v31.16b}, v11.16b
+	.endm
+
+	.macro		mul_by_x_2x, out0, out1, in0, in1, tmp0, tmp1, const
+	sshr		\tmp0\().16b, \in0\().16b, #7
+	shl		\out0\().16b, \in0\().16b, #1
+	sshr		\tmp1\().16b, \in1\().16b, #7
+	and		\tmp0\().16b, \tmp0\().16b, \const\().16b
+	shl		\out1\().16b, \in1\().16b, #1
+	and		\tmp1\().16b, \tmp1\().16b, \const\().16b
+	eor		\out0\().16b, \out0\().16b, \tmp0\().16b
+	eor		\out1\().16b, \out1\().16b, \tmp1\().16b
+	.endm
+
+	.macro		mul_by_x2_2x, out0, out1, in0, in1, tmp0, tmp1, const
+	ushr		\tmp0\().16b, \in0\().16b, #6
+	shl		\out0\().16b, \in0\().16b, #2
+	ushr		\tmp1\().16b, \in1\().16b, #6
+	pmul		\tmp0\().16b, \tmp0\().16b, \const\().16b
+	shl		\out1\().16b, \in1\().16b, #2
+	pmul		\tmp1\().16b, \tmp1\().16b, \const\().16b
+	eor		\out0\().16b, \out0\().16b, \tmp0\().16b
+	eor		\out1\().16b, \out1\().16b, \tmp1\().16b
+	.endm
+
+	.macro		mix_columns_2x, in0, in1, enc
+	.if		\enc == 0
+	/* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */
+	mul_by_x2_2x	v8, v9, \in0, \in1, v10, v11, v12
+	eor		\in0\().16b, \in0\().16b, v8.16b
+	rev32		v8.8h, v8.8h
+	eor		\in1\().16b, \in1\().16b, v9.16b
+	rev32		v9.8h, v9.8h
+	eor		\in0\().16b, \in0\().16b, v8.16b
+	eor		\in1\().16b, \in1\().16b, v9.16b
+	.endif
+
+	mul_by_x_2x	v8, v9, \in0, \in1, v10, v11, v12
+	rev32		v10.8h, \in0\().8h
+	rev32		v11.8h, \in1\().8h
+	eor		v10.16b, v10.16b, v8.16b
+	eor		v11.16b, v11.16b, v9.16b
+	eor		\in0\().16b, \in0\().16b, v10.16b
+	eor		\in1\().16b, \in1\().16b, v11.16b
+	tbl		\in0\().16b, {\in0\().16b}, v14.16b
+	tbl		\in1\().16b, {\in1\().16b}, v14.16b
+	eor		\in0\().16b, \in0\().16b, v10.16b
+	eor		\in1\().16b, \in1\().16b, v11.16b
+	.endm
+
+	.macro		do_block_4x, enc, in0, in1, in2, in3, rounds, rk, rkp, i
+	ld1		{v15.4s}, [\rk]
+	add		\rkp, \rk, #16
+	mov		\i, \rounds
+.La\@:	eor		\in0\().16b, \in0\().16b, v15.16b	/* ^round key */
+	eor		\in1\().16b, \in1\().16b, v15.16b	/* ^round key */
+	eor		\in2\().16b, \in2\().16b, v15.16b	/* ^round key */
+	eor		\in3\().16b, \in3\().16b, v15.16b	/* ^round key */
+	movi		v15.16b, #0x40
+	tbl		\in0\().16b, {\in0\().16b}, v13.16b	/* ShiftRows */
+	tbl		\in1\().16b, {\in1\().16b}, v13.16b	/* ShiftRows */
+	tbl		\in2\().16b, {\in2\().16b}, v13.16b	/* ShiftRows */
+	tbl		\in3\().16b, {\in3\().16b}, v13.16b	/* ShiftRows */
+	sub_bytes_4x	\in0, \in1, \in2, \in3
+	sub		\i, \i, #1
+	ld1		{v15.4s}, [\rkp], #16
+	cbz		\i, .Lb\@
+	mix_columns_2x	\in0, \in1, \enc
+	mix_columns_2x	\in2, \in3, \enc
+	b		.La\@
+.Lb\@:	eor		\in0\().16b, \in0\().16b, v15.16b	/* ^round key */
+	eor		\in1\().16b, \in1\().16b, v15.16b	/* ^round key */
+	eor		\in2\().16b, \in2\().16b, v15.16b	/* ^round key */
+	eor		\in3\().16b, \in3\().16b, v15.16b	/* ^round key */
+	.endm
+
+	.macro		encrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i
+	do_block_4x	1, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i
+	.endm
+
+	.macro		decrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i
+	do_block_4x	0, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i
+	.endm
+
+#include "aes-modes.S"
+
+	.section	".rodata", "a"
+	.align		4
+.LForward_ShiftRows:
+	.octa		0x0b06010c07020d08030e09040f0a0500
+
+.LReverse_ShiftRows:
+	.octa		0x0306090c0f0205080b0e0104070a0d00
+
+.Lror32by8:
+	.octa		0x0c0f0e0d080b0a090407060500030201
diff --git a/lib/crypto/arm64/aes.h b/lib/crypto/arm64/aes.h
index 63eea6271ef9..9e9e45a6f787 100644
--- a/lib/crypto/arm64/aes.h
+++ b/lib/crypto/arm64/aes.h
@@ -11,6 +11,7 @@
 #include <linux/unaligned.h>
 #include <linux/cpufeature.h>
 
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aes);
 
 struct aes_block {
@@ -28,6 +29,9 @@ asmlinkage void __aes_ce_decrypt(const u32 inv_rk[], u8 out[AES_BLOCK_SIZE],
 asmlinkage u32 __aes_ce_sub(u32 l);
 asmlinkage void __aes_ce_invert(struct aes_block *out,
 				const struct aes_block *in);
+asmlinkage void neon_aes_mac_update(u8 const in[], u32 const rk[], int rounds,
+				    size_t blocks, u8 dg[], int enc_before,
+				    int enc_after);
 
 /*
  * Expand an AES key using the crypto extensions if supported and usable or
@@ -48,8 +52,7 @@ static void aes_expandkey_arm64(u32 rndkeys[], u32 *inv_rndkeys,
 	struct aes_block *key_enc, *key_dec;
 	int i, j;
 
-	if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) ||
-	    !static_branch_likely(&have_aes) || unlikely(!may_use_simd())) {
+	if (!static_branch_likely(&have_aes) || unlikely(!may_use_simd())) {
 		aes_expandkey_generic(rndkeys, inv_rndkeys, in_key, key_len);
 		return;
 	}
@@ -126,12 +129,40 @@ int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
 }
 EXPORT_SYMBOL(ce_aes_expandkey);
 
+EXPORT_SYMBOL_NS_GPL(neon_aes_ecb_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_ecb_decrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_decrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_cts_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_cts_decrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_ctr_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_xctr_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_xts_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_xts_decrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL");
+
+EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_decrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_decrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_cts_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_cts_decrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_ctr_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_xctr_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_xts_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_xts_decrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL");
+#if IS_MODULE(CONFIG_CRYPTO_AES_ARM64_CE_CCM)
+EXPORT_SYMBOL_NS_GPL(ce_aes_mac_update, "CRYPTO_INTERNAL");
+#endif
+
 static void aes_encrypt_arch(const struct aes_enckey *key,
 			     u8 out[AES_BLOCK_SIZE],
 			     const u8 in[AES_BLOCK_SIZE])
 {
-	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
-	    static_branch_likely(&have_aes) && likely(may_use_simd())) {
+	if (static_branch_likely(&have_aes) && likely(may_use_simd())) {
 		scoped_ksimd()
 			__aes_ce_encrypt(key->k.rndkeys, out, in, key->nrounds);
 	} else {
@@ -143,8 +174,7 @@ static void aes_decrypt_arch(const struct aes_key *key,
 			     u8 out[AES_BLOCK_SIZE],
 			     const u8 in[AES_BLOCK_SIZE])
 {
-	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
-	    static_branch_likely(&have_aes) && likely(may_use_simd())) {
+	if (static_branch_likely(&have_aes) && likely(may_use_simd())) {
 		scoped_ksimd()
 			__aes_ce_decrypt(key->inv_k.inv_rndkeys, out, in,
 					 key->nrounds);
@@ -154,11 +184,36 @@ static void aes_decrypt_arch(const struct aes_key *key,
 	}
 }
 
-#ifdef CONFIG_KERNEL_MODE_NEON
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS)
+#define aes_cbcmac_blocks_arch aes_cbcmac_blocks_arch
+static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE],
+				   const struct aes_enckey *key, const u8 *data,
+				   size_t nblocks, bool enc_before,
+				   bool enc_after)
+{
+	if (static_branch_likely(&have_neon) && likely(may_use_simd())) {
+		scoped_ksimd() {
+			if (static_branch_likely(&have_aes))
+				ce_aes_mac_update(data, key->k.rndkeys,
+						  key->nrounds, nblocks, h,
+						  enc_before, enc_after);
+			else
+				neon_aes_mac_update(data, key->k.rndkeys,
+						    key->nrounds, nblocks, h,
+						    enc_before, enc_after);
+		}
+		return true;
+	}
+	return false;
+}
+#endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */
+
 #define aes_mod_init_arch aes_mod_init_arch
 static void aes_mod_init_arch(void)
 {
-	if (cpu_have_named_feature(AES))
-		static_branch_enable(&have_aes);
+	if (cpu_have_named_feature(ASIMD)) {
+		static_branch_enable(&have_neon);
+		if (cpu_have_named_feature(AES))
+			static_branch_enable(&have_aes);
+	}
 }
-#endif /* CONFIG_KERNEL_MODE_NEON */
diff --git a/lib/crypto/arm64/chacha-neon-core.S b/lib/crypto/arm64/chacha-neon-core.S
index 80079586ecc7..cb18eec968bd 100644
--- a/lib/crypto/arm64/chacha-neon-core.S
+++ b/lib/crypto/arm64/chacha-neon-core.S
@@ -531,10 +531,6 @@ SYM_FUNC_START(chacha_4block_xor_neon)
 	add		v3.4s, v3.4s, v19.4s
 	  add		a2, a2, w8
 	  add		a3, a3, w9
-CPU_BE(	  rev		a0, a0		)
-CPU_BE(	  rev		a1, a1		)
-CPU_BE(	  rev		a2, a2		)
-CPU_BE(	  rev		a3, a3		)
 
 	ld4r		{v24.4s-v27.4s}, [x0], #16
 	ld4r		{v28.4s-v31.4s}, [x0]
@@ -555,10 +551,6 @@ CPU_BE(	  rev		a3, a3		)
 	add		v7.4s, v7.4s, v23.4s
 	  add		a6, a6, w8
 	  add		a7, a7, w9
-CPU_BE(	  rev		a4, a4		)
-CPU_BE(	  rev		a5, a5		)
-CPU_BE(	  rev		a6, a6		)
-CPU_BE(	  rev		a7, a7		)
 
 	// x8[0-3] += s2[0]
 	// x9[0-3] += s2[1]
@@ -576,10 +568,6 @@ CPU_BE(	  rev		a7, a7		)
 	add		v11.4s, v11.4s, v27.4s
 	  add		a10, a10, w8
 	  add		a11, a11, w9
-CPU_BE(	  rev		a8, a8		)
-CPU_BE(	  rev		a9, a9		)
-CPU_BE(	  rev		a10, a10	)
-CPU_BE(	  rev		a11, a11	)
 
 	// x12[0-3] += s3[0]
 	// x13[0-3] += s3[1]
@@ -597,10 +585,6 @@ CPU_BE(	  rev		a11, a11	)
 	add		v15.4s, v15.4s, v31.4s
 	  add		a14, a14, w8
 	  add		a15, a15, w9
-CPU_BE(	  rev		a12, a12	)
-CPU_BE(	  rev		a13, a13	)
-CPU_BE(	  rev		a14, a14	)
-CPU_BE(	  rev		a15, a15	)
 
 	// interleave 32-bit words in state n, n+1
 	  ldp		w6, w7, [x2], #64
diff --git a/lib/crypto/arm64/chacha.h b/lib/crypto/arm64/chacha.h
index ca8c6a8b0578..c6f8ddf98e2d 100644
--- a/lib/crypto/arm64/chacha.h
+++ b/lib/crypto/arm64/chacha.h
@@ -36,9 +36,9 @@ asmlinkage void hchacha_block_neon(const struct chacha_state *state,
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
 
 static void chacha_doneon(struct chacha_state *state, u8 *dst, const u8 *src,
-			  int bytes, int nrounds)
+			  unsigned int bytes, int nrounds)
 {
-	while (bytes > 0) {
+	while (bytes) {
 		int l = min(bytes, CHACHA_BLOCK_SIZE * 5);
 
 		if (l <= CHACHA_BLOCK_SIZE) {
@@ -76,16 +76,8 @@ static void chacha_crypt_arch(struct chacha_state *state, u8 *dst,
 	    !crypto_simd_usable())
 		return chacha_crypt_generic(state, dst, src, bytes, nrounds);
 
-	do {
-		unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
-
-		scoped_ksimd()
-			chacha_doneon(state, dst, src, todo, nrounds);
-
-		bytes -= todo;
-		src += todo;
-		dst += todo;
-	} while (bytes);
+	scoped_ksimd()
+		chacha_doneon(state, dst, src, bytes, nrounds);
 }
 
 #define chacha_mod_init_arch chacha_mod_init_arch
diff --git a/lib/crypto/arm64/gf128hash.h b/lib/crypto/arm64/gf128hash.h
new file mode 100644
index 000000000000..1d1179f87f8d
--- /dev/null
+++ b/lib/crypto/arm64/gf128hash.h
@@ -0,0 +1,121 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * GHASH and POLYVAL, arm64 optimized
+ *
+ * Copyright 2025 Google LLC
+ */
+#include <asm/simd.h>
+#include <linux/cpufeature.h>
+
+#define NUM_H_POWERS 8
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_asimd);
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull);
+
+asmlinkage void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg,
+				      const u8 *src,
+				      const struct polyval_elem *h);
+asmlinkage void polyval_mul_pmull(struct polyval_elem *a,
+				  const struct polyval_elem *b);
+asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc,
+				     const struct polyval_key *key,
+				     const u8 *data, size_t nblocks);
+
+#define polyval_preparekey_arch polyval_preparekey_arch
+static void polyval_preparekey_arch(struct polyval_key *key,
+				    const u8 raw_key[POLYVAL_BLOCK_SIZE])
+{
+	static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS);
+	memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE);
+	if (static_branch_likely(&have_pmull) && may_use_simd()) {
+		scoped_ksimd() {
+			for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
+				key->h_powers[i] = key->h_powers[i + 1];
+				polyval_mul_pmull(
+					&key->h_powers[i],
+					&key->h_powers[NUM_H_POWERS - 1]);
+			}
+		}
+	} else {
+		for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
+			key->h_powers[i] = key->h_powers[i + 1];
+			polyval_mul_generic(&key->h_powers[i],
+					    &key->h_powers[NUM_H_POWERS - 1]);
+		}
+	}
+}
+
+static void polyval_mul_arm64(struct polyval_elem *a,
+			      const struct polyval_elem *b)
+{
+	if (static_branch_likely(&have_asimd) && may_use_simd()) {
+		static const u8 zeroes[GHASH_BLOCK_SIZE];
+
+		scoped_ksimd() {
+			if (static_branch_likely(&have_pmull)) {
+				polyval_mul_pmull(a, b);
+			} else {
+				/*
+				 * Note that this is indeed equivalent to a
+				 * POLYVAL multiplication, since it takes the
+				 * accumulator and key in POLYVAL format, and
+				 * byte-swapping a block of zeroes is a no-op.
+				 */
+				pmull_ghash_update_p8(1, a, zeroes, b);
+			}
+		}
+	} else {
+		polyval_mul_generic(a, b);
+	}
+}
+
+#define ghash_mul_arch ghash_mul_arch
+static void ghash_mul_arch(struct polyval_elem *acc,
+			   const struct ghash_key *key)
+{
+	polyval_mul_arm64(acc, &key->h);
+}
+
+#define polyval_mul_arch polyval_mul_arch
+static void polyval_mul_arch(struct polyval_elem *acc,
+			     const struct polyval_key *key)
+{
+	polyval_mul_arm64(acc, &key->h_powers[NUM_H_POWERS - 1]);
+}
+
+#define ghash_blocks_arch ghash_blocks_arch
+static void ghash_blocks_arch(struct polyval_elem *acc,
+			      const struct ghash_key *key,
+			      const u8 *data, size_t nblocks)
+{
+	if (static_branch_likely(&have_asimd) && may_use_simd()) {
+		scoped_ksimd()
+			pmull_ghash_update_p8(nblocks, acc, data, &key->h);
+	} else {
+		ghash_blocks_generic(acc, &key->h, data, nblocks);
+	}
+}
+
+#define polyval_blocks_arch polyval_blocks_arch
+static void polyval_blocks_arch(struct polyval_elem *acc,
+				const struct polyval_key *key,
+				const u8 *data, size_t nblocks)
+{
+	if (static_branch_likely(&have_pmull) && may_use_simd()) {
+		scoped_ksimd()
+			polyval_blocks_pmull(acc, key, data, nblocks);
+	} else {
+		polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1],
+				       data, nblocks);
+	}
+}
+
+#define gf128hash_mod_init_arch gf128hash_mod_init_arch
+static void gf128hash_mod_init_arch(void)
+{
+	if (cpu_have_named_feature(ASIMD)) {
+		static_branch_enable(&have_asimd);
+		if (cpu_have_named_feature(PMULL))
+			static_branch_enable(&have_pmull);
+	}
+}
diff --git a/lib/crypto/arm64/ghash-neon-core.S b/lib/crypto/arm64/ghash-neon-core.S
new file mode 100644
index 000000000000..4c5799172b49
--- /dev/null
+++ b/lib/crypto/arm64/ghash-neon-core.S
@@ -0,0 +1,220 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Accelerated GHASH implementation with ARMv8 ASIMD instructions.
+ *
+ * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+	SHASH		.req	v0
+	SHASH2		.req	v1
+	T1		.req	v2
+	T2		.req	v3
+	XM		.req	v5
+	XL		.req	v6
+	XH		.req	v7
+	IN1		.req	v7
+
+	k00_16		.req	v8
+	k32_48		.req	v9
+
+	t3		.req	v10
+	t4		.req	v11
+	t5		.req	v12
+	t6		.req	v13
+	t7		.req	v14
+	t8		.req	v15
+	t9		.req	v16
+
+	perm1		.req	v17
+	perm2		.req	v18
+	perm3		.req	v19
+
+	sh1		.req	v20
+	sh2		.req	v21
+	sh3		.req	v22
+	sh4		.req	v23
+
+	ss1		.req	v24
+	ss2		.req	v25
+	ss3		.req	v26
+	ss4		.req	v27
+
+	.text
+
+	.macro		__pmull_p8, rq, ad, bd
+	ext		t3.8b, \ad\().8b, \ad\().8b, #1		// A1
+	ext		t5.8b, \ad\().8b, \ad\().8b, #2		// A2
+	ext		t7.8b, \ad\().8b, \ad\().8b, #3		// A3
+
+	__pmull_p8_\bd	\rq, \ad
+	.endm
+
+	.macro		__pmull2_p8, rq, ad, bd
+	tbl		t3.16b, {\ad\().16b}, perm1.16b		// A1
+	tbl		t5.16b, {\ad\().16b}, perm2.16b		// A2
+	tbl		t7.16b, {\ad\().16b}, perm3.16b		// A3
+
+	__pmull2_p8_\bd	\rq, \ad
+	.endm
+
+	.macro		__pmull_p8_SHASH, rq, ad
+	__pmull_p8_tail	\rq, \ad\().8b, SHASH.8b, 8b,, sh1, sh2, sh3, sh4
+	.endm
+
+	.macro		__pmull_p8_SHASH2, rq, ad
+	__pmull_p8_tail	\rq, \ad\().8b, SHASH2.8b, 8b,, ss1, ss2, ss3, ss4
+	.endm
+
+	.macro		__pmull2_p8_SHASH, rq, ad
+	__pmull_p8_tail	\rq, \ad\().16b, SHASH.16b, 16b, 2, sh1, sh2, sh3, sh4
+	.endm
+
+	.macro		__pmull_p8_tail, rq, ad, bd, nb, t, b1, b2, b3, b4
+	pmull\t		t3.8h, t3.\nb, \bd			// F = A1*B
+	pmull\t		t4.8h, \ad, \b1\().\nb			// E = A*B1
+	pmull\t		t5.8h, t5.\nb, \bd			// H = A2*B
+	pmull\t		t6.8h, \ad, \b2\().\nb			// G = A*B2
+	pmull\t		t7.8h, t7.\nb, \bd			// J = A3*B
+	pmull\t		t8.8h, \ad, \b3\().\nb			// I = A*B3
+	pmull\t		t9.8h, \ad, \b4\().\nb			// K = A*B4
+	pmull\t		\rq\().8h, \ad, \bd			// D = A*B
+
+	eor		t3.16b, t3.16b, t4.16b			// L = E + F
+	eor		t5.16b, t5.16b, t6.16b			// M = G + H
+	eor		t7.16b, t7.16b, t8.16b			// N = I + J
+
+	uzp1		t4.2d, t3.2d, t5.2d
+	uzp2		t3.2d, t3.2d, t5.2d
+	uzp1		t6.2d, t7.2d, t9.2d
+	uzp2		t7.2d, t7.2d, t9.2d
+
+	// t3 = (L) (P0 + P1) << 8
+	// t5 = (M) (P2 + P3) << 16
+	eor		t4.16b, t4.16b, t3.16b
+	and		t3.16b, t3.16b, k32_48.16b
+
+	// t7 = (N) (P4 + P5) << 24
+	// t9 = (K) (P6 + P7) << 32
+	eor		t6.16b, t6.16b, t7.16b
+	and		t7.16b, t7.16b, k00_16.16b
+
+	eor		t4.16b, t4.16b, t3.16b
+	eor		t6.16b, t6.16b, t7.16b
+
+	zip2		t5.2d, t4.2d, t3.2d
+	zip1		t3.2d, t4.2d, t3.2d
+	zip2		t9.2d, t6.2d, t7.2d
+	zip1		t7.2d, t6.2d, t7.2d
+
+	ext		t3.16b, t3.16b, t3.16b, #15
+	ext		t5.16b, t5.16b, t5.16b, #14
+	ext		t7.16b, t7.16b, t7.16b, #13
+	ext		t9.16b, t9.16b, t9.16b, #12
+
+	eor		t3.16b, t3.16b, t5.16b
+	eor		t7.16b, t7.16b, t9.16b
+	eor		\rq\().16b, \rq\().16b, t3.16b
+	eor		\rq\().16b, \rq\().16b, t7.16b
+	.endm
+
+	.macro		__pmull_pre_p8
+	ext		SHASH2.16b, SHASH.16b, SHASH.16b, #8
+	eor		SHASH2.16b, SHASH2.16b, SHASH.16b
+
+	// k00_16 := 0x0000000000000000_000000000000ffff
+	// k32_48 := 0x00000000ffffffff_0000ffffffffffff
+	movi		k32_48.2d, #0xffffffff
+	mov		k32_48.h[2], k32_48.h[0]
+	ushr		k00_16.2d, k32_48.2d, #32
+
+	// prepare the permutation vectors
+	mov_q		x5, 0x080f0e0d0c0b0a09
+	movi		T1.8b, #8
+	dup		perm1.2d, x5
+	eor		perm1.16b, perm1.16b, T1.16b
+	ushr		perm2.2d, perm1.2d, #8
+	ushr		perm3.2d, perm1.2d, #16
+	ushr		T1.2d, perm1.2d, #24
+	sli		perm2.2d, perm1.2d, #56
+	sli		perm3.2d, perm1.2d, #48
+	sli		T1.2d, perm1.2d, #40
+
+	// precompute loop invariants
+	tbl		sh1.16b, {SHASH.16b}, perm1.16b
+	tbl		sh2.16b, {SHASH.16b}, perm2.16b
+	tbl		sh3.16b, {SHASH.16b}, perm3.16b
+	tbl		sh4.16b, {SHASH.16b}, T1.16b
+	ext		ss1.8b, SHASH2.8b, SHASH2.8b, #1
+	ext		ss2.8b, SHASH2.8b, SHASH2.8b, #2
+	ext		ss3.8b, SHASH2.8b, SHASH2.8b, #3
+	ext		ss4.8b, SHASH2.8b, SHASH2.8b, #4
+	.endm
+
+	.macro		__pmull_reduce_p8
+	eor		XM.16b, XM.16b, T1.16b
+
+	mov		XL.d[1], XM.d[0]
+	mov		XH.d[0], XM.d[1]
+
+	shl		T1.2d, XL.2d, #57
+	shl		T2.2d, XL.2d, #62
+	eor		T2.16b, T2.16b, T1.16b
+	shl		T1.2d, XL.2d, #63
+	eor		T2.16b, T2.16b, T1.16b
+	ext		T1.16b, XL.16b, XH.16b, #8
+	eor		T2.16b, T2.16b, T1.16b
+
+	mov		XL.d[1], T2.d[0]
+	mov		XH.d[0], T2.d[1]
+
+	ushr		T2.2d, XL.2d, #1
+	eor		XH.16b, XH.16b, XL.16b
+	eor		XL.16b, XL.16b, T2.16b
+	ushr		T2.2d, T2.2d, #6
+	ushr		XL.2d, XL.2d, #1
+	.endm
+
+	/*
+	 * void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg,
+	 *			      const u8 *src,
+	 *			      const struct polyval_elem *h)
+	 */
+SYM_FUNC_START(pmull_ghash_update_p8)
+	ld1		{SHASH.2d}, [x3]
+	ld1		{XL.2d}, [x1]
+
+	__pmull_pre_p8
+
+0:	ld1		{T1.2d}, [x2], #16
+	sub		x0, x0, #1
+
+	/* multiply XL by SHASH in GF(2^128) */
+	rev64		T1.16b, T1.16b
+
+	ext		T2.16b, XL.16b, XL.16b, #8
+	ext		IN1.16b, T1.16b, T1.16b, #8
+	eor		T1.16b, T1.16b, T2.16b
+	eor		XL.16b, XL.16b, IN1.16b
+
+	__pmull2_p8	XH, XL, SHASH			// a1 * b1
+	eor		T1.16b, T1.16b, XL.16b
+	__pmull_p8 	XL, XL, SHASH			// a0 * b0
+	__pmull_p8	XM, T1, SHASH2			// (a1 + a0)(b1 + b0)
+
+	eor		T2.16b, XL.16b, XH.16b
+	ext		T1.16b, XL.16b, XH.16b, #8
+	eor		XM.16b, XM.16b, T2.16b
+
+	__pmull_reduce_p8
+
+	eor		T2.16b, T2.16b, XH.16b
+	eor		XL.16b, XL.16b, T2.16b
+
+	cbnz		x0, 0b
+
+	st1		{XL.2d}, [x1]
+	ret
+SYM_FUNC_END(pmull_ghash_update_p8)
diff --git a/lib/crypto/arm64/poly1305.h b/lib/crypto/arm64/poly1305.h
index b77669767cd6..3d4bde857699 100644
--- a/lib/crypto/arm64/poly1305.h
+++ b/lib/crypto/arm64/poly1305.h
@@ -27,17 +27,11 @@ static void poly1305_blocks(struct poly1305_block_state *state, const u8 *src,
 			    unsigned int len, u32 padbit)
 {
 	if (static_branch_likely(&have_neon) && likely(may_use_simd())) {
-		do {
-			unsigned int todo = min_t(unsigned int, len, SZ_4K);
-
-			scoped_ksimd()
-				poly1305_blocks_neon(state, src, todo, padbit);
-
-			len -= todo;
-			src += todo;
-		} while (len);
-	} else
+		scoped_ksimd()
+			poly1305_blocks_neon(state, src, len, padbit);
+	} else {
 		poly1305_blocks_arm64(state, src, len, padbit);
+	}
 }
 
 #define poly1305_mod_init_arch poly1305_mod_init_arch
diff --git a/lib/crypto/arm64/polyval.h b/lib/crypto/arm64/polyval.h
deleted file mode 100644
index a39763395e9b..000000000000
--- a/lib/crypto/arm64/polyval.h
+++ /dev/null
@@ -1,80 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * POLYVAL library functions, arm64 optimized
- *
- * Copyright 2025 Google LLC
- */
-#include <asm/simd.h>
-#include <linux/cpufeature.h>
-
-#define NUM_H_POWERS 8
-
-static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull);
-
-asmlinkage void polyval_mul_pmull(struct polyval_elem *a,
-				  const struct polyval_elem *b);
-asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc,
-				     const struct polyval_key *key,
-				     const u8 *data, size_t nblocks);
-
-static void polyval_preparekey_arch(struct polyval_key *key,
-				    const u8 raw_key[POLYVAL_BLOCK_SIZE])
-{
-	static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS);
-	memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE);
-	if (static_branch_likely(&have_pmull) && may_use_simd()) {
-		scoped_ksimd() {
-			for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
-				key->h_powers[i] = key->h_powers[i + 1];
-				polyval_mul_pmull(
-					&key->h_powers[i],
-					&key->h_powers[NUM_H_POWERS - 1]);
-			}
-		}
-	} else {
-		for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
-			key->h_powers[i] = key->h_powers[i + 1];
-			polyval_mul_generic(&key->h_powers[i],
-					    &key->h_powers[NUM_H_POWERS - 1]);
-		}
-	}
-}
-
-static void polyval_mul_arch(struct polyval_elem *acc,
-			     const struct polyval_key *key)
-{
-	if (static_branch_likely(&have_pmull) && may_use_simd()) {
-		scoped_ksimd()
-			polyval_mul_pmull(acc, &key->h_powers[NUM_H_POWERS - 1]);
-	} else {
-		polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]);
-	}
-}
-
-static void polyval_blocks_arch(struct polyval_elem *acc,
-				const struct polyval_key *key,
-				const u8 *data, size_t nblocks)
-{
-	if (static_branch_likely(&have_pmull) && may_use_simd()) {
-		do {
-			/* Allow rescheduling every 4 KiB. */
-			size_t n = min_t(size_t, nblocks,
-					 4096 / POLYVAL_BLOCK_SIZE);
-
-			scoped_ksimd()
-				polyval_blocks_pmull(acc, key, data, n);
-			data += n * POLYVAL_BLOCK_SIZE;
-			nblocks -= n;
-		} while (nblocks);
-	} else {
-		polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1],
-				       data, nblocks);
-	}
-}
-
-#define polyval_mod_init_arch polyval_mod_init_arch
-static void polyval_mod_init_arch(void)
-{
-	if (cpu_have_named_feature(PMULL))
-		static_branch_enable(&have_pmull);
-}
diff --git a/lib/crypto/arm64/sha1-ce-core.S b/lib/crypto/arm64/sha1-ce-core.S
index 8fbd4767f0f0..40f2a6c8d0c5 100644
--- a/lib/crypto/arm64/sha1-ce-core.S
+++ b/lib/crypto/arm64/sha1-ce-core.S
@@ -62,10 +62,10 @@
 	.endm
 
 	/*
-	 * size_t __sha1_ce_transform(struct sha1_block_state *state,
-	 *			      const u8 *data, size_t nblocks);
+	 * void sha1_ce_transform(struct sha1_block_state *state,
+	 *			  const u8 *data, size_t nblocks);
 	 */
-SYM_FUNC_START(__sha1_ce_transform)
+SYM_FUNC_START(sha1_ce_transform)
 	/* load round constants */
 	loadrc		k0.4s, 0x5a827999, w6
 	loadrc		k1.4s, 0x6ed9eba1, w6
@@ -80,10 +80,10 @@ SYM_FUNC_START(__sha1_ce_transform)
 0:	ld1		{v8.4s-v11.4s}, [x1], #64
 	sub		x2, x2, #1
 
-CPU_LE(	rev32		v8.16b, v8.16b		)
-CPU_LE(	rev32		v9.16b, v9.16b		)
-CPU_LE(	rev32		v10.16b, v10.16b	)
-CPU_LE(	rev32		v11.16b, v11.16b	)
+	rev32		v8.16b, v8.16b
+	rev32		v9.16b, v9.16b
+	rev32		v10.16b, v10.16b
+	rev32		v11.16b, v11.16b
 
 	add		t0.4s, v8.4s, k0.4s
 	mov		dg0v.16b, dgav.16b
@@ -116,15 +116,11 @@ CPU_LE(	rev32		v11.16b, v11.16b	)
 	add		dgbv.2s, dgbv.2s, dg1v.2s
 	add		dgav.4s, dgav.4s, dg0v.4s
 
-	/* return early if voluntary preemption is needed */
-	cond_yield	1f, x5, x6
-
 	/* handled all input blocks? */
 	cbnz		x2, 0b
 
 	/* store new state */
-1:	st1		{dgav.4s}, [x0]
+	st1		{dgav.4s}, [x0]
 	str		dgb, [x0, #16]
-	mov		x0, x2
 	ret
-SYM_FUNC_END(__sha1_ce_transform)
+SYM_FUNC_END(sha1_ce_transform)
diff --git a/lib/crypto/arm64/sha1.h b/lib/crypto/arm64/sha1.h
index bc7071f1be09..112c5d443c56 100644
--- a/lib/crypto/arm64/sha1.h
+++ b/lib/crypto/arm64/sha1.h
@@ -9,22 +9,15 @@
 
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce);
 
-asmlinkage size_t __sha1_ce_transform(struct sha1_block_state *state,
-				      const u8 *data, size_t nblocks);
+asmlinkage void sha1_ce_transform(struct sha1_block_state *state,
+				  const u8 *data, size_t nblocks);
 
 static void sha1_blocks(struct sha1_block_state *state,
 			const u8 *data, size_t nblocks)
 {
 	if (static_branch_likely(&have_ce) && likely(may_use_simd())) {
-		do {
-			size_t rem;
-
-			scoped_ksimd()
-				rem = __sha1_ce_transform(state, data, nblocks);
-
-			data += (nblocks - rem) * SHA1_BLOCK_SIZE;
-			nblocks = rem;
-		} while (nblocks);
+		scoped_ksimd()
+			sha1_ce_transform(state, data, nblocks);
 	} else {
 		sha1_blocks_generic(state, data, nblocks);
 	}
diff --git a/lib/crypto/arm64/sha256-ce.S b/lib/crypto/arm64/sha256-ce.S
index e4bfe42a61a9..8fdbf0a9ff89 100644
--- a/lib/crypto/arm64/sha256-ce.S
+++ b/lib/crypto/arm64/sha256-ce.S
@@ -79,11 +79,11 @@
 	.endm
 
 	/*
-	 * size_t __sha256_ce_transform(struct sha256_block_state *state,
-	 *				const u8 *data, size_t nblocks);
+	 * void sha256_ce_transform(struct sha256_block_state *state,
+	 *			    const u8 *data, size_t nblocks);
 	 */
 	.text
-SYM_FUNC_START(__sha256_ce_transform)
+SYM_FUNC_START(sha256_ce_transform)
 
 	load_round_constants	x8
 
@@ -94,10 +94,10 @@ SYM_FUNC_START(__sha256_ce_transform)
 0:	ld1		{v16.4s-v19.4s}, [x1], #64
 	sub		x2, x2, #1
 
-CPU_LE(	rev32		v16.16b, v16.16b	)
-CPU_LE(	rev32		v17.16b, v17.16b	)
-CPU_LE(	rev32		v18.16b, v18.16b	)
-CPU_LE(	rev32		v19.16b, v19.16b	)
+	rev32		v16.16b, v16.16b
+	rev32		v17.16b, v17.16b
+	rev32		v18.16b, v18.16b
+	rev32		v19.16b, v19.16b
 
 	add		t0.4s, v16.4s, v0.4s
 	mov		dg0v.16b, dgav.16b
@@ -127,17 +127,13 @@ CPU_LE(	rev32		v19.16b, v19.16b	)
 	add		dgav.4s, dgav.4s, dg0v.4s
 	add		dgbv.4s, dgbv.4s, dg1v.4s
 
-	/* return early if voluntary preemption is needed */
-	cond_yield	1f, x5, x6
-
 	/* handled all input blocks? */
 	cbnz		x2, 0b
 
 	/* store new state */
-1:	st1		{dgav.4s, dgbv.4s}, [x0]
-	mov		x0, x2
+	st1		{dgav.4s, dgbv.4s}, [x0]
 	ret
-SYM_FUNC_END(__sha256_ce_transform)
+SYM_FUNC_END(sha256_ce_transform)
 
 	.unreq dga
 	.unreq dgav
@@ -293,14 +289,14 @@ SYM_FUNC_START(sha256_ce_finup2x)
 	ld1		{v20.4s-v23.4s}, [data2], #64
 .Lfinup2x_loop_have_data:
 	// Convert the words of the data blocks from big endian.
-CPU_LE(	rev32		v16.16b, v16.16b	)
-CPU_LE(	rev32		v17.16b, v17.16b	)
-CPU_LE(	rev32		v18.16b, v18.16b	)
-CPU_LE(	rev32		v19.16b, v19.16b	)
-CPU_LE(	rev32		v20.16b, v20.16b	)
-CPU_LE(	rev32		v21.16b, v21.16b	)
-CPU_LE(	rev32		v22.16b, v22.16b	)
-CPU_LE(	rev32		v23.16b, v23.16b	)
+	rev32		v16.16b, v16.16b
+	rev32		v17.16b, v17.16b
+	rev32		v18.16b, v18.16b
+	rev32		v19.16b, v19.16b
+	rev32		v20.16b, v20.16b
+	rev32		v21.16b, v21.16b
+	rev32		v22.16b, v22.16b
+	rev32		v23.16b, v23.16b
 .Lfinup2x_loop_have_bswapped_data:
 
 	// Save the original state for each block.
@@ -340,11 +336,8 @@ CPU_LE(	rev32		v23.16b, v23.16b	)
 	sub		w8, len, #64		// w8 = len - 64
 	add		data1, data1, w8, sxtw	// data1 += len - 64
 	add		data2, data2, w8, sxtw	// data2 += len - 64
-CPU_LE(	mov		x9, #0x80		)
-CPU_LE(	fmov		d16, x9			)
-CPU_BE(	movi		v16.16b, #0		)
-CPU_BE(	mov		x9, #0x8000000000000000	)
-CPU_BE(	mov		v16.d[1], x9		)
+	mov		x9, #0x80
+	fmov		d16, x9
 	movi		v17.16b, #0
 	stp		q16, q17, [sp, #64]
 	stp		q17, q17, [sp, #96]
@@ -352,7 +345,7 @@ CPU_BE(	mov		v16.d[1], x9		)
 	cmp		len, #56
 	b.ge		1f		// will count spill into its own block?
 	lsl		count, count, #3
-CPU_LE(	rev		count, count		)
+	rev		count, count
 	str		count, [x9, #56]
 	mov		final_step, #2	// won't need count-only block
 	b		2f
@@ -397,10 +390,10 @@ CPU_LE(	rev		count, count		)
 
 .Lfinup2x_done:
 	// Write the two digests with all bytes in the correct order.
-CPU_LE(	rev32		state0_a.16b, state0_a.16b	)
-CPU_LE(	rev32		state1_a.16b, state1_a.16b	)
-CPU_LE(	rev32		state0_b.16b, state0_b.16b	)
-CPU_LE(	rev32		state1_b.16b, state1_b.16b	)
+	rev32		state0_a.16b, state0_a.16b
+	rev32		state1_a.16b, state1_a.16b
+	rev32		state0_b.16b, state0_b.16b
+	rev32		state1_b.16b, state1_b.16b
 	st1		{state0_a.4s-state1_a.4s}, [out1]
 	st1		{state0_b.4s-state1_b.4s}, [out2]
 	add		sp, sp, #128
diff --git a/lib/crypto/arm64/sha256.h b/lib/crypto/arm64/sha256.h
index 568dff0f276a..b4353d3c4dd0 100644
--- a/lib/crypto/arm64/sha256.h
+++ b/lib/crypto/arm64/sha256.h
@@ -14,27 +14,17 @@ asmlinkage void sha256_block_data_order(struct sha256_block_state *state,
 					const u8 *data, size_t nblocks);
 asmlinkage void sha256_block_neon(struct sha256_block_state *state,
 				  const u8 *data, size_t nblocks);
-asmlinkage size_t __sha256_ce_transform(struct sha256_block_state *state,
-					const u8 *data, size_t nblocks);
+asmlinkage void sha256_ce_transform(struct sha256_block_state *state,
+				    const u8 *data, size_t nblocks);
 
 static void sha256_blocks(struct sha256_block_state *state,
 			  const u8 *data, size_t nblocks)
 {
-	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
-	    static_branch_likely(&have_neon) && likely(may_use_simd())) {
-		if (static_branch_likely(&have_ce)) {
-			do {
-				size_t rem;
-
-				scoped_ksimd()
-					rem = __sha256_ce_transform(state, data,
-								    nblocks);
-
-				data += (nblocks - rem) * SHA256_BLOCK_SIZE;
-				nblocks = rem;
-			} while (nblocks);
-		} else {
-			scoped_ksimd()
+	if (static_branch_likely(&have_neon) && likely(may_use_simd())) {
+		scoped_ksimd() {
+			if (static_branch_likely(&have_ce))
+				sha256_ce_transform(state, data, nblocks);
+			else
 				sha256_block_neon(state, data, nblocks);
 		}
 	} else {
@@ -56,14 +46,9 @@ static bool sha256_finup_2x_arch(const struct __sha256_ctx *ctx,
 				 u8 out1[SHA256_DIGEST_SIZE],
 				 u8 out2[SHA256_DIGEST_SIZE])
 {
-	/*
-	 * The assembly requires len >= SHA256_BLOCK_SIZE && len <= INT_MAX.
-	 * Further limit len to 65536 to avoid spending too long with preemption
-	 * disabled.  (Of course, in practice len is nearly always 4096 anyway.)
-	 */
-	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
-	    static_branch_likely(&have_ce) && len >= SHA256_BLOCK_SIZE &&
-	    len <= 65536 && likely(may_use_simd())) {
+	/* The assembly requires len >= SHA256_BLOCK_SIZE && len <= INT_MAX. */
+	if (static_branch_likely(&have_ce) && len >= SHA256_BLOCK_SIZE &&
+	    len <= INT_MAX && likely(may_use_simd())) {
 		scoped_ksimd()
 			sha256_ce_finup2x(ctx, data1, data2, len, out1, out2);
 		kmsan_unpoison_memory(out1, SHA256_DIGEST_SIZE);
@@ -78,7 +63,6 @@ static bool sha256_finup_2x_is_optimized_arch(void)
 	return static_key_enabled(&have_ce);
 }
 
-#ifdef CONFIG_KERNEL_MODE_NEON
 #define sha256_mod_init_arch sha256_mod_init_arch
 static void sha256_mod_init_arch(void)
 {
@@ -88,4 +72,3 @@ static void sha256_mod_init_arch(void)
 			static_branch_enable(&have_ce);
 	}
 }
-#endif /* CONFIG_KERNEL_MODE_NEON */
diff --git a/lib/crypto/arm64/sha3-ce-core.S b/lib/crypto/arm64/sha3-ce-core.S
index ace90b506490..b8ab01987ae0 100644
--- a/lib/crypto/arm64/sha3-ce-core.S
+++ b/lib/crypto/arm64/sha3-ce-core.S
@@ -37,8 +37,8 @@
 	.endm
 
 	/*
-	 * size_t sha3_ce_transform(struct sha3_state *state, const u8 *data,
-	 *			    size_t nblocks, size_t block_size)
+	 * void sha3_ce_transform(struct sha3_state *state, const u8 *data,
+	 *			  size_t nblocks, size_t block_size)
 	 *
 	 * block_size is assumed to be one of 72 (SHA3-512), 104 (SHA3-384), 136
 	 * (SHA3-256 and SHAKE256), 144 (SHA3-224), or 168 (SHAKE128).
@@ -185,18 +185,16 @@ SYM_FUNC_START(sha3_ce_transform)
 	eor	 v0.16b,  v0.16b, v31.16b
 
 	cbnz	w8, 3b
-	cond_yield 4f, x8, x9
 	cbnz	x2, 0b
 
 	/* save state */
-4:	st1	{ v0.1d- v3.1d}, [x0], #32
+	st1	{ v0.1d- v3.1d}, [x0], #32
 	st1	{ v4.1d- v7.1d}, [x0], #32
 	st1	{ v8.1d-v11.1d}, [x0], #32
 	st1	{v12.1d-v15.1d}, [x0], #32
 	st1	{v16.1d-v19.1d}, [x0], #32
 	st1	{v20.1d-v23.1d}, [x0], #32
 	st1	{v24.1d}, [x0]
-	mov	x0, x2
 	ret
 SYM_FUNC_END(sha3_ce_transform)
 
diff --git a/lib/crypto/arm64/sha3.h b/lib/crypto/arm64/sha3.h
index b602f1b3b282..eaaba3224acc 100644
--- a/lib/crypto/arm64/sha3.h
+++ b/lib/crypto/arm64/sha3.h
@@ -12,22 +12,15 @@
 
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3);
 
-asmlinkage size_t sha3_ce_transform(struct sha3_state *state, const u8 *data,
-				    size_t nblocks, size_t block_size);
+asmlinkage void sha3_ce_transform(struct sha3_state *state, const u8 *data,
+				  size_t nblocks, size_t block_size);
 
 static void sha3_absorb_blocks(struct sha3_state *state, const u8 *data,
 			       size_t nblocks, size_t block_size)
 {
 	if (static_branch_likely(&have_sha3) && likely(may_use_simd())) {
-		do {
-			size_t rem;
-
-			scoped_ksimd()
-				rem = sha3_ce_transform(state, data, nblocks,
-							block_size);
-			data += (nblocks - rem) * block_size;
-			nblocks = rem;
-		} while (nblocks);
+		scoped_ksimd()
+			sha3_ce_transform(state, data, nblocks, block_size);
 	} else {
 		sha3_absorb_blocks_generic(state, data, nblocks, block_size);
 	}
diff --git a/lib/crypto/arm64/sha512-ce-core.S b/lib/crypto/arm64/sha512-ce-core.S
index ffd51acfd1ee..daa2c63a20c0 100644
--- a/lib/crypto/arm64/sha512-ce-core.S
+++ b/lib/crypto/arm64/sha512-ce-core.S
@@ -93,11 +93,11 @@
 	.endm
 
 	/*
-	 * size_t __sha512_ce_transform(struct sha512_block_state *state,
-	 *				const u8 *data, size_t nblocks);
+	 * void sha512_ce_transform(struct sha512_block_state *state,
+	 *			    const u8 *data, size_t nblocks);
 	 */
 	.text
-SYM_FUNC_START(__sha512_ce_transform)
+SYM_FUNC_START(sha512_ce_transform)
 	/* load state */
 	ld1		{v8.2d-v11.2d}, [x0]
 
@@ -110,14 +110,14 @@ SYM_FUNC_START(__sha512_ce_transform)
 	ld1		{v16.2d-v19.2d}, [x1], #64
 	sub		x2, x2, #1
 
-CPU_LE(	rev64		v12.16b, v12.16b	)
-CPU_LE(	rev64		v13.16b, v13.16b	)
-CPU_LE(	rev64		v14.16b, v14.16b	)
-CPU_LE(	rev64		v15.16b, v15.16b	)
-CPU_LE(	rev64		v16.16b, v16.16b	)
-CPU_LE(	rev64		v17.16b, v17.16b	)
-CPU_LE(	rev64		v18.16b, v18.16b	)
-CPU_LE(	rev64		v19.16b, v19.16b	)
+	rev64		v12.16b, v12.16b
+	rev64		v13.16b, v13.16b
+	rev64		v14.16b, v14.16b
+	rev64		v15.16b, v15.16b
+	rev64		v16.16b, v16.16b
+	rev64		v17.16b, v17.16b
+	rev64		v18.16b, v18.16b
+	rev64		v19.16b, v19.16b
 
 	mov		x4, x3				// rc pointer
 
@@ -186,12 +186,10 @@ CPU_LE(	rev64		v19.16b, v19.16b	)
 	add		v10.2d, v10.2d, v2.2d
 	add		v11.2d, v11.2d, v3.2d
 
-	cond_yield	3f, x4, x5
 	/* handled all input blocks? */
 	cbnz		x2, 0b
 
 	/* store new state */
-3:	st1		{v8.2d-v11.2d}, [x0]
-	mov		x0, x2
+	st1		{v8.2d-v11.2d}, [x0]
 	ret
-SYM_FUNC_END(__sha512_ce_transform)
+SYM_FUNC_END(sha512_ce_transform)
diff --git a/lib/crypto/arm64/sha512.h b/lib/crypto/arm64/sha512.h
index 7eb7ef04d268..5da27e6e23ea 100644
--- a/lib/crypto/arm64/sha512.h
+++ b/lib/crypto/arm64/sha512.h
@@ -12,34 +12,24 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha512_insns);
 
 asmlinkage void sha512_block_data_order(struct sha512_block_state *state,
 					const u8 *data, size_t nblocks);
-asmlinkage size_t __sha512_ce_transform(struct sha512_block_state *state,
-					const u8 *data, size_t nblocks);
+asmlinkage void sha512_ce_transform(struct sha512_block_state *state,
+				    const u8 *data, size_t nblocks);
 
 static void sha512_blocks(struct sha512_block_state *state,
 			  const u8 *data, size_t nblocks)
 {
-	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
-	    static_branch_likely(&have_sha512_insns) &&
+	if (static_branch_likely(&have_sha512_insns) &&
 	    likely(may_use_simd())) {
-		do {
-			size_t rem;
-
-			scoped_ksimd()
-				rem = __sha512_ce_transform(state, data, nblocks);
-
-			data += (nblocks - rem) * SHA512_BLOCK_SIZE;
-			nblocks = rem;
-		} while (nblocks);
+		scoped_ksimd()
+			sha512_ce_transform(state, data, nblocks);
 	} else {
 		sha512_block_data_order(state, data, nblocks);
 	}
 }
 
-#ifdef CONFIG_KERNEL_MODE_NEON
 #define sha512_mod_init_arch sha512_mod_init_arch
 static void sha512_mod_init_arch(void)
 {
 	if (cpu_have_named_feature(SHA512))
 		static_branch_enable(&have_sha512_insns);
 }
-#endif /* CONFIG_KERNEL_MODE_NEON */
diff --git a/lib/crypto/arm64/sm3-ce-core.S b/lib/crypto/arm64/sm3-ce-core.S
new file mode 100644
index 000000000000..ee7f900d7cff
--- /dev/null
+++ b/lib/crypto/arm64/sm3-ce-core.S
@@ -0,0 +1,138 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * sm3-ce-core.S - SM3 secure hash using ARMv8.2 Crypto Extensions
+ *
+ * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+	.irp		b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12
+	.set		.Lv\b\().4s, \b
+	.endr
+
+	.macro		sm3partw1, rd, rn, rm
+	.inst		0xce60c000 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
+	.endm
+
+	.macro		sm3partw2, rd, rn, rm
+	.inst		0xce60c400 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
+	.endm
+
+	.macro		sm3ss1, rd, rn, rm, ra
+	.inst		0xce400000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16)
+	.endm
+
+	.macro		sm3tt1a, rd, rn, rm, imm2
+	.inst		0xce408000 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
+	.endm
+
+	.macro		sm3tt1b, rd, rn, rm, imm2
+	.inst		0xce408400 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
+	.endm
+
+	.macro		sm3tt2a, rd, rn, rm, imm2
+	.inst		0xce408800 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
+	.endm
+
+	.macro		sm3tt2b, rd, rn, rm, imm2
+	.inst		0xce408c00 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
+	.endm
+
+	.macro		round, ab, s0, t0, t1, i
+	sm3ss1		v5.4s, v8.4s, \t0\().4s, v9.4s
+	shl		\t1\().4s, \t0\().4s, #1
+	sri		\t1\().4s, \t0\().4s, #31
+	sm3tt1\ab	v8.4s, v5.4s, v10.4s, \i
+	sm3tt2\ab	v9.4s, v5.4s, \s0\().4s, \i
+	.endm
+
+	.macro		qround, ab, s0, s1, s2, s3, s4
+	.ifnb		\s4
+	ext		\s4\().16b, \s1\().16b, \s2\().16b, #12
+	ext		v6.16b, \s0\().16b, \s1\().16b, #12
+	ext		v7.16b, \s2\().16b, \s3\().16b, #8
+	sm3partw1	\s4\().4s, \s0\().4s, \s3\().4s
+	.endif
+
+	eor		v10.16b, \s0\().16b, \s1\().16b
+
+	round		\ab, \s0, v11, v12, 0
+	round		\ab, \s0, v12, v11, 1
+	round		\ab, \s0, v11, v12, 2
+	round		\ab, \s0, v12, v11, 3
+
+	.ifnb		\s4
+	sm3partw2	\s4\().4s, v7.4s, v6.4s
+	.endif
+	.endm
+
+	/*
+	 * void sm3_ce_transform(struct sm3_block_state *state,
+	 *			 const u8 *data, size_t nblocks)
+	 */
+	.text
+SYM_FUNC_START(sm3_ce_transform)
+	/* load state */
+	ld1		{v8.4s-v9.4s}, [x0]
+	rev64		v8.4s, v8.4s
+	rev64		v9.4s, v9.4s
+	ext		v8.16b, v8.16b, v8.16b, #8
+	ext		v9.16b, v9.16b, v9.16b, #8
+
+	adr_l		x8, .Lt
+	ldp		s13, s14, [x8]
+
+	/* load input */
+0:	ld1		{v0.16b-v3.16b}, [x1], #64
+	sub		x2, x2, #1
+
+	mov		v15.16b, v8.16b
+	mov		v16.16b, v9.16b
+
+	rev32		v0.16b, v0.16b
+	rev32		v1.16b, v1.16b
+	rev32		v2.16b, v2.16b
+	rev32		v3.16b, v3.16b
+
+	ext		v11.16b, v13.16b, v13.16b, #4
+
+	qround		a, v0, v1, v2, v3, v4
+	qround		a, v1, v2, v3, v4, v0
+	qround		a, v2, v3, v4, v0, v1
+	qround		a, v3, v4, v0, v1, v2
+
+	ext		v11.16b, v14.16b, v14.16b, #4
+
+	qround		b, v4, v0, v1, v2, v3
+	qround		b, v0, v1, v2, v3, v4
+	qround		b, v1, v2, v3, v4, v0
+	qround		b, v2, v3, v4, v0, v1
+	qround		b, v3, v4, v0, v1, v2
+	qround		b, v4, v0, v1, v2, v3
+	qround		b, v0, v1, v2, v3, v4
+	qround		b, v1, v2, v3, v4, v0
+	qround		b, v2, v3, v4, v0, v1
+	qround		b, v3, v4
+	qround		b, v4, v0
+	qround		b, v0, v1
+
+	eor		v8.16b, v8.16b, v15.16b
+	eor		v9.16b, v9.16b, v16.16b
+
+	/* handled all input blocks? */
+	cbnz		x2, 0b
+
+	/* save state */
+	rev64		v8.4s, v8.4s
+	rev64		v9.4s, v9.4s
+	ext		v8.16b, v8.16b, v8.16b, #8
+	ext		v9.16b, v9.16b, v9.16b, #8
+	st1		{v8.4s-v9.4s}, [x0]
+	ret
+SYM_FUNC_END(sm3_ce_transform)
+
+	.section	".rodata", "a"
+	.align		3
+.Lt:	.word		0x79cc4519, 0x9d8a7a87
diff --git a/lib/crypto/arm64/sm3-neon-core.S b/lib/crypto/arm64/sm3-neon-core.S
new file mode 100644
index 000000000000..ad874af13802
--- /dev/null
+++ b/lib/crypto/arm64/sm3-neon-core.S
@@ -0,0 +1,600 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * sm3-neon-core.S - SM3 secure hash using NEON instructions
+ *
+ * Linux/arm64 port of the libgcrypt SM3 implementation for AArch64
+ *
+ * Copyright (C) 2021 Jussi Kivilinna <jussi.kivilinna@iki.fi>
+ * Copyright (c) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+/* Context structure */
+
+#define state_h0 0
+#define state_h1 4
+#define state_h2 8
+#define state_h3 12
+#define state_h4 16
+#define state_h5 20
+#define state_h6 24
+#define state_h7 28
+
+/* Stack structure */
+
+#define STACK_W_SIZE        (32 * 2 * 3)
+
+#define STACK_W             (0)
+#define STACK_SIZE          (STACK_W + STACK_W_SIZE)
+
+/* Register macros */
+
+#define RSTATE x0
+#define RDATA  x1
+#define RNBLKS x2
+#define RKPTR  x28
+#define RFRAME x29
+
+#define ra w3
+#define rb w4
+#define rc w5
+#define rd w6
+#define re w7
+#define rf w8
+#define rg w9
+#define rh w10
+
+#define t0 w11
+#define t1 w12
+#define t2 w13
+#define t3 w14
+#define t4 w15
+#define t5 w16
+#define t6 w17
+
+#define k_even w19
+#define k_odd w20
+
+#define addr0 x21
+#define addr1 x22
+
+#define s0 w23
+#define s1 w24
+#define s2 w25
+#define s3 w26
+
+#define W0 v0
+#define W1 v1
+#define W2 v2
+#define W3 v3
+#define W4 v4
+#define W5 v5
+
+#define XTMP0 v6
+#define XTMP1 v7
+#define XTMP2 v16
+#define XTMP3 v17
+#define XTMP4 v18
+#define XTMP5 v19
+#define XTMP6 v20
+
+/* Helper macros. */
+
+#define _(...) /*_*/
+
+#define clear_vec(x) \
+	movi	x.8h, #0;
+
+#define rolw(o, a, n) \
+	ror	o, a, #(32 - n);
+
+/* Round function macros. */
+
+#define GG1_1(x, y, z, o, t) \
+	eor	o, x, y;
+#define GG1_2(x, y, z, o, t) \
+	eor	o, o, z;
+#define GG1_3(x, y, z, o, t)
+
+#define FF1_1(x, y, z, o, t) GG1_1(x, y, z, o, t)
+#define FF1_2(x, y, z, o, t)
+#define FF1_3(x, y, z, o, t) GG1_2(x, y, z, o, t)
+
+#define GG2_1(x, y, z, o, t) \
+	bic	o, z, x;
+#define GG2_2(x, y, z, o, t) \
+	and	t, y, x;
+#define GG2_3(x, y, z, o, t) \
+	eor	o, o, t;
+
+#define FF2_1(x, y, z, o, t) \
+	eor	o, x, y;
+#define FF2_2(x, y, z, o, t) \
+	and	t, x, y; \
+	and	o, o, z;
+#define FF2_3(x, y, z, o, t) \
+	eor	o, o, t;
+
+#define R(i, a, b, c, d, e, f, g, h, k, K_LOAD, round, widx, wtype, IOP, iop_param) \
+	K_LOAD(round);                                                        \
+	ldr	t5, [sp, #(wtype##_W1_ADDR(round, widx))];                    \
+	rolw(t0, a, 12);                              /* rol(a, 12) => t0 */  \
+      IOP(1, iop_param);                                                      \
+	FF##i##_1(a, b, c, t1, t2);                                           \
+	ldr	t6, [sp, #(wtype##_W1W2_ADDR(round, widx))];                  \
+	add	k, k, e;                                                      \
+      IOP(2, iop_param);                                                      \
+	GG##i##_1(e, f, g, t3, t4);                                           \
+	FF##i##_2(a, b, c, t1, t2);                                           \
+      IOP(3, iop_param);                                                      \
+	add	k, k, t0;                                                     \
+	add	h, h, t5;                                                     \
+	add	d, d, t6;                     /* w1w2 + d => d */             \
+      IOP(4, iop_param);                                                      \
+	rolw(k, k, 7);                        /* rol (t0 + e + t), 7) => k */ \
+	GG##i##_2(e, f, g, t3, t4);                                           \
+	add	h, h, k;                      /* h + w1 + k => h */           \
+      IOP(5, iop_param);                                                      \
+	FF##i##_3(a, b, c, t1, t2);                                           \
+	eor	t0, t0, k;                    /* k ^ t0 => t0 */              \
+	GG##i##_3(e, f, g, t3, t4);                                           \
+	add	d, d, t1;                     /* FF(a,b,c) + d => d */        \
+      IOP(6, iop_param);                                                      \
+	add	t3, t3, h;                    /* GG(e,f,g) + h => t3 */       \
+	rolw(b, b, 9);                        /* rol(b, 9) => b */            \
+	eor	h, t3, t3, ror #(32-9);                                       \
+      IOP(7, iop_param);                                                      \
+	add	d, d, t0;                     /* t0 + d => d */               \
+	rolw(f, f, 19);                       /* rol(f, 19) => f */           \
+      IOP(8, iop_param);                                                      \
+	eor	h, h, t3, ror #(32-17);       /* P0(t3) => h */
+
+#define R1(a, b, c, d, e, f, g, h, k, K_LOAD, round, widx, wtype, IOP, iop_param) \
+	R(1, ##a, ##b, ##c, ##d, ##e, ##f, ##g, ##h, ##k, K_LOAD, round, widx, wtype, IOP, iop_param)
+
+#define R2(a, b, c, d, e, f, g, h, k, K_LOAD, round, widx, wtype, IOP, iop_param) \
+	R(2, ##a, ##b, ##c, ##d, ##e, ##f, ##g, ##h, ##k, K_LOAD, round, widx, wtype, IOP, iop_param)
+
+#define KL(round) \
+	ldp	k_even, k_odd, [RKPTR, #(4*(round))];
+
+/* Input expansion macros. */
+
+/* Byte-swapped input address. */
+#define IW_W_ADDR(round, widx, offs) \
+	(STACK_W + ((round) / 4) * 64 + (offs) + ((widx) * 4))
+
+/* Expanded input address. */
+#define XW_W_ADDR(round, widx, offs) \
+	(STACK_W + ((((round) / 3) - 4) % 2) * 64 + (offs) + ((widx) * 4))
+
+/* Rounds 1-12, byte-swapped input block addresses. */
+#define IW_W1_ADDR(round, widx)   IW_W_ADDR(round, widx, 32)
+#define IW_W1W2_ADDR(round, widx) IW_W_ADDR(round, widx, 48)
+
+/* Rounds 1-12, expanded input block addresses. */
+#define XW_W1_ADDR(round, widx)   XW_W_ADDR(round, widx, 0)
+#define XW_W1W2_ADDR(round, widx) XW_W_ADDR(round, widx, 16)
+
+/* Input block loading.
+ * Interleaving within round function needed for in-order CPUs. */
+#define LOAD_W_VEC_1_1() \
+	add	addr0, sp, #IW_W1_ADDR(0, 0);
+#define LOAD_W_VEC_1_2() \
+	add	addr1, sp, #IW_W1_ADDR(4, 0);
+#define LOAD_W_VEC_1_3() \
+	ld1	{W0.16b}, [RDATA], #16;
+#define LOAD_W_VEC_1_4() \
+	ld1	{W1.16b}, [RDATA], #16;
+#define LOAD_W_VEC_1_5() \
+	ld1	{W2.16b}, [RDATA], #16;
+#define LOAD_W_VEC_1_6() \
+	ld1	{W3.16b}, [RDATA], #16;
+#define LOAD_W_VEC_1_7() \
+	rev32	XTMP0.16b, W0.16b;
+#define LOAD_W_VEC_1_8() \
+	rev32	XTMP1.16b, W1.16b;
+#define LOAD_W_VEC_2_1() \
+	rev32	XTMP2.16b, W2.16b;
+#define LOAD_W_VEC_2_2() \
+	rev32	XTMP3.16b, W3.16b;
+#define LOAD_W_VEC_2_3() \
+	eor	XTMP4.16b, XTMP1.16b, XTMP0.16b;
+#define LOAD_W_VEC_2_4() \
+	eor	XTMP5.16b, XTMP2.16b, XTMP1.16b;
+#define LOAD_W_VEC_2_5() \
+	st1	{XTMP0.16b}, [addr0], #16;
+#define LOAD_W_VEC_2_6() \
+	st1	{XTMP4.16b}, [addr0]; \
+	add	addr0, sp, #IW_W1_ADDR(8, 0);
+#define LOAD_W_VEC_2_7() \
+	eor	XTMP6.16b, XTMP3.16b, XTMP2.16b;
+#define LOAD_W_VEC_2_8() \
+	ext	W0.16b, XTMP0.16b, XTMP0.16b, #8;  /* W0: xx, w0, xx, xx */
+#define LOAD_W_VEC_3_1() \
+	mov	W2.16b, XTMP1.16b;                 /* W2: xx, w6, w5, w4 */
+#define LOAD_W_VEC_3_2() \
+	st1	{XTMP1.16b}, [addr1], #16;
+#define LOAD_W_VEC_3_3() \
+	st1	{XTMP5.16b}, [addr1]; \
+	ext	W1.16b, XTMP0.16b, XTMP0.16b, #4;  /* W1: xx, w3, w2, w1 */
+#define LOAD_W_VEC_3_4() \
+	ext	W3.16b, XTMP1.16b, XTMP2.16b, #12; /* W3: xx, w9, w8, w7 */
+#define LOAD_W_VEC_3_5() \
+	ext	W4.16b, XTMP2.16b, XTMP3.16b, #8;  /* W4: xx, w12, w11, w10 */
+#define LOAD_W_VEC_3_6() \
+	st1	{XTMP2.16b}, [addr0], #16;
+#define LOAD_W_VEC_3_7() \
+	st1	{XTMP6.16b}, [addr0];
+#define LOAD_W_VEC_3_8() \
+	ext	W5.16b, XTMP3.16b, XTMP3.16b, #4;  /* W5: xx, w15, w14, w13 */
+
+#define LOAD_W_VEC_1(iop_num, ...) \
+	LOAD_W_VEC_1_##iop_num()
+#define LOAD_W_VEC_2(iop_num, ...) \
+	LOAD_W_VEC_2_##iop_num()
+#define LOAD_W_VEC_3(iop_num, ...) \
+	LOAD_W_VEC_3_##iop_num()
+
+/* Message scheduling. Note: 3 words per vector register.
+ * Interleaving within round function needed for in-order CPUs. */
+#define SCHED_W_1_1(round, w0, w1, w2, w3, w4, w5) \
+	/* Load (w[i - 16]) => XTMP0 */            \
+	/* Load (w[i - 13]) => XTMP5 */            \
+	ext	XTMP0.16b, w0.16b, w0.16b, #12;    /* XTMP0: w0, xx, xx, xx */
+#define SCHED_W_1_2(round, w0, w1, w2, w3, w4, w5) \
+	ext	XTMP5.16b, w1.16b, w1.16b, #12;
+#define SCHED_W_1_3(round, w0, w1, w2, w3, w4, w5) \
+	ext	XTMP0.16b, XTMP0.16b, w1.16b, #12; /* XTMP0: xx, w2, w1, w0 */
+#define SCHED_W_1_4(round, w0, w1, w2, w3, w4, w5) \
+	ext	XTMP5.16b, XTMP5.16b, w2.16b, #12;
+#define SCHED_W_1_5(round, w0, w1, w2, w3, w4, w5) \
+	/* w[i - 9] == w3 */                       \
+	/* W3 ^ XTMP0 => XTMP0 */                  \
+	eor	XTMP0.16b, XTMP0.16b, w3.16b;
+#define SCHED_W_1_6(round, w0, w1, w2, w3, w4, w5) \
+	/* w[i - 3] == w5 */                       \
+	/* rol(XMM5, 15) ^ XTMP0 => XTMP0 */       \
+	/* rol(XTMP5, 7) => XTMP1 */               \
+	add	addr0, sp, #XW_W1_ADDR((round), 0); \
+	shl	XTMP2.4s, w5.4s, #15;
+#define SCHED_W_1_7(round, w0, w1, w2, w3, w4, w5) \
+	shl	XTMP1.4s, XTMP5.4s, #7;
+#define SCHED_W_1_8(round, w0, w1, w2, w3, w4, w5) \
+	sri	XTMP2.4s, w5.4s, #(32-15);
+#define SCHED_W_2_1(round, w0, w1, w2, w3, w4, w5) \
+	sri	XTMP1.4s, XTMP5.4s, #(32-7);
+#define SCHED_W_2_2(round, w0, w1, w2, w3, w4, w5) \
+	eor	XTMP0.16b, XTMP0.16b, XTMP2.16b;
+#define SCHED_W_2_3(round, w0, w1, w2, w3, w4, w5) \
+	/* w[i - 6] == W4 */                       \
+	/* W4 ^ XTMP1 => XTMP1 */                  \
+	eor	XTMP1.16b, XTMP1.16b, w4.16b;
+#define SCHED_W_2_4(round, w0, w1, w2, w3, w4, w5) \
+	/* P1(XTMP0) ^ XTMP1 => W0 */              \
+	shl	XTMP3.4s, XTMP0.4s, #15;
+#define SCHED_W_2_5(round, w0, w1, w2, w3, w4, w5) \
+	shl	XTMP4.4s, XTMP0.4s, #23;
+#define SCHED_W_2_6(round, w0, w1, w2, w3, w4, w5) \
+	eor	w0.16b, XTMP1.16b, XTMP0.16b;
+#define SCHED_W_2_7(round, w0, w1, w2, w3, w4, w5) \
+	sri	XTMP3.4s, XTMP0.4s, #(32-15);
+#define SCHED_W_2_8(round, w0, w1, w2, w3, w4, w5) \
+	sri	XTMP4.4s, XTMP0.4s, #(32-23);
+#define SCHED_W_3_1(round, w0, w1, w2, w3, w4, w5) \
+	eor	w0.16b, w0.16b, XTMP3.16b;
+#define SCHED_W_3_2(round, w0, w1, w2, w3, w4, w5) \
+	/* Load (w[i - 3]) => XTMP2 */             \
+	ext	XTMP2.16b, w4.16b, w4.16b, #12;
+#define SCHED_W_3_3(round, w0, w1, w2, w3, w4, w5) \
+	eor	w0.16b, w0.16b, XTMP4.16b;
+#define SCHED_W_3_4(round, w0, w1, w2, w3, w4, w5) \
+	ext	XTMP2.16b, XTMP2.16b, w5.16b, #12;
+#define SCHED_W_3_5(round, w0, w1, w2, w3, w4, w5) \
+	/* W1 ^ W2 => XTMP3 */                     \
+	eor	XTMP3.16b, XTMP2.16b, w0.16b;
+#define SCHED_W_3_6(round, w0, w1, w2, w3, w4, w5)
+#define SCHED_W_3_7(round, w0, w1, w2, w3, w4, w5) \
+	st1	{XTMP2.16b-XTMP3.16b}, [addr0];
+#define SCHED_W_3_8(round, w0, w1, w2, w3, w4, w5)
+
+#define SCHED_W_W0W1W2W3W4W5_1(iop_num, round) \
+	SCHED_W_1_##iop_num(round, W0, W1, W2, W3, W4, W5)
+#define SCHED_W_W0W1W2W3W4W5_2(iop_num, round) \
+	SCHED_W_2_##iop_num(round, W0, W1, W2, W3, W4, W5)
+#define SCHED_W_W0W1W2W3W4W5_3(iop_num, round) \
+	SCHED_W_3_##iop_num(round, W0, W1, W2, W3, W4, W5)
+
+#define SCHED_W_W1W2W3W4W5W0_1(iop_num, round) \
+	SCHED_W_1_##iop_num(round, W1, W2, W3, W4, W5, W0)
+#define SCHED_W_W1W2W3W4W5W0_2(iop_num, round) \
+	SCHED_W_2_##iop_num(round, W1, W2, W3, W4, W5, W0)
+#define SCHED_W_W1W2W3W4W5W0_3(iop_num, round) \
+	SCHED_W_3_##iop_num(round, W1, W2, W3, W4, W5, W0)
+
+#define SCHED_W_W2W3W4W5W0W1_1(iop_num, round) \
+	SCHED_W_1_##iop_num(round, W2, W3, W4, W5, W0, W1)
+#define SCHED_W_W2W3W4W5W0W1_2(iop_num, round) \
+	SCHED_W_2_##iop_num(round, W2, W3, W4, W5, W0, W1)
+#define SCHED_W_W2W3W4W5W0W1_3(iop_num, round) \
+	SCHED_W_3_##iop_num(round, W2, W3, W4, W5, W0, W1)
+
+#define SCHED_W_W3W4W5W0W1W2_1(iop_num, round) \
+	SCHED_W_1_##iop_num(round, W3, W4, W5, W0, W1, W2)
+#define SCHED_W_W3W4W5W0W1W2_2(iop_num, round) \
+	SCHED_W_2_##iop_num(round, W3, W4, W5, W0, W1, W2)
+#define SCHED_W_W3W4W5W0W1W2_3(iop_num, round) \
+	SCHED_W_3_##iop_num(round, W3, W4, W5, W0, W1, W2)
+
+#define SCHED_W_W4W5W0W1W2W3_1(iop_num, round) \
+	SCHED_W_1_##iop_num(round, W4, W5, W0, W1, W2, W3)
+#define SCHED_W_W4W5W0W1W2W3_2(iop_num, round) \
+	SCHED_W_2_##iop_num(round, W4, W5, W0, W1, W2, W3)
+#define SCHED_W_W4W5W0W1W2W3_3(iop_num, round) \
+	SCHED_W_3_##iop_num(round, W4, W5, W0, W1, W2, W3)
+
+#define SCHED_W_W5W0W1W2W3W4_1(iop_num, round) \
+	SCHED_W_1_##iop_num(round, W5, W0, W1, W2, W3, W4)
+#define SCHED_W_W5W0W1W2W3W4_2(iop_num, round) \
+	SCHED_W_2_##iop_num(round, W5, W0, W1, W2, W3, W4)
+#define SCHED_W_W5W0W1W2W3W4_3(iop_num, round) \
+	SCHED_W_3_##iop_num(round, W5, W0, W1, W2, W3, W4)
+
+
+	/*
+	 * Transform nblocks*64 bytes (nblocks*16 32-bit words) at 'data'.
+	 *
+	 * void sm3_neon_transform(struct sm3_block_state *state,
+	 *			   const u8 *data, size_t nblocks)
+	 */
+	.text
+.align 3
+SYM_FUNC_START(sm3_neon_transform)
+	ldp		ra, rb, [RSTATE, #0]
+	ldp		rc, rd, [RSTATE, #8]
+	ldp		re, rf, [RSTATE, #16]
+	ldp		rg, rh, [RSTATE, #24]
+
+	stp		x28, x29, [sp, #-16]!
+	stp		x19, x20, [sp, #-16]!
+	stp		x21, x22, [sp, #-16]!
+	stp		x23, x24, [sp, #-16]!
+	stp		x25, x26, [sp, #-16]!
+	mov		RFRAME, sp
+
+	sub		addr0, sp, #STACK_SIZE
+	adr_l		RKPTR, .LKtable
+	and		sp, addr0, #(~63)
+
+	/* Preload first block. */
+	LOAD_W_VEC_1(1, 0)
+	LOAD_W_VEC_1(2, 0)
+	LOAD_W_VEC_1(3, 0)
+	LOAD_W_VEC_1(4, 0)
+	LOAD_W_VEC_1(5, 0)
+	LOAD_W_VEC_1(6, 0)
+	LOAD_W_VEC_1(7, 0)
+	LOAD_W_VEC_1(8, 0)
+	LOAD_W_VEC_2(1, 0)
+	LOAD_W_VEC_2(2, 0)
+	LOAD_W_VEC_2(3, 0)
+	LOAD_W_VEC_2(4, 0)
+	LOAD_W_VEC_2(5, 0)
+	LOAD_W_VEC_2(6, 0)
+	LOAD_W_VEC_2(7, 0)
+	LOAD_W_VEC_2(8, 0)
+	LOAD_W_VEC_3(1, 0)
+	LOAD_W_VEC_3(2, 0)
+	LOAD_W_VEC_3(3, 0)
+	LOAD_W_VEC_3(4, 0)
+	LOAD_W_VEC_3(5, 0)
+	LOAD_W_VEC_3(6, 0)
+	LOAD_W_VEC_3(7, 0)
+	LOAD_W_VEC_3(8, 0)
+
+.balign 16
+.Loop:
+	/* Transform 0-3 */
+	R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 0, 0, IW, _, 0)
+	R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  1, 1, IW, _, 0)
+	R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 2, 2, IW, _, 0)
+	R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  3, 3, IW, _, 0)
+
+	/* Transform 4-7 + Precalc 12-14 */
+	R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 4, 0, IW, _, 0)
+	R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  5, 1, IW, _, 0)
+	R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 6, 2, IW, SCHED_W_W0W1W2W3W4W5_1, 12)
+	R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  7, 3, IW, SCHED_W_W0W1W2W3W4W5_2, 12)
+
+	/* Transform 8-11 + Precalc 12-17 */
+	R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 8, 0, IW, SCHED_W_W0W1W2W3W4W5_3, 12)
+	R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  9, 1, IW, SCHED_W_W1W2W3W4W5W0_1, 15)
+	R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 10, 2, IW, SCHED_W_W1W2W3W4W5W0_2, 15)
+	R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  11, 3, IW, SCHED_W_W1W2W3W4W5W0_3, 15)
+
+	/* Transform 12-14 + Precalc 18-20 */
+	R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 12, 0, XW, SCHED_W_W2W3W4W5W0W1_1, 18)
+	R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  13, 1, XW, SCHED_W_W2W3W4W5W0W1_2, 18)
+	R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 14, 2, XW, SCHED_W_W2W3W4W5W0W1_3, 18)
+
+	/* Transform 15-17 + Precalc 21-23 */
+	R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  15, 0, XW, SCHED_W_W3W4W5W0W1W2_1, 21)
+	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 16, 1, XW, SCHED_W_W3W4W5W0W1W2_2, 21)
+	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  17, 2, XW, SCHED_W_W3W4W5W0W1W2_3, 21)
+
+	/* Transform 18-20 + Precalc 24-26 */
+	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 18, 0, XW, SCHED_W_W4W5W0W1W2W3_1, 24)
+	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  19, 1, XW, SCHED_W_W4W5W0W1W2W3_2, 24)
+	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 20, 2, XW, SCHED_W_W4W5W0W1W2W3_3, 24)
+
+	/* Transform 21-23 + Precalc 27-29 */
+	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  21, 0, XW, SCHED_W_W5W0W1W2W3W4_1, 27)
+	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 22, 1, XW, SCHED_W_W5W0W1W2W3W4_2, 27)
+	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  23, 2, XW, SCHED_W_W5W0W1W2W3W4_3, 27)
+
+	/* Transform 24-26 + Precalc 30-32 */
+	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 24, 0, XW, SCHED_W_W0W1W2W3W4W5_1, 30)
+	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  25, 1, XW, SCHED_W_W0W1W2W3W4W5_2, 30)
+	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 26, 2, XW, SCHED_W_W0W1W2W3W4W5_3, 30)
+
+	/* Transform 27-29 + Precalc 33-35 */
+	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  27, 0, XW, SCHED_W_W1W2W3W4W5W0_1, 33)
+	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 28, 1, XW, SCHED_W_W1W2W3W4W5W0_2, 33)
+	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  29, 2, XW, SCHED_W_W1W2W3W4W5W0_3, 33)
+
+	/* Transform 30-32 + Precalc 36-38 */
+	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 30, 0, XW, SCHED_W_W2W3W4W5W0W1_1, 36)
+	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  31, 1, XW, SCHED_W_W2W3W4W5W0W1_2, 36)
+	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 32, 2, XW, SCHED_W_W2W3W4W5W0W1_3, 36)
+
+	/* Transform 33-35 + Precalc 39-41 */
+	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  33, 0, XW, SCHED_W_W3W4W5W0W1W2_1, 39)
+	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 34, 1, XW, SCHED_W_W3W4W5W0W1W2_2, 39)
+	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  35, 2, XW, SCHED_W_W3W4W5W0W1W2_3, 39)
+
+	/* Transform 36-38 + Precalc 42-44 */
+	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 36, 0, XW, SCHED_W_W4W5W0W1W2W3_1, 42)
+	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  37, 1, XW, SCHED_W_W4W5W0W1W2W3_2, 42)
+	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 38, 2, XW, SCHED_W_W4W5W0W1W2W3_3, 42)
+
+	/* Transform 39-41 + Precalc 45-47 */
+	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  39, 0, XW, SCHED_W_W5W0W1W2W3W4_1, 45)
+	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 40, 1, XW, SCHED_W_W5W0W1W2W3W4_2, 45)
+	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  41, 2, XW, SCHED_W_W5W0W1W2W3W4_3, 45)
+
+	/* Transform 42-44 + Precalc 48-50 */
+	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 42, 0, XW, SCHED_W_W0W1W2W3W4W5_1, 48)
+	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  43, 1, XW, SCHED_W_W0W1W2W3W4W5_2, 48)
+	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 44, 2, XW, SCHED_W_W0W1W2W3W4W5_3, 48)
+
+	/* Transform 45-47 + Precalc 51-53 */
+	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  45, 0, XW, SCHED_W_W1W2W3W4W5W0_1, 51)
+	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 46, 1, XW, SCHED_W_W1W2W3W4W5W0_2, 51)
+	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  47, 2, XW, SCHED_W_W1W2W3W4W5W0_3, 51)
+
+	/* Transform 48-50 + Precalc 54-56 */
+	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 48, 0, XW, SCHED_W_W2W3W4W5W0W1_1, 54)
+	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  49, 1, XW, SCHED_W_W2W3W4W5W0W1_2, 54)
+	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 50, 2, XW, SCHED_W_W2W3W4W5W0W1_3, 54)
+
+	/* Transform 51-53 + Precalc 57-59 */
+	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  51, 0, XW, SCHED_W_W3W4W5W0W1W2_1, 57)
+	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 52, 1, XW, SCHED_W_W3W4W5W0W1W2_2, 57)
+	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  53, 2, XW, SCHED_W_W3W4W5W0W1W2_3, 57)
+
+	/* Transform 54-56 + Precalc 60-62 */
+	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 54, 0, XW, SCHED_W_W4W5W0W1W2W3_1, 60)
+	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  55, 1, XW, SCHED_W_W4W5W0W1W2W3_2, 60)
+	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 56, 2, XW, SCHED_W_W4W5W0W1W2W3_3, 60)
+
+	/* Transform 57-59 + Precalc 63 */
+	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  57, 0, XW, SCHED_W_W5W0W1W2W3W4_1, 63)
+	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 58, 1, XW, SCHED_W_W5W0W1W2W3W4_2, 63)
+	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  59, 2, XW, SCHED_W_W5W0W1W2W3W4_3, 63)
+
+	/* Transform 60 */
+	R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 60, 0, XW, _, _)
+	subs		RNBLKS, RNBLKS, #1
+	b.eq		.Lend
+
+	/* Transform 61-63 + Preload next block */
+	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  61, 1, XW, LOAD_W_VEC_1, _)
+	ldp		s0, s1, [RSTATE, #0]
+	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 62, 2, XW, LOAD_W_VEC_2, _)
+	ldp		s2, s3, [RSTATE, #8]
+	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  63, 0, XW, LOAD_W_VEC_3, _)
+
+	/* Update the chaining variables. */
+	eor		ra, ra, s0
+	eor		rb, rb, s1
+	ldp		s0, s1, [RSTATE, #16]
+	eor		rc, rc, s2
+	ldp		k_even, k_odd, [RSTATE, #24]
+	eor		rd, rd, s3
+	eor		re, re, s0
+	stp		ra, rb, [RSTATE, #0]
+	eor		rf, rf, s1
+	stp		rc, rd, [RSTATE, #8]
+	eor		rg, rg, k_even
+	stp		re, rf, [RSTATE, #16]
+	eor		rh, rh, k_odd
+	stp		rg, rh, [RSTATE, #24]
+	b		.Loop
+
+.Lend:
+	/* Transform 61-63 */
+	R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd,  _,  61, 1, XW, _, _)
+	ldp		s0, s1, [RSTATE, #0]
+	R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 62, 2, XW, _, _)
+	ldp		s2, s3, [RSTATE, #8]
+	R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd,  _,  63, 0, XW, _, _)
+
+	/* Update the chaining variables. */
+	eor		ra, ra, s0
+	clear_vec(W0)
+	eor		rb, rb, s1
+	clear_vec(W1)
+	ldp		s0, s1, [RSTATE, #16]
+	clear_vec(W2)
+	eor		rc, rc, s2
+	clear_vec(W3)
+	ldp		k_even, k_odd, [RSTATE, #24]
+	clear_vec(W4)
+	eor		rd, rd, s3
+	clear_vec(W5)
+	eor		re, re, s0
+	clear_vec(XTMP0)
+	stp		ra, rb, [RSTATE, #0]
+	clear_vec(XTMP1)
+	eor		rf, rf, s1
+	clear_vec(XTMP2)
+	stp		rc, rd, [RSTATE, #8]
+	clear_vec(XTMP3)
+	eor		rg, rg, k_even
+	clear_vec(XTMP4)
+	stp		re, rf, [RSTATE, #16]
+	clear_vec(XTMP5)
+	eor		rh, rh, k_odd
+	clear_vec(XTMP6)
+	stp		rg, rh, [RSTATE, #24]
+
+	/* Clear message expansion area */
+	add		addr0, sp, #STACK_W
+	st1		{W0.16b-W3.16b}, [addr0], #64
+	st1		{W0.16b-W3.16b}, [addr0], #64
+	st1		{W0.16b-W3.16b}, [addr0]
+
+	mov		sp, RFRAME
+
+	ldp		x25, x26, [sp], #16
+	ldp		x23, x24, [sp], #16
+	ldp		x21, x22, [sp], #16
+	ldp		x19, x20, [sp], #16
+	ldp		x28, x29, [sp], #16
+
+	ret
+SYM_FUNC_END(sm3_neon_transform)
+
+
+	.section	".rodata", "a"
+
+	.align 4
+.LKtable:
+	.long 0x79cc4519, 0xf3988a32, 0xe7311465, 0xce6228cb
+	.long 0x9cc45197, 0x3988a32f, 0x7311465e, 0xe6228cbc
+	.long 0xcc451979, 0x988a32f3, 0x311465e7, 0x6228cbce
+	.long 0xc451979c, 0x88a32f39, 0x11465e73, 0x228cbce6
+	.long 0x9d8a7a87, 0x3b14f50f, 0x7629ea1e, 0xec53d43c
+	.long 0xd8a7a879, 0xb14f50f3, 0x629ea1e7, 0xc53d43ce
+	.long 0x8a7a879d, 0x14f50f3b, 0x29ea1e76, 0x53d43cec
+	.long 0xa7a879d8, 0x4f50f3b1, 0x9ea1e762, 0x3d43cec5
+	.long 0x7a879d8a, 0xf50f3b14, 0xea1e7629, 0xd43cec53
+	.long 0xa879d8a7, 0x50f3b14f, 0xa1e7629e, 0x43cec53d
+	.long 0x879d8a7a, 0x0f3b14f5, 0x1e7629ea, 0x3cec53d4
+	.long 0x79d8a7a8, 0xf3b14f50, 0xe7629ea1, 0xcec53d43
+	.long 0x9d8a7a87, 0x3b14f50f, 0x7629ea1e, 0xec53d43c
+	.long 0xd8a7a879, 0xb14f50f3, 0x629ea1e7, 0xc53d43ce
+	.long 0x8a7a879d, 0x14f50f3b, 0x29ea1e76, 0x53d43cec
+	.long 0xa7a879d8, 0x4f50f3b1, 0x9ea1e762, 0x3d43cec5
diff --git a/lib/crypto/arm64/sm3.h b/lib/crypto/arm64/sm3.h
new file mode 100644
index 000000000000..beb9cd82bb7d
--- /dev/null
+++ b/lib/crypto/arm64/sm3.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * SM3 optimized for ARM64
+ *
+ * Copyright 2026 Google LLC
+ */
+#include <asm/simd.h>
+#include <linux/cpufeature.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce);
+
+asmlinkage void sm3_neon_transform(struct sm3_block_state *state,
+				   const u8 *data, size_t nblocks);
+asmlinkage void sm3_ce_transform(struct sm3_block_state *state,
+				 const u8 *data, size_t nblocks);
+
+static void sm3_blocks(struct sm3_block_state *state,
+		       const u8 *data, size_t nblocks)
+{
+	if (static_branch_likely(&have_neon) && likely(may_use_simd())) {
+		scoped_ksimd() {
+			if (static_branch_likely(&have_ce))
+				sm3_ce_transform(state, data, nblocks);
+			else
+				sm3_neon_transform(state, data, nblocks);
+		}
+	} else {
+		sm3_blocks_generic(state, data, nblocks);
+	}
+}
+
+#define sm3_mod_init_arch sm3_mod_init_arch
+static void sm3_mod_init_arch(void)
+{
+	if (cpu_have_named_feature(ASIMD)) {
+		static_branch_enable(&have_neon);
+		if (cpu_have_named_feature(SM3))
+			static_branch_enable(&have_ce);
+	}
+}
diff --git a/lib/crypto/chacha.c b/lib/crypto/chacha.c
index e0c7cb4af318..86e5d382a4e0 100644
--- a/lib/crypto/chacha.c
+++ b/lib/crypto/chacha.c
@@ -5,8 +5,8 @@
  * Copyright (C) 2015 Martin Willi
  */
 
-#include <crypto/algapi.h> // for crypto_xor_cpy
 #include <crypto/chacha.h>
+#include <crypto/utils.h>
 #include <linux/export.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
diff --git a/lib/crypto/chacha20poly1305.c b/lib/crypto/chacha20poly1305.c
index 212ce33562af..ea42a28f4ff7 100644
--- a/lib/crypto/chacha20poly1305.c
+++ b/lib/crypto/chacha20poly1305.c
@@ -356,20 +356,6 @@ bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len
 }
 EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace);
 
-static int __init chacha20poly1305_init(void)
-{
-	if (IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) &&
-	    WARN_ON(!chacha20poly1305_selftest()))
-		return -ENODEV;
-	return 0;
-}
-
-static void __exit chacha20poly1305_exit(void)
-{
-}
-
-module_init(chacha20poly1305_init);
-module_exit(chacha20poly1305_exit);
 MODULE_LICENSE("GPL v2");
 MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction");
 MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
diff --git a/lib/crypto/fips.h b/lib/crypto/fips.h
index 023410c2e0db..9fc49747db64 100644
--- a/lib/crypto/fips.h
+++ b/lib/crypto/fips.h
@@ -43,3 +43,8 @@ static const u8 fips_test_sha3_256_value[] __initconst __maybe_unused = {
 	0xba, 0x9b, 0xb6, 0xaa, 0x32, 0xa7, 0x97, 0x00,
 	0x98, 0xdb, 0xff, 0xe7, 0xc6, 0xde, 0xb5, 0x82,
 };
+
+static const u8 fips_test_aes_cmac_value[] __initconst __maybe_unused = {
+	0xc5, 0x88, 0x28, 0x55, 0xd7, 0x2c, 0x00, 0xb6,
+	0x6a, 0xa7, 0xfc, 0x82, 0x90, 0x81, 0xcf, 0x18,
+};
diff --git a/lib/crypto/polyval.c b/lib/crypto/gf128hash.c
index 5796275f574a..2650603d8ba8 100644
--- a/lib/crypto/polyval.c
+++ b/lib/crypto/gf128hash.c
@@ -1,34 +1,41 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
- * POLYVAL library functions
+ * GF(2^128) polynomial hashing: GHASH and POLYVAL
  *
  * Copyright 2025 Google LLC
  */
 
-#include <crypto/polyval.h>
+#include <crypto/gf128hash.h>
 #include <linux/export.h>
 #include <linux/module.h>
 #include <linux/string.h>
 #include <linux/unaligned.h>
 
 /*
- * POLYVAL is an almost-XOR-universal hash function.  Similar to GHASH, POLYVAL
- * interprets the message as the coefficients of a polynomial in GF(2^128) and
- * evaluates that polynomial at a secret point.  POLYVAL has a simple
- * mathematical relationship with GHASH, but it uses a better field convention
- * which makes it easier and faster to implement.
+ * GHASH and POLYVAL are almost-XOR-universal hash functions.  They interpret
+ * the message as the coefficients of a polynomial in the finite field GF(2^128)
+ * and evaluate that polynomial at a secret point.
  *
- * POLYVAL is not a cryptographic hash function, and it should be used only by
- * algorithms that are specifically designed to use it.
+ * Neither GHASH nor POLYVAL is a cryptographic hash function.  They should be
+ * used only by algorithms that are specifically designed to use them.
  *
- * POLYVAL is specified by "AES-GCM-SIV: Nonce Misuse-Resistant Authenticated
- * Encryption" (https://datatracker.ietf.org/doc/html/rfc8452)
+ * GHASH is the older variant, defined as part of GCM in NIST SP 800-38D
+ * (https://nvlpubs.nist.gov/nistpubs/legacy/sp/nistspecialpublication800-38d.pdf).
+ * GHASH is hard to implement directly, due to its backwards mapping between
+ * bits and polynomial coefficients.  GHASH implementations typically pre and
+ * post-process the inputs and outputs (mainly by byte-swapping) to convert the
+ * GHASH computation into an equivalent computation over a different,
+ * easier-to-use representation of GF(2^128).
  *
- * POLYVAL is also used by HCTR2.  See "Length-preserving encryption with HCTR2"
- * (https://eprint.iacr.org/2021/1441.pdf).
+ * POLYVAL is a newer GF(2^128) polynomial hash, originally defined as part of
+ * AES-GCM-SIV (https://datatracker.ietf.org/doc/html/rfc8452) and also used by
+ * HCTR2 (https://eprint.iacr.org/2021/1441.pdf).  It uses that easier-to-use
+ * field representation directly, eliminating the data conversion steps.
  *
- * This file provides a library API for POLYVAL.  This API can delegate to
- * either a generic implementation or an architecture-optimized implementation.
+ * This file provides library APIs for GHASH and POLYVAL.  These APIs can
+ * delegate to either a generic implementation or an architecture-optimized
+ * implementation.  Due to the mathematical relationship between GHASH and
+ * POLYVAL, in some cases code for one is reused with the other.
  *
  * For the generic implementation, we don't use the traditional table approach
  * to GF(2^128) multiplication.  That approach is not constant-time and requires
@@ -205,6 +212,19 @@ polyval_mul_generic(struct polyval_elem *a, const struct polyval_elem *b)
 	a->hi = cpu_to_le64(c3);
 }
 
+static void __maybe_unused ghash_blocks_generic(struct polyval_elem *acc,
+						const struct polyval_elem *key,
+						const u8 *data, size_t nblocks)
+{
+	do {
+		acc->lo ^=
+			cpu_to_le64(get_unaligned_be64((__be64 *)(data + 8)));
+		acc->hi ^= cpu_to_le64(get_unaligned_be64((__be64 *)data));
+		polyval_mul_generic(acc, key);
+		data += GHASH_BLOCK_SIZE;
+	} while (--nblocks);
+}
+
 static void __maybe_unused
 polyval_blocks_generic(struct polyval_elem *acc, const struct polyval_elem *key,
 		       const u8 *data, size_t nblocks)
@@ -217,16 +237,118 @@ polyval_blocks_generic(struct polyval_elem *acc, const struct polyval_elem *key,
 	} while (--nblocks);
 }
 
-/* Include the arch-optimized implementation of POLYVAL, if one is available. */
-#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH
-#include "polyval.h" /* $(SRCARCH)/polyval.h */
+/* Convert the key from GHASH format to POLYVAL format. */
+static void __maybe_unused ghash_key_to_polyval(const u8 in[GHASH_BLOCK_SIZE],
+						struct polyval_elem *out)
+{
+	u64 hi = get_unaligned_be64(&in[0]);
+	u64 lo = get_unaligned_be64(&in[8]);
+	u64 mask = (s64)hi >> 63;
+
+	hi = (hi << 1) ^ (lo >> 63) ^ (mask & ((u64)0xc2 << 56));
+	lo = (lo << 1) ^ (mask & 1);
+	out->lo = cpu_to_le64(lo);
+	out->hi = cpu_to_le64(hi);
+}
+
+/* Convert the accumulator from POLYVAL format to GHASH format. */
+static void polyval_acc_to_ghash(const struct polyval_elem *in,
+				 u8 out[GHASH_BLOCK_SIZE])
+{
+	put_unaligned_be64(le64_to_cpu(in->hi), &out[0]);
+	put_unaligned_be64(le64_to_cpu(in->lo), &out[8]);
+}
+
+/* Convert the accumulator from GHASH format to POLYVAL format. */
+static void __maybe_unused ghash_acc_to_polyval(const u8 in[GHASH_BLOCK_SIZE],
+						struct polyval_elem *out)
+{
+	out->lo = cpu_to_le64(get_unaligned_be64(&in[8]));
+	out->hi = cpu_to_le64(get_unaligned_be64(&in[0]));
+}
+
+#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH
+#include "gf128hash.h" /* $(SRCARCH)/gf128hash.h */
+#endif
+
+void ghash_preparekey(struct ghash_key *key, const u8 raw_key[GHASH_BLOCK_SIZE])
+{
+#ifdef ghash_preparekey_arch
+	ghash_preparekey_arch(key, raw_key);
+#else
+	ghash_key_to_polyval(raw_key, &key->h);
+#endif
+}
+EXPORT_SYMBOL_GPL(ghash_preparekey);
+
+static void ghash_mul(struct ghash_ctx *ctx)
+{
+#ifdef ghash_mul_arch
+	ghash_mul_arch(&ctx->acc, ctx->key);
+#elif defined(ghash_blocks_arch)
+	static const u8 zeroes[GHASH_BLOCK_SIZE];
+
+	ghash_blocks_arch(&ctx->acc, ctx->key, zeroes, 1);
+#else
+	polyval_mul_generic(&ctx->acc, &ctx->key->h);
+#endif
+}
+
+/* nblocks is always >= 1. */
+static void ghash_blocks(struct ghash_ctx *ctx, const u8 *data, size_t nblocks)
+{
+#ifdef ghash_blocks_arch
+	ghash_blocks_arch(&ctx->acc, ctx->key, data, nblocks);
+#else
+	ghash_blocks_generic(&ctx->acc, &ctx->key->h, data, nblocks);
+#endif
+}
+
+void ghash_update(struct ghash_ctx *ctx, const u8 *data, size_t len)
+{
+	if (unlikely(ctx->partial)) {
+		size_t n = min(len, GHASH_BLOCK_SIZE - ctx->partial);
+
+		len -= n;
+		while (n--)
+			ctx->acc.bytes[GHASH_BLOCK_SIZE - 1 - ctx->partial++] ^=
+				*data++;
+		if (ctx->partial < GHASH_BLOCK_SIZE)
+			return;
+		ghash_mul(ctx);
+	}
+	if (len >= GHASH_BLOCK_SIZE) {
+		size_t nblocks = len / GHASH_BLOCK_SIZE;
+
+		ghash_blocks(ctx, data, nblocks);
+		data += len & ~(GHASH_BLOCK_SIZE - 1);
+		len &= GHASH_BLOCK_SIZE - 1;
+	}
+	for (size_t i = 0; i < len; i++)
+		ctx->acc.bytes[GHASH_BLOCK_SIZE - 1 - i] ^= data[i];
+	ctx->partial = len;
+}
+EXPORT_SYMBOL_GPL(ghash_update);
+
+void ghash_final(struct ghash_ctx *ctx, u8 out[GHASH_BLOCK_SIZE])
+{
+	if (unlikely(ctx->partial))
+		ghash_mul(ctx);
+	polyval_acc_to_ghash(&ctx->acc, out);
+	memzero_explicit(ctx, sizeof(*ctx));
+}
+EXPORT_SYMBOL_GPL(ghash_final);
+
 void polyval_preparekey(struct polyval_key *key,
 			const u8 raw_key[POLYVAL_BLOCK_SIZE])
 {
+#ifdef polyval_preparekey_arch
 	polyval_preparekey_arch(key, raw_key);
+#else
+	memcpy(key->h.bytes, raw_key, POLYVAL_BLOCK_SIZE);
+#endif
 }
 EXPORT_SYMBOL_GPL(polyval_preparekey);
-#endif /* Else, polyval_preparekey() is an inline function. */
 
 /*
  * polyval_mul_generic() and polyval_blocks_generic() take the key as a
@@ -238,17 +360,22 @@ EXPORT_SYMBOL_GPL(polyval_preparekey);
 
 static void polyval_mul(struct polyval_ctx *ctx)
 {
-#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH
+#ifdef polyval_mul_arch
 	polyval_mul_arch(&ctx->acc, ctx->key);
+#elif defined(polyval_blocks_arch)
+	static const u8 zeroes[POLYVAL_BLOCK_SIZE];
+
+	polyval_blocks_arch(&ctx->acc, ctx->key, zeroes, 1);
 #else
 	polyval_mul_generic(&ctx->acc, &ctx->key->h);
 #endif
 }
 
+/* nblocks is always >= 1. */
 static void polyval_blocks(struct polyval_ctx *ctx,
 			   const u8 *data, size_t nblocks)
 {
-#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH
+#ifdef polyval_blocks_arch
 	polyval_blocks_arch(&ctx->acc, ctx->key, data, nblocks);
 #else
 	polyval_blocks_generic(&ctx->acc, &ctx->key->h, data, nblocks);
@@ -289,19 +416,19 @@ void polyval_final(struct polyval_ctx *ctx, u8 out[POLYVAL_BLOCK_SIZE])
 }
 EXPORT_SYMBOL_GPL(polyval_final);
 
-#ifdef polyval_mod_init_arch
-static int __init polyval_mod_init(void)
+#ifdef gf128hash_mod_init_arch
+static int __init gf128hash_mod_init(void)
 {
-	polyval_mod_init_arch();
+	gf128hash_mod_init_arch();
 	return 0;
 }
-subsys_initcall(polyval_mod_init);
+subsys_initcall(gf128hash_mod_init);
 
-static void __exit polyval_mod_exit(void)
+static void __exit gf128hash_mod_exit(void)
 {
 }
-module_exit(polyval_mod_exit);
+module_exit(gf128hash_mod_exit);
 #endif
 
-MODULE_DESCRIPTION("POLYVAL almost-XOR-universal hash function");
+MODULE_DESCRIPTION("GF(2^128) polynomial hashing: GHASH and POLYVAL");
 MODULE_LICENSE("GPL");
diff --git a/lib/crypto/gf128mul.c b/lib/crypto/gf128mul.c
index e5a727b15f07..7ebf07ce1168 100644
--- a/lib/crypto/gf128mul.c
+++ b/lib/crypto/gf128mul.c
@@ -127,27 +127,9 @@
 	(i & 0x02 ? 0x0384 : 0) ^ (i & 0x01 ? 0x01c2 : 0) \
 )
 
-static const u16 gf128mul_table_le[256] = gf128mul_dat(xda_le);
 static const u16 gf128mul_table_be[256] = gf128mul_dat(xda_be);
 
-/*
- * The following functions multiply a field element by x^8 in
- * the polynomial field representation.  They use 64-bit word operations
- * to gain speed but compensate for machine endianness and hence work
- * correctly on both styles of machine.
- */
-
-static void gf128mul_x8_lle(be128 *x)
-{
-	u64 a = be64_to_cpu(x->a);
-	u64 b = be64_to_cpu(x->b);
-	u64 _tt = gf128mul_table_le[b & 0xff];
-
-	x->b = cpu_to_be64((b >> 8) | (a << 56));
-	x->a = cpu_to_be64((a >> 8) ^ (_tt << 48));
-}
-
-/* time invariant version of gf128mul_x8_lle */
+/* A table-less implementation of multiplying by x^8 */
 static void gf128mul_x8_lle_ti(be128 *x)
 {
 	u64 a = be64_to_cpu(x->a);
@@ -305,58 +287,5 @@ void gf128mul_64k_bbe(be128 *a, const struct gf128mul_64k *t)
 }
 EXPORT_SYMBOL(gf128mul_64k_bbe);
 
-/*      This version uses 4k bytes of table space.
-    A 16 byte buffer has to be multiplied by a 16 byte key
-    value in GF(2^128).  If we consider a GF(2^128) value in a
-    single byte, we can construct a table of the 256 16 byte
-    values that result from the 256 values of this byte.
-    This requires 4096 bytes. If we take the highest byte in
-    the buffer and use this table to get the result, we then
-    have to multiply by x^120 to get the final value. For the
-    next highest byte the result has to be multiplied by x^112
-    and so on. But we can do this by accumulating the result
-    in an accumulator starting with the result for the top
-    byte.  We repeatedly multiply the accumulator value by
-    x^8 and then add in (i.e. xor) the 16 bytes of the next
-    lower byte in the buffer, stopping when we reach the
-    lowest byte. This requires a 4096 byte table.
-*/
-struct gf128mul_4k *gf128mul_init_4k_lle(const be128 *g)
-{
-	struct gf128mul_4k *t;
-	int j, k;
-
-	t = kzalloc_obj(*t);
-	if (!t)
-		goto out;
-
-	t->t[128] = *g;
-	for (j = 64; j > 0; j >>= 1)
-		gf128mul_x_lle(&t->t[j], &t->t[j+j]);
-
-	for (j = 2; j < 256; j += j)
-		for (k = 1; k < j; ++k)
-			be128_xor(&t->t[j + k], &t->t[j], &t->t[k]);
-
-out:
-	return t;
-}
-EXPORT_SYMBOL(gf128mul_init_4k_lle);
-
-void gf128mul_4k_lle(be128 *a, const struct gf128mul_4k *t)
-{
-	u8 *ap = (u8 *)a;
-	be128 r[1];
-	int i = 15;
-
-	*r = t->t[ap[15]];
-	while (i--) {
-		gf128mul_x8_lle(r);
-		be128_xor(r, r, &t->t[ap[i]]);
-	}
-	*a = *r;
-}
-EXPORT_SYMBOL(gf128mul_4k_lle);
-
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("Functions for multiplying elements of GF(2^128)");
diff --git a/lib/crypto/memneq.c b/lib/crypto/memneq.c
index 44daacb8cb51..08924acd44bc 100644
--- a/lib/crypto/memneq.c
+++ b/lib/crypto/memneq.c
@@ -59,7 +59,7 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include <crypto/algapi.h>
+#include <crypto/utils.h>
 #include <linux/export.h>
 #include <linux/module.h>
 #include <linux/unaligned.h>
@@ -159,7 +159,7 @@ static inline unsigned long __crypto_memneq_16(const void *a, const void *b)
 /* Compare two areas of memory without leaking timing information,
  * and with special optimizations for common sizes.  Users should
  * not call this function directly, but should instead use
- * crypto_memneq defined in crypto/algapi.h.
+ * crypto_memneq defined in crypto/utils.h.
  */
 noinline unsigned long __crypto_memneq(const void *a, const void *b,
 				       size_t size)
diff --git a/lib/crypto/mips/md5.h b/lib/crypto/mips/md5.h
deleted file mode 100644
index e08e28aeffa4..000000000000
--- a/lib/crypto/mips/md5.h
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Cryptographic API.
- *
- * MD5 Message Digest Algorithm (RFC1321).
- *
- * Adapted for OCTEON by Aaro Koskinen <aaro.koskinen@iki.fi>.
- *
- * Based on crypto/md5.c, which is:
- *
- * Derived from cryptoapi implementation, originally based on the
- * public domain implementation written by Colin Plumb in 1993.
- *
- * Copyright (c) Cryptoapi developers.
- * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 2 of the License, or (at your option)
- * any later version.
- */
-
-#include <asm/octeon/crypto.h>
-#include <asm/octeon/octeon.h>
-
-/*
- * We pass everything as 64-bit. OCTEON can handle misaligned data.
- */
-
-static void md5_blocks(struct md5_block_state *state,
-		       const u8 *data, size_t nblocks)
-{
-	struct octeon_cop2_state cop2_state;
-	u64 *state64 = (u64 *)state;
-	unsigned long flags;
-
-	if (!octeon_has_crypto())
-		return md5_blocks_generic(state, data, nblocks);
-
-	cpu_to_le32_array(state->h, ARRAY_SIZE(state->h));
-
-	flags = octeon_crypto_enable(&cop2_state);
-	write_octeon_64bit_hash_dword(state64[0], 0);
-	write_octeon_64bit_hash_dword(state64[1], 1);
-
-	do {
-		const u64 *block = (const u64 *)data;
-
-		write_octeon_64bit_block_dword(block[0], 0);
-		write_octeon_64bit_block_dword(block[1], 1);
-		write_octeon_64bit_block_dword(block[2], 2);
-		write_octeon_64bit_block_dword(block[3], 3);
-		write_octeon_64bit_block_dword(block[4], 4);
-		write_octeon_64bit_block_dword(block[5], 5);
-		write_octeon_64bit_block_dword(block[6], 6);
-		octeon_md5_start(block[7]);
-
-		data += MD5_BLOCK_SIZE;
-	} while (--nblocks);
-
-	state64[0] = read_octeon_64bit_hash_dword(0);
-	state64[1] = read_octeon_64bit_hash_dword(1);
-	octeon_crypto_disable(&cop2_state, flags);
-
-	le32_to_cpu_array(state->h, ARRAY_SIZE(state->h));
-}
diff --git a/lib/crypto/powerpc/.gitignore b/lib/crypto/powerpc/.gitignore
index 598ca7aff6b1..7aa71d83f739 100644
--- a/lib/crypto/powerpc/.gitignore
+++ b/lib/crypto/powerpc/.gitignore
@@ -1,2 +1,3 @@
 # SPDX-License-Identifier: GPL-2.0-only
 aesp8-ppc.S
+ghashp8-ppc.S
diff --git a/lib/crypto/powerpc/gf128hash.h b/lib/crypto/powerpc/gf128hash.h
new file mode 100644
index 000000000000..629cd325d0c7
--- /dev/null
+++ b/lib/crypto/powerpc/gf128hash.h
@@ -0,0 +1,109 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * GHASH routines supporting VMX instructions on the Power 8
+ *
+ * Copyright (C) 2015, 2019 International Business Machines Inc.
+ * Copyright (C) 2014 - 2018 Linaro Ltd.
+ * Copyright 2026 Google LLC
+ */
+
+#include <asm/simd.h>
+#include <asm/switch_to.h>
+#include <linux/cpufeature.h>
+#include <linux/jump_label.h>
+#include <linux/preempt.h>
+#include <linux/uaccess.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto);
+
+void gcm_init_p8(u64 htable[4][2], const u8 h[16]);
+void gcm_gmult_p8(u8 Xi[16], const u64 htable[4][2]);
+void gcm_ghash_p8(u8 Xi[16], const u64 htable[4][2], const u8 *in, size_t len);
+
+#define ghash_preparekey_arch ghash_preparekey_arch
+static void ghash_preparekey_arch(struct ghash_key *key,
+				  const u8 raw_key[GHASH_BLOCK_SIZE])
+{
+	ghash_key_to_polyval(raw_key, &key->h);
+
+	if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) {
+		preempt_disable();
+		pagefault_disable();
+		enable_kernel_vsx();
+		gcm_init_p8(key->htable, raw_key);
+		disable_kernel_vsx();
+		pagefault_enable();
+		preempt_enable();
+	} else {
+		/* This reproduces gcm_init_p8() on both LE and BE systems. */
+		key->htable[0][0] = 0;
+		key->htable[0][1] = 0xc200000000000000;
+
+		key->htable[1][0] = 0;
+		key->htable[1][1] = le64_to_cpu(key->h.lo);
+
+		key->htable[2][0] = le64_to_cpu(key->h.lo);
+		key->htable[2][1] = le64_to_cpu(key->h.hi);
+
+		key->htable[3][0] = le64_to_cpu(key->h.hi);
+		key->htable[3][1] = 0;
+	}
+}
+
+#define ghash_mul_arch ghash_mul_arch
+static void ghash_mul_arch(struct polyval_elem *acc,
+			   const struct ghash_key *key)
+{
+	if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) {
+		u8 ghash_acc[GHASH_BLOCK_SIZE];
+
+		polyval_acc_to_ghash(acc, ghash_acc);
+
+		preempt_disable();
+		pagefault_disable();
+		enable_kernel_vsx();
+		gcm_gmult_p8(ghash_acc, key->htable);
+		disable_kernel_vsx();
+		pagefault_enable();
+		preempt_enable();
+
+		ghash_acc_to_polyval(ghash_acc, acc);
+		memzero_explicit(ghash_acc, sizeof(ghash_acc));
+	} else {
+		polyval_mul_generic(acc, &key->h);
+	}
+}
+
+#define ghash_blocks_arch ghash_blocks_arch
+static void ghash_blocks_arch(struct polyval_elem *acc,
+			      const struct ghash_key *key,
+			      const u8 *data, size_t nblocks)
+{
+	if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) {
+		u8 ghash_acc[GHASH_BLOCK_SIZE];
+
+		polyval_acc_to_ghash(acc, ghash_acc);
+
+		preempt_disable();
+		pagefault_disable();
+		enable_kernel_vsx();
+		gcm_ghash_p8(ghash_acc, key->htable, data,
+			     nblocks * GHASH_BLOCK_SIZE);
+		disable_kernel_vsx();
+		pagefault_enable();
+		preempt_enable();
+
+		ghash_acc_to_polyval(ghash_acc, acc);
+		memzero_explicit(ghash_acc, sizeof(ghash_acc));
+	} else {
+		ghash_blocks_generic(acc, &key->h, data, nblocks);
+	}
+}
+
+#define gf128hash_mod_init_arch gf128hash_mod_init_arch
+static void gf128hash_mod_init_arch(void)
+{
+	if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
+	    (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
+		static_branch_enable(&have_vec_crypto);
+}
diff --git a/lib/crypto/powerpc/ghashp8-ppc.pl b/lib/crypto/powerpc/ghashp8-ppc.pl
new file mode 100644
index 000000000000..7c38eedc02cc
--- /dev/null
+++ b/lib/crypto/powerpc/ghashp8-ppc.pl
@@ -0,0 +1,244 @@
+#!/usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0
+
+# This code is taken from the OpenSSL project but the author (Andy Polyakov)
+# has relicensed it under the GPLv2. Therefore this program is free software;
+# you can redistribute it and/or modify it under the terms of the GNU General
+# Public License version 2 as published by the Free Software Foundation.
+#
+# The original headers, including the original license headers, are
+# included below for completeness.
+
+# ====================================================================
+# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+# project. The module is, however, dual licensed under OpenSSL and
+# CRYPTOGAMS licenses depending on where you obtain it. For further
+# details see https://www.openssl.org/~appro/cryptogams/.
+# ====================================================================
+#
+# GHASH for PowerISA v2.07.
+#
+# July 2014
+#
+# Accurate performance measurements are problematic, because it's
+# always virtualized setup with possibly throttled processor.
+# Relative comparison is therefore more informative. This initial
+# version is ~2.1x slower than hardware-assisted AES-128-CTR, ~12x
+# faster than "4-bit" integer-only compiler-generated 64-bit code.
+# "Initial version" means that there is room for futher improvement.
+
+$flavour=shift;
+$output =shift;
+
+if ($flavour =~ /64/) {
+	$SIZE_T=8;
+	$LRSAVE=2*$SIZE_T;
+	$STU="stdu";
+	$POP="ld";
+	$PUSH="std";
+} elsif ($flavour =~ /32/) {
+	$SIZE_T=4;
+	$LRSAVE=$SIZE_T;
+	$STU="stwu";
+	$POP="lwz";
+	$PUSH="stw";
+} else { die "nonsense $flavour"; }
+
+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
+( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
+( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
+( $xlate="${dir}../../../arch/powerpc/crypto/ppc-xlate.pl" and -f $xlate) or
+die "can't locate ppc-xlate.pl";
+
+open STDOUT,"| $^X $xlate $flavour $output" || die "can't call $xlate: $!";
+
+my ($Xip,$Htbl,$inp,$len)=map("r$_",(3..6));	# argument block
+
+my ($Xl,$Xm,$Xh,$IN)=map("v$_",(0..3));
+my ($zero,$t0,$t1,$t2,$xC2,$H,$Hh,$Hl,$lemask)=map("v$_",(4..12));
+my $vrsave="r12";
+
+$code=<<___;
+.machine	"any"
+
+.text
+
+.globl	.gcm_init_p8
+	lis		r0,0xfff0
+	li		r8,0x10
+	mfspr		$vrsave,256
+	li		r9,0x20
+	mtspr		256,r0
+	li		r10,0x30
+	lvx_u		$H,0,r4			# load H
+	le?xor		r7,r7,r7
+	le?addi		r7,r7,0x8		# need a vperm start with 08
+	le?lvsr		5,0,r7
+	le?vspltisb	6,0x0f
+	le?vxor		5,5,6			# set a b-endian mask
+	le?vperm	$H,$H,$H,5
+
+	vspltisb	$xC2,-16		# 0xf0
+	vspltisb	$t0,1			# one
+	vaddubm		$xC2,$xC2,$xC2		# 0xe0
+	vxor		$zero,$zero,$zero
+	vor		$xC2,$xC2,$t0		# 0xe1
+	vsldoi		$xC2,$xC2,$zero,15	# 0xe1...
+	vsldoi		$t1,$zero,$t0,1		# ...1
+	vaddubm		$xC2,$xC2,$xC2		# 0xc2...
+	vspltisb	$t2,7
+	vor		$xC2,$xC2,$t1		# 0xc2....01
+	vspltb		$t1,$H,0		# most significant byte
+	vsl		$H,$H,$t0		# H<<=1
+	vsrab		$t1,$t1,$t2		# broadcast carry bit
+	vand		$t1,$t1,$xC2
+	vxor		$H,$H,$t1		# twisted H
+
+	vsldoi		$H,$H,$H,8		# twist even more ...
+	vsldoi		$xC2,$zero,$xC2,8	# 0xc2.0
+	vsldoi		$Hl,$zero,$H,8		# ... and split
+	vsldoi		$Hh,$H,$zero,8
+
+	stvx_u		$xC2,0,r3		# save pre-computed table
+	stvx_u		$Hl,r8,r3
+	stvx_u		$H, r9,r3
+	stvx_u		$Hh,r10,r3
+
+	mtspr		256,$vrsave
+	blr
+	.long		0
+	.byte		0,12,0x14,0,0,0,2,0
+	.long		0
+.size	.gcm_init_p8,.-.gcm_init_p8
+
+.globl	.gcm_gmult_p8
+	lis		r0,0xfff8
+	li		r8,0x10
+	mfspr		$vrsave,256
+	li		r9,0x20
+	mtspr		256,r0
+	li		r10,0x30
+	lvx_u		$IN,0,$Xip		# load Xi
+
+	lvx_u		$Hl,r8,$Htbl		# load pre-computed table
+	 le?lvsl	$lemask,r0,r0
+	lvx_u		$H, r9,$Htbl
+	 le?vspltisb	$t0,0x07
+	lvx_u		$Hh,r10,$Htbl
+	 le?vxor	$lemask,$lemask,$t0
+	lvx_u		$xC2,0,$Htbl
+	 le?vperm	$IN,$IN,$IN,$lemask
+	vxor		$zero,$zero,$zero
+
+	vpmsumd		$Xl,$IN,$Hl		# H.lo·Xi.lo
+	vpmsumd		$Xm,$IN,$H		# H.hi·Xi.lo+H.lo·Xi.hi
+	vpmsumd		$Xh,$IN,$Hh		# H.hi·Xi.hi
+
+	vpmsumd		$t2,$Xl,$xC2		# 1st phase
+
+	vsldoi		$t0,$Xm,$zero,8
+	vsldoi		$t1,$zero,$Xm,8
+	vxor		$Xl,$Xl,$t0
+	vxor		$Xh,$Xh,$t1
+
+	vsldoi		$Xl,$Xl,$Xl,8
+	vxor		$Xl,$Xl,$t2
+
+	vsldoi		$t1,$Xl,$Xl,8		# 2nd phase
+	vpmsumd		$Xl,$Xl,$xC2
+	vxor		$t1,$t1,$Xh
+	vxor		$Xl,$Xl,$t1
+
+	le?vperm	$Xl,$Xl,$Xl,$lemask
+	stvx_u		$Xl,0,$Xip		# write out Xi
+
+	mtspr		256,$vrsave
+	blr
+	.long		0
+	.byte		0,12,0x14,0,0,0,2,0
+	.long		0
+.size	.gcm_gmult_p8,.-.gcm_gmult_p8
+
+.globl	.gcm_ghash_p8
+	lis		r0,0xfff8
+	li		r8,0x10
+	mfspr		$vrsave,256
+	li		r9,0x20
+	mtspr		256,r0
+	li		r10,0x30
+	lvx_u		$Xl,0,$Xip		# load Xi
+
+	lvx_u		$Hl,r8,$Htbl		# load pre-computed table
+	 le?lvsl	$lemask,r0,r0
+	lvx_u		$H, r9,$Htbl
+	 le?vspltisb	$t0,0x07
+	lvx_u		$Hh,r10,$Htbl
+	 le?vxor	$lemask,$lemask,$t0
+	lvx_u		$xC2,0,$Htbl
+	 le?vperm	$Xl,$Xl,$Xl,$lemask
+	vxor		$zero,$zero,$zero
+
+	lvx_u		$IN,0,$inp
+	addi		$inp,$inp,16
+	subi		$len,$len,16
+	 le?vperm	$IN,$IN,$IN,$lemask
+	vxor		$IN,$IN,$Xl
+	b		Loop
+
+.align	5
+Loop:
+	 subic		$len,$len,16
+	vpmsumd		$Xl,$IN,$Hl		# H.lo·Xi.lo
+	 subfe.		r0,r0,r0		# borrow?-1:0
+	vpmsumd		$Xm,$IN,$H		# H.hi·Xi.lo+H.lo·Xi.hi
+	 and		r0,r0,$len
+	vpmsumd		$Xh,$IN,$Hh		# H.hi·Xi.hi
+	 add		$inp,$inp,r0
+
+	vpmsumd		$t2,$Xl,$xC2		# 1st phase
+
+	vsldoi		$t0,$Xm,$zero,8
+	vsldoi		$t1,$zero,$Xm,8
+	vxor		$Xl,$Xl,$t0
+	vxor		$Xh,$Xh,$t1
+
+	vsldoi		$Xl,$Xl,$Xl,8
+	vxor		$Xl,$Xl,$t2
+	 lvx_u		$IN,0,$inp
+	 addi		$inp,$inp,16
+
+	vsldoi		$t1,$Xl,$Xl,8		# 2nd phase
+	vpmsumd		$Xl,$Xl,$xC2
+	 le?vperm	$IN,$IN,$IN,$lemask
+	vxor		$t1,$t1,$Xh
+	vxor		$IN,$IN,$t1
+	vxor		$IN,$IN,$Xl
+	beq		Loop			# did $len-=16 borrow?
+
+	vxor		$Xl,$Xl,$t1
+	le?vperm	$Xl,$Xl,$Xl,$lemask
+	stvx_u		$Xl,0,$Xip		# write out Xi
+
+	mtspr		256,$vrsave
+	blr
+	.long		0
+	.byte		0,12,0x14,0,0,0,4,0
+	.long		0
+.size	.gcm_ghash_p8,.-.gcm_ghash_p8
+
+.asciz  "GHASH for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>"
+.align  2
+___
+
+foreach (split("\n",$code)) {
+	if ($flavour =~ /le$/o) {	# little-endian
+	    s/le\?//o		or
+	    s/be\?/#be#/o;
+	} else {
+	    s/le\?/#le#/o	or
+	    s/be\?//o;
+	}
+	print $_,"\n";
+}
+
+close STDOUT; # enforce flush
diff --git a/lib/crypto/riscv/gf128hash.h b/lib/crypto/riscv/gf128hash.h
new file mode 100644
index 000000000000..4301a0384f60
--- /dev/null
+++ b/lib/crypto/riscv/gf128hash.h
@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * GHASH, RISC-V optimized
+ *
+ * Copyright (C) 2023 VRULL GmbH
+ * Copyright (C) 2023 SiFive, Inc.
+ * Copyright 2026 Google LLC
+ */
+
+#include <asm/simd.h>
+#include <asm/vector.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkg);
+
+asmlinkage void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE],
+			   const u8 key[GHASH_BLOCK_SIZE],
+			   const u8 *data, size_t nblocks);
+
+#define ghash_preparekey_arch ghash_preparekey_arch
+static void ghash_preparekey_arch(struct ghash_key *key,
+				  const u8 raw_key[GHASH_BLOCK_SIZE])
+{
+	/* Save key in POLYVAL format for fallback */
+	ghash_key_to_polyval(raw_key, &key->h);
+
+	/* Save key in GHASH format for zvkg */
+	memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE);
+}
+
+#define ghash_blocks_arch ghash_blocks_arch
+static void ghash_blocks_arch(struct polyval_elem *acc,
+			      const struct ghash_key *key,
+			      const u8 *data, size_t nblocks)
+{
+	if (static_branch_likely(&have_zvkg) && likely(may_use_simd())) {
+		u8 ghash_acc[GHASH_BLOCK_SIZE];
+
+		polyval_acc_to_ghash(acc, ghash_acc);
+
+		kernel_vector_begin();
+		ghash_zvkg(ghash_acc, key->h_raw, data, nblocks);
+		kernel_vector_end();
+
+		ghash_acc_to_polyval(ghash_acc, acc);
+		memzero_explicit(ghash_acc, sizeof(ghash_acc));
+	} else {
+		ghash_blocks_generic(acc, &key->h, data, nblocks);
+	}
+}
+
+#define gf128hash_mod_init_arch gf128hash_mod_init_arch
+static void gf128hash_mod_init_arch(void)
+{
+	if (riscv_isa_extension_available(NULL, ZVKG) &&
+	    riscv_vector_vlen() >= 128)
+		static_branch_enable(&have_zvkg);
+}
diff --git a/lib/crypto/riscv/ghash-riscv64-zvkg.S b/lib/crypto/riscv/ghash-riscv64-zvkg.S
new file mode 100644
index 000000000000..6a2a2f2bc7c8
--- /dev/null
+++ b/lib/crypto/riscv/ghash-riscv64-zvkg.S
@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */
+//
+// This file is dual-licensed, meaning that you can use it under your
+// choice of either of the following two licenses:
+//
+// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
+//
+// Licensed under the Apache License 2.0 (the "License"). You can obtain
+// a copy in the file LICENSE in the source distribution or at
+// https://www.openssl.org/source/license.html
+//
+// or
+//
+// Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu>
+// Copyright (c) 2023, Jerry Shih <jerry.shih@sifive.com>
+// Copyright 2024 Google LLC
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The generated code of this file depends on the following RISC-V extensions:
+// - RV64I
+// - RISC-V Vector ('V') with VLEN >= 128
+// - RISC-V Vector GCM/GMAC extension ('Zvkg')
+
+#include <linux/linkage.h>
+
+.text
+.option arch, +zvkg
+
+#define ACCUMULATOR	a0
+#define KEY		a1
+#define DATA		a2
+#define NBLOCKS		a3
+
+// void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE],
+//		   const u8 key[GHASH_BLOCK_SIZE],
+//		   const u8 *data, size_t nblocks);
+//
+// |nblocks| must be nonzero.
+SYM_FUNC_START(ghash_zvkg)
+	vsetivli	zero, 4, e32, m1, ta, ma
+	vle32.v		v1, (ACCUMULATOR)
+	vle32.v		v2, (KEY)
+.Lnext_block:
+	vle32.v		v3, (DATA)
+	vghsh.vv	v1, v2, v3
+	addi		DATA, DATA, 16
+	addi		NBLOCKS, NBLOCKS, -1
+	bnez		NBLOCKS, .Lnext_block
+
+	vse32.v		v1, (ACCUMULATOR)
+	ret
+SYM_FUNC_END(ghash_zvkg)
diff --git a/lib/crypto/riscv/sm3-riscv64-zvksh-zvkb.S b/lib/crypto/riscv/sm3-riscv64-zvksh-zvkb.S
new file mode 100644
index 000000000000..a1d4468b0485
--- /dev/null
+++ b/lib/crypto/riscv/sm3-riscv64-zvksh-zvkb.S
@@ -0,0 +1,124 @@
+/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */
+//
+// This file is dual-licensed, meaning that you can use it under your
+// choice of either of the following two licenses:
+//
+// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
+//
+// Licensed under the Apache License 2.0 (the "License"). You can obtain
+// a copy in the file LICENSE in the source distribution or at
+// https://www.openssl.org/source/license.html
+//
+// or
+//
+// Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu>
+// Copyright (c) 2023, Jerry Shih <jerry.shih@sifive.com>
+// Copyright 2024 Google LLC
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The generated code of this file depends on the following RISC-V extensions:
+// - RV64I
+// - RISC-V Vector ('V') with VLEN >= 128
+// - RISC-V Vector SM3 Secure Hash extension ('Zvksh')
+// - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb')
+
+#include <linux/linkage.h>
+
+.text
+.option arch, +zvksh, +zvkb
+
+#define STATEP		a0
+#define DATA		a1
+#define NUM_BLOCKS	a2
+
+#define STATE		v0	// LMUL=2
+#define PREV_STATE	v2	// LMUL=2
+#define W0		v4	// LMUL=2
+#define W1		v6	// LMUL=2
+#define VTMP		v8	// LMUL=2
+
+.macro	sm3_8rounds	i, w0, w1
+	// Do 4 rounds using W_{0+i}..W_{7+i}.
+	vsm3c.vi	STATE, \w0, \i + 0
+	vslidedown.vi	VTMP, \w0, 2
+	vsm3c.vi	STATE, VTMP, \i + 1
+
+	// Compute W_{4+i}..W_{11+i}.
+	vslidedown.vi	VTMP, \w0, 4
+	vslideup.vi	VTMP, \w1, 4
+
+	// Do 4 rounds using W_{4+i}..W_{11+i}.
+	vsm3c.vi	STATE, VTMP, \i + 2
+	vslidedown.vi	VTMP, VTMP, 2
+	vsm3c.vi	STATE, VTMP, \i + 3
+
+.if \i < 28
+	// Compute W_{16+i}..W_{23+i}.
+	vsm3me.vv	\w0, \w1, \w0
+.endif
+	// For the next 8 rounds, w0 and w1 are swapped.
+.endm
+
+// void sm3_transform_zvksh_zvkb(struct sm3_block_state *state,
+//				 const u8 *data, size_t nblocks);
+SYM_FUNC_START(sm3_transform_zvksh_zvkb)
+
+	// Load the state and endian-swap each 32-bit word.
+	vsetivli	zero, 8, e32, m2, ta, ma
+	vle32.v		STATE, (STATEP)
+	vrev8.v		STATE, STATE
+
+.Lnext_block:
+	addi		NUM_BLOCKS, NUM_BLOCKS, -1
+
+	// Save the previous state, as it's needed later.
+	vmv.v.v		PREV_STATE, STATE
+
+	// Load the next 512-bit message block into W0-W1.
+	vle32.v		W0, (DATA)
+	addi		DATA, DATA, 32
+	vle32.v		W1, (DATA)
+	addi		DATA, DATA, 32
+
+	// Do the 64 rounds of SM3.
+	sm3_8rounds	0, W0, W1
+	sm3_8rounds	4, W1, W0
+	sm3_8rounds	8, W0, W1
+	sm3_8rounds	12, W1, W0
+	sm3_8rounds	16, W0, W1
+	sm3_8rounds	20, W1, W0
+	sm3_8rounds	24, W0, W1
+	sm3_8rounds	28, W1, W0
+
+	// XOR in the previous state.
+	vxor.vv		STATE, STATE, PREV_STATE
+
+	// Repeat if more blocks remain.
+	bnez		NUM_BLOCKS, .Lnext_block
+
+	// Store the new state and return.
+	vrev8.v		STATE, STATE
+	vse32.v		STATE, (STATEP)
+	ret
+SYM_FUNC_END(sm3_transform_zvksh_zvkb)
diff --git a/lib/crypto/riscv/sm3.h b/lib/crypto/riscv/sm3.h
new file mode 100644
index 000000000000..c1fbee7094e6
--- /dev/null
+++ b/lib/crypto/riscv/sm3.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * SM3 using the RISC-V vector crypto extensions
+ *
+ * Copyright (C) 2023 VRULL GmbH
+ * Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
+ *
+ * Copyright (C) 2023 SiFive, Inc.
+ * Author: Jerry Shih <jerry.shih@sifive.com>
+ */
+
+#include <asm/simd.h>
+#include <asm/vector.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions);
+
+asmlinkage void sm3_transform_zvksh_zvkb(struct sm3_block_state *state,
+					 const u8 *data, size_t nblocks);
+
+static void sm3_blocks(struct sm3_block_state *state,
+		       const u8 *data, size_t nblocks)
+{
+	if (static_branch_likely(&have_extensions) && likely(may_use_simd())) {
+		kernel_vector_begin();
+		sm3_transform_zvksh_zvkb(state, data, nblocks);
+		kernel_vector_end();
+	} else {
+		sm3_blocks_generic(state, data, nblocks);
+	}
+}
+
+#define sm3_mod_init_arch sm3_mod_init_arch
+static void sm3_mod_init_arch(void)
+{
+	if (riscv_isa_extension_available(NULL, ZVKSH) &&
+	    riscv_isa_extension_available(NULL, ZVKB) &&
+	    riscv_vector_vlen() >= 128)
+		static_branch_enable(&have_extensions);
+}
diff --git a/lib/crypto/s390/gf128hash.h b/lib/crypto/s390/gf128hash.h
new file mode 100644
index 000000000000..1e46ce4bca40
--- /dev/null
+++ b/lib/crypto/s390/gf128hash.h
@@ -0,0 +1,54 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * GHASH optimized using the CP Assist for Cryptographic Functions (CPACF)
+ *
+ * Copyright 2026 Google LLC
+ */
+#include <asm/cpacf.h>
+#include <linux/cpufeature.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_ghash);
+
+#define ghash_preparekey_arch ghash_preparekey_arch
+static void ghash_preparekey_arch(struct ghash_key *key,
+				  const u8 raw_key[GHASH_BLOCK_SIZE])
+{
+	/* Save key in POLYVAL format for fallback */
+	ghash_key_to_polyval(raw_key, &key->h);
+
+	/* Save key in GHASH format for CPACF_KIMD_GHASH */
+	memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE);
+}
+
+#define ghash_blocks_arch ghash_blocks_arch
+static void ghash_blocks_arch(struct polyval_elem *acc,
+			      const struct ghash_key *key,
+			      const u8 *data, size_t nblocks)
+{
+	if (static_branch_likely(&have_cpacf_ghash)) {
+		/*
+		 * CPACF_KIMD_GHASH requires the accumulator and key in a single
+		 * buffer, each using the GHASH convention.
+		 */
+		u8 ctx[2][GHASH_BLOCK_SIZE] __aligned(8);
+
+		polyval_acc_to_ghash(acc, ctx[0]);
+		memcpy(ctx[1], key->h_raw, GHASH_BLOCK_SIZE);
+
+		cpacf_kimd(CPACF_KIMD_GHASH, ctx, data,
+			   nblocks * GHASH_BLOCK_SIZE);
+
+		ghash_acc_to_polyval(ctx[0], acc);
+		memzero_explicit(ctx, sizeof(ctx));
+	} else {
+		ghash_blocks_generic(acc, &key->h, data, nblocks);
+	}
+}
+
+#define gf128hash_mod_init_arch gf128hash_mod_init_arch
+static void gf128hash_mod_init_arch(void)
+{
+	if (cpu_have_feature(S390_CPU_FEATURE_MSA) &&
+	    cpacf_query_func(CPACF_KIMD, CPACF_KIMD_GHASH))
+		static_branch_enable(&have_cpacf_ghash);
+}
diff --git a/lib/crypto/sm3.c b/lib/crypto/sm3.c
index c6b9ad8a3ac6..b02b8a247adf 100644
--- a/lib/crypto/sm3.c
+++ b/lib/crypto/sm3.c
@@ -15,6 +15,13 @@
 #include <linux/string.h>
 #include <linux/unaligned.h>
 
+static const struct sm3_block_state sm3_iv = {
+	.h = {
+		SM3_IVA, SM3_IVB, SM3_IVC, SM3_IVD,
+		SM3_IVE, SM3_IVF, SM3_IVG, SM3_IVH,
+	},
+};
+
 static const u32 ____cacheline_aligned K[64] = {
 	0x79cc4519, 0xf3988a32, 0xe7311465, 0xce6228cb,
 	0x9cc45197, 0x3988a32f, 0x7311465e, 0xe6228cbc,
@@ -72,18 +79,19 @@ static const u32 ____cacheline_aligned K[64] = {
 		^ rol32(W[(i-13) & 0x0f], 7)		\
 		^ W[(i-6) & 0x0f])
 
-static void sm3_transform(struct sm3_state *sctx, u8 const *data, u32 W[16])
+static void sm3_block_generic(struct sm3_block_state *state,
+			      const u8 data[SM3_BLOCK_SIZE], u32 W[16])
 {
 	u32 a, b, c, d, e, f, g, h, ss1, ss2;
 
-	a = sctx->state[0];
-	b = sctx->state[1];
-	c = sctx->state[2];
-	d = sctx->state[3];
-	e = sctx->state[4];
-	f = sctx->state[5];
-	g = sctx->state[6];
-	h = sctx->state[7];
+	a = state->h[0];
+	b = state->h[1];
+	c = state->h[2];
+	d = state->h[3];
+	e = state->h[4];
+	f = state->h[5];
+	g = state->h[6];
+	h = state->h[7];
 
 	R1(a, b, c, d, e, f, g, h, K[0], I(0), I(4));
 	R1(d, a, b, c, h, e, f, g, K[1], I(1), I(5));
@@ -153,14 +161,14 @@ static void sm3_transform(struct sm3_state *sctx, u8 const *data, u32 W[16])
 	R2(c, d, a, b, g, h, e, f, K[62], W1(62), W2(66));
 	R2(b, c, d, a, f, g, h, e, K[63], W1(63), W2(67));
 
-	sctx->state[0] ^= a;
-	sctx->state[1] ^= b;
-	sctx->state[2] ^= c;
-	sctx->state[3] ^= d;
-	sctx->state[4] ^= e;
-	sctx->state[5] ^= f;
-	sctx->state[6] ^= g;
-	sctx->state[7] ^= h;
+	state->h[0] ^= a;
+	state->h[1] ^= b;
+	state->h[2] ^= c;
+	state->h[3] ^= d;
+	state->h[4] ^= e;
+	state->h[5] ^= f;
+	state->h[6] ^= g;
+	state->h[7] ^= h;
 }
 #undef R
 #undef R1
@@ -169,18 +177,114 @@ static void sm3_transform(struct sm3_state *sctx, u8 const *data, u32 W[16])
 #undef W1
 #undef W2
 
-void sm3_block_generic(struct sm3_state *sctx, u8 const *data, int blocks)
+static void __maybe_unused sm3_blocks_generic(struct sm3_block_state *state,
+					      const u8 *data, size_t nblocks)
 {
 	u32 W[16];
 
 	do {
-		sm3_transform(sctx, data, W);
+		sm3_block_generic(state, data, W);
 		data += SM3_BLOCK_SIZE;
-	} while (--blocks);
+	} while (--nblocks);
 
 	memzero_explicit(W, sizeof(W));
 }
-EXPORT_SYMBOL_GPL(sm3_block_generic);
 
-MODULE_DESCRIPTION("Generic SM3 library");
+#ifdef CONFIG_CRYPTO_LIB_SM3_ARCH
+#include "sm3.h" /* $(SRCARCH)/sm3.h */
+#else
+#define sm3_blocks sm3_blocks_generic
+#endif
+
+void sm3_init(struct sm3_ctx *ctx)
+{
+	ctx->state = sm3_iv;
+	ctx->bytecount = 0;
+}
+EXPORT_SYMBOL_GPL(sm3_init);
+
+void sm3_update(struct sm3_ctx *ctx, const u8 *data, size_t len)
+{
+	size_t partial = ctx->bytecount % SM3_BLOCK_SIZE;
+
+	ctx->bytecount += len;
+
+	if (partial + len >= SM3_BLOCK_SIZE) {
+		size_t nblocks;
+
+		if (partial) {
+			size_t l = SM3_BLOCK_SIZE - partial;
+
+			memcpy(&ctx->buf[partial], data, l);
+			data += l;
+			len -= l;
+
+			sm3_blocks(&ctx->state, ctx->buf, 1);
+		}
+
+		nblocks = len / SM3_BLOCK_SIZE;
+		len %= SM3_BLOCK_SIZE;
+
+		if (nblocks) {
+			sm3_blocks(&ctx->state, data, nblocks);
+			data += nblocks * SM3_BLOCK_SIZE;
+		}
+		partial = 0;
+	}
+	if (len)
+		memcpy(&ctx->buf[partial], data, len);
+}
+EXPORT_SYMBOL_GPL(sm3_update);
+
+static void __sm3_final(struct sm3_ctx *ctx, u8 out[SM3_DIGEST_SIZE])
+{
+	u64 bitcount = ctx->bytecount << 3;
+	size_t partial = ctx->bytecount % SM3_BLOCK_SIZE;
+
+	ctx->buf[partial++] = 0x80;
+	if (partial > SM3_BLOCK_SIZE - 8) {
+		memset(&ctx->buf[partial], 0, SM3_BLOCK_SIZE - partial);
+		sm3_blocks(&ctx->state, ctx->buf, 1);
+		partial = 0;
+	}
+	memset(&ctx->buf[partial], 0, SM3_BLOCK_SIZE - 8 - partial);
+	*(__be64 *)&ctx->buf[SM3_BLOCK_SIZE - 8] = cpu_to_be64(bitcount);
+	sm3_blocks(&ctx->state, ctx->buf, 1);
+
+	for (size_t i = 0; i < SM3_DIGEST_SIZE; i += 4)
+		put_unaligned_be32(ctx->state.h[i / 4], out + i);
+}
+
+void sm3_final(struct sm3_ctx *ctx, u8 out[SM3_DIGEST_SIZE])
+{
+	__sm3_final(ctx, out);
+	memzero_explicit(ctx, sizeof(*ctx));
+}
+EXPORT_SYMBOL_GPL(sm3_final);
+
+void sm3(const u8 *data, size_t len, u8 out[SM3_DIGEST_SIZE])
+{
+	struct sm3_ctx ctx;
+
+	sm3_init(&ctx);
+	sm3_update(&ctx, data, len);
+	sm3_final(&ctx, out);
+}
+EXPORT_SYMBOL_GPL(sm3);
+
+#ifdef sm3_mod_init_arch
+static int __init sm3_mod_init(void)
+{
+	sm3_mod_init_arch();
+	return 0;
+}
+subsys_initcall(sm3_mod_init);
+
+static void __exit sm3_mod_exit(void)
+{
+}
+module_exit(sm3_mod_exit);
+#endif
+
+MODULE_DESCRIPTION("SM3 library functions");
 MODULE_LICENSE("GPL v2");
diff --git a/lib/crypto/sparc/md5.h b/lib/crypto/sparc/md5.h
deleted file mode 100644
index 3995f3e075eb..000000000000
--- a/lib/crypto/sparc/md5.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * MD5 accelerated using the sparc64 crypto opcodes
- *
- * Copyright (c) Alan Smithee.
- * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
- * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
- * Copyright (c) Mathias Krause <minipli@googlemail.com>
- * Copyright (c) Cryptoapi developers.
- * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
- */
-
-#include <asm/elf.h>
-#include <asm/opcodes.h>
-#include <asm/pstate.h>
-
-static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_md5_opcodes);
-
-asmlinkage void md5_sparc64_transform(struct md5_block_state *state,
-				      const u8 *data, size_t nblocks);
-
-static void md5_blocks(struct md5_block_state *state,
-		       const u8 *data, size_t nblocks)
-{
-	if (static_branch_likely(&have_md5_opcodes)) {
-		cpu_to_le32_array(state->h, ARRAY_SIZE(state->h));
-		md5_sparc64_transform(state, data, nblocks);
-		le32_to_cpu_array(state->h, ARRAY_SIZE(state->h));
-	} else {
-		md5_blocks_generic(state, data, nblocks);
-	}
-}
-
-#define md5_mod_init_arch md5_mod_init_arch
-static void md5_mod_init_arch(void)
-{
-	unsigned long cfr;
-
-	if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
-		return;
-
-	__asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
-	if (!(cfr & CFR_MD5))
-		return;
-
-	static_branch_enable(&have_md5_opcodes);
-	pr_info("Using sparc64 md5 opcode optimized MD5 implementation\n");
-}
diff --git a/lib/crypto/sparc/md5_asm.S b/lib/crypto/sparc/md5_asm.S
deleted file mode 100644
index 60b544e4d205..000000000000
--- a/lib/crypto/sparc/md5_asm.S
+++ /dev/null
@@ -1,70 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#include <linux/linkage.h>
-#include <asm/opcodes.h>
-#include <asm/visasm.h>
-
-ENTRY(md5_sparc64_transform)
-	/* %o0 = digest, %o1 = data, %o2 = rounds */
-	VISEntryHalf
-	ld	[%o0 + 0x00], %f0
-	ld	[%o0 + 0x04], %f1
-	andcc	%o1, 0x7, %g0
-	ld	[%o0 + 0x08], %f2
-	bne,pn	%xcc, 10f
-	 ld	[%o0 + 0x0c], %f3
-
-1:
-	ldd	[%o1 + 0x00], %f8
-	ldd	[%o1 + 0x08], %f10
-	ldd	[%o1 + 0x10], %f12
-	ldd	[%o1 + 0x18], %f14
-	ldd	[%o1 + 0x20], %f16
-	ldd	[%o1 + 0x28], %f18
-	ldd	[%o1 + 0x30], %f20
-	ldd	[%o1 + 0x38], %f22
-
-	MD5
-
-	subcc	%o2, 1, %o2
-	bne,pt	%xcc, 1b
-	 add	%o1, 0x40, %o1
-
-5:
-	st	%f0, [%o0 + 0x00]
-	st	%f1, [%o0 + 0x04]
-	st	%f2, [%o0 + 0x08]
-	st	%f3, [%o0 + 0x0c]
-	retl
-	 VISExitHalf
-10:
-	alignaddr %o1, %g0, %o1
-
-	ldd	[%o1 + 0x00], %f10
-1:
-	ldd	[%o1 + 0x08], %f12
-	ldd	[%o1 + 0x10], %f14
-	ldd	[%o1 + 0x18], %f16
-	ldd	[%o1 + 0x20], %f18
-	ldd	[%o1 + 0x28], %f20
-	ldd	[%o1 + 0x30], %f22
-	ldd	[%o1 + 0x38], %f24
-	ldd	[%o1 + 0x40], %f26
-
-	faligndata %f10, %f12, %f8
-	faligndata %f12, %f14, %f10
-	faligndata %f14, %f16, %f12
-	faligndata %f16, %f18, %f14
-	faligndata %f18, %f20, %f16
-	faligndata %f20, %f22, %f18
-	faligndata %f22, %f24, %f20
-	faligndata %f24, %f26, %f22
-
-	MD5
-
-	subcc	%o2, 1, %o2
-	fsrc2	%f26, %f10
-	bne,pt	%xcc, 1b
-	 add	%o1, 0x40, %o1
-
-	ba,a,pt	%xcc, 5b
-ENDPROC(md5_sparc64_transform)
diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig
index 0de289b429a9..9409c1a935c3 100644
--- a/lib/crypto/tests/Kconfig
+++ b/lib/crypto/tests/Kconfig
@@ -1,9 +1,18 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
+config CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST
+	tristate "KUnit tests for AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_AES_CBC_MACS
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC message
+	  authentication codes.
+
 config CRYPTO_LIB_BLAKE2B_KUNIT_TEST
 	tristate "KUnit tests for BLAKE2b" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_BLAKE2B
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	help
 	  KUnit tests for the BLAKE2b cryptographic hash function.
@@ -11,25 +20,42 @@ config CRYPTO_LIB_BLAKE2B_KUNIT_TEST
 config CRYPTO_LIB_BLAKE2S_KUNIT_TEST
 	tristate "KUnit tests for BLAKE2s" if !KUNIT_ALL_TESTS
 	depends on KUNIT
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	# No need to depend on CRYPTO_LIB_BLAKE2S here, as that option doesn't
 	# exist; the BLAKE2s code is always built-in for the /dev/random driver.
 	help
 	  KUnit tests for the BLAKE2s cryptographic hash function.
 
+config CRYPTO_LIB_CHACHA20POLY1305_KUNIT_TEST
+	tristate "KUnit tests for ChaCha20Poly1305" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_CHACHA20POLY1305
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the ChaCha20Poly1305 authenticated encryption
+	  algorithm.
+
 config CRYPTO_LIB_CURVE25519_KUNIT_TEST
 	tristate "KUnit tests for Curve25519" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_CURVE25519
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	help
 	  KUnit tests for the Curve25519 Diffie-Hellman function.
 
+config CRYPTO_LIB_GHASH_KUNIT_TEST
+	tristate "KUnit tests for GHASH" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_GF128HASH
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the GHASH library functions.
+
 config CRYPTO_LIB_MD5_KUNIT_TEST
 	tristate "KUnit tests for MD5" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_MD5
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	help
 	  KUnit tests for the MD5 cryptographic hash function and its
@@ -38,7 +64,7 @@ config CRYPTO_LIB_MD5_KUNIT_TEST
 config CRYPTO_LIB_MLDSA_KUNIT_TEST
 	tristate "KUnit tests for ML-DSA" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_MLDSA
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	help
 	  KUnit tests for the ML-DSA digital signature algorithm.
@@ -46,22 +72,22 @@ config CRYPTO_LIB_MLDSA_KUNIT_TEST
 config CRYPTO_LIB_NH_KUNIT_TEST
 	tristate "KUnit tests for NH" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_NH
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	help
 	  KUnit tests for the NH almost-universal hash function.
 
 config CRYPTO_LIB_POLY1305_KUNIT_TEST
 	tristate "KUnit tests for Poly1305" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_POLY1305
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	help
 	  KUnit tests for the Poly1305 library functions.
 
 config CRYPTO_LIB_POLYVAL_KUNIT_TEST
 	tristate "KUnit tests for POLYVAL" if !KUNIT_ALL_TESTS
-	depends on KUNIT && CRYPTO_LIB_POLYVAL
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	depends on KUNIT && CRYPTO_LIB_GF128HASH
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	help
 	  KUnit tests for the POLYVAL library functions.
@@ -69,7 +95,7 @@ config CRYPTO_LIB_POLYVAL_KUNIT_TEST
 config CRYPTO_LIB_SHA1_KUNIT_TEST
 	tristate "KUnit tests for SHA-1" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_SHA1
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	help
 	  KUnit tests for the SHA-1 cryptographic hash function and its
@@ -80,7 +106,7 @@ config CRYPTO_LIB_SHA1_KUNIT_TEST
 config CRYPTO_LIB_SHA256_KUNIT_TEST
 	tristate "KUnit tests for SHA-224 and SHA-256" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_SHA256
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	help
 	  KUnit tests for the SHA-224 and SHA-256 cryptographic hash functions
@@ -91,7 +117,7 @@ config CRYPTO_LIB_SHA256_KUNIT_TEST
 config CRYPTO_LIB_SHA512_KUNIT_TEST
 	tristate "KUnit tests for SHA-384 and SHA-512" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_SHA512
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	help
 	  KUnit tests for the SHA-384 and SHA-512 cryptographic hash functions
@@ -100,13 +126,47 @@ config CRYPTO_LIB_SHA512_KUNIT_TEST
 config CRYPTO_LIB_SHA3_KUNIT_TEST
 	tristate "KUnit tests for SHA-3" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_SHA3
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	help
 	  KUnit tests for the SHA3 cryptographic hash and XOF functions,
 	  including SHA3-224, SHA3-256, SHA3-384, SHA3-512, SHAKE128 and
 	  SHAKE256.
 
+config CRYPTO_LIB_SM3_KUNIT_TEST
+	tristate "KUnit tests for SM3" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_SM3
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the SM3 cryptographic hash function.
+
+config CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT
+	tristate "Enable all crypto library code for KUnit tests"
+	depends on KUNIT
+	select CRYPTO_LIB_AES_CBC_MACS
+	select CRYPTO_LIB_BLAKE2B
+	select CRYPTO_LIB_CHACHA20POLY1305
+	select CRYPTO_LIB_CURVE25519
+	select CRYPTO_LIB_GF128HASH
+	select CRYPTO_LIB_MD5
+	select CRYPTO_LIB_MLDSA
+	select CRYPTO_LIB_NH
+	select CRYPTO_LIB_POLY1305
+	select CRYPTO_LIB_SHA1
+	select CRYPTO_LIB_SHA256
+	select CRYPTO_LIB_SHA512
+	select CRYPTO_LIB_SHA3
+	select CRYPTO_LIB_SM3
+	help
+	  Enable all the crypto library code that has KUnit tests.
+
+	  Enable this only if you'd like to test all the crypto library code,
+	  even code that wouldn't otherwise need to be built.
+
+	  You'll still need to enable the tests themselves, either individually
+	  or using KUNIT_ALL_TESTS.
+
 config CRYPTO_LIB_BENCHMARK_VISIBLE
 	bool
 
diff --git a/lib/crypto/tests/Makefile b/lib/crypto/tests/Makefile
index f4262379f56c..a739413500b6 100644
--- a/lib/crypto/tests/Makefile
+++ b/lib/crypto/tests/Makefile
@@ -1,8 +1,11 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
+obj-$(CONFIG_CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST) += aes_cbc_macs_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST) += blake2b_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST) += blake2s_kunit.o
+obj-$(CONFIG_CRYPTO_LIB_CHACHA20POLY1305_KUNIT_TEST) += chacha20poly1305_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_CURVE25519_KUNIT_TEST) += curve25519_kunit.o
+obj-$(CONFIG_CRYPTO_LIB_GHASH_KUNIT_TEST) += ghash_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_MD5_KUNIT_TEST) += md5_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_MLDSA_KUNIT_TEST) += mldsa_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_NH_KUNIT_TEST) += nh_kunit.o
@@ -12,3 +15,4 @@ obj-$(CONFIG_CRYPTO_LIB_SHA1_KUNIT_TEST) += sha1_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_SHA256_KUNIT_TEST) += sha224_kunit.o sha256_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_SHA512_KUNIT_TEST) += sha384_kunit.o sha512_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_SHA3_KUNIT_TEST) += sha3_kunit.o
+obj-$(CONFIG_CRYPTO_LIB_SM3_KUNIT_TEST) += sm3_kunit.o
diff --git a/lib/crypto/tests/aes-cmac-testvecs.h b/lib/crypto/tests/aes-cmac-testvecs.h
new file mode 100644
index 000000000000..8177862ba5a3
--- /dev/null
+++ b/lib/crypto/tests/aes-cmac-testvecs.h
@@ -0,0 +1,181 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* This file was generated by: ./scripts/crypto/gen-hash-testvecs.py aes-cmac */
+
+static const struct {
+	size_t data_len;
+	u8 digest[AES_BLOCK_SIZE];
+} hash_testvecs[] = {
+	{
+		.data_len = 0,
+		.digest = {
+			0x9a, 0xeb, 0x94, 0xc1, 0xe9, 0xc1, 0x57, 0x49,
+			0x7e, 0xba, 0x66, 0x47, 0x9f, 0x03, 0x2c, 0x5b,
+		},
+	},
+	{
+		.data_len = 1,
+		.digest = {
+			0x52, 0xef, 0x65, 0xda, 0x7b, 0x92, 0x0c, 0x0f,
+			0xdd, 0xd6, 0xb9, 0x68, 0x3f, 0xcc, 0x5f, 0xea,
+		},
+	},
+	{
+		.data_len = 2,
+		.digest = {
+			0xc3, 0x95, 0x15, 0xea, 0x16, 0x33, 0xbe, 0x5a,
+			0xd4, 0x2c, 0x25, 0x06, 0x15, 0xc6, 0x10, 0x19,
+		},
+	},
+	{
+		.data_len = 3,
+		.digest = {
+			0x82, 0x41, 0x41, 0xd5, 0x33, 0x26, 0x0b, 0xb6,
+			0xc8, 0xf7, 0x8d, 0x76, 0x8a, 0xfc, 0x0e, 0xe4,
+		},
+	},
+	{
+		.data_len = 16,
+		.digest = {
+			0x94, 0x09, 0x80, 0x07, 0xba, 0x7c, 0xed, 0xd2,
+			0x74, 0x72, 0x30, 0x26, 0xb5, 0x11, 0x64, 0xc1,
+		},
+	},
+	{
+		.data_len = 32,
+		.digest = {
+			0xeb, 0xcf, 0x1e, 0x67, 0x21, 0x64, 0x93, 0xa0,
+			0xea, 0xc4, 0xb9, 0x2d, 0x55, 0xc8, 0xac, 0x99,
+		},
+	},
+	{
+		.data_len = 48,
+		.digest = {
+			0xd0, 0xd6, 0xdb, 0xe2, 0x45, 0x98, 0x6a, 0x7a,
+			0x5f, 0xd6, 0xcd, 0x9d, 0x12, 0x26, 0x20, 0x87,
+		},
+	},
+	{
+		.data_len = 49,
+		.digest = {
+			0x63, 0x25, 0x3c, 0xe2, 0x2a, 0xfa, 0xe3, 0x1e,
+			0x54, 0x10, 0x18, 0x28, 0xc6, 0xb8, 0xcb, 0x58,
+		},
+	},
+	{
+		.data_len = 63,
+		.digest = {
+			0x4d, 0xab, 0xae, 0x99, 0x90, 0x13, 0x3f, 0x4f,
+			0x42, 0x0f, 0x19, 0x94, 0xa2, 0x52, 0xfd, 0xaf,
+		},
+	},
+	{
+		.data_len = 64,
+		.digest = {
+			0xf7, 0x49, 0xb9, 0xa7, 0xf9, 0x3e, 0xa0, 0xca,
+			0xb2, 0x6c, 0xd7, 0x87, 0x7d, 0x1e, 0xd2, 0xcb,
+		},
+	},
+	{
+		.data_len = 65,
+		.digest = {
+			0x27, 0x2c, 0xb7, 0xc8, 0xdd, 0x26, 0xa9, 0xfe,
+			0x37, 0x64, 0x84, 0x38, 0xa5, 0x7e, 0xbc, 0x04,
+		},
+	},
+	{
+		.data_len = 127,
+		.digest = {
+			0xfd, 0x1f, 0x01, 0xa4, 0xea, 0x9b, 0xbd, 0xef,
+			0x09, 0x97, 0x57, 0x60, 0x95, 0x23, 0xcc, 0x71,
+		},
+	},
+	{
+		.data_len = 128,
+		.digest = {
+			0x28, 0x21, 0xee, 0x56, 0x9f, 0x38, 0xd6, 0x0e,
+			0xe3, 0x22, 0x06, 0x20, 0xad, 0xd8, 0x33, 0x74,
+		},
+	},
+	{
+		.data_len = 129,
+		.digest = {
+			0x07, 0x28, 0x4a, 0x2a, 0xd3, 0x85, 0xa6, 0x87,
+			0x5c, 0x01, 0x8c, 0xb9, 0xd3, 0x4b, 0xce, 0x20,
+		},
+	},
+	{
+		.data_len = 256,
+		.digest = {
+			0xe6, 0x12, 0x25, 0x6b, 0xf9, 0x69, 0x4d, 0x5a,
+			0x1a, 0xb0, 0xe6, 0x11, 0x46, 0x24, 0x08, 0xdf,
+		},
+	},
+	{
+		.data_len = 511,
+		.digest = {
+			0xce, 0x28, 0x1f, 0x14, 0xb9, 0xcc, 0x7e, 0x1f,
+			0xb5, 0x13, 0x2b, 0x45, 0x04, 0x54, 0xe9, 0x5f,
+		},
+	},
+	{
+		.data_len = 513,
+		.digest = {
+			0x63, 0x12, 0xbd, 0x85, 0x60, 0x1b, 0x99, 0x7e,
+			0x0a, 0xf7, 0x0f, 0xc1, 0xb5, 0x66, 0xf8, 0x9a,
+		},
+	},
+	{
+		.data_len = 1000,
+		.digest = {
+			0xbd, 0x49, 0x5e, 0x21, 0xc6, 0x58, 0x74, 0x6b,
+			0x21, 0xc2, 0x62, 0x6a, 0x15, 0xca, 0x1d, 0x8a,
+		},
+	},
+	{
+		.data_len = 3333,
+		.digest = {
+			0xfe, 0x6b, 0xfa, 0xfc, 0x4c, 0x0b, 0x63, 0x0d,
+			0x41, 0x7f, 0xa9, 0xd8, 0xba, 0xe3, 0xce, 0xce,
+		},
+	},
+	{
+		.data_len = 4096,
+		.digest = {
+			0x41, 0x7c, 0xbc, 0x2e, 0x2f, 0xff, 0xdf, 0x09,
+			0x31, 0xc5, 0x79, 0x0a, 0x1d, 0x6e, 0x46, 0xec,
+		},
+	},
+	{
+		.data_len = 4128,
+		.digest = {
+			0x6a, 0x9d, 0x86, 0xa8, 0xab, 0xa5, 0xc1, 0xc5,
+			0x0d, 0x54, 0xf3, 0x51, 0x10, 0x46, 0x25, 0x5a,
+		},
+	},
+	{
+		.data_len = 4160,
+		.digest = {
+			0x4c, 0x50, 0xff, 0x2a, 0xe9, 0xca, 0x9e, 0x07,
+			0x8a, 0x86, 0x67, 0x5e, 0xe5, 0x0a, 0xfd, 0x69,
+		},
+	},
+	{
+		.data_len = 4224,
+		.digest = {
+			0x3a, 0xfa, 0x80, 0x9d, 0x80, 0xe3, 0x1e, 0x95,
+			0x53, 0x93, 0x39, 0x17, 0xd3, 0xda, 0x49, 0x15,
+		},
+	},
+	{
+		.data_len = 16384,
+		.digest = {
+			0x48, 0xf4, 0x4f, 0x2d, 0x5d, 0xf2, 0x02, 0xcf,
+			0x51, 0x3c, 0x1b, 0x12, 0x80, 0x8f, 0xb0, 0xd6,
+		},
+	},
+};
+
+static const u8 hash_testvec_consolidated[AES_BLOCK_SIZE] = {
+	0x41, 0xad, 0x25, 0xa1, 0xeb, 0xce, 0x6b, 0x9c,
+	0x06, 0xdf, 0x47, 0xc4, 0x3a, 0x59, 0x50, 0x07,
+};
diff --git a/lib/crypto/tests/aes_cbc_macs_kunit.c b/lib/crypto/tests/aes_cbc_macs_kunit.c
new file mode 100644
index 000000000000..ae3745212f03
--- /dev/null
+++ b/lib/crypto/tests/aes_cbc_macs_kunit.c
@@ -0,0 +1,228 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright 2026 Google LLC
+ */
+#include <crypto/aes-cbc-macs.h>
+#include "aes-cmac-testvecs.h"
+
+/*
+ * A fixed key used when presenting AES-CMAC as an unkeyed hash function in
+ * order to reuse hash-test-template.h.  At the beginning of the test suite,
+ * this is initialized to a key prepared from bytes generated from a fixed seed.
+ */
+static struct aes_cmac_key test_key;
+
+static void aes_cmac_init_withtestkey(struct aes_cmac_ctx *ctx)
+{
+	aes_cmac_init(ctx, &test_key);
+}
+
+static void aes_cmac_withtestkey(const u8 *data, size_t data_len,
+				 u8 out[AES_BLOCK_SIZE])
+{
+	aes_cmac(&test_key, data, data_len, out);
+}
+
+#define HASH aes_cmac_withtestkey
+#define HASH_CTX aes_cmac_ctx
+#define HASH_SIZE AES_BLOCK_SIZE
+#define HASH_INIT aes_cmac_init_withtestkey
+#define HASH_UPDATE aes_cmac_update
+#define HASH_FINAL aes_cmac_final
+#include "hash-test-template.h"
+
+static int aes_cbc_macs_suite_init(struct kunit_suite *suite)
+{
+	u8 raw_key[AES_KEYSIZE_256];
+	int err;
+
+	rand_bytes_seeded_from_len(raw_key, sizeof(raw_key));
+	err = aes_cmac_preparekey(&test_key, raw_key, sizeof(raw_key));
+	if (err)
+		return err;
+	return hash_suite_init(suite);
+}
+
+static void aes_cbc_macs_suite_exit(struct kunit_suite *suite)
+{
+	hash_suite_exit(suite);
+}
+
+/* Verify compatibility of the AES-CMAC implementation with RFC 4493. */
+static void test_aes_cmac_rfc4493(struct kunit *test)
+{
+	static const u8 raw_key[AES_KEYSIZE_128] = {
+		0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6,
+		0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,
+	};
+	static const struct {
+		size_t data_len;
+		const u8 data[40];
+		const u8 mac[AES_BLOCK_SIZE];
+	} testvecs[] = {
+		{
+			/* Example 1 from RFC 4493 */
+			.data_len = 0,
+			.mac = {
+				0xbb, 0x1d, 0x69, 0x29, 0xe9, 0x59, 0x37, 0x28,
+				0x7f, 0xa3, 0x7d, 0x12, 0x9b, 0x75, 0x67, 0x46,
+			},
+
+		},
+		{
+			/* Example 2 from RFC 4493 */
+			.data = {
+				0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96,
+				0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a,
+			},
+			.data_len = 16,
+			.mac = {
+				0x07, 0x0a, 0x16, 0xb4, 0x6b, 0x4d, 0x41, 0x44,
+				0xf7, 0x9b, 0xdd, 0x9d, 0xd0, 0x4a, 0x28, 0x7c,
+			},
+		},
+		{
+			/* Example 3 from RFC 4493 */
+			.data = {
+				0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96,
+				0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a,
+				0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c,
+				0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51,
+				0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11,
+			},
+			.data_len = 40,
+			.mac = {
+				0xdf, 0xa6, 0x67, 0x47, 0xde, 0x9a, 0xe6, 0x30,
+				0x30, 0xca, 0x32, 0x61, 0x14, 0x97, 0xc8, 0x27,
+			},
+		},
+	};
+	struct aes_cmac_key key;
+	int err;
+
+	err = aes_cmac_preparekey(&key, raw_key, sizeof(raw_key));
+	KUNIT_ASSERT_EQ(test, err, 0);
+
+	for (size_t i = 0; i < ARRAY_SIZE(testvecs); i++) {
+		u8 mac[AES_BLOCK_SIZE];
+
+		aes_cmac(&key, testvecs[i].data, testvecs[i].data_len, mac);
+		KUNIT_ASSERT_MEMEQ(test, mac, testvecs[i].mac, AES_BLOCK_SIZE);
+	}
+}
+
+/*
+ * Verify compatibility of the AES-XCBC-MAC implementation with RFC 3566.
+ *
+ * Additional AES-XCBC-MAC tests are not necessary, since the AES-XCBC-MAC
+ * implementation is well covered by the AES-CMAC tests already.  Only the key
+ * preparation function differs; the rest of the code is shared.
+ */
+static void test_aes_xcbcmac_rfc3566(struct kunit *test)
+{
+	struct aes_cmac_key key;
+	/* AES-XCBC-MAC Test Case #4 from RFC 3566 */
+	static const u8 raw_key[AES_KEYSIZE_128] = {
+		0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+		0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+	};
+	static const u8 message[20] = {
+		0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09,
+		0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13,
+	};
+	static const u8 expected_mac[AES_BLOCK_SIZE] = {
+		0x47, 0xf5, 0x1b, 0x45, 0x64, 0x96, 0x62, 0x15,
+		0xb8, 0x98, 0x5c, 0x63, 0x05, 0x5e, 0xd3, 0x08,
+	};
+	u8 actual_mac[AES_BLOCK_SIZE];
+
+	aes_xcbcmac_preparekey(&key, raw_key);
+	aes_cmac(&key, message, sizeof(message), actual_mac);
+	KUNIT_ASSERT_MEMEQ(test, actual_mac, expected_mac, AES_BLOCK_SIZE);
+}
+
+static void test_aes_cbcmac_rfc3610(struct kunit *test)
+{
+	/*
+	 * The following AES-CBC-MAC test vector is extracted from RFC 3610
+	 * Packet Vector #11.  It required some rearrangement to get the actual
+	 * input to AES-CBC-MAC from the values given.
+	 */
+	static const u8 raw_key[AES_KEYSIZE_128] = {
+		0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
+		0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf,
+	};
+	const size_t unpadded_data_len = 52;
+	static const u8 data[64] = {
+		/* clang-format off */
+		/* CCM header */
+		0x61, 0x00, 0x00, 0x00, 0x0d, 0x0c, 0x0b, 0x0a,
+		0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0x00, 0x14,
+		/* CCM additional authentication blocks */
+		0x00, 0x0c, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05,
+		0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x00, 0x00,
+		/* CCM message blocks */
+		0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13,
+		0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b,
+		0x1c, 0x1d, 0x1e, 0x1f, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		/* clang-format on */
+	};
+	static const u8 expected_mac[AES_BLOCK_SIZE] = {
+		0x6b, 0x5e, 0x24, 0x34, 0x12, 0xcc, 0xc2, 0xad,
+		0x6f, 0x1b, 0x11, 0xc3, 0xa1, 0xa9, 0xd8, 0xbc,
+	};
+	struct aes_enckey key;
+	struct aes_cbcmac_ctx ctx;
+	u8 actual_mac[AES_BLOCK_SIZE];
+	int err;
+
+	err = aes_prepareenckey(&key, raw_key, sizeof(raw_key));
+	KUNIT_ASSERT_EQ(test, err, 0);
+
+	/*
+	 * Trailing zeroes should not affect the CBC-MAC value, up to the next
+	 * AES block boundary.
+	 */
+	for (size_t data_len = unpadded_data_len; data_len <= sizeof(data);
+	     data_len++) {
+		aes_cbcmac_init(&ctx, &key);
+		aes_cbcmac_update(&ctx, data, data_len);
+		aes_cbcmac_final(&ctx, actual_mac);
+		KUNIT_ASSERT_MEMEQ(test, actual_mac, expected_mac,
+				   AES_BLOCK_SIZE);
+
+		/* Incremental computations should produce the same result. */
+		for (size_t part1_len = 0; part1_len <= data_len; part1_len++) {
+			aes_cbcmac_init(&ctx, &key);
+			aes_cbcmac_update(&ctx, data, part1_len);
+			aes_cbcmac_update(&ctx, &data[part1_len],
+					  data_len - part1_len);
+			aes_cbcmac_final(&ctx, actual_mac);
+			KUNIT_ASSERT_MEMEQ(test, actual_mac, expected_mac,
+					   AES_BLOCK_SIZE);
+		}
+	}
+}
+
+static struct kunit_case aes_cbc_macs_test_cases[] = {
+	HASH_KUNIT_CASES,
+	KUNIT_CASE(test_aes_cmac_rfc4493),
+	KUNIT_CASE(test_aes_xcbcmac_rfc3566),
+	KUNIT_CASE(test_aes_cbcmac_rfc3610),
+	KUNIT_CASE(benchmark_hash),
+	{},
+};
+
+static struct kunit_suite aes_cbc_macs_test_suite = {
+	.name = "aes_cbc_macs",
+	.test_cases = aes_cbc_macs_test_cases,
+	.suite_init = aes_cbc_macs_suite_init,
+	.suite_exit = aes_cbc_macs_suite_exit,
+};
+kunit_test_suite(aes_cbc_macs_test_suite);
+
+MODULE_DESCRIPTION(
+	"KUnit tests and benchmark for AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC");
+MODULE_IMPORT_NS("CRYPTO_INTERNAL");
+MODULE_LICENSE("GPL");
diff --git a/lib/crypto/chacha20poly1305-selftest.c b/lib/crypto/tests/chacha20poly1305_kunit.c
index e4c85bc5a6d7..97a68fab88a7 100644
--- a/lib/crypto/chacha20poly1305-selftest.c
+++ b/lib/crypto/tests/chacha20poly1305_kunit.c
@@ -6,9 +6,8 @@
 #include <crypto/chacha20poly1305.h>
 #include <crypto/chacha.h>
 #include <crypto/poly1305.h>
-
+#include <kunit/test.h>
 #include <linux/unaligned.h>
-#include <linux/bug.h>
 #include <linux/init.h>
 #include <linux/mm.h>
 #include <linux/kernel.h>
@@ -27,7 +26,7 @@ struct chacha20poly1305_testvec {
  * chapoly construction.
  */
 
-static const u8 enc_input001[] __initconst = {
+static const u8 enc_input001[] = {
 	0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74,
 	0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20,
 	0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66,
@@ -63,7 +62,7 @@ static const u8 enc_input001[] __initconst = {
 	0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80,
 	0x9d
 };
-static const u8 enc_output001[] __initconst = {
+static const u8 enc_output001[] = {
 	0x64, 0xa0, 0x86, 0x15, 0x75, 0x86, 0x1a, 0xf4,
 	0x60, 0xf0, 0x62, 0xc7, 0x9b, 0xe6, 0x43, 0xbd,
 	0x5e, 0x80, 0x5c, 0xfd, 0x34, 0x5c, 0xf3, 0x89,
@@ -101,95 +100,95 @@ static const u8 enc_output001[] __initconst = {
 	0x22, 0x39, 0x23, 0x36, 0xfe, 0xa1, 0x85, 0x1f,
 	0x38
 };
-static const u8 enc_assoc001[] __initconst = {
+static const u8 enc_assoc001[] = {
 	0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x4e, 0x91
 };
-static const u8 enc_nonce001[] __initconst = {
+static const u8 enc_nonce001[] = {
 	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08
 };
-static const u8 enc_key001[] __initconst = {
+static const u8 enc_key001[] = {
 	0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a,
 	0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0,
 	0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09,
 	0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0
 };
 
-static const u8 enc_input002[] __initconst = { };
-static const u8 enc_output002[] __initconst = {
+static const u8 enc_input002[] = { };
+static const u8 enc_output002[] = {
 	0xea, 0xe0, 0x1e, 0x9e, 0x2c, 0x91, 0xaa, 0xe1,
 	0xdb, 0x5d, 0x99, 0x3f, 0x8a, 0xf7, 0x69, 0x92
 };
-static const u8 enc_assoc002[] __initconst = { };
-static const u8 enc_nonce002[] __initconst = {
+static const u8 enc_assoc002[] = { };
+static const u8 enc_nonce002[] = {
 	0xca, 0xbf, 0x33, 0x71, 0x32, 0x45, 0x77, 0x8e
 };
-static const u8 enc_key002[] __initconst = {
+static const u8 enc_key002[] = {
 	0x4c, 0xf5, 0x96, 0x83, 0x38, 0xe6, 0xae, 0x7f,
 	0x2d, 0x29, 0x25, 0x76, 0xd5, 0x75, 0x27, 0x86,
 	0x91, 0x9a, 0x27, 0x7a, 0xfb, 0x46, 0xc5, 0xef,
 	0x94, 0x81, 0x79, 0x57, 0x14, 0x59, 0x40, 0x68
 };
 
-static const u8 enc_input003[] __initconst = { };
-static const u8 enc_output003[] __initconst = {
+static const u8 enc_input003[] = { };
+static const u8 enc_output003[] = {
 	0xdd, 0x6b, 0x3b, 0x82, 0xce, 0x5a, 0xbd, 0xd6,
 	0xa9, 0x35, 0x83, 0xd8, 0x8c, 0x3d, 0x85, 0x77
 };
-static const u8 enc_assoc003[] __initconst = {
+static const u8 enc_assoc003[] = {
 	0x33, 0x10, 0x41, 0x12, 0x1f, 0xf3, 0xd2, 0x6b
 };
-static const u8 enc_nonce003[] __initconst = {
+static const u8 enc_nonce003[] = {
 	0x3d, 0x86, 0xb5, 0x6b, 0xc8, 0xa3, 0x1f, 0x1d
 };
-static const u8 enc_key003[] __initconst = {
+static const u8 enc_key003[] = {
 	0x2d, 0xb0, 0x5d, 0x40, 0xc8, 0xed, 0x44, 0x88,
 	0x34, 0xd1, 0x13, 0xaf, 0x57, 0xa1, 0xeb, 0x3a,
 	0x2a, 0x80, 0x51, 0x36, 0xec, 0x5b, 0xbc, 0x08,
 	0x93, 0x84, 0x21, 0xb5, 0x13, 0x88, 0x3c, 0x0d
 };
 
-static const u8 enc_input004[] __initconst = {
+static const u8 enc_input004[] = {
 	0xa4
 };
-static const u8 enc_output004[] __initconst = {
+static const u8 enc_output004[] = {
 	0xb7, 0x1b, 0xb0, 0x73, 0x59, 0xb0, 0x84, 0xb2,
 	0x6d, 0x8e, 0xab, 0x94, 0x31, 0xa1, 0xae, 0xac,
 	0x89
 };
-static const u8 enc_assoc004[] __initconst = {
+static const u8 enc_assoc004[] = {
 	0x6a, 0xe2, 0xad, 0x3f, 0x88, 0x39, 0x5a, 0x40
 };
-static const u8 enc_nonce004[] __initconst = {
+static const u8 enc_nonce004[] = {
 	0xd2, 0x32, 0x1f, 0x29, 0x28, 0xc6, 0xc4, 0xc4
 };
-static const u8 enc_key004[] __initconst = {
+static const u8 enc_key004[] = {
 	0x4b, 0x28, 0x4b, 0xa3, 0x7b, 0xbe, 0xe9, 0xf8,
 	0x31, 0x80, 0x82, 0xd7, 0xd8, 0xe8, 0xb5, 0xa1,
 	0xe2, 0x18, 0x18, 0x8a, 0x9c, 0xfa, 0xa3, 0x3d,
 	0x25, 0x71, 0x3e, 0x40, 0xbc, 0x54, 0x7a, 0x3e
 };
 
-static const u8 enc_input005[] __initconst = {
+static const u8 enc_input005[] = {
 	0x2d
 };
-static const u8 enc_output005[] __initconst = {
+static const u8 enc_output005[] = {
 	0xbf, 0xe1, 0x5b, 0x0b, 0xdb, 0x6b, 0xf5, 0x5e,
 	0x6c, 0x5d, 0x84, 0x44, 0x39, 0x81, 0xc1, 0x9c,
 	0xac
 };
-static const u8 enc_assoc005[] __initconst = { };
-static const u8 enc_nonce005[] __initconst = {
+static const u8 enc_assoc005[] = { };
+static const u8 enc_nonce005[] = {
 	0x20, 0x1c, 0xaa, 0x5f, 0x9c, 0xbf, 0x92, 0x30
 };
-static const u8 enc_key005[] __initconst = {
+static const u8 enc_key005[] = {
 	0x66, 0xca, 0x9c, 0x23, 0x2a, 0x4b, 0x4b, 0x31,
 	0x0e, 0x92, 0x89, 0x8b, 0xf4, 0x93, 0xc7, 0x87,
 	0x98, 0xa3, 0xd8, 0x39, 0xf8, 0xf4, 0xa7, 0x01,
 	0xc0, 0x2e, 0x0a, 0xa6, 0x7e, 0x5a, 0x78, 0x87
 };
 
-static const u8 enc_input006[] __initconst = {
+static const u8 enc_input006[] = {
 	0x33, 0x2f, 0x94, 0xc1, 0xa4, 0xef, 0xcc, 0x2a,
 	0x5b, 0xa6, 0xe5, 0x8f, 0x1d, 0x40, 0xf0, 0x92,
 	0x3c, 0xd9, 0x24, 0x11, 0xa9, 0x71, 0xf9, 0x37,
@@ -208,7 +207,7 @@ static const u8 enc_input006[] __initconst = {
 	0x69, 0x21, 0x70, 0xd8, 0xa4, 0x4b, 0xc8, 0xde,
 	0x8f
 };
-static const u8 enc_output006[] __initconst = {
+static const u8 enc_output006[] = {
 	0x8b, 0x06, 0xd3, 0x31, 0xb0, 0x93, 0x45, 0xb1,
 	0x75, 0x6e, 0x26, 0xf9, 0x67, 0xbc, 0x90, 0x15,
 	0x81, 0x2c, 0xb5, 0xf0, 0xc6, 0x2b, 0xc7, 0x8c,
@@ -229,20 +228,20 @@ static const u8 enc_output006[] __initconst = {
 	0x8d, 0x31, 0xf7, 0x7a, 0x39, 0x4d, 0x8f, 0x9a,
 	0xeb
 };
-static const u8 enc_assoc006[] __initconst = {
+static const u8 enc_assoc006[] = {
 	0x70, 0xd3, 0x33, 0xf3, 0x8b, 0x18, 0x0b
 };
-static const u8 enc_nonce006[] __initconst = {
+static const u8 enc_nonce006[] = {
 	0xdf, 0x51, 0x84, 0x82, 0x42, 0x0c, 0x75, 0x9c
 };
-static const u8 enc_key006[] __initconst = {
+static const u8 enc_key006[] = {
 	0x68, 0x7b, 0x8d, 0x8e, 0xe3, 0xc4, 0xdd, 0xae,
 	0xdf, 0x72, 0x7f, 0x53, 0x72, 0x25, 0x1e, 0x78,
 	0x91, 0xcb, 0x69, 0x76, 0x1f, 0x49, 0x93, 0xf9,
 	0x6f, 0x21, 0xcc, 0x39, 0x9c, 0xad, 0xb1, 0x01
 };
 
-static const u8 enc_input007[] __initconst = {
+static const u8 enc_input007[] = {
 	0x9b, 0x18, 0xdb, 0xdd, 0x9a, 0x0f, 0x3e, 0xa5,
 	0x15, 0x17, 0xde, 0xdf, 0x08, 0x9d, 0x65, 0x0a,
 	0x67, 0x30, 0x12, 0xe2, 0x34, 0x77, 0x4b, 0xc1,
@@ -276,7 +275,7 @@ static const u8 enc_input007[] __initconst = {
 	0x40, 0x5f, 0x99, 0xb7, 0x73, 0xec, 0x9b, 0x2b,
 	0xf0, 0x65, 0x11, 0xc8, 0xd0, 0x0a, 0x9f, 0xd3
 };
-static const u8 enc_output007[] __initconst = {
+static const u8 enc_output007[] = {
 	0x85, 0x04, 0xc2, 0xed, 0x8d, 0xfd, 0x97, 0x5c,
 	0xd2, 0xb7, 0xe2, 0xc1, 0x6b, 0xa3, 0xba, 0xf8,
 	0xc9, 0x50, 0xc3, 0xc6, 0xa5, 0xe3, 0xa4, 0x7c,
@@ -312,18 +311,18 @@ static const u8 enc_output007[] __initconst = {
 	0x7a, 0x4b, 0xbc, 0xb0, 0x10, 0x5c, 0x96, 0x42,
 	0x3a, 0x00, 0x98, 0xcd, 0x15, 0xe8, 0xb7, 0x53
 };
-static const u8 enc_assoc007[] __initconst = { };
-static const u8 enc_nonce007[] __initconst = {
+static const u8 enc_assoc007[] = { };
+static const u8 enc_nonce007[] = {
 	0xde, 0x7b, 0xef, 0xc3, 0x65, 0x1b, 0x68, 0xb0
 };
-static const u8 enc_key007[] __initconst = {
+static const u8 enc_key007[] = {
 	0x8d, 0xb8, 0x91, 0x48, 0xf0, 0xe7, 0x0a, 0xbd,
 	0xf9, 0x3f, 0xcd, 0xd9, 0xa0, 0x1e, 0x42, 0x4c,
 	0xe7, 0xde, 0x25, 0x3d, 0xa3, 0xd7, 0x05, 0x80,
 	0x8d, 0xf2, 0x82, 0xac, 0x44, 0x16, 0x51, 0x01
 };
 
-static const u8 enc_input008[] __initconst = {
+static const u8 enc_input008[] = {
 	0xc3, 0x09, 0x94, 0x62, 0xe6, 0x46, 0x2e, 0x10,
 	0xbe, 0x00, 0xe4, 0xfc, 0xf3, 0x40, 0xa3, 0xe2,
 	0x0f, 0xc2, 0x8b, 0x28, 0xdc, 0xba, 0xb4, 0x3c,
@@ -389,7 +388,7 @@ static const u8 enc_input008[] __initconst = {
 	0x2e, 0x79, 0xb5, 0xe2, 0xb8, 0xe8, 0xb9, 0x7b,
 	0xd5, 0x10, 0xcb, 0xff, 0x5d, 0x14, 0x73, 0xf3
 };
-static const u8 enc_output008[] __initconst = {
+static const u8 enc_output008[] = {
 	0x14, 0xf6, 0x41, 0x37, 0xa6, 0xd4, 0x27, 0xcd,
 	0xdb, 0x06, 0x3e, 0x9a, 0x4e, 0xab, 0xd5, 0xb1,
 	0x1e, 0x6b, 0xd2, 0xbc, 0x11, 0xf4, 0x28, 0x93,
@@ -457,18 +456,18 @@ static const u8 enc_output008[] __initconst = {
 	0x54, 0xfd, 0x0d, 0x98, 0x1c, 0x5a, 0x6f, 0x1f,
 	0x9a, 0x40, 0xcd, 0xa2, 0xff, 0x6a, 0xf1, 0x54
 };
-static const u8 enc_assoc008[] __initconst = { };
-static const u8 enc_nonce008[] __initconst = {
+static const u8 enc_assoc008[] = { };
+static const u8 enc_nonce008[] = {
 	0x0e, 0x0d, 0x57, 0xbb, 0x7b, 0x40, 0x54, 0x02
 };
-static const u8 enc_key008[] __initconst = {
+static const u8 enc_key008[] = {
 	0xf2, 0xaa, 0x4f, 0x99, 0xfd, 0x3e, 0xa8, 0x53,
 	0xc1, 0x44, 0xe9, 0x81, 0x18, 0xdc, 0xf5, 0xf0,
 	0x3e, 0x44, 0x15, 0x59, 0xe0, 0xc5, 0x44, 0x86,
 	0xc3, 0x91, 0xa8, 0x75, 0xc0, 0x12, 0x46, 0xba
 };
 
-static const u8 enc_input009[] __initconst = {
+static const u8 enc_input009[] = {
 	0xe6, 0xc3, 0xdb, 0x63, 0x55, 0x15, 0xe3, 0x5b,
 	0xb7, 0x4b, 0x27, 0x8b, 0x5a, 0xdd, 0xc2, 0xe8,
 	0x3a, 0x6b, 0xd7, 0x81, 0x96, 0x35, 0x97, 0xca,
@@ -535,7 +534,7 @@ static const u8 enc_input009[] __initconst = {
 	0x74, 0xb9, 0xe2, 0xd6, 0x1c, 0x80, 0x2c, 0x52,
 	0x65
 };
-static const u8 enc_output009[] __initconst = {
+static const u8 enc_output009[] = {
 	0xfd, 0x81, 0x8d, 0xd0, 0x3d, 0xb4, 0xd5, 0xdf,
 	0xd3, 0x42, 0x47, 0x5a, 0x6d, 0x19, 0x27, 0x66,
 	0x4b, 0x2e, 0x0c, 0x27, 0x9c, 0x96, 0x4c, 0x72,
@@ -604,21 +603,21 @@ static const u8 enc_output009[] __initconst = {
 	0x43, 0xb5, 0xd9, 0x90, 0xe1, 0x85, 0xf5, 0xa8,
 	0xae
 };
-static const u8 enc_assoc009[] __initconst = {
+static const u8 enc_assoc009[] = {
 	0x5a, 0x27, 0xff, 0xeb, 0xdf, 0x84, 0xb2, 0x9e,
 	0xef
 };
-static const u8 enc_nonce009[] __initconst = {
+static const u8 enc_nonce009[] = {
 	0xef, 0x2d, 0x63, 0xee, 0x6b, 0x80, 0x8b, 0x78
 };
-static const u8 enc_key009[] __initconst = {
+static const u8 enc_key009[] = {
 	0xea, 0xbc, 0x56, 0x99, 0xe3, 0x50, 0xff, 0xc5,
 	0xcc, 0x1a, 0xd7, 0xc1, 0x57, 0x72, 0xea, 0x86,
 	0x5b, 0x89, 0x88, 0x61, 0x3d, 0x2f, 0x9b, 0xb2,
 	0xe7, 0x9c, 0xec, 0x74, 0x6e, 0x3e, 0xf4, 0x3b
 };
 
-static const u8 enc_input010[] __initconst = {
+static const u8 enc_input010[] = {
 	0x42, 0x93, 0xe4, 0xeb, 0x97, 0xb0, 0x57, 0xbf,
 	0x1a, 0x8b, 0x1f, 0xe4, 0x5f, 0x36, 0x20, 0x3c,
 	0xef, 0x0a, 0xa9, 0x48, 0x5f, 0x5f, 0x37, 0x22,
@@ -748,7 +747,7 @@ static const u8 enc_input010[] __initconst = {
 	0x5a, 0xc0, 0x33, 0x1f, 0xcb, 0x05, 0x6d, 0x5c,
 	0x06, 0x87, 0x52, 0xa2, 0x8f, 0x26, 0xd5, 0x4f
 };
-static const u8 enc_output010[] __initconst = {
+static const u8 enc_output010[] = {
 	0xe5, 0x26, 0xa4, 0x3d, 0xbd, 0x33, 0xd0, 0x4b,
 	0x6f, 0x05, 0xa7, 0x6e, 0x12, 0x7a, 0xd2, 0x74,
 	0xa6, 0xdd, 0xbd, 0x95, 0xeb, 0xf9, 0xa4, 0xf1,
@@ -880,21 +879,21 @@ static const u8 enc_output010[] __initconst = {
 	0x80, 0x6b, 0xf6, 0x1f, 0xc3, 0xcc, 0x97, 0xc9,
 	0x24, 0x9f, 0xf3, 0xaf, 0x43, 0x14, 0xd5, 0xa0
 };
-static const u8 enc_assoc010[] __initconst = {
+static const u8 enc_assoc010[] = {
 	0xd2, 0xa1, 0x70, 0xdb, 0x7a, 0xf8, 0xfa, 0x27,
 	0xba, 0x73, 0x0f, 0xbf, 0x3d, 0x1e, 0x82, 0xb2
 };
-static const u8 enc_nonce010[] __initconst = {
+static const u8 enc_nonce010[] = {
 	0xdb, 0x92, 0x0f, 0x7f, 0x17, 0x54, 0x0c, 0x30
 };
-static const u8 enc_key010[] __initconst = {
+static const u8 enc_key010[] = {
 	0x47, 0x11, 0xeb, 0x86, 0x2b, 0x2c, 0xab, 0x44,
 	0x34, 0xda, 0x7f, 0x57, 0x03, 0x39, 0x0c, 0xaf,
 	0x2c, 0x14, 0xfd, 0x65, 0x23, 0xe9, 0x8e, 0x74,
 	0xd5, 0x08, 0x68, 0x08, 0xe7, 0xb4, 0x72, 0xd7
 };
 
-static const u8 enc_input011[] __initconst = {
+static const u8 enc_input011[] = {
 	0x7a, 0x57, 0xf2, 0xc7, 0x06, 0x3f, 0x50, 0x7b,
 	0x36, 0x1a, 0x66, 0x5c, 0xb9, 0x0e, 0x5e, 0x3b,
 	0x45, 0x60, 0xbe, 0x9a, 0x31, 0x9f, 0xff, 0x5d,
@@ -1138,7 +1137,7 @@ static const u8 enc_input011[] __initconst = {
 	0x5c, 0x42, 0x16, 0xb8, 0x28, 0xeb, 0x1b, 0x61,
 	0x10, 0x1e, 0xbf, 0xec, 0xa8
 };
-static const u8 enc_output011[] __initconst = {
+static const u8 enc_output011[] = {
 	0x6a, 0xfc, 0x4b, 0x25, 0xdf, 0xc0, 0xe4, 0xe8,
 	0x17, 0x4d, 0x4c, 0xc9, 0x7e, 0xde, 0x3a, 0xcc,
 	0x3c, 0xba, 0x6a, 0x77, 0x47, 0xdb, 0xe3, 0x74,
@@ -1384,20 +1383,20 @@ static const u8 enc_output011[] __initconst = {
 	0x86, 0xc4, 0xfc, 0x97, 0xae, 0x3f, 0x8f, 0x1e,
 	0x2b, 0xdf, 0xcd, 0xf9, 0x3c
 };
-static const u8 enc_assoc011[] __initconst = {
+static const u8 enc_assoc011[] = {
 	0xd6, 0x31, 0xda, 0x5d, 0x42, 0x5e, 0xd7
 };
-static const u8 enc_nonce011[] __initconst = {
+static const u8 enc_nonce011[] = {
 	0xfd, 0x87, 0xd4, 0xd8, 0x62, 0xfd, 0xec, 0xaa
 };
-static const u8 enc_key011[] __initconst = {
+static const u8 enc_key011[] = {
 	0x35, 0x4e, 0xb5, 0x70, 0x50, 0x42, 0x8a, 0x85,
 	0xf2, 0xfb, 0xed, 0x7b, 0xd0, 0x9e, 0x97, 0xca,
 	0xfa, 0x98, 0x66, 0x63, 0xee, 0x37, 0xcc, 0x52,
 	0xfe, 0xd1, 0xdf, 0x95, 0x15, 0x34, 0x29, 0x38
 };
 
-static const u8 enc_input012[] __initconst = {
+static const u8 enc_input012[] = {
 	0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0,
 	0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5,
 	0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57,
@@ -1651,7 +1650,7 @@ static const u8 enc_input012[] __initconst = {
 	0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d,
 	0x78, 0xec, 0x00
 };
-static const u8 enc_output012[] __initconst = {
+static const u8 enc_output012[] = {
 	0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3,
 	0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf,
 	0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1,
@@ -1907,7 +1906,7 @@ static const u8 enc_output012[] __initconst = {
 	0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62,
 	0x70, 0xcf, 0xd6
 };
-static const u8 enc_assoc012[] __initconst = {
+static const u8 enc_assoc012[] = {
 	0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8,
 	0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce,
 	0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c,
@@ -1917,10 +1916,10 @@ static const u8 enc_assoc012[] __initconst = {
 	0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b,
 	0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9
 };
-static const u8 enc_nonce012[] __initconst = {
+static const u8 enc_nonce012[] = {
 	0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06
 };
-static const u8 enc_key012[] __initconst = {
+static const u8 enc_key012[] = {
 	0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e,
 	0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d,
 	0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e,
@@ -1928,7 +1927,7 @@ static const u8 enc_key012[] __initconst = {
 };
 
 /* wycheproof - rfc7539 */
-static const u8 enc_input013[] __initconst = {
+static const u8 enc_input013[] = {
 	0x4c, 0x61, 0x64, 0x69, 0x65, 0x73, 0x20, 0x61,
 	0x6e, 0x64, 0x20, 0x47, 0x65, 0x6e, 0x74, 0x6c,
 	0x65, 0x6d, 0x65, 0x6e, 0x20, 0x6f, 0x66, 0x20,
@@ -1945,7 +1944,7 @@ static const u8 enc_input013[] __initconst = {
 	0x75, 0x6c, 0x64, 0x20, 0x62, 0x65, 0x20, 0x69,
 	0x74, 0x2e
 };
-static const u8 enc_output013[] __initconst = {
+static const u8 enc_output013[] = {
 	0xd3, 0x1a, 0x8d, 0x34, 0x64, 0x8e, 0x60, 0xdb,
 	0x7b, 0x86, 0xaf, 0xbc, 0x53, 0xef, 0x7e, 0xc2,
 	0xa4, 0xad, 0xed, 0x51, 0x29, 0x6e, 0x08, 0xfe,
@@ -1964,15 +1963,15 @@ static const u8 enc_output013[] __initconst = {
 	0xe2, 0x6a, 0x7e, 0x90, 0x2e, 0xcb, 0xd0, 0x60,
 	0x06, 0x91
 };
-static const u8 enc_assoc013[] __initconst = {
+static const u8 enc_assoc013[] = {
 	0x50, 0x51, 0x52, 0x53, 0xc0, 0xc1, 0xc2, 0xc3,
 	0xc4, 0xc5, 0xc6, 0xc7
 };
-static const u8 enc_nonce013[] __initconst = {
+static const u8 enc_nonce013[] = {
 	0x07, 0x00, 0x00, 0x00, 0x40, 0x41, 0x42, 0x43,
 	0x44, 0x45, 0x46, 0x47
 };
-static const u8 enc_key013[] __initconst = {
+static const u8 enc_key013[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -1980,17 +1979,17 @@ static const u8 enc_key013[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input014[] __initconst = { };
-static const u8 enc_output014[] __initconst = {
+static const u8 enc_input014[] = { };
+static const u8 enc_output014[] = {
 	0x76, 0xac, 0xb3, 0x42, 0xcf, 0x31, 0x66, 0xa5,
 	0xb6, 0x3c, 0x0c, 0x0e, 0xa1, 0x38, 0x3c, 0x8d
 };
-static const u8 enc_assoc014[] __initconst = { };
-static const u8 enc_nonce014[] __initconst = {
+static const u8 enc_assoc014[] = { };
+static const u8 enc_nonce014[] = {
 	0x4d, 0xa5, 0xbf, 0x8d, 0xfd, 0x58, 0x52, 0xc1,
 	0xea, 0x12, 0x37, 0x9d
 };
-static const u8 enc_key014[] __initconst = {
+static const u8 enc_key014[] = {
 	0x80, 0xba, 0x31, 0x92, 0xc8, 0x03, 0xce, 0x96,
 	0x5e, 0xa3, 0x71, 0xd5, 0xff, 0x07, 0x3c, 0xf0,
 	0xf4, 0x3b, 0x6a, 0x2a, 0xb5, 0x76, 0xb2, 0x08,
@@ -1998,19 +1997,19 @@ static const u8 enc_key014[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input015[] __initconst = { };
-static const u8 enc_output015[] __initconst = {
+static const u8 enc_input015[] = { };
+static const u8 enc_output015[] = {
 	0x90, 0x6f, 0xa6, 0x28, 0x4b, 0x52, 0xf8, 0x7b,
 	0x73, 0x59, 0xcb, 0xaa, 0x75, 0x63, 0xc7, 0x09
 };
-static const u8 enc_assoc015[] __initconst = {
+static const u8 enc_assoc015[] = {
 	0xbd, 0x50, 0x67, 0x64, 0xf2, 0xd2, 0xc4, 0x10
 };
-static const u8 enc_nonce015[] __initconst = {
+static const u8 enc_nonce015[] = {
 	0xa9, 0x2e, 0xf0, 0xac, 0x99, 0x1d, 0xd5, 0x16,
 	0xa3, 0xc6, 0xf6, 0x89
 };
-static const u8 enc_key015[] __initconst = {
+static const u8 enc_key015[] = {
 	0x7a, 0x4c, 0xd7, 0x59, 0x17, 0x2e, 0x02, 0xeb,
 	0x20, 0x4d, 0xb2, 0xc3, 0xf5, 0xc7, 0x46, 0x22,
 	0x7d, 0xf5, 0x84, 0xfc, 0x13, 0x45, 0x19, 0x63,
@@ -2018,20 +2017,20 @@ static const u8 enc_key015[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input016[] __initconst = {
+static const u8 enc_input016[] = {
 	0x2a
 };
-static const u8 enc_output016[] __initconst = {
+static const u8 enc_output016[] = {
 	0x3a, 0xca, 0xc2, 0x7d, 0xec, 0x09, 0x68, 0x80,
 	0x1e, 0x9f, 0x6e, 0xde, 0xd6, 0x9d, 0x80, 0x75,
 	0x22
 };
-static const u8 enc_assoc016[] __initconst = { };
-static const u8 enc_nonce016[] __initconst = {
+static const u8 enc_assoc016[] = { };
+static const u8 enc_nonce016[] = {
 	0x99, 0xe2, 0x3e, 0xc4, 0x89, 0x85, 0xbc, 0xcd,
 	0xee, 0xab, 0x60, 0xf1
 };
-static const u8 enc_key016[] __initconst = {
+static const u8 enc_key016[] = {
 	0xcc, 0x56, 0xb6, 0x80, 0x55, 0x2e, 0xb7, 0x50,
 	0x08, 0xf5, 0x48, 0x4b, 0x4c, 0xb8, 0x03, 0xfa,
 	0x50, 0x63, 0xeb, 0xd6, 0xea, 0xb9, 0x1f, 0x6a,
@@ -2039,22 +2038,22 @@ static const u8 enc_key016[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input017[] __initconst = {
+static const u8 enc_input017[] = {
 	0x51
 };
-static const u8 enc_output017[] __initconst = {
+static const u8 enc_output017[] = {
 	0xc4, 0x16, 0x83, 0x10, 0xca, 0x45, 0xb1, 0xf7,
 	0xc6, 0x6c, 0xad, 0x4e, 0x99, 0xe4, 0x3f, 0x72,
 	0xb9
 };
-static const u8 enc_assoc017[] __initconst = {
+static const u8 enc_assoc017[] = {
 	0x91, 0xca, 0x6c, 0x59, 0x2c, 0xbc, 0xca, 0x53
 };
-static const u8 enc_nonce017[] __initconst = {
+static const u8 enc_nonce017[] = {
 	0xab, 0x0d, 0xca, 0x71, 0x6e, 0xe0, 0x51, 0xd2,
 	0x78, 0x2f, 0x44, 0x03
 };
-static const u8 enc_key017[] __initconst = {
+static const u8 enc_key017[] = {
 	0x46, 0xf0, 0x25, 0x49, 0x65, 0xf7, 0x69, 0xd5,
 	0x2b, 0xdb, 0x4a, 0x70, 0xb4, 0x43, 0x19, 0x9f,
 	0x8e, 0xf2, 0x07, 0x52, 0x0d, 0x12, 0x20, 0xc5,
@@ -2062,20 +2061,20 @@ static const u8 enc_key017[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input018[] __initconst = {
+static const u8 enc_input018[] = {
 	0x5c, 0x60
 };
-static const u8 enc_output018[] __initconst = {
+static const u8 enc_output018[] = {
 	0x4d, 0x13, 0x91, 0xe8, 0xb6, 0x1e, 0xfb, 0x39,
 	0xc1, 0x22, 0x19, 0x54, 0x53, 0x07, 0x7b, 0x22,
 	0xe5, 0xe2
 };
-static const u8 enc_assoc018[] __initconst = { };
-static const u8 enc_nonce018[] __initconst = {
+static const u8 enc_assoc018[] = { };
+static const u8 enc_nonce018[] = {
 	0x46, 0x1a, 0xf1, 0x22, 0xe9, 0xf2, 0xe0, 0x34,
 	0x7e, 0x03, 0xf2, 0xdb
 };
-static const u8 enc_key018[] __initconst = {
+static const u8 enc_key018[] = {
 	0x2f, 0x7f, 0x7e, 0x4f, 0x59, 0x2b, 0xb3, 0x89,
 	0x19, 0x49, 0x89, 0x74, 0x35, 0x07, 0xbf, 0x3e,
 	0xe9, 0xcb, 0xde, 0x17, 0x86, 0xb6, 0x69, 0x5f,
@@ -2083,22 +2082,22 @@ static const u8 enc_key018[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input019[] __initconst = {
+static const u8 enc_input019[] = {
 	0xdd, 0xf2
 };
-static const u8 enc_output019[] __initconst = {
+static const u8 enc_output019[] = {
 	0xb6, 0x0d, 0xea, 0xd0, 0xfd, 0x46, 0x97, 0xec,
 	0x2e, 0x55, 0x58, 0x23, 0x77, 0x19, 0xd0, 0x24,
 	0x37, 0xa2
 };
-static const u8 enc_assoc019[] __initconst = {
+static const u8 enc_assoc019[] = {
 	0x88, 0x36, 0x4f, 0xc8, 0x06, 0x05, 0x18, 0xbf
 };
-static const u8 enc_nonce019[] __initconst = {
+static const u8 enc_nonce019[] = {
 	0x61, 0x54, 0x6b, 0xa5, 0xf1, 0x72, 0x05, 0x90,
 	0xb6, 0x04, 0x0a, 0xc6
 };
-static const u8 enc_key019[] __initconst = {
+static const u8 enc_key019[] = {
 	0xc8, 0x83, 0x3d, 0xce, 0x5e, 0xa9, 0xf2, 0x48,
 	0xaa, 0x20, 0x30, 0xea, 0xcf, 0xe7, 0x2b, 0xff,
 	0xe6, 0x9a, 0x62, 0x0c, 0xaf, 0x79, 0x33, 0x44,
@@ -2106,20 +2105,20 @@ static const u8 enc_key019[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input020[] __initconst = {
+static const u8 enc_input020[] = {
 	0xab, 0x85, 0xe9, 0xc1, 0x57, 0x17, 0x31
 };
-static const u8 enc_output020[] __initconst = {
+static const u8 enc_output020[] = {
 	0x5d, 0xfe, 0x34, 0x40, 0xdb, 0xb3, 0xc3, 0xed,
 	0x7a, 0x43, 0x4e, 0x26, 0x02, 0xd3, 0x94, 0x28,
 	0x1e, 0x0a, 0xfa, 0x9f, 0xb7, 0xaa, 0x42
 };
-static const u8 enc_assoc020[] __initconst = { };
-static const u8 enc_nonce020[] __initconst = {
+static const u8 enc_assoc020[] = { };
+static const u8 enc_nonce020[] = {
 	0x3c, 0x4e, 0x65, 0x4d, 0x66, 0x3f, 0xa4, 0x59,
 	0x6d, 0xc5, 0x5b, 0xb7
 };
-static const u8 enc_key020[] __initconst = {
+static const u8 enc_key020[] = {
 	0x55, 0x56, 0x81, 0x58, 0xd3, 0xa6, 0x48, 0x3f,
 	0x1f, 0x70, 0x21, 0xea, 0xb6, 0x9b, 0x70, 0x3f,
 	0x61, 0x42, 0x51, 0xca, 0xdc, 0x1a, 0xf5, 0xd3,
@@ -2127,22 +2126,22 @@ static const u8 enc_key020[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input021[] __initconst = {
+static const u8 enc_input021[] = {
 	0x4e, 0xe5, 0xcd, 0xa2, 0x0d, 0x42, 0x90
 };
-static const u8 enc_output021[] __initconst = {
+static const u8 enc_output021[] = {
 	0x4b, 0xd4, 0x72, 0x12, 0x94, 0x1c, 0xe3, 0x18,
 	0x5f, 0x14, 0x08, 0xee, 0x7f, 0xbf, 0x18, 0xf5,
 	0xab, 0xad, 0x6e, 0x22, 0x53, 0xa1, 0xba
 };
-static const u8 enc_assoc021[] __initconst = {
+static const u8 enc_assoc021[] = {
 	0x84, 0xe4, 0x6b, 0xe8, 0xc0, 0x91, 0x90, 0x53
 };
-static const u8 enc_nonce021[] __initconst = {
+static const u8 enc_nonce021[] = {
 	0x58, 0x38, 0x93, 0x75, 0xc6, 0x9e, 0xe3, 0x98,
 	0xde, 0x94, 0x83, 0x96
 };
-static const u8 enc_key021[] __initconst = {
+static const u8 enc_key021[] = {
 	0xe3, 0xc0, 0x9e, 0x7f, 0xab, 0x1a, 0xef, 0xb5,
 	0x16, 0xda, 0x6a, 0x33, 0x02, 0x2a, 0x1d, 0xd4,
 	0xeb, 0x27, 0x2c, 0x80, 0xd5, 0x40, 0xc5, 0xda,
@@ -2150,20 +2149,20 @@ static const u8 enc_key021[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input022[] __initconst = {
+static const u8 enc_input022[] = {
 	0xbe, 0x33, 0x08, 0xf7, 0x2a, 0x2c, 0x6a, 0xed
 };
-static const u8 enc_output022[] __initconst = {
+static const u8 enc_output022[] = {
 	0x8e, 0x94, 0x39, 0xa5, 0x6e, 0xee, 0xc8, 0x17,
 	0xfb, 0xe8, 0xa6, 0xed, 0x8f, 0xab, 0xb1, 0x93,
 	0x75, 0x39, 0xdd, 0x6c, 0x00, 0xe9, 0x00, 0x21
 };
-static const u8 enc_assoc022[] __initconst = { };
-static const u8 enc_nonce022[] __initconst = {
+static const u8 enc_assoc022[] = { };
+static const u8 enc_nonce022[] = {
 	0x4f, 0x07, 0xaf, 0xed, 0xfd, 0xc3, 0xb6, 0xc2,
 	0x36, 0x18, 0x23, 0xd3
 };
-static const u8 enc_key022[] __initconst = {
+static const u8 enc_key022[] = {
 	0x51, 0xe4, 0xbf, 0x2b, 0xad, 0x92, 0xb7, 0xaf,
 	0xf1, 0xa4, 0xbc, 0x05, 0x55, 0x0b, 0xa8, 0x1d,
 	0xf4, 0xb9, 0x6f, 0xab, 0xf4, 0x1c, 0x12, 0xc7,
@@ -2171,22 +2170,22 @@ static const u8 enc_key022[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input023[] __initconst = {
+static const u8 enc_input023[] = {
 	0xa4, 0xc9, 0xc2, 0x80, 0x1b, 0x71, 0xf7, 0xdf
 };
-static const u8 enc_output023[] __initconst = {
+static const u8 enc_output023[] = {
 	0xb9, 0xb9, 0x10, 0x43, 0x3a, 0xf0, 0x52, 0xb0,
 	0x45, 0x30, 0xf5, 0x1a, 0xee, 0xe0, 0x24, 0xe0,
 	0xa4, 0x45, 0xa6, 0x32, 0x8f, 0xa6, 0x7a, 0x18
 };
-static const u8 enc_assoc023[] __initconst = {
+static const u8 enc_assoc023[] = {
 	0x66, 0xc0, 0xae, 0x70, 0x07, 0x6c, 0xb1, 0x4d
 };
-static const u8 enc_nonce023[] __initconst = {
+static const u8 enc_nonce023[] = {
 	0xb4, 0xea, 0x66, 0x6e, 0xe1, 0x19, 0x56, 0x33,
 	0x66, 0x48, 0x4a, 0x78
 };
-static const u8 enc_key023[] __initconst = {
+static const u8 enc_key023[] = {
 	0x11, 0x31, 0xc1, 0x41, 0x85, 0x77, 0xa0, 0x54,
 	0xde, 0x7a, 0x4a, 0xc5, 0x51, 0x95, 0x0f, 0x1a,
 	0x05, 0x3f, 0x9a, 0xe4, 0x6e, 0x5b, 0x75, 0xfe,
@@ -2194,22 +2193,22 @@ static const u8 enc_key023[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input024[] __initconst = {
+static const u8 enc_input024[] = {
 	0x42, 0xba, 0xae, 0x59, 0x78, 0xfe, 0xaf, 0x5c,
 	0x36, 0x8d, 0x14, 0xe0
 };
-static const u8 enc_output024[] __initconst = {
+static const u8 enc_output024[] = {
 	0xff, 0x7d, 0xc2, 0x03, 0xb2, 0x6c, 0x46, 0x7a,
 	0x6b, 0x50, 0xdb, 0x33, 0x57, 0x8c, 0x0f, 0x27,
 	0x58, 0xc2, 0xe1, 0x4e, 0x36, 0xd4, 0xfc, 0x10,
 	0x6d, 0xcb, 0x29, 0xb4
 };
-static const u8 enc_assoc024[] __initconst = { };
-static const u8 enc_nonce024[] __initconst = {
+static const u8 enc_assoc024[] = { };
+static const u8 enc_nonce024[] = {
 	0x9a, 0x59, 0xfc, 0xe2, 0x6d, 0xf0, 0x00, 0x5e,
 	0x07, 0x53, 0x86, 0x56
 };
-static const u8 enc_key024[] __initconst = {
+static const u8 enc_key024[] = {
 	0x99, 0xb6, 0x2b, 0xd5, 0xaf, 0xbe, 0x3f, 0xb0,
 	0x15, 0xbd, 0xe9, 0x3f, 0x0a, 0xbf, 0x48, 0x39,
 	0x57, 0xa1, 0xc3, 0xeb, 0x3c, 0xa5, 0x9c, 0xb5,
@@ -2217,24 +2216,24 @@ static const u8 enc_key024[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input025[] __initconst = {
+static const u8 enc_input025[] = {
 	0xfd, 0xc8, 0x5b, 0x94, 0xa4, 0xb2, 0xa6, 0xb7,
 	0x59, 0xb1, 0xa0, 0xda
 };
-static const u8 enc_output025[] __initconst = {
+static const u8 enc_output025[] = {
 	0x9f, 0x88, 0x16, 0xde, 0x09, 0x94, 0xe9, 0x38,
 	0xd9, 0xe5, 0x3f, 0x95, 0xd0, 0x86, 0xfc, 0x6c,
 	0x9d, 0x8f, 0xa9, 0x15, 0xfd, 0x84, 0x23, 0xa7,
 	0xcf, 0x05, 0x07, 0x2f
 };
-static const u8 enc_assoc025[] __initconst = {
+static const u8 enc_assoc025[] = {
 	0xa5, 0x06, 0xe1, 0xa5, 0xc6, 0x90, 0x93, 0xf9
 };
-static const u8 enc_nonce025[] __initconst = {
+static const u8 enc_nonce025[] = {
 	0x58, 0xdb, 0xd4, 0xad, 0x2c, 0x4a, 0xd3, 0x5d,
 	0xd9, 0x06, 0xe9, 0xce
 };
-static const u8 enc_key025[] __initconst = {
+static const u8 enc_key025[] = {
 	0x85, 0xf3, 0x5b, 0x62, 0x82, 0xcf, 0xf4, 0x40,
 	0xbc, 0x10, 0x20, 0xc8, 0x13, 0x6f, 0xf2, 0x70,
 	0x31, 0x11, 0x0f, 0xa6, 0x3e, 0xc1, 0x6f, 0x1e,
@@ -2242,22 +2241,22 @@ static const u8 enc_key025[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input026[] __initconst = {
+static const u8 enc_input026[] = {
 	0x51, 0xf8, 0xc1, 0xf7, 0x31, 0xea, 0x14, 0xac,
 	0xdb, 0x21, 0x0a, 0x6d, 0x97, 0x3e, 0x07
 };
-static const u8 enc_output026[] __initconst = {
+static const u8 enc_output026[] = {
 	0x0b, 0x29, 0x63, 0x8e, 0x1f, 0xbd, 0xd6, 0xdf,
 	0x53, 0x97, 0x0b, 0xe2, 0x21, 0x00, 0x42, 0x2a,
 	0x91, 0x34, 0x08, 0x7d, 0x67, 0xa4, 0x6e, 0x79,
 	0x17, 0x8d, 0x0a, 0x93, 0xf5, 0xe1, 0xd2
 };
-static const u8 enc_assoc026[] __initconst = { };
-static const u8 enc_nonce026[] __initconst = {
+static const u8 enc_assoc026[] = { };
+static const u8 enc_nonce026[] = {
 	0x68, 0xab, 0x7f, 0xdb, 0xf6, 0x19, 0x01, 0xda,
 	0xd4, 0x61, 0xd2, 0x3c
 };
-static const u8 enc_key026[] __initconst = {
+static const u8 enc_key026[] = {
 	0x67, 0x11, 0x96, 0x27, 0xbd, 0x98, 0x8e, 0xda,
 	0x90, 0x62, 0x19, 0xe0, 0x8c, 0x0d, 0x0d, 0x77,
 	0x9a, 0x07, 0xd2, 0x08, 0xce, 0x8a, 0x4f, 0xe0,
@@ -2265,24 +2264,24 @@ static const u8 enc_key026[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input027[] __initconst = {
+static const u8 enc_input027[] = {
 	0x97, 0x46, 0x9d, 0xa6, 0x67, 0xd6, 0x11, 0x0f,
 	0x9c, 0xbd, 0xa1, 0xd1, 0xa2, 0x06, 0x73
 };
-static const u8 enc_output027[] __initconst = {
+static const u8 enc_output027[] = {
 	0x32, 0xdb, 0x66, 0xc4, 0xa3, 0x81, 0x9d, 0x81,
 	0x55, 0x74, 0x55, 0xe5, 0x98, 0x0f, 0xed, 0xfe,
 	0xae, 0x30, 0xde, 0xc9, 0x4e, 0x6a, 0xd3, 0xa9,
 	0xee, 0xa0, 0x6a, 0x0d, 0x70, 0x39, 0x17
 };
-static const u8 enc_assoc027[] __initconst = {
+static const u8 enc_assoc027[] = {
 	0x64, 0x53, 0xa5, 0x33, 0x84, 0x63, 0x22, 0x12
 };
-static const u8 enc_nonce027[] __initconst = {
+static const u8 enc_nonce027[] = {
 	0xd9, 0x5b, 0x32, 0x43, 0xaf, 0xae, 0xf7, 0x14,
 	0xc5, 0x03, 0x5b, 0x6a
 };
-static const u8 enc_key027[] __initconst = {
+static const u8 enc_key027[] = {
 	0xe6, 0xf1, 0x11, 0x8d, 0x41, 0xe4, 0xb4, 0x3f,
 	0xb5, 0x82, 0x21, 0xb7, 0xed, 0x79, 0x67, 0x38,
 	0x34, 0xe0, 0xd8, 0xac, 0x5c, 0x4f, 0xa6, 0x0b,
@@ -2290,22 +2289,22 @@ static const u8 enc_key027[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input028[] __initconst = {
+static const u8 enc_input028[] = {
 	0x54, 0x9b, 0x36, 0x5a, 0xf9, 0x13, 0xf3, 0xb0,
 	0x81, 0x13, 0x1c, 0xcb, 0x6b, 0x82, 0x55, 0x88
 };
-static const u8 enc_output028[] __initconst = {
+static const u8 enc_output028[] = {
 	0xe9, 0x11, 0x0e, 0x9f, 0x56, 0xab, 0x3c, 0xa4,
 	0x83, 0x50, 0x0c, 0xea, 0xba, 0xb6, 0x7a, 0x13,
 	0x83, 0x6c, 0xca, 0xbf, 0x15, 0xa6, 0xa2, 0x2a,
 	0x51, 0xc1, 0x07, 0x1c, 0xfa, 0x68, 0xfa, 0x0c
 };
-static const u8 enc_assoc028[] __initconst = { };
-static const u8 enc_nonce028[] __initconst = {
+static const u8 enc_assoc028[] = { };
+static const u8 enc_nonce028[] = {
 	0x2f, 0xcb, 0x1b, 0x38, 0xa9, 0x9e, 0x71, 0xb8,
 	0x47, 0x40, 0xad, 0x9b
 };
-static const u8 enc_key028[] __initconst = {
+static const u8 enc_key028[] = {
 	0x59, 0xd4, 0xea, 0xfb, 0x4d, 0xe0, 0xcf, 0xc7,
 	0xd3, 0xdb, 0x99, 0xa8, 0xf5, 0x4b, 0x15, 0xd7,
 	0xb3, 0x9f, 0x0a, 0xcc, 0x8d, 0xa6, 0x97, 0x63,
@@ -2313,24 +2312,24 @@ static const u8 enc_key028[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input029[] __initconst = {
+static const u8 enc_input029[] = {
 	0x55, 0xa4, 0x65, 0x64, 0x4f, 0x5b, 0x65, 0x09,
 	0x28, 0xcb, 0xee, 0x7c, 0x06, 0x32, 0x14, 0xd6
 };
-static const u8 enc_output029[] __initconst = {
+static const u8 enc_output029[] = {
 	0xe4, 0xb1, 0x13, 0xcb, 0x77, 0x59, 0x45, 0xf3,
 	0xd3, 0xa8, 0xae, 0x9e, 0xc1, 0x41, 0xc0, 0x0c,
 	0x7c, 0x43, 0xf1, 0x6c, 0xe0, 0x96, 0xd0, 0xdc,
 	0x27, 0xc9, 0x58, 0x49, 0xdc, 0x38, 0x3b, 0x7d
 };
-static const u8 enc_assoc029[] __initconst = {
+static const u8 enc_assoc029[] = {
 	0x03, 0x45, 0x85, 0x62, 0x1a, 0xf8, 0xd7, 0xff
 };
-static const u8 enc_nonce029[] __initconst = {
+static const u8 enc_nonce029[] = {
 	0x11, 0x8a, 0x69, 0x64, 0xc2, 0xd3, 0xe3, 0x80,
 	0x07, 0x1f, 0x52, 0x66
 };
-static const u8 enc_key029[] __initconst = {
+static const u8 enc_key029[] = {
 	0xb9, 0x07, 0xa4, 0x50, 0x75, 0x51, 0x3f, 0xe8,
 	0xa8, 0x01, 0x9e, 0xde, 0xe3, 0xf2, 0x59, 0x14,
 	0x87, 0xb2, 0xa0, 0x30, 0xb0, 0x3c, 0x6e, 0x1d,
@@ -2338,24 +2337,24 @@ static const u8 enc_key029[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input030[] __initconst = {
+static const u8 enc_input030[] = {
 	0x3f, 0xf1, 0x51, 0x4b, 0x1c, 0x50, 0x39, 0x15,
 	0x91, 0x8f, 0x0c, 0x0c, 0x31, 0x09, 0x4a, 0x6e,
 	0x1f
 };
-static const u8 enc_output030[] __initconst = {
+static const u8 enc_output030[] = {
 	0x02, 0xcc, 0x3a, 0xcb, 0x5e, 0xe1, 0xfc, 0xdd,
 	0x12, 0xa0, 0x3b, 0xb8, 0x57, 0x97, 0x64, 0x74,
 	0xd3, 0xd8, 0x3b, 0x74, 0x63, 0xa2, 0xc3, 0x80,
 	0x0f, 0xe9, 0x58, 0xc2, 0x8e, 0xaa, 0x29, 0x08,
 	0x13
 };
-static const u8 enc_assoc030[] __initconst = { };
-static const u8 enc_nonce030[] __initconst = {
+static const u8 enc_assoc030[] = { };
+static const u8 enc_nonce030[] = {
 	0x45, 0xaa, 0xa3, 0xe5, 0xd1, 0x6d, 0x2d, 0x42,
 	0xdc, 0x03, 0x44, 0x5d
 };
-static const u8 enc_key030[] __initconst = {
+static const u8 enc_key030[] = {
 	0x3b, 0x24, 0x58, 0xd8, 0x17, 0x6e, 0x16, 0x21,
 	0xc0, 0xcc, 0x24, 0xc0, 0xc0, 0xe2, 0x4c, 0x1e,
 	0x80, 0xd7, 0x2f, 0x7e, 0xe9, 0x14, 0x9a, 0x4b,
@@ -2363,26 +2362,26 @@ static const u8 enc_key030[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input031[] __initconst = {
+static const u8 enc_input031[] = {
 	0x63, 0x85, 0x8c, 0xa3, 0xe2, 0xce, 0x69, 0x88,
 	0x7b, 0x57, 0x8a, 0x3c, 0x16, 0x7b, 0x42, 0x1c,
 	0x9c
 };
-static const u8 enc_output031[] __initconst = {
+static const u8 enc_output031[] = {
 	0x35, 0x76, 0x64, 0x88, 0xd2, 0xbc, 0x7c, 0x2b,
 	0x8d, 0x17, 0xcb, 0xbb, 0x9a, 0xbf, 0xad, 0x9e,
 	0x6d, 0x1f, 0x39, 0x1e, 0x65, 0x7b, 0x27, 0x38,
 	0xdd, 0xa0, 0x84, 0x48, 0xcb, 0xa2, 0x81, 0x1c,
 	0xeb
 };
-static const u8 enc_assoc031[] __initconst = {
+static const u8 enc_assoc031[] = {
 	0x9a, 0xaf, 0x29, 0x9e, 0xee, 0xa7, 0x8f, 0x79
 };
-static const u8 enc_nonce031[] __initconst = {
+static const u8 enc_nonce031[] = {
 	0xf0, 0x38, 0x4f, 0xb8, 0x76, 0x12, 0x14, 0x10,
 	0x63, 0x3d, 0x99, 0x3d
 };
-static const u8 enc_key031[] __initconst = {
+static const u8 enc_key031[] = {
 	0xf6, 0x0c, 0x6a, 0x1b, 0x62, 0x57, 0x25, 0xf7,
 	0x6c, 0x70, 0x37, 0xb4, 0x8f, 0xe3, 0x57, 0x7f,
 	0xa7, 0xf7, 0xb8, 0x7b, 0x1b, 0xd5, 0xa9, 0x82,
@@ -2390,24 +2389,24 @@ static const u8 enc_key031[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input032[] __initconst = {
+static const u8 enc_input032[] = {
 	0x10, 0xf1, 0xec, 0xf9, 0xc6, 0x05, 0x84, 0x66,
 	0x5d, 0x9a, 0xe5, 0xef, 0xe2, 0x79, 0xe7, 0xf7,
 	0x37, 0x7e, 0xea, 0x69, 0x16, 0xd2, 0xb1, 0x11
 };
-static const u8 enc_output032[] __initconst = {
+static const u8 enc_output032[] = {
 	0x42, 0xf2, 0x6c, 0x56, 0xcb, 0x4b, 0xe2, 0x1d,
 	0x9d, 0x8d, 0x0c, 0x80, 0xfc, 0x99, 0xdd, 0xe0,
 	0x0d, 0x75, 0xf3, 0x80, 0x74, 0xbf, 0xe7, 0x64,
 	0x54, 0xaa, 0x7e, 0x13, 0xd4, 0x8f, 0xff, 0x7d,
 	0x75, 0x57, 0x03, 0x94, 0x57, 0x04, 0x0a, 0x3a
 };
-static const u8 enc_assoc032[] __initconst = { };
-static const u8 enc_nonce032[] __initconst = {
+static const u8 enc_assoc032[] = { };
+static const u8 enc_nonce032[] = {
 	0xe6, 0xb1, 0xad, 0xf2, 0xfd, 0x58, 0xa8, 0x76,
 	0x2c, 0x65, 0xf3, 0x1b
 };
-static const u8 enc_key032[] __initconst = {
+static const u8 enc_key032[] = {
 	0x02, 0x12, 0xa8, 0xde, 0x50, 0x07, 0xed, 0x87,
 	0xb3, 0x3f, 0x1a, 0x70, 0x90, 0xb6, 0x11, 0x4f,
 	0x9e, 0x08, 0xce, 0xfd, 0x96, 0x07, 0xf2, 0xc2,
@@ -2415,26 +2414,26 @@ static const u8 enc_key032[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input033[] __initconst = {
+static const u8 enc_input033[] = {
 	0x92, 0x22, 0xf9, 0x01, 0x8e, 0x54, 0xfd, 0x6d,
 	0xe1, 0x20, 0x08, 0x06, 0xa9, 0xee, 0x8e, 0x4c,
 	0xc9, 0x04, 0xd2, 0x9f, 0x25, 0xcb, 0xa1, 0x93
 };
-static const u8 enc_output033[] __initconst = {
+static const u8 enc_output033[] = {
 	0x12, 0x30, 0x32, 0x43, 0x7b, 0x4b, 0xfd, 0x69,
 	0x20, 0xe8, 0xf7, 0xe7, 0xe0, 0x08, 0x7a, 0xe4,
 	0x88, 0x9e, 0xbe, 0x7a, 0x0a, 0xd0, 0xe9, 0x00,
 	0x3c, 0xf6, 0x8f, 0x17, 0x95, 0x50, 0xda, 0x63,
 	0xd3, 0xb9, 0x6c, 0x2d, 0x55, 0x41, 0x18, 0x65
 };
-static const u8 enc_assoc033[] __initconst = {
+static const u8 enc_assoc033[] = {
 	0x3e, 0x8b, 0xc5, 0xad, 0xe1, 0x82, 0xff, 0x08
 };
-static const u8 enc_nonce033[] __initconst = {
+static const u8 enc_nonce033[] = {
 	0x6b, 0x28, 0x2e, 0xbe, 0xcc, 0x54, 0x1b, 0xcd,
 	0x78, 0x34, 0xed, 0x55
 };
-static const u8 enc_key033[] __initconst = {
+static const u8 enc_key033[] = {
 	0xc5, 0xbc, 0x09, 0x56, 0x56, 0x46, 0xe7, 0xed,
 	0xda, 0x95, 0x4f, 0x1f, 0x73, 0x92, 0x23, 0xda,
 	0xda, 0x20, 0xb9, 0x5c, 0x44, 0xab, 0x03, 0x3d,
@@ -2442,13 +2441,13 @@ static const u8 enc_key033[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input034[] __initconst = {
+static const u8 enc_input034[] = {
 	0xb0, 0x53, 0x99, 0x92, 0x86, 0xa2, 0x82, 0x4f,
 	0x42, 0xcc, 0x8c, 0x20, 0x3a, 0xb2, 0x4e, 0x2c,
 	0x97, 0xa6, 0x85, 0xad, 0xcc, 0x2a, 0xd3, 0x26,
 	0x62, 0x55, 0x8e, 0x55, 0xa5, 0xc7, 0x29
 };
-static const u8 enc_output034[] __initconst = {
+static const u8 enc_output034[] = {
 	0x45, 0xc7, 0xd6, 0xb5, 0x3a, 0xca, 0xd4, 0xab,
 	0xb6, 0x88, 0x76, 0xa6, 0xe9, 0x6a, 0x48, 0xfb,
 	0x59, 0x52, 0x4d, 0x2c, 0x92, 0xc9, 0xd8, 0xa1,
@@ -2456,12 +2455,12 @@ static const u8 enc_output034[] __initconst = {
 	0x6d, 0x3c, 0xa1, 0x0e, 0x31, 0x1b, 0x69, 0x5f,
 	0x3e, 0xae, 0x15, 0x51, 0x65, 0x24, 0x93
 };
-static const u8 enc_assoc034[] __initconst = { };
-static const u8 enc_nonce034[] __initconst = {
+static const u8 enc_assoc034[] = { };
+static const u8 enc_nonce034[] = {
 	0x04, 0xa9, 0xbe, 0x03, 0x50, 0x8a, 0x5f, 0x31,
 	0x37, 0x1a, 0x6f, 0xd2
 };
-static const u8 enc_key034[] __initconst = {
+static const u8 enc_key034[] = {
 	0x2e, 0xb5, 0x1c, 0x46, 0x9a, 0xa8, 0xeb, 0x9e,
 	0x6c, 0x54, 0xa8, 0x34, 0x9b, 0xae, 0x50, 0xa2,
 	0x0f, 0x0e, 0x38, 0x27, 0x11, 0xbb, 0xa1, 0x15,
@@ -2469,13 +2468,13 @@ static const u8 enc_key034[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input035[] __initconst = {
+static const u8 enc_input035[] = {
 	0xf4, 0x52, 0x06, 0xab, 0xc2, 0x55, 0x52, 0xb2,
 	0xab, 0xc9, 0xab, 0x7f, 0xa2, 0x43, 0x03, 0x5f,
 	0xed, 0xaa, 0xdd, 0xc3, 0xb2, 0x29, 0x39, 0x56,
 	0xf1, 0xea, 0x6e, 0x71, 0x56, 0xe7, 0xeb
 };
-static const u8 enc_output035[] __initconst = {
+static const u8 enc_output035[] = {
 	0x46, 0xa8, 0x0c, 0x41, 0x87, 0x02, 0x47, 0x20,
 	0x08, 0x46, 0x27, 0x58, 0x00, 0x80, 0xdd, 0xe5,
 	0xa3, 0xf4, 0xa1, 0x10, 0x93, 0xa7, 0x07, 0x6e,
@@ -2483,14 +2482,14 @@ static const u8 enc_output035[] __initconst = {
 	0x4d, 0x4a, 0xa2, 0x83, 0x5a, 0x52, 0xe7, 0x2d,
 	0x14, 0xdf, 0x0e, 0x4f, 0x47, 0xf2, 0x5f
 };
-static const u8 enc_assoc035[] __initconst = {
+static const u8 enc_assoc035[] = {
 	0x37, 0x46, 0x18, 0xa0, 0x6e, 0xa9, 0x8a, 0x48
 };
-static const u8 enc_nonce035[] __initconst = {
+static const u8 enc_nonce035[] = {
 	0x47, 0x0a, 0x33, 0x9e, 0xcb, 0x32, 0x19, 0xb8,
 	0xb8, 0x1a, 0x1f, 0x8b
 };
-static const u8 enc_key035[] __initconst = {
+static const u8 enc_key035[] = {
 	0x7f, 0x5b, 0x74, 0xc0, 0x7e, 0xd1, 0xb4, 0x0f,
 	0xd1, 0x43, 0x58, 0xfe, 0x2f, 0xf2, 0xa7, 0x40,
 	0xc1, 0x16, 0xc7, 0x70, 0x65, 0x10, 0xe6, 0xa4,
@@ -2498,13 +2497,13 @@ static const u8 enc_key035[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input036[] __initconst = {
+static const u8 enc_input036[] = {
 	0xb9, 0xc5, 0x54, 0xcb, 0xc3, 0x6a, 0xc1, 0x8a,
 	0xe8, 0x97, 0xdf, 0x7b, 0xee, 0xca, 0xc1, 0xdb,
 	0xeb, 0x4e, 0xaf, 0xa1, 0x56, 0xbb, 0x60, 0xce,
 	0x2e, 0x5d, 0x48, 0xf0, 0x57, 0x15, 0xe6, 0x78
 };
-static const u8 enc_output036[] __initconst = {
+static const u8 enc_output036[] = {
 	0xea, 0x29, 0xaf, 0xa4, 0x9d, 0x36, 0xe8, 0x76,
 	0x0f, 0x5f, 0xe1, 0x97, 0x23, 0xb9, 0x81, 0x1e,
 	0xd5, 0xd5, 0x19, 0x93, 0x4a, 0x44, 0x0f, 0x50,
@@ -2512,12 +2511,12 @@ static const u8 enc_output036[] __initconst = {
 	0x22, 0x25, 0x41, 0xaf, 0x46, 0xb8, 0x65, 0x33,
 	0xc6, 0xb6, 0x8d, 0x2f, 0xf1, 0x08, 0xa7, 0xea
 };
-static const u8 enc_assoc036[] __initconst = { };
-static const u8 enc_nonce036[] __initconst = {
+static const u8 enc_assoc036[] = { };
+static const u8 enc_nonce036[] = {
 	0x72, 0xcf, 0xd9, 0x0e, 0xf3, 0x02, 0x6c, 0xa2,
 	0x2b, 0x7e, 0x6e, 0x6a
 };
-static const u8 enc_key036[] __initconst = {
+static const u8 enc_key036[] = {
 	0xe1, 0x73, 0x1d, 0x58, 0x54, 0xe1, 0xb7, 0x0c,
 	0xb3, 0xff, 0xe8, 0xb7, 0x86, 0xa2, 0xb3, 0xeb,
 	0xf0, 0x99, 0x43, 0x70, 0x95, 0x47, 0x57, 0xb9,
@@ -2525,13 +2524,13 @@ static const u8 enc_key036[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input037[] __initconst = {
+static const u8 enc_input037[] = {
 	0x6b, 0x26, 0x04, 0x99, 0x6c, 0xd3, 0x0c, 0x14,
 	0xa1, 0x3a, 0x52, 0x57, 0xed, 0x6c, 0xff, 0xd3,
 	0xbc, 0x5e, 0x29, 0xd6, 0xb9, 0x7e, 0xb1, 0x79,
 	0x9e, 0xb3, 0x35, 0xe2, 0x81, 0xea, 0x45, 0x1e
 };
-static const u8 enc_output037[] __initconst = {
+static const u8 enc_output037[] = {
 	0x6d, 0xad, 0x63, 0x78, 0x97, 0x54, 0x4d, 0x8b,
 	0xf6, 0xbe, 0x95, 0x07, 0xed, 0x4d, 0x1b, 0xb2,
 	0xe9, 0x54, 0xbc, 0x42, 0x7e, 0x5d, 0xe7, 0x29,
@@ -2539,14 +2538,14 @@ static const u8 enc_output037[] __initconst = {
 	0x7b, 0x99, 0x7d, 0x93, 0xc9, 0x82, 0x18, 0x9d,
 	0x70, 0x95, 0xdc, 0x79, 0x4c, 0x74, 0x62, 0x32
 };
-static const u8 enc_assoc037[] __initconst = {
+static const u8 enc_assoc037[] = {
 	0x23, 0x33, 0xe5, 0xce, 0x0f, 0x93, 0xb0, 0x59
 };
-static const u8 enc_nonce037[] __initconst = {
+static const u8 enc_nonce037[] = {
 	0x26, 0x28, 0x80, 0xd4, 0x75, 0xf3, 0xda, 0xc5,
 	0x34, 0x0d, 0xd1, 0xb8
 };
-static const u8 enc_key037[] __initconst = {
+static const u8 enc_key037[] = {
 	0x27, 0xd8, 0x60, 0x63, 0x1b, 0x04, 0x85, 0xa4,
 	0x10, 0x70, 0x2f, 0xea, 0x61, 0xbc, 0x87, 0x3f,
 	0x34, 0x42, 0x26, 0x0c, 0xad, 0xed, 0x4a, 0xbd,
@@ -2554,7 +2553,7 @@ static const u8 enc_key037[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input038[] __initconst = {
+static const u8 enc_input038[] = {
 	0x97, 0x3d, 0x0c, 0x75, 0x38, 0x26, 0xba, 0xe4,
 	0x66, 0xcf, 0x9a, 0xbb, 0x34, 0x93, 0x15, 0x2e,
 	0x9d, 0xe7, 0x81, 0x9e, 0x2b, 0xd0, 0xc7, 0x11,
@@ -2562,7 +2561,7 @@ static const u8 enc_input038[] __initconst = {
 	0x1a, 0xa3, 0xce, 0xdc, 0x0d, 0xfd, 0x7b, 0x46,
 	0x7e, 0x26, 0x22, 0x8b, 0xc8, 0x6c, 0x9a
 };
-static const u8 enc_output038[] __initconst = {
+static const u8 enc_output038[] = {
 	0xfb, 0xa7, 0x8a, 0xe4, 0xf9, 0xd8, 0x08, 0xa6,
 	0x2e, 0x3d, 0xa4, 0x0b, 0xe2, 0xcb, 0x77, 0x00,
 	0xc3, 0x61, 0x3d, 0x9e, 0xb2, 0xc5, 0x29, 0xc6,
@@ -2572,12 +2571,12 @@ static const u8 enc_output038[] __initconst = {
 	0x04, 0x69, 0x56, 0xdb, 0x3a, 0x51, 0x29, 0x08,
 	0xbd, 0x7a, 0xfc, 0x8f, 0x2a, 0xb0, 0xa9
 };
-static const u8 enc_assoc038[] __initconst = { };
-static const u8 enc_nonce038[] __initconst = {
+static const u8 enc_assoc038[] = { };
+static const u8 enc_nonce038[] = {
 	0xe7, 0x4a, 0x51, 0x5e, 0x7e, 0x21, 0x02, 0xb9,
 	0x0b, 0xef, 0x55, 0xd2
 };
-static const u8 enc_key038[] __initconst = {
+static const u8 enc_key038[] = {
 	0xcf, 0x0d, 0x40, 0xa4, 0x64, 0x4e, 0x5f, 0x51,
 	0x81, 0x51, 0x65, 0xd5, 0x30, 0x1b, 0x22, 0x63,
 	0x1f, 0x45, 0x44, 0xc4, 0x9a, 0x18, 0x78, 0xe3,
@@ -2585,7 +2584,7 @@ static const u8 enc_key038[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input039[] __initconst = {
+static const u8 enc_input039[] = {
 	0xa9, 0x89, 0x95, 0x50, 0x4d, 0xf1, 0x6f, 0x74,
 	0x8b, 0xfb, 0x77, 0x85, 0xff, 0x91, 0xee, 0xb3,
 	0xb6, 0x60, 0xea, 0x9e, 0xd3, 0x45, 0x0c, 0x3d,
@@ -2593,7 +2592,7 @@ static const u8 enc_input039[] __initconst = {
 	0xa9, 0x97, 0x8d, 0x75, 0x54, 0x2e, 0xf9, 0x1c,
 	0x45, 0x67, 0x62, 0x21, 0x56, 0x40, 0xb9
 };
-static const u8 enc_output039[] __initconst = {
+static const u8 enc_output039[] = {
 	0xa1, 0xff, 0xed, 0x80, 0x76, 0x18, 0x29, 0xec,
 	0xce, 0x24, 0x2e, 0x0e, 0x88, 0xb1, 0x38, 0x04,
 	0x90, 0x16, 0xbc, 0xa0, 0x18, 0xda, 0x2b, 0x6e,
@@ -2603,14 +2602,14 @@ static const u8 enc_output039[] __initconst = {
 	0xcb, 0xf0, 0xbe, 0xfd, 0xa0, 0xb7, 0x02, 0x42,
 	0xc6, 0x40, 0xd7, 0xcd, 0x02, 0xd7, 0xa3
 };
-static const u8 enc_assoc039[] __initconst = {
+static const u8 enc_assoc039[] = {
 	0xb3, 0xe4, 0x06, 0x46, 0x83, 0xb0, 0x2d, 0x84
 };
-static const u8 enc_nonce039[] __initconst = {
+static const u8 enc_nonce039[] = {
 	0xd4, 0xd8, 0x07, 0x34, 0x16, 0x83, 0x82, 0x5b,
 	0x31, 0xcd, 0x4d, 0x95
 };
-static const u8 enc_key039[] __initconst = {
+static const u8 enc_key039[] = {
 	0x6c, 0xbf, 0xd7, 0x1c, 0x64, 0x5d, 0x18, 0x4c,
 	0xf5, 0xd2, 0x3c, 0x40, 0x2b, 0xdb, 0x0d, 0x25,
 	0xec, 0x54, 0x89, 0x8c, 0x8a, 0x02, 0x73, 0xd4,
@@ -2618,7 +2617,7 @@ static const u8 enc_key039[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input040[] __initconst = {
+static const u8 enc_input040[] = {
 	0xd0, 0x96, 0x80, 0x31, 0x81, 0xbe, 0xef, 0x9e,
 	0x00, 0x8f, 0xf8, 0x5d, 0x5d, 0xdc, 0x38, 0xdd,
 	0xac, 0xf0, 0xf0, 0x9e, 0xe5, 0xf7, 0xe0, 0x7f,
@@ -2628,7 +2627,7 @@ static const u8 enc_input040[] __initconst = {
 	0x18, 0xf1, 0x18, 0x55, 0x86, 0xbf, 0xea, 0x9d,
 	0x4c, 0x68, 0x5d, 0x50, 0xe4, 0xbb, 0x9a, 0x82
 };
-static const u8 enc_output040[] __initconst = {
+static const u8 enc_output040[] = {
 	0x9a, 0x4e, 0xf2, 0x2b, 0x18, 0x16, 0x77, 0xb5,
 	0x75, 0x5c, 0x08, 0xf7, 0x47, 0xc0, 0xf8, 0xd8,
 	0xe8, 0xd4, 0xc1, 0x8a, 0x9c, 0xc2, 0x40, 0x5c,
@@ -2640,12 +2639,12 @@ static const u8 enc_output040[] __initconst = {
 	0x9f, 0xf3, 0x42, 0x7a, 0x0f, 0x32, 0xfa, 0x56,
 	0x6d, 0x9c, 0xa0, 0xa7, 0x8a, 0xef, 0xc0, 0x13
 };
-static const u8 enc_assoc040[] __initconst = { };
-static const u8 enc_nonce040[] __initconst = {
+static const u8 enc_assoc040[] = { };
+static const u8 enc_nonce040[] = {
 	0xd6, 0x10, 0x40, 0xa3, 0x13, 0xed, 0x49, 0x28,
 	0x23, 0xcc, 0x06, 0x5b
 };
-static const u8 enc_key040[] __initconst = {
+static const u8 enc_key040[] = {
 	0x5b, 0x1d, 0x10, 0x35, 0xc0, 0xb1, 0x7e, 0xe0,
 	0xb0, 0x44, 0x47, 0x67, 0xf8, 0x0a, 0x25, 0xb8,
 	0xc1, 0xb7, 0x41, 0xf4, 0xb5, 0x0a, 0x4d, 0x30,
@@ -2653,7 +2652,7 @@ static const u8 enc_key040[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input041[] __initconst = {
+static const u8 enc_input041[] = {
 	0x94, 0xee, 0x16, 0x6d, 0x6d, 0x6e, 0xcf, 0x88,
 	0x32, 0x43, 0x71, 0x36, 0xb4, 0xae, 0x80, 0x5d,
 	0x42, 0x88, 0x64, 0x35, 0x95, 0x86, 0xd9, 0x19,
@@ -2663,7 +2662,7 @@ static const u8 enc_input041[] __initconst = {
 	0x4a, 0x10, 0x8c, 0x7d, 0x7c, 0xe3, 0x4e, 0x6c,
 	0x6f, 0x8e, 0xa1, 0xbe, 0xc0, 0x56, 0x73, 0x17
 };
-static const u8 enc_output041[] __initconst = {
+static const u8 enc_output041[] = {
 	0x5f, 0xbb, 0xde, 0xcc, 0x34, 0xbe, 0x20, 0x16,
 	0x14, 0xf6, 0x36, 0x03, 0x1e, 0xeb, 0x42, 0xf1,
 	0xca, 0xce, 0x3c, 0x79, 0xa1, 0x2c, 0xff, 0xd8,
@@ -2675,14 +2674,14 @@ static const u8 enc_output041[] __initconst = {
 	0x09, 0x26, 0x3d, 0xa7, 0xb4, 0xcb, 0x92, 0x14,
 	0x52, 0xf9, 0x7d, 0xca, 0x40, 0xf5, 0x80, 0xec
 };
-static const u8 enc_assoc041[] __initconst = {
+static const u8 enc_assoc041[] = {
 	0x71, 0x93, 0xf6, 0x23, 0x66, 0x33, 0x21, 0xa2
 };
-static const u8 enc_nonce041[] __initconst = {
+static const u8 enc_nonce041[] = {
 	0xd3, 0x1c, 0x21, 0xab, 0xa1, 0x75, 0xb7, 0x0d,
 	0xe4, 0xeb, 0xb1, 0x9c
 };
-static const u8 enc_key041[] __initconst = {
+static const u8 enc_key041[] = {
 	0x97, 0xd6, 0x35, 0xc4, 0xf4, 0x75, 0x74, 0xd9,
 	0x99, 0x8a, 0x90, 0x87, 0x5d, 0xa1, 0xd3, 0xa2,
 	0x84, 0xb7, 0x55, 0xb2, 0xd3, 0x92, 0x97, 0xa5,
@@ -2690,7 +2689,7 @@ static const u8 enc_key041[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input042[] __initconst = {
+static const u8 enc_input042[] = {
 	0xb4, 0x29, 0xeb, 0x80, 0xfb, 0x8f, 0xe8, 0xba,
 	0xed, 0xa0, 0xc8, 0x5b, 0x9c, 0x33, 0x34, 0x58,
 	0xe7, 0xc2, 0x99, 0x2e, 0x55, 0x84, 0x75, 0x06,
@@ -2705,7 +2704,7 @@ static const u8 enc_input042[] __initconst = {
 	0xc3, 0xea, 0x46, 0xc8, 0x76, 0x1d, 0x50, 0xf5,
 	0x92
 };
-static const u8 enc_output042[] __initconst = {
+static const u8 enc_output042[] = {
 	0xd0, 0x10, 0x2f, 0x6c, 0x25, 0x8b, 0xf4, 0x97,
 	0x42, 0xce, 0xc3, 0x4c, 0xf2, 0xd0, 0xfe, 0xdf,
 	0x23, 0xd1, 0x05, 0xfb, 0x4c, 0x84, 0xcf, 0x98,
@@ -2722,12 +2721,12 @@ static const u8 enc_output042[] __initconst = {
 	0x19, 0x61, 0x21, 0x27, 0xce, 0x49, 0x99, 0x3b,
 	0xb0
 };
-static const u8 enc_assoc042[] __initconst = { };
-static const u8 enc_nonce042[] __initconst = {
+static const u8 enc_assoc042[] = { };
+static const u8 enc_nonce042[] = {
 	0x17, 0xc8, 0x6a, 0x8a, 0xbb, 0xb7, 0xe0, 0x03,
 	0xac, 0xde, 0x27, 0x99
 };
-static const u8 enc_key042[] __initconst = {
+static const u8 enc_key042[] = {
 	0xfe, 0x6e, 0x55, 0xbd, 0xae, 0xd1, 0xf7, 0x28,
 	0x4c, 0xa5, 0xfc, 0x0f, 0x8c, 0x5f, 0x2b, 0x8d,
 	0xf5, 0x6d, 0xc0, 0xf4, 0x9e, 0x8c, 0xa6, 0x6a,
@@ -2735,7 +2734,7 @@ static const u8 enc_key042[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input043[] __initconst = {
+static const u8 enc_input043[] = {
 	0xce, 0xb5, 0x34, 0xce, 0x50, 0xdc, 0x23, 0xff,
 	0x63, 0x8a, 0xce, 0x3e, 0xf6, 0x3a, 0xb2, 0xcc,
 	0x29, 0x73, 0xee, 0xad, 0xa8, 0x07, 0x85, 0xfc,
@@ -2750,7 +2749,7 @@ static const u8 enc_input043[] __initconst = {
 	0xdc, 0x47, 0x97, 0x0d, 0x84, 0xa6, 0xad, 0x31,
 	0x76
 };
-static const u8 enc_output043[] __initconst = {
+static const u8 enc_output043[] = {
 	0x75, 0x45, 0x39, 0x1b, 0x51, 0xde, 0x01, 0xd5,
 	0xc5, 0x3d, 0xfa, 0xca, 0x77, 0x79, 0x09, 0x06,
 	0x3e, 0x58, 0xed, 0xee, 0x4b, 0xb1, 0x22, 0x7e,
@@ -2767,14 +2766,14 @@ static const u8 enc_output043[] __initconst = {
 	0xc7, 0xbb, 0xf1, 0x0d, 0xca, 0xdd, 0x7f, 0xac,
 	0xf6
 };
-static const u8 enc_assoc043[] __initconst = {
+static const u8 enc_assoc043[] = {
 	0xa1, 0x1c, 0x40, 0xb6, 0x03, 0x76, 0x73, 0x30
 };
-static const u8 enc_nonce043[] __initconst = {
+static const u8 enc_nonce043[] = {
 	0x46, 0x36, 0x2f, 0x45, 0xd6, 0x37, 0x9e, 0x63,
 	0xe5, 0x22, 0x94, 0x60
 };
-static const u8 enc_key043[] __initconst = {
+static const u8 enc_key043[] = {
 	0xaa, 0xbc, 0x06, 0x34, 0x74, 0xe6, 0x5c, 0x4c,
 	0x3e, 0x9b, 0xdc, 0x48, 0x0d, 0xea, 0x97, 0xb4,
 	0x51, 0x10, 0xc8, 0x61, 0x88, 0x46, 0xff, 0x6b,
@@ -2782,24 +2781,24 @@ static const u8 enc_key043[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input044[] __initconst = {
+static const u8 enc_input044[] = {
 	0xe5, 0xcc, 0xaa, 0x44, 0x1b, 0xc8, 0x14, 0x68,
 	0x8f, 0x8f, 0x6e, 0x8f, 0x28, 0xb5, 0x00, 0xb2
 };
-static const u8 enc_output044[] __initconst = {
+static const u8 enc_output044[] = {
 	0x7e, 0x72, 0xf5, 0xa1, 0x85, 0xaf, 0x16, 0xa6,
 	0x11, 0x92, 0x1b, 0x43, 0x8f, 0x74, 0x9f, 0x0b,
 	0x12, 0x42, 0xc6, 0x70, 0x73, 0x23, 0x34, 0x02,
 	0x9a, 0xdf, 0xe1, 0xc5, 0x00, 0x16, 0x51, 0xe4
 };
-static const u8 enc_assoc044[] __initconst = {
+static const u8 enc_assoc044[] = {
 	0x02
 };
-static const u8 enc_nonce044[] __initconst = {
+static const u8 enc_nonce044[] = {
 	0x87, 0x34, 0x5f, 0x10, 0x55, 0xfd, 0x9e, 0x21,
 	0x02, 0xd5, 0x06, 0x56
 };
-static const u8 enc_key044[] __initconst = {
+static const u8 enc_key044[] = {
 	0x7d, 0x00, 0xb4, 0x80, 0x95, 0xad, 0xfa, 0x32,
 	0x72, 0x05, 0x06, 0x07, 0xb2, 0x64, 0x18, 0x50,
 	0x02, 0xba, 0x99, 0x95, 0x7c, 0x49, 0x8b, 0xe0,
@@ -2807,24 +2806,24 @@ static const u8 enc_key044[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input045[] __initconst = {
+static const u8 enc_input045[] = {
 	0x02, 0xcd, 0xe1, 0x68, 0xfb, 0xa3, 0xf5, 0x44,
 	0xbb, 0xd0, 0x33, 0x2f, 0x7a, 0xde, 0xad, 0xa8
 };
-static const u8 enc_output045[] __initconst = {
+static const u8 enc_output045[] = {
 	0x85, 0xf2, 0x9a, 0x71, 0x95, 0x57, 0xcd, 0xd1,
 	0x4d, 0x1f, 0x8f, 0xff, 0xab, 0x6d, 0x9e, 0x60,
 	0x73, 0x2c, 0xa3, 0x2b, 0xec, 0xd5, 0x15, 0xa1,
 	0xed, 0x35, 0x3f, 0x54, 0x2e, 0x99, 0x98, 0x58
 };
-static const u8 enc_assoc045[] __initconst = {
+static const u8 enc_assoc045[] = {
 	0xb6, 0x48
 };
-static const u8 enc_nonce045[] __initconst = {
+static const u8 enc_nonce045[] = {
 	0x87, 0xa3, 0x16, 0x3e, 0xc0, 0x59, 0x8a, 0xd9,
 	0x5b, 0x3a, 0xa7, 0x13
 };
-static const u8 enc_key045[] __initconst = {
+static const u8 enc_key045[] = {
 	0x64, 0x32, 0x71, 0x7f, 0x1d, 0xb8, 0x5e, 0x41,
 	0xac, 0x78, 0x36, 0xbc, 0xe2, 0x51, 0x85, 0xa0,
 	0x80, 0xd5, 0x76, 0x2b, 0x9e, 0x2b, 0x18, 0x44,
@@ -2832,25 +2831,25 @@ static const u8 enc_key045[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input046[] __initconst = {
+static const u8 enc_input046[] = {
 	0x16, 0xdd, 0xd2, 0x3f, 0xf5, 0x3f, 0x3d, 0x23,
 	0xc0, 0x63, 0x34, 0x48, 0x70, 0x40, 0xeb, 0x47
 };
-static const u8 enc_output046[] __initconst = {
+static const u8 enc_output046[] = {
 	0xc1, 0xb2, 0x95, 0x93, 0x6d, 0x56, 0xfa, 0xda,
 	0xc0, 0x3e, 0x5f, 0x74, 0x2b, 0xff, 0x73, 0xa1,
 	0x39, 0xc4, 0x57, 0xdb, 0xab, 0x66, 0x38, 0x2b,
 	0xab, 0xb3, 0xb5, 0x58, 0x00, 0xcd, 0xa5, 0xb8
 };
-static const u8 enc_assoc046[] __initconst = {
+static const u8 enc_assoc046[] = {
 	0xbd, 0x4c, 0xd0, 0x2f, 0xc7, 0x50, 0x2b, 0xbd,
 	0xbd, 0xf6, 0xc9, 0xa3, 0xcb, 0xe8, 0xf0
 };
-static const u8 enc_nonce046[] __initconst = {
+static const u8 enc_nonce046[] = {
 	0x6f, 0x57, 0x3a, 0xa8, 0x6b, 0xaa, 0x49, 0x2b,
 	0xa4, 0x65, 0x96, 0xdf
 };
-static const u8 enc_key046[] __initconst = {
+static const u8 enc_key046[] = {
 	0x8e, 0x34, 0xcf, 0x73, 0xd2, 0x45, 0xa1, 0x08,
 	0x2a, 0x92, 0x0b, 0x86, 0x36, 0x4e, 0xb8, 0x96,
 	0xc4, 0x94, 0x64, 0x67, 0xbc, 0xb3, 0xd5, 0x89,
@@ -2858,25 +2857,25 @@ static const u8 enc_key046[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input047[] __initconst = {
+static const u8 enc_input047[] = {
 	0x62, 0x3b, 0x78, 0x50, 0xc3, 0x21, 0xe2, 0xcf,
 	0x0c, 0x6f, 0xbc, 0xc8, 0xdf, 0xd1, 0xaf, 0xf2
 };
-static const u8 enc_output047[] __initconst = {
+static const u8 enc_output047[] = {
 	0xc8, 0x4c, 0x9b, 0xb7, 0xc6, 0x1c, 0x1b, 0xcb,
 	0x17, 0x77, 0x2a, 0x1c, 0x50, 0x0c, 0x50, 0x95,
 	0xdb, 0xad, 0xf7, 0xa5, 0x13, 0x8c, 0xa0, 0x34,
 	0x59, 0xa2, 0xcd, 0x65, 0x83, 0x1e, 0x09, 0x2f
 };
-static const u8 enc_assoc047[] __initconst = {
+static const u8 enc_assoc047[] = {
 	0x89, 0xcc, 0xe9, 0xfb, 0x47, 0x44, 0x1d, 0x07,
 	0xe0, 0x24, 0x5a, 0x66, 0xfe, 0x8b, 0x77, 0x8b
 };
-static const u8 enc_nonce047[] __initconst = {
+static const u8 enc_nonce047[] = {
 	0x1a, 0x65, 0x18, 0xf0, 0x2e, 0xde, 0x1d, 0xa6,
 	0x80, 0x92, 0x66, 0xd9
 };
-static const u8 enc_key047[] __initconst = {
+static const u8 enc_key047[] = {
 	0xcb, 0x55, 0x75, 0xf5, 0xc7, 0xc4, 0x5c, 0x91,
 	0xcf, 0x32, 0x0b, 0x13, 0x9f, 0xb5, 0x94, 0x23,
 	0x75, 0x60, 0xd0, 0xa3, 0xe6, 0xf8, 0x65, 0xa6,
@@ -2884,26 +2883,26 @@ static const u8 enc_key047[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input048[] __initconst = {
+static const u8 enc_input048[] = {
 	0x87, 0xb3, 0xa4, 0xd7, 0xb2, 0x6d, 0x8d, 0x32,
 	0x03, 0xa0, 0xde, 0x1d, 0x64, 0xef, 0x82, 0xe3
 };
-static const u8 enc_output048[] __initconst = {
+static const u8 enc_output048[] = {
 	0x94, 0xbc, 0x80, 0x62, 0x1e, 0xd1, 0xe7, 0x1b,
 	0x1f, 0xd2, 0xb5, 0xc3, 0xa1, 0x5e, 0x35, 0x68,
 	0x33, 0x35, 0x11, 0x86, 0x17, 0x96, 0x97, 0x84,
 	0x01, 0x59, 0x8b, 0x96, 0x37, 0x22, 0xf5, 0xb3
 };
-static const u8 enc_assoc048[] __initconst = {
+static const u8 enc_assoc048[] = {
 	0xd1, 0x9f, 0x2d, 0x98, 0x90, 0x95, 0xf7, 0xab,
 	0x03, 0xa5, 0xfd, 0xe8, 0x44, 0x16, 0xe0, 0x0c,
 	0x0e
 };
-static const u8 enc_nonce048[] __initconst = {
+static const u8 enc_nonce048[] = {
 	0x56, 0x4d, 0xee, 0x49, 0xab, 0x00, 0xd2, 0x40,
 	0xfc, 0x10, 0x68, 0xc3
 };
-static const u8 enc_key048[] __initconst = {
+static const u8 enc_key048[] = {
 	0xa5, 0x56, 0x9e, 0x72, 0x9a, 0x69, 0xb2, 0x4b,
 	0xa6, 0xe0, 0xff, 0x15, 0xc4, 0x62, 0x78, 0x97,
 	0x43, 0x68, 0x24, 0xc9, 0x41, 0xe9, 0xd0, 0x0b,
@@ -2911,27 +2910,27 @@ static const u8 enc_key048[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input049[] __initconst = {
+static const u8 enc_input049[] = {
 	0xe6, 0x01, 0xb3, 0x85, 0x57, 0x79, 0x7d, 0xa2,
 	0xf8, 0xa4, 0x10, 0x6a, 0x08, 0x9d, 0x1d, 0xa6
 };
-static const u8 enc_output049[] __initconst = {
+static const u8 enc_output049[] = {
 	0x29, 0x9b, 0x5d, 0x3f, 0x3d, 0x03, 0xc0, 0x87,
 	0x20, 0x9a, 0x16, 0xe2, 0x85, 0x14, 0x31, 0x11,
 	0x4b, 0x45, 0x4e, 0xd1, 0x98, 0xde, 0x11, 0x7e,
 	0x83, 0xec, 0x49, 0xfa, 0x8d, 0x85, 0x08, 0xd6
 };
-static const u8 enc_assoc049[] __initconst = {
+static const u8 enc_assoc049[] = {
 	0x5e, 0x64, 0x70, 0xfa, 0xcd, 0x99, 0xc1, 0xd8,
 	0x1e, 0x37, 0xcd, 0x44, 0x01, 0x5f, 0xe1, 0x94,
 	0x80, 0xa2, 0xa4, 0xd3, 0x35, 0x2a, 0x4f, 0xf5,
 	0x60, 0xc0, 0x64, 0x0f, 0xdb, 0xda
 };
-static const u8 enc_nonce049[] __initconst = {
+static const u8 enc_nonce049[] = {
 	0xdf, 0x87, 0x13, 0xe8, 0x7e, 0xc3, 0xdb, 0xcf,
 	0xad, 0x14, 0xd5, 0x3e
 };
-static const u8 enc_key049[] __initconst = {
+static const u8 enc_key049[] = {
 	0x56, 0x20, 0x74, 0x65, 0xb4, 0xe4, 0x8e, 0x6d,
 	0x04, 0x63, 0x0f, 0x4a, 0x42, 0xf3, 0x5c, 0xfc,
 	0x16, 0x3a, 0xb2, 0x89, 0xc2, 0x2a, 0x2b, 0x47,
@@ -2939,27 +2938,27 @@ static const u8 enc_key049[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input050[] __initconst = {
+static const u8 enc_input050[] = {
 	0xdc, 0x9e, 0x9e, 0xaf, 0x11, 0xe3, 0x14, 0x18,
 	0x2d, 0xf6, 0xa4, 0xeb, 0xa1, 0x7a, 0xec, 0x9c
 };
-static const u8 enc_output050[] __initconst = {
+static const u8 enc_output050[] = {
 	0x60, 0x5b, 0xbf, 0x90, 0xae, 0xb9, 0x74, 0xf6,
 	0x60, 0x2b, 0xc7, 0x78, 0x05, 0x6f, 0x0d, 0xca,
 	0x38, 0xea, 0x23, 0xd9, 0x90, 0x54, 0xb4, 0x6b,
 	0x42, 0xff, 0xe0, 0x04, 0x12, 0x9d, 0x22, 0x04
 };
-static const u8 enc_assoc050[] __initconst = {
+static const u8 enc_assoc050[] = {
 	0xba, 0x44, 0x6f, 0x6f, 0x9a, 0x0c, 0xed, 0x22,
 	0x45, 0x0f, 0xeb, 0x10, 0x73, 0x7d, 0x90, 0x07,
 	0xfd, 0x69, 0xab, 0xc1, 0x9b, 0x1d, 0x4d, 0x90,
 	0x49, 0xa5, 0x55, 0x1e, 0x86, 0xec, 0x2b, 0x37
 };
-static const u8 enc_nonce050[] __initconst = {
+static const u8 enc_nonce050[] = {
 	0x8d, 0xf4, 0xb1, 0x5a, 0x88, 0x8c, 0x33, 0x28,
 	0x6a, 0x7b, 0x76, 0x51
 };
-static const u8 enc_key050[] __initconst = {
+static const u8 enc_key050[] = {
 	0x39, 0x37, 0x98, 0x6a, 0xf8, 0x6d, 0xaf, 0xc1,
 	0xba, 0x0c, 0x46, 0x72, 0xd8, 0xab, 0xc4, 0x6c,
 	0x20, 0x70, 0x62, 0x68, 0x2d, 0x9c, 0x26, 0x4a,
@@ -2967,28 +2966,28 @@ static const u8 enc_key050[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input051[] __initconst = {
+static const u8 enc_input051[] = {
 	0x81, 0xce, 0x84, 0xed, 0xe9, 0xb3, 0x58, 0x59,
 	0xcc, 0x8c, 0x49, 0xa8, 0xf6, 0xbe, 0x7d, 0xc6
 };
-static const u8 enc_output051[] __initconst = {
+static const u8 enc_output051[] = {
 	0x7b, 0x7c, 0xe0, 0xd8, 0x24, 0x80, 0x9a, 0x70,
 	0xde, 0x32, 0x56, 0x2c, 0xcf, 0x2c, 0x2b, 0xbd,
 	0x15, 0xd4, 0x4a, 0x00, 0xce, 0x0d, 0x19, 0xb4,
 	0x23, 0x1f, 0x92, 0x1e, 0x22, 0xbc, 0x0a, 0x43
 };
-static const u8 enc_assoc051[] __initconst = {
+static const u8 enc_assoc051[] = {
 	0xd4, 0x1a, 0x82, 0x8d, 0x5e, 0x71, 0x82, 0x92,
 	0x47, 0x02, 0x19, 0x05, 0x40, 0x2e, 0xa2, 0x57,
 	0xdc, 0xcb, 0xc3, 0xb8, 0x0f, 0xcd, 0x56, 0x75,
 	0x05, 0x6b, 0x68, 0xbb, 0x59, 0xe6, 0x2e, 0x88,
 	0x73
 };
-static const u8 enc_nonce051[] __initconst = {
+static const u8 enc_nonce051[] = {
 	0xbe, 0x40, 0xe5, 0xf1, 0xa1, 0x18, 0x17, 0xa0,
 	0xa8, 0xfa, 0x89, 0x49
 };
-static const u8 enc_key051[] __initconst = {
+static const u8 enc_key051[] = {
 	0x36, 0x37, 0x2a, 0xbc, 0xdb, 0x78, 0xe0, 0x27,
 	0x96, 0x46, 0xac, 0x3d, 0x17, 0x6b, 0x96, 0x74,
 	0xe9, 0x15, 0x4e, 0xec, 0xf0, 0xd5, 0x46, 0x9c,
@@ -2996,17 +2995,17 @@ static const u8 enc_key051[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input052[] __initconst = {
+static const u8 enc_input052[] = {
 	0xa6, 0x67, 0x47, 0xc8, 0x9e, 0x85, 0x7a, 0xf3,
 	0xa1, 0x8e, 0x2c, 0x79, 0x50, 0x00, 0x87, 0xed
 };
-static const u8 enc_output052[] __initconst = {
+static const u8 enc_output052[] = {
 	0xca, 0x82, 0xbf, 0xf3, 0xe2, 0xf3, 0x10, 0xcc,
 	0xc9, 0x76, 0x67, 0x2c, 0x44, 0x15, 0xe6, 0x9b,
 	0x57, 0x63, 0x8c, 0x62, 0xa5, 0xd8, 0x5d, 0xed,
 	0x77, 0x4f, 0x91, 0x3c, 0x81, 0x3e, 0xa0, 0x32
 };
-static const u8 enc_assoc052[] __initconst = {
+static const u8 enc_assoc052[] = {
 	0x3f, 0x2d, 0xd4, 0x9b, 0xbf, 0x09, 0xd6, 0x9a,
 	0x78, 0xa3, 0xd8, 0x0e, 0xa2, 0x56, 0x66, 0x14,
 	0xfc, 0x37, 0x94, 0x74, 0x19, 0x6c, 0x1a, 0xae,
@@ -3014,11 +3013,11 @@ static const u8 enc_assoc052[] __initconst = {
 	0x6f, 0x42, 0xca, 0x42, 0x05, 0x6a, 0x97, 0x92,
 	0xcc, 0x1b, 0x9f, 0xb3, 0xc7, 0xd2, 0x61
 };
-static const u8 enc_nonce052[] __initconst = {
+static const u8 enc_nonce052[] = {
 	0x84, 0xc8, 0x7d, 0xae, 0x4e, 0xee, 0x27, 0x73,
 	0x0e, 0xc3, 0x5d, 0x12
 };
-static const u8 enc_key052[] __initconst = {
+static const u8 enc_key052[] = {
 	0x9f, 0x14, 0x79, 0xed, 0x09, 0x7d, 0x7f, 0xe5,
 	0x29, 0xc1, 0x1f, 0x2f, 0x5a, 0xdd, 0x9a, 0xaf,
 	0xf4, 0xa1, 0xca, 0x0b, 0x68, 0x99, 0x7a, 0x2c,
@@ -3026,13 +3025,13 @@ static const u8 enc_key052[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input053[] __initconst = {
+static const u8 enc_input053[] = {
 	0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83,
 	0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8,
 	0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b,
 	0x80, 0xe0, 0x30, 0xbe, 0xeb, 0xd3, 0x29, 0xbe
 };
-static const u8 enc_output053[] __initconst = {
+static const u8 enc_output053[] = {
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@@ -3040,14 +3039,14 @@ static const u8 enc_output053[] __initconst = {
 	0xe6, 0xd3, 0xd7, 0x32, 0x4a, 0x1c, 0xbb, 0xa7,
 	0x77, 0xbb, 0xb0, 0xec, 0xdd, 0xa3, 0x78, 0x07
 };
-static const u8 enc_assoc053[] __initconst = {
+static const u8 enc_assoc053[] = {
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
 };
-static const u8 enc_nonce053[] __initconst = {
+static const u8 enc_nonce053[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
 };
-static const u8 enc_key053[] __initconst = {
+static const u8 enc_key053[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -3055,7 +3054,7 @@ static const u8 enc_key053[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input054[] __initconst = {
+static const u8 enc_input054[] = {
 	0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83,
 	0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8,
 	0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b,
@@ -3065,7 +3064,7 @@ static const u8 enc_input054[] __initconst = {
 	0x08, 0x69, 0xff, 0xd2, 0xec, 0x5e, 0x26, 0xe5,
 	0x53, 0xb7, 0xb2, 0x27, 0xfe, 0x87, 0xfd, 0xbd
 };
-static const u8 enc_output054[] __initconst = {
+static const u8 enc_output054[] = {
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@@ -3077,14 +3076,14 @@ static const u8 enc_output054[] __initconst = {
 	0x06, 0x2d, 0xe6, 0x79, 0x5f, 0x27, 0x4f, 0xd2,
 	0xa3, 0x05, 0xd7, 0x69, 0x80, 0xbc, 0x9c, 0xce
 };
-static const u8 enc_assoc054[] __initconst = {
+static const u8 enc_assoc054[] = {
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
 };
-static const u8 enc_nonce054[] __initconst = {
+static const u8 enc_nonce054[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
 };
-static const u8 enc_key054[] __initconst = {
+static const u8 enc_key054[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -3092,7 +3091,7 @@ static const u8 enc_key054[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input055[] __initconst = {
+static const u8 enc_input055[] = {
 	0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83,
 	0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8,
 	0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b,
@@ -3110,7 +3109,7 @@ static const u8 enc_input055[] __initconst = {
 	0x77, 0xd3, 0x0b, 0xc5, 0x76, 0x92, 0xed, 0x38,
 	0xfb, 0xac, 0x01, 0x88, 0x38, 0x04, 0x88, 0xc7
 };
-static const u8 enc_output055[] __initconst = {
+static const u8 enc_output055[] = {
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@@ -3130,14 +3129,14 @@ static const u8 enc_output055[] __initconst = {
 	0xd8, 0xb4, 0x79, 0x02, 0xba, 0xae, 0xaf, 0xb3,
 	0x42, 0x03, 0x05, 0x15, 0x29, 0xaf, 0x28, 0x2e
 };
-static const u8 enc_assoc055[] __initconst = {
+static const u8 enc_assoc055[] = {
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
 };
-static const u8 enc_nonce055[] __initconst = {
+static const u8 enc_nonce055[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
 };
-static const u8 enc_key055[] __initconst = {
+static const u8 enc_key055[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -3145,13 +3144,13 @@ static const u8 enc_key055[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input056[] __initconst = {
+static const u8 enc_input056[] = {
 	0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c,
 	0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17,
 	0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84,
 	0x7f, 0x1f, 0xcf, 0x41, 0x14, 0x2c, 0xd6, 0x41
 };
-static const u8 enc_output056[] __initconst = {
+static const u8 enc_output056[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -3159,14 +3158,14 @@ static const u8 enc_output056[] __initconst = {
 	0xb3, 0x89, 0x1c, 0x84, 0x9c, 0xb5, 0x2c, 0x27,
 	0x74, 0x7e, 0xdf, 0xcf, 0x31, 0x21, 0x3b, 0xb6
 };
-static const u8 enc_assoc056[] __initconst = {
+static const u8 enc_assoc056[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce056[] __initconst = {
+static const u8 enc_nonce056[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
 };
-static const u8 enc_key056[] __initconst = {
+static const u8 enc_key056[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -3174,7 +3173,7 @@ static const u8 enc_key056[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input057[] __initconst = {
+static const u8 enc_input057[] = {
 	0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c,
 	0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17,
 	0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84,
@@ -3184,7 +3183,7 @@ static const u8 enc_input057[] __initconst = {
 	0xf7, 0x96, 0x00, 0x2d, 0x13, 0xa1, 0xd9, 0x1a,
 	0xac, 0x48, 0x4d, 0xd8, 0x01, 0x78, 0x02, 0x42
 };
-static const u8 enc_output057[] __initconst = {
+static const u8 enc_output057[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -3196,14 +3195,14 @@ static const u8 enc_output057[] __initconst = {
 	0xf0, 0xc1, 0x2d, 0x26, 0xef, 0x03, 0x02, 0x9b,
 	0x62, 0xc0, 0x08, 0xda, 0x27, 0xc5, 0xdc, 0x68
 };
-static const u8 enc_assoc057[] __initconst = {
+static const u8 enc_assoc057[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce057[] __initconst = {
+static const u8 enc_nonce057[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
 };
-static const u8 enc_key057[] __initconst = {
+static const u8 enc_key057[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -3211,7 +3210,7 @@ static const u8 enc_key057[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input058[] __initconst = {
+static const u8 enc_input058[] = {
 	0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c,
 	0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17,
 	0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84,
@@ -3229,7 +3228,7 @@ static const u8 enc_input058[] __initconst = {
 	0x88, 0x2c, 0xf4, 0x3a, 0x89, 0x6d, 0x12, 0xc7,
 	0x04, 0x53, 0xfe, 0x77, 0xc7, 0xfb, 0x77, 0x38
 };
-static const u8 enc_output058[] __initconst = {
+static const u8 enc_output058[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -3249,14 +3248,14 @@ static const u8 enc_output058[] __initconst = {
 	0xee, 0x65, 0x78, 0x30, 0x01, 0xc2, 0x56, 0x91,
 	0xfa, 0x28, 0xd0, 0xf5, 0xf1, 0xc1, 0xd7, 0x62
 };
-static const u8 enc_assoc058[] __initconst = {
+static const u8 enc_assoc058[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce058[] __initconst = {
+static const u8 enc_nonce058[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
 };
-static const u8 enc_key058[] __initconst = {
+static const u8 enc_key058[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -3264,13 +3263,13 @@ static const u8 enc_key058[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input059[] __initconst = {
+static const u8 enc_input059[] = {
 	0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03,
 	0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68,
 	0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb,
 	0x80, 0xe0, 0x30, 0x3e, 0xeb, 0xd3, 0x29, 0x3e
 };
-static const u8 enc_output059[] __initconst = {
+static const u8 enc_output059[] = {
 	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
 	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
 	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
@@ -3278,14 +3277,14 @@ static const u8 enc_output059[] __initconst = {
 	0x79, 0xba, 0x7a, 0x29, 0xf5, 0xa7, 0xbb, 0x75,
 	0x79, 0x7a, 0xf8, 0x7a, 0x61, 0x01, 0x29, 0xa4
 };
-static const u8 enc_assoc059[] __initconst = {
+static const u8 enc_assoc059[] = {
 	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
 	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80
 };
-static const u8 enc_nonce059[] __initconst = {
+static const u8 enc_nonce059[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
 };
-static const u8 enc_key059[] __initconst = {
+static const u8 enc_key059[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -3293,7 +3292,7 @@ static const u8 enc_key059[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input060[] __initconst = {
+static const u8 enc_input060[] = {
 	0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03,
 	0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68,
 	0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb,
@@ -3303,7 +3302,7 @@ static const u8 enc_input060[] __initconst = {
 	0x08, 0x69, 0xff, 0x52, 0xec, 0x5e, 0x26, 0x65,
 	0x53, 0xb7, 0xb2, 0xa7, 0xfe, 0x87, 0xfd, 0x3d
 };
-static const u8 enc_output060[] __initconst = {
+static const u8 enc_output060[] = {
 	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
 	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
 	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
@@ -3315,14 +3314,14 @@ static const u8 enc_output060[] __initconst = {
 	0x36, 0xb1, 0x74, 0x38, 0x19, 0xe1, 0xb9, 0xba,
 	0x15, 0x51, 0xe8, 0xed, 0x92, 0x2a, 0x95, 0x9a
 };
-static const u8 enc_assoc060[] __initconst = {
+static const u8 enc_assoc060[] = {
 	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
 	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80
 };
-static const u8 enc_nonce060[] __initconst = {
+static const u8 enc_nonce060[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
 };
-static const u8 enc_key060[] __initconst = {
+static const u8 enc_key060[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -3330,7 +3329,7 @@ static const u8 enc_key060[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input061[] __initconst = {
+static const u8 enc_input061[] = {
 	0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03,
 	0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68,
 	0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb,
@@ -3348,7 +3347,7 @@ static const u8 enc_input061[] __initconst = {
 	0x77, 0xd3, 0x0b, 0x45, 0x76, 0x92, 0xed, 0xb8,
 	0xfb, 0xac, 0x01, 0x08, 0x38, 0x04, 0x88, 0x47
 };
-static const u8 enc_output061[] __initconst = {
+static const u8 enc_output061[] = {
 	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
 	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
 	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
@@ -3368,14 +3367,14 @@ static const u8 enc_output061[] __initconst = {
 	0xfe, 0xac, 0x49, 0x55, 0x55, 0x4e, 0x80, 0x6f,
 	0x3a, 0x19, 0x02, 0xe2, 0x44, 0x32, 0xc0, 0x8a
 };
-static const u8 enc_assoc061[] __initconst = {
+static const u8 enc_assoc061[] = {
 	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
 	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80
 };
-static const u8 enc_nonce061[] __initconst = {
+static const u8 enc_nonce061[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
 };
-static const u8 enc_key061[] __initconst = {
+static const u8 enc_key061[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -3383,13 +3382,13 @@ static const u8 enc_key061[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input062[] __initconst = {
+static const u8 enc_input062[] = {
 	0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc,
 	0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97,
 	0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04,
 	0x7f, 0x1f, 0xcf, 0xc1, 0x14, 0x2c, 0xd6, 0xc1
 };
-static const u8 enc_output062[] __initconst = {
+static const u8 enc_output062[] = {
 	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
 	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
 	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
@@ -3397,14 +3396,14 @@ static const u8 enc_output062[] __initconst = {
 	0x20, 0xa3, 0x79, 0x8d, 0xf1, 0x29, 0x2c, 0x59,
 	0x72, 0xbf, 0x97, 0x41, 0xae, 0xc3, 0x8a, 0x19
 };
-static const u8 enc_assoc062[] __initconst = {
+static const u8 enc_assoc062[] = {
 	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
 	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f
 };
-static const u8 enc_nonce062[] __initconst = {
+static const u8 enc_nonce062[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
 };
-static const u8 enc_key062[] __initconst = {
+static const u8 enc_key062[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -3412,7 +3411,7 @@ static const u8 enc_key062[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input063[] __initconst = {
+static const u8 enc_input063[] = {
 	0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc,
 	0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97,
 	0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04,
@@ -3422,7 +3421,7 @@ static const u8 enc_input063[] __initconst = {
 	0xf7, 0x96, 0x00, 0xad, 0x13, 0xa1, 0xd9, 0x9a,
 	0xac, 0x48, 0x4d, 0x58, 0x01, 0x78, 0x02, 0xc2
 };
-static const u8 enc_output063[] __initconst = {
+static const u8 enc_output063[] = {
 	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
 	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
 	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
@@ -3434,14 +3433,14 @@ static const u8 enc_output063[] __initconst = {
 	0xc0, 0x3d, 0x9f, 0x67, 0x35, 0x4a, 0x97, 0xb2,
 	0xf0, 0x74, 0xf7, 0x55, 0x15, 0x57, 0xe4, 0x9c
 };
-static const u8 enc_assoc063[] __initconst = {
+static const u8 enc_assoc063[] = {
 	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
 	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f
 };
-static const u8 enc_nonce063[] __initconst = {
+static const u8 enc_nonce063[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
 };
-static const u8 enc_key063[] __initconst = {
+static const u8 enc_key063[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -3449,7 +3448,7 @@ static const u8 enc_key063[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input064[] __initconst = {
+static const u8 enc_input064[] = {
 	0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc,
 	0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97,
 	0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04,
@@ -3467,7 +3466,7 @@ static const u8 enc_input064[] __initconst = {
 	0x88, 0x2c, 0xf4, 0xba, 0x89, 0x6d, 0x12, 0x47,
 	0x04, 0x53, 0xfe, 0xf7, 0xc7, 0xfb, 0x77, 0xb8
 };
-static const u8 enc_output064[] __initconst = {
+static const u8 enc_output064[] = {
 	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
 	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
 	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
@@ -3487,14 +3486,14 @@ static const u8 enc_output064[] __initconst = {
 	0xc8, 0x6d, 0xa8, 0xdd, 0x65, 0x22, 0x86, 0xd5,
 	0x02, 0x13, 0xd3, 0x28, 0xd6, 0x3e, 0x40, 0x06
 };
-static const u8 enc_assoc064[] __initconst = {
+static const u8 enc_assoc064[] = {
 	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
 	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f
 };
-static const u8 enc_nonce064[] __initconst = {
+static const u8 enc_nonce064[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
 };
-static const u8 enc_key064[] __initconst = {
+static const u8 enc_key064[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -3502,13 +3501,13 @@ static const u8 enc_key064[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input065[] __initconst = {
+static const u8 enc_input065[] = {
 	0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c,
 	0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17,
 	0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84,
 	0xff, 0x1f, 0xcf, 0x41, 0x94, 0x2c, 0xd6, 0x41
 };
-static const u8 enc_output065[] __initconst = {
+static const u8 enc_output065[] = {
 	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
 	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
 	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
@@ -3516,14 +3515,14 @@ static const u8 enc_output065[] __initconst = {
 	0xbe, 0xde, 0x90, 0x83, 0xce, 0xb3, 0x6d, 0xdf,
 	0xe5, 0xfa, 0x81, 0x1f, 0x95, 0x47, 0x1c, 0x67
 };
-static const u8 enc_assoc065[] __initconst = {
+static const u8 enc_assoc065[] = {
 	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
 	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce065[] __initconst = {
+static const u8 enc_nonce065[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
 };
-static const u8 enc_key065[] __initconst = {
+static const u8 enc_key065[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -3531,7 +3530,7 @@ static const u8 enc_key065[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input066[] __initconst = {
+static const u8 enc_input066[] = {
 	0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c,
 	0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17,
 	0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84,
@@ -3541,7 +3540,7 @@ static const u8 enc_input066[] __initconst = {
 	0x77, 0x96, 0x00, 0x2d, 0x93, 0xa1, 0xd9, 0x1a,
 	0x2c, 0x48, 0x4d, 0xd8, 0x81, 0x78, 0x02, 0x42
 };
-static const u8 enc_output066[] __initconst = {
+static const u8 enc_output066[] = {
 	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
 	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
 	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
@@ -3553,14 +3552,14 @@ static const u8 enc_output066[] __initconst = {
 	0x30, 0x08, 0x74, 0xbb, 0x06, 0x92, 0xb6, 0x89,
 	0xde, 0xad, 0x9a, 0xe1, 0x5b, 0x06, 0x73, 0x90
 };
-static const u8 enc_assoc066[] __initconst = {
+static const u8 enc_assoc066[] = {
 	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
 	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce066[] __initconst = {
+static const u8 enc_nonce066[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
 };
-static const u8 enc_key066[] __initconst = {
+static const u8 enc_key066[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -3568,7 +3567,7 @@ static const u8 enc_key066[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input067[] __initconst = {
+static const u8 enc_input067[] = {
 	0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c,
 	0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17,
 	0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84,
@@ -3586,7 +3585,7 @@ static const u8 enc_input067[] __initconst = {
 	0x08, 0x2c, 0xf4, 0x3a, 0x09, 0x6d, 0x12, 0xc7,
 	0x84, 0x53, 0xfe, 0x77, 0x47, 0xfb, 0x77, 0x38
 };
-static const u8 enc_output067[] __initconst = {
+static const u8 enc_output067[] = {
 	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
 	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
 	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
@@ -3606,14 +3605,14 @@ static const u8 enc_output067[] __initconst = {
 	0x99, 0xca, 0xd8, 0x5f, 0x45, 0xca, 0x40, 0x94,
 	0x2d, 0x0d, 0x4d, 0x5e, 0x95, 0x0a, 0xde, 0x22
 };
-static const u8 enc_assoc067[] __initconst = {
+static const u8 enc_assoc067[] = {
 	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff,
 	0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce067[] __initconst = {
+static const u8 enc_nonce067[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
 };
-static const u8 enc_key067[] __initconst = {
+static const u8 enc_key067[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -3621,13 +3620,13 @@ static const u8 enc_key067[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input068[] __initconst = {
+static const u8 enc_input068[] = {
 	0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c,
 	0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17,
 	0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84,
 	0x80, 0xe0, 0x30, 0xbe, 0x14, 0x2c, 0xd6, 0x41
 };
-static const u8 enc_output068[] __initconst = {
+static const u8 enc_output068[] = {
 	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
 	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
 	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
@@ -3635,14 +3634,14 @@ static const u8 enc_output068[] __initconst = {
 	0x8b, 0xbe, 0x14, 0x52, 0x72, 0xe7, 0xc2, 0xd9,
 	0xa1, 0x89, 0x1a, 0x3a, 0xb0, 0x98, 0x3d, 0x9d
 };
-static const u8 enc_assoc068[] __initconst = {
+static const u8 enc_assoc068[] = {
 	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
 	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce068[] __initconst = {
+static const u8 enc_nonce068[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
 };
-static const u8 enc_key068[] __initconst = {
+static const u8 enc_key068[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -3650,7 +3649,7 @@ static const u8 enc_key068[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input069[] __initconst = {
+static const u8 enc_input069[] = {
 	0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c,
 	0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17,
 	0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84,
@@ -3660,7 +3659,7 @@ static const u8 enc_input069[] __initconst = {
 	0x08, 0x69, 0xff, 0xd2, 0x13, 0xa1, 0xd9, 0x1a,
 	0x53, 0xb7, 0xb2, 0x27, 0x01, 0x78, 0x02, 0x42
 };
-static const u8 enc_output069[] __initconst = {
+static const u8 enc_output069[] = {
 	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
 	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
 	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
@@ -3672,14 +3671,14 @@ static const u8 enc_output069[] __initconst = {
 	0x3b, 0x41, 0x86, 0x19, 0x13, 0xa8, 0xf6, 0xde,
 	0x7f, 0x61, 0xe2, 0x25, 0x63, 0x1b, 0xc3, 0x82
 };
-static const u8 enc_assoc069[] __initconst = {
+static const u8 enc_assoc069[] = {
 	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
 	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce069[] __initconst = {
+static const u8 enc_nonce069[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
 };
-static const u8 enc_key069[] __initconst = {
+static const u8 enc_key069[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -3687,7 +3686,7 @@ static const u8 enc_key069[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input070[] __initconst = {
+static const u8 enc_input070[] = {
 	0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c,
 	0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17,
 	0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84,
@@ -3705,7 +3704,7 @@ static const u8 enc_input070[] __initconst = {
 	0x77, 0xd3, 0x0b, 0xc5, 0x89, 0x6d, 0x12, 0xc7,
 	0xfb, 0xac, 0x01, 0x88, 0xc7, 0xfb, 0x77, 0x38
 };
-static const u8 enc_output070[] __initconst = {
+static const u8 enc_output070[] = {
 	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
 	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
 	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
@@ -3725,14 +3724,14 @@ static const u8 enc_output070[] __initconst = {
 	0x84, 0x28, 0xbc, 0xf0, 0x23, 0xec, 0x6b, 0xf3,
 	0x1f, 0xd9, 0xef, 0xb2, 0x03, 0xff, 0x08, 0x71
 };
-static const u8 enc_assoc070[] __initconst = {
+static const u8 enc_assoc070[] = {
 	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
 	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce070[] __initconst = {
+static const u8 enc_nonce070[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
 };
-static const u8 enc_key070[] __initconst = {
+static const u8 enc_key070[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -3740,13 +3739,13 @@ static const u8 enc_key070[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input071[] __initconst = {
+static const u8 enc_input071[] = {
 	0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83,
 	0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8,
 	0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b,
 	0x7f, 0x1f, 0xcf, 0x41, 0xeb, 0xd3, 0x29, 0xbe
 };
-static const u8 enc_output071[] __initconst = {
+static const u8 enc_output071[] = {
 	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
 	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
 	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
@@ -3754,14 +3753,14 @@ static const u8 enc_output071[] __initconst = {
 	0x13, 0x9f, 0xdf, 0x64, 0x74, 0xea, 0x24, 0xf5,
 	0x49, 0xb0, 0x75, 0x82, 0x5f, 0x2c, 0x76, 0x20
 };
-static const u8 enc_assoc071[] __initconst = {
+static const u8 enc_assoc071[] = {
 	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
 	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00
 };
-static const u8 enc_nonce071[] __initconst = {
+static const u8 enc_nonce071[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
 };
-static const u8 enc_key071[] __initconst = {
+static const u8 enc_key071[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -3769,7 +3768,7 @@ static const u8 enc_key071[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input072[] __initconst = {
+static const u8 enc_input072[] = {
 	0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83,
 	0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8,
 	0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b,
@@ -3779,7 +3778,7 @@ static const u8 enc_input072[] __initconst = {
 	0xf7, 0x96, 0x00, 0x2d, 0xec, 0x5e, 0x26, 0xe5,
 	0xac, 0x48, 0x4d, 0xd8, 0xfe, 0x87, 0xfd, 0xbd
 };
-static const u8 enc_output072[] __initconst = {
+static const u8 enc_output072[] = {
 	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
 	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
 	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
@@ -3791,14 +3790,14 @@ static const u8 enc_output072[] __initconst = {
 	0xbb, 0xad, 0x8d, 0x86, 0x3b, 0x83, 0x5a, 0x8e,
 	0x86, 0x64, 0xfd, 0x1d, 0x45, 0x66, 0xb6, 0xb4
 };
-static const u8 enc_assoc072[] __initconst = {
+static const u8 enc_assoc072[] = {
 	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
 	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00
 };
-static const u8 enc_nonce072[] __initconst = {
+static const u8 enc_nonce072[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
 };
-static const u8 enc_key072[] __initconst = {
+static const u8 enc_key072[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -3806,7 +3805,7 @@ static const u8 enc_key072[] __initconst = {
 };
 
 /* wycheproof - misc */
-static const u8 enc_input073[] __initconst = {
+static const u8 enc_input073[] = {
 	0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83,
 	0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8,
 	0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b,
@@ -3824,7 +3823,7 @@ static const u8 enc_input073[] __initconst = {
 	0x88, 0x2c, 0xf4, 0x3a, 0x76, 0x92, 0xed, 0x38,
 	0x04, 0x53, 0xfe, 0x77, 0x38, 0x04, 0x88, 0xc7
 };
-static const u8 enc_output073[] __initconst = {
+static const u8 enc_output073[] = {
 	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
 	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
 	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
@@ -3844,14 +3843,14 @@ static const u8 enc_output073[] __initconst = {
 	0x42, 0xf2, 0x35, 0x42, 0x97, 0x84, 0x9a, 0x51,
 	0x1d, 0x53, 0xe5, 0x57, 0x17, 0x72, 0xf7, 0x1f
 };
-static const u8 enc_assoc073[] __initconst = {
+static const u8 enc_assoc073[] = {
 	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
 	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00
 };
-static const u8 enc_nonce073[] __initconst = {
+static const u8 enc_nonce073[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00
 };
-static const u8 enc_key073[] __initconst = {
+static const u8 enc_key073[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -3859,7 +3858,7 @@ static const u8 enc_key073[] __initconst = {
 };
 
 /* wycheproof - checking for int overflows */
-static const u8 enc_input074[] __initconst = {
+static const u8 enc_input074[] = {
 	0xd4, 0x50, 0x0b, 0xf0, 0x09, 0x49, 0x35, 0x51,
 	0xc3, 0x80, 0xad, 0xf5, 0x2c, 0x57, 0x3a, 0x69,
 	0xdf, 0x7e, 0x8b, 0x76, 0x24, 0x63, 0x33, 0x0f,
@@ -3877,7 +3876,7 @@ static const u8 enc_input074[] __initconst = {
 	0x93, 0x76, 0x71, 0xa0, 0x62, 0x9b, 0xd9, 0x5c,
 	0x99, 0x15, 0xc7, 0x85, 0x55, 0x77, 0x1e, 0x7a
 };
-static const u8 enc_output074[] __initconst = {
+static const u8 enc_output074[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -3897,7 +3896,7 @@ static const u8 enc_output074[] __initconst = {
 	0x0b, 0x30, 0x0d, 0x8d, 0xa5, 0x6c, 0x21, 0x85,
 	0x75, 0x52, 0x79, 0x55, 0x3c, 0x4c, 0x82, 0xca
 };
-static const u8 enc_assoc074[] __initconst = {
+static const u8 enc_assoc074[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -3907,11 +3906,11 @@ static const u8 enc_assoc074[] __initconst = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce074[] __initconst = {
+static const u8 enc_nonce074[] = {
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
 	0x00, 0x02, 0x50, 0x6e
 };
-static const u8 enc_key074[] __initconst = {
+static const u8 enc_key074[] = {
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
@@ -3919,7 +3918,7 @@ static const u8 enc_key074[] __initconst = {
 };
 
 /* wycheproof - checking for int overflows */
-static const u8 enc_input075[] __initconst = {
+static const u8 enc_input075[] = {
 	0x7d, 0xe8, 0x7f, 0x67, 0x29, 0x94, 0x52, 0x75,
 	0xd0, 0x65, 0x5d, 0xa4, 0xc7, 0xfd, 0xe4, 0x56,
 	0x9e, 0x16, 0xf1, 0x11, 0xb5, 0xeb, 0x26, 0xc2,
@@ -3937,7 +3936,7 @@ static const u8 enc_input075[] __initconst = {
 	0x3a, 0xc1, 0xee, 0x54, 0xc2, 0x9e, 0xe4, 0xc1,
 	0x70, 0xde, 0x40, 0x8f, 0x66, 0x69, 0x21, 0x94
 };
-static const u8 enc_output075[] __initconst = {
+static const u8 enc_output075[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -3957,7 +3956,7 @@ static const u8 enc_output075[] __initconst = {
 	0xc5, 0x78, 0xe2, 0xaa, 0x44, 0xd3, 0x09, 0xb7,
 	0xb6, 0xa5, 0x19, 0x3b, 0xdc, 0x61, 0x18, 0xf5
 };
-static const u8 enc_assoc075[] __initconst = {
+static const u8 enc_assoc075[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -3967,11 +3966,11 @@ static const u8 enc_assoc075[] __initconst = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce075[] __initconst = {
+static const u8 enc_nonce075[] = {
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
 	0x00, 0x03, 0x18, 0xa5
 };
-static const u8 enc_key075[] __initconst = {
+static const u8 enc_key075[] = {
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
@@ -3979,7 +3978,7 @@ static const u8 enc_key075[] __initconst = {
 };
 
 /* wycheproof - checking for int overflows */
-static const u8 enc_input076[] __initconst = {
+static const u8 enc_input076[] = {
 	0x1b, 0x99, 0x6f, 0x9a, 0x3c, 0xcc, 0x67, 0x85,
 	0xde, 0x22, 0xff, 0x5b, 0x8a, 0xdd, 0x95, 0x02,
 	0xce, 0x03, 0xa0, 0xfa, 0xf5, 0x99, 0x2a, 0x09,
@@ -3997,7 +3996,7 @@ static const u8 enc_input076[] __initconst = {
 	0xc3, 0x53, 0xd0, 0xcf, 0x93, 0x8d, 0xcc, 0xb9,
 	0xef, 0xad, 0x8f, 0xed, 0xbe, 0x46, 0xda, 0xa5
 };
-static const u8 enc_output076[] __initconst = {
+static const u8 enc_output076[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4017,7 +4016,7 @@ static const u8 enc_output076[] __initconst = {
 	0x4b, 0x0b, 0xda, 0x8a, 0xd0, 0x43, 0x83, 0x0d,
 	0x83, 0x19, 0xab, 0x82, 0xc5, 0x0c, 0x76, 0x63
 };
-static const u8 enc_assoc076[] __initconst = {
+static const u8 enc_assoc076[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4027,10 +4026,10 @@ static const u8 enc_assoc076[] __initconst = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce076[] __initconst = {
+static const u8 enc_nonce076[] = {
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xb4, 0xf0
 };
-static const u8 enc_key076[] __initconst = {
+static const u8 enc_key076[] = {
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
@@ -4038,7 +4037,7 @@ static const u8 enc_key076[] __initconst = {
 };
 
 /* wycheproof - checking for int overflows */
-static const u8 enc_input077[] __initconst = {
+static const u8 enc_input077[] = {
 	0x86, 0xcb, 0xac, 0xae, 0x4d, 0x3f, 0x74, 0xae,
 	0x01, 0x21, 0x3e, 0x05, 0x51, 0xcc, 0x15, 0x16,
 	0x0e, 0xa1, 0xbe, 0x84, 0x08, 0xe3, 0xd5, 0xd7,
@@ -4056,7 +4055,7 @@ static const u8 enc_input077[] __initconst = {
 	0x0e, 0x76, 0x2b, 0x43, 0x0c, 0x4d, 0x51, 0x7c,
 	0x97, 0x10, 0x70, 0x68, 0xf4, 0x98, 0xef, 0x7f
 };
-static const u8 enc_output077[] __initconst = {
+static const u8 enc_output077[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4076,7 +4075,7 @@ static const u8 enc_output077[] __initconst = {
 	0x4b, 0xc9, 0x8f, 0x72, 0xc4, 0x94, 0xc2, 0xa4,
 	0x3c, 0x2b, 0x15, 0xa1, 0x04, 0x3f, 0x1c, 0xfa
 };
-static const u8 enc_assoc077[] __initconst = {
+static const u8 enc_assoc077[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4086,10 +4085,10 @@ static const u8 enc_assoc077[] __initconst = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce077[] __initconst = {
+static const u8 enc_nonce077[] = {
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0xfb, 0x66
 };
-static const u8 enc_key077[] __initconst = {
+static const u8 enc_key077[] = {
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
@@ -4097,7 +4096,7 @@ static const u8 enc_key077[] __initconst = {
 };
 
 /* wycheproof - checking for int overflows */
-static const u8 enc_input078[] __initconst = {
+static const u8 enc_input078[] = {
 	0xfa, 0xb1, 0xcd, 0xdf, 0x4f, 0xe1, 0x98, 0xef,
 	0x63, 0xad, 0xd8, 0x81, 0xd6, 0xea, 0xd6, 0xc5,
 	0x76, 0x37, 0xbb, 0xe9, 0x20, 0x18, 0xca, 0x7c,
@@ -4115,7 +4114,7 @@ static const u8 enc_input078[] __initconst = {
 	0xc3, 0x24, 0x60, 0x41, 0x43, 0x21, 0x43, 0xe9,
 	0xab, 0x3a, 0x6d, 0x2c, 0xcc, 0x2f, 0x4d, 0x62
 };
-static const u8 enc_output078[] __initconst = {
+static const u8 enc_output078[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4135,7 +4134,7 @@ static const u8 enc_output078[] __initconst = {
 	0xf7, 0xe9, 0xe1, 0x51, 0xb0, 0x25, 0x33, 0xc7,
 	0x46, 0x58, 0xbf, 0xc7, 0x73, 0x7c, 0x68, 0x0d
 };
-static const u8 enc_assoc078[] __initconst = {
+static const u8 enc_assoc078[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4145,10 +4144,10 @@ static const u8 enc_assoc078[] __initconst = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce078[] __initconst = {
+static const u8 enc_nonce078[] = {
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0xbb, 0x90
 };
-static const u8 enc_key078[] __initconst = {
+static const u8 enc_key078[] = {
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
@@ -4156,7 +4155,7 @@ static const u8 enc_key078[] __initconst = {
 };
 
 /* wycheproof - checking for int overflows */
-static const u8 enc_input079[] __initconst = {
+static const u8 enc_input079[] = {
 	0x22, 0x72, 0x02, 0xbe, 0x7f, 0x35, 0x15, 0xe9,
 	0xd1, 0xc0, 0x2e, 0xea, 0x2f, 0x19, 0x50, 0xb6,
 	0x48, 0x1b, 0x04, 0x8a, 0x4c, 0x91, 0x50, 0x6c,
@@ -4174,7 +4173,7 @@ static const u8 enc_input079[] __initconst = {
 	0xfb, 0xe1, 0xee, 0x45, 0xb1, 0xb2, 0x1f, 0x71,
 	0x62, 0xe2, 0xfc, 0xaa, 0x74, 0x2a, 0xbe, 0xfd
 };
-static const u8 enc_output079[] __initconst = {
+static const u8 enc_output079[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4194,7 +4193,7 @@ static const u8 enc_output079[] __initconst = {
 	0x79, 0x5b, 0xcf, 0xf6, 0x47, 0xc5, 0x53, 0xc2,
 	0xe4, 0xeb, 0x6e, 0x0e, 0xaf, 0xd9, 0xe0, 0x4e
 };
-static const u8 enc_assoc079[] __initconst = {
+static const u8 enc_assoc079[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4204,10 +4203,10 @@ static const u8 enc_assoc079[] __initconst = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce079[] __initconst = {
+static const u8 enc_nonce079[] = {
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x48, 0x4a
 };
-static const u8 enc_key079[] __initconst = {
+static const u8 enc_key079[] = {
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
@@ -4215,7 +4214,7 @@ static const u8 enc_key079[] __initconst = {
 };
 
 /* wycheproof - checking for int overflows */
-static const u8 enc_input080[] __initconst = {
+static const u8 enc_input080[] = {
 	0xfa, 0xe5, 0x83, 0x45, 0xc1, 0x6c, 0xb0, 0xf5,
 	0xcc, 0x53, 0x7f, 0x2b, 0x1b, 0x34, 0x69, 0xc9,
 	0x69, 0x46, 0x3b, 0x3e, 0xa7, 0x1b, 0xcf, 0x6b,
@@ -4233,7 +4232,7 @@ static const u8 enc_input080[] __initconst = {
 	0x18, 0x01, 0xce, 0x33, 0xc4, 0xe4, 0xa7, 0x7d,
 	0x83, 0x1d, 0x3c, 0xe3, 0x4e, 0x84, 0x10, 0xe1
 };
-static const u8 enc_output080[] __initconst = {
+static const u8 enc_output080[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4253,7 +4252,7 @@ static const u8 enc_output080[] __initconst = {
 	0x19, 0x46, 0xd6, 0x53, 0x96, 0x0f, 0x94, 0x7a,
 	0x74, 0xd3, 0xe8, 0x09, 0x3c, 0xf4, 0x85, 0x02
 };
-static const u8 enc_assoc080[] __initconst = {
+static const u8 enc_assoc080[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4263,10 +4262,10 @@ static const u8 enc_assoc080[] __initconst = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce080[] __initconst = {
+static const u8 enc_nonce080[] = {
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0x2f, 0x40
 };
-static const u8 enc_key080[] __initconst = {
+static const u8 enc_key080[] = {
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
@@ -4274,7 +4273,7 @@ static const u8 enc_key080[] __initconst = {
 };
 
 /* wycheproof - checking for int overflows */
-static const u8 enc_input081[] __initconst = {
+static const u8 enc_input081[] = {
 	0xeb, 0xb2, 0x16, 0xdd, 0xd7, 0xca, 0x70, 0x92,
 	0x15, 0xf5, 0x03, 0xdf, 0x9c, 0xe6, 0x3c, 0x5c,
 	0xd2, 0x19, 0x4e, 0x7d, 0x90, 0x99, 0xe8, 0xa9,
@@ -4292,7 +4291,7 @@ static const u8 enc_input081[] __initconst = {
 	0x83, 0xfd, 0xca, 0x39, 0xd2, 0xe1, 0x4f, 0x23,
 	0xd0, 0x0a, 0x58, 0x26, 0x64, 0xf4, 0xec, 0xb1
 };
-static const u8 enc_output081[] __initconst = {
+static const u8 enc_output081[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4312,7 +4311,7 @@ static const u8 enc_output081[] __initconst = {
 	0x36, 0xc3, 0x00, 0x29, 0x85, 0xdd, 0x21, 0xba,
 	0xf8, 0x95, 0xd6, 0x33, 0x57, 0x3f, 0x12, 0xc0
 };
-static const u8 enc_assoc081[] __initconst = {
+static const u8 enc_assoc081[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4322,10 +4321,10 @@ static const u8 enc_assoc081[] __initconst = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce081[] __initconst = {
+static const u8 enc_nonce081[] = {
 	0x00, 0x00, 0x00, 0x00, 0x00, 0xe2, 0x93, 0x35
 };
-static const u8 enc_key081[] __initconst = {
+static const u8 enc_key081[] = {
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
 	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
@@ -4333,7 +4332,7 @@ static const u8 enc_key081[] __initconst = {
 };
 
 /* wycheproof - checking for int overflows */
-static const u8 enc_input082[] __initconst = {
+static const u8 enc_input082[] = {
 	0x40, 0x8a, 0xe6, 0xef, 0x1c, 0x7e, 0xf0, 0xfb,
 	0x2c, 0x2d, 0x61, 0x08, 0x16, 0xfc, 0x78, 0x49,
 	0xef, 0xa5, 0x8f, 0x78, 0x27, 0x3f, 0x5f, 0x16,
@@ -4351,7 +4350,7 @@ static const u8 enc_input082[] __initconst = {
 	0x2b, 0x68, 0x9f, 0x93, 0x55, 0xd9, 0xc1, 0x83,
 	0x80, 0x1f, 0x6a, 0xcc, 0x31, 0x3f, 0x89, 0x07
 };
-static const u8 enc_output082[] __initconst = {
+static const u8 enc_output082[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4371,7 +4370,7 @@ static const u8 enc_output082[] __initconst = {
 	0x65, 0x14, 0x51, 0x8e, 0x0a, 0x26, 0x41, 0x42,
 	0xe0, 0xb7, 0x35, 0x1f, 0x96, 0x7f, 0xc2, 0xae
 };
-static const u8 enc_assoc082[] __initconst = {
+static const u8 enc_assoc082[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4381,10 +4380,10 @@ static const u8 enc_assoc082[] __initconst = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce082[] __initconst = {
+static const u8 enc_nonce082[] = {
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0xf7, 0xd5
 };
-static const u8 enc_key082[] __initconst = {
+static const u8 enc_key082[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -4392,7 +4391,7 @@ static const u8 enc_key082[] __initconst = {
 };
 
 /* wycheproof - checking for int overflows */
-static const u8 enc_input083[] __initconst = {
+static const u8 enc_input083[] = {
 	0x0a, 0x0a, 0x24, 0x49, 0x9b, 0xca, 0xde, 0x58,
 	0xcf, 0x15, 0x76, 0xc3, 0x12, 0xac, 0xa9, 0x84,
 	0x71, 0x8c, 0xb4, 0xcc, 0x7e, 0x01, 0x53, 0xf5,
@@ -4410,7 +4409,7 @@ static const u8 enc_input083[] __initconst = {
 	0xb3, 0x06, 0xa2, 0x1b, 0x42, 0xd4, 0xc3, 0xba,
 	0x6e, 0x6f, 0x0c, 0xbc, 0xc8, 0x1e, 0x87, 0x7a
 };
-static const u8 enc_output083[] __initconst = {
+static const u8 enc_output083[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4430,7 +4429,7 @@ static const u8 enc_output083[] __initconst = {
 	0x4c, 0x19, 0x4d, 0xa6, 0xa9, 0x9f, 0xd6, 0x5b,
 	0x40, 0xe9, 0xca, 0xd7, 0x98, 0xf4, 0x4b, 0x19
 };
-static const u8 enc_assoc083[] __initconst = {
+static const u8 enc_assoc083[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4440,10 +4439,10 @@ static const u8 enc_assoc083[] __initconst = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce083[] __initconst = {
+static const u8 enc_nonce083[] = {
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0xfc, 0xe4
 };
-static const u8 enc_key083[] __initconst = {
+static const u8 enc_key083[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -4451,7 +4450,7 @@ static const u8 enc_key083[] __initconst = {
 };
 
 /* wycheproof - checking for int overflows */
-static const u8 enc_input084[] __initconst = {
+static const u8 enc_input084[] = {
 	0x4a, 0x0a, 0xaf, 0xf8, 0x49, 0x47, 0x29, 0x18,
 	0x86, 0x91, 0x70, 0x13, 0x40, 0xf3, 0xce, 0x2b,
 	0x8a, 0x78, 0xee, 0xd3, 0xa0, 0xf0, 0x65, 0x99,
@@ -4469,7 +4468,7 @@ static const u8 enc_input084[] __initconst = {
 	0x5d, 0x7e, 0x51, 0x3b, 0xe5, 0xb8, 0xea, 0x97,
 	0x13, 0x10, 0xd5, 0xbf, 0x16, 0xba, 0x7a, 0xee
 };
-static const u8 enc_output084[] __initconst = {
+static const u8 enc_output084[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4489,7 +4488,7 @@ static const u8 enc_output084[] __initconst = {
 	0xc8, 0xae, 0x77, 0x88, 0xcd, 0x28, 0x74, 0xab,
 	0xc1, 0x38, 0x54, 0x1e, 0x11, 0xfd, 0x05, 0x87
 };
-static const u8 enc_assoc084[] __initconst = {
+static const u8 enc_assoc084[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4499,10 +4498,10 @@ static const u8 enc_assoc084[] __initconst = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce084[] __initconst = {
+static const u8 enc_nonce084[] = {
 	0x00, 0x00, 0x00, 0x00, 0x01, 0x84, 0x86, 0xa8
 };
-static const u8 enc_key084[] __initconst = {
+static const u8 enc_key084[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -4510,7 +4509,7 @@ static const u8 enc_key084[] __initconst = {
 };
 
 /* wycheproof - checking for int overflows */
-static const u8 enc_input085[] __initconst = {
+static const u8 enc_input085[] = {
 	0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
 	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
 	0x78, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
@@ -4528,7 +4527,7 @@ static const u8 enc_input085[] __initconst = {
 	0xf7, 0x4d, 0x5b, 0xf8, 0x67, 0x1c, 0x5a, 0x8a,
 	0x50, 0x92, 0xf6, 0x1d, 0x54, 0xc9, 0xaa, 0x5b
 };
-static const u8 enc_output085[] __initconst = {
+static const u8 enc_output085[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4548,7 +4547,7 @@ static const u8 enc_output085[] __initconst = {
 	0x93, 0x3a, 0x51, 0x63, 0xc7, 0xf6, 0x23, 0x68,
 	0x32, 0x7b, 0x3f, 0xbc, 0x10, 0x36, 0xc9, 0x43
 };
-static const u8 enc_assoc085[] __initconst = {
+static const u8 enc_assoc085[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4558,10 +4557,10 @@ static const u8 enc_assoc085[] __initconst = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce085[] __initconst = {
+static const u8 enc_nonce085[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key085[] __initconst = {
+static const u8 enc_key085[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -4569,7 +4568,7 @@ static const u8 enc_key085[] __initconst = {
 };
 
 /* wycheproof - special case tag */
-static const u8 enc_input086[] __initconst = {
+static const u8 enc_input086[] = {
 	0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6,
 	0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd,
 	0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b,
@@ -4579,7 +4578,7 @@ static const u8 enc_input086[] __initconst = {
 	0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63,
 	0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d
 };
-static const u8 enc_output086[] __initconst = {
+static const u8 enc_output086[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4591,17 +4590,17 @@ static const u8 enc_output086[] __initconst = {
 	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
 	0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
 };
-static const u8 enc_assoc086[] __initconst = {
+static const u8 enc_assoc086[] = {
 	0x85, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xa6, 0x90, 0x2f, 0xcb, 0xc8, 0x83, 0xbb, 0xc1,
 	0x80, 0xb2, 0x56, 0xae, 0x34, 0xad, 0x7f, 0x00
 };
-static const u8 enc_nonce086[] __initconst = {
+static const u8 enc_nonce086[] = {
 	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
 	0x08, 0x09, 0x0a, 0x0b
 };
-static const u8 enc_key086[] __initconst = {
+static const u8 enc_key086[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -4609,7 +4608,7 @@ static const u8 enc_key086[] __initconst = {
 };
 
 /* wycheproof - special case tag */
-static const u8 enc_input087[] __initconst = {
+static const u8 enc_input087[] = {
 	0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6,
 	0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd,
 	0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b,
@@ -4619,7 +4618,7 @@ static const u8 enc_input087[] __initconst = {
 	0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63,
 	0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d
 };
-static const u8 enc_output087[] __initconst = {
+static const u8 enc_output087[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4631,17 +4630,17 @@ static const u8 enc_output087[] __initconst = {
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
 };
-static const u8 enc_assoc087[] __initconst = {
+static const u8 enc_assoc087[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0x24, 0x7e, 0x50, 0x64, 0x2a, 0x1c, 0x0a, 0x2f,
 	0x8f, 0x77, 0x21, 0x96, 0x09, 0xdb, 0xa9, 0x58
 };
-static const u8 enc_nonce087[] __initconst = {
+static const u8 enc_nonce087[] = {
 	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
 	0x08, 0x09, 0x0a, 0x0b
 };
-static const u8 enc_key087[] __initconst = {
+static const u8 enc_key087[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -4649,7 +4648,7 @@ static const u8 enc_key087[] __initconst = {
 };
 
 /* wycheproof - special case tag */
-static const u8 enc_input088[] __initconst = {
+static const u8 enc_input088[] = {
 	0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6,
 	0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd,
 	0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b,
@@ -4659,7 +4658,7 @@ static const u8 enc_input088[] __initconst = {
 	0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63,
 	0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d
 };
-static const u8 enc_output088[] __initconst = {
+static const u8 enc_output088[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4671,17 +4670,17 @@ static const u8 enc_output088[] __initconst = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_assoc088[] __initconst = {
+static const u8 enc_assoc088[] = {
 	0x7c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xd9, 0xe7, 0x2c, 0x06, 0x4a, 0xc8, 0x96, 0x1f,
 	0x3f, 0xa5, 0x85, 0xe0, 0xe2, 0xab, 0xd6, 0x00
 };
-static const u8 enc_nonce088[] __initconst = {
+static const u8 enc_nonce088[] = {
 	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
 	0x08, 0x09, 0x0a, 0x0b
 };
-static const u8 enc_key088[] __initconst = {
+static const u8 enc_key088[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -4689,7 +4688,7 @@ static const u8 enc_key088[] __initconst = {
 };
 
 /* wycheproof - special case tag */
-static const u8 enc_input089[] __initconst = {
+static const u8 enc_input089[] = {
 	0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6,
 	0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd,
 	0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b,
@@ -4699,7 +4698,7 @@ static const u8 enc_input089[] __initconst = {
 	0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63,
 	0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d
 };
-static const u8 enc_output089[] __initconst = {
+static const u8 enc_output089[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4711,17 +4710,17 @@ static const u8 enc_output089[] __initconst = {
 	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
 	0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80
 };
-static const u8 enc_assoc089[] __initconst = {
+static const u8 enc_assoc089[] = {
 	0x65, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0x95, 0xaf, 0x0f, 0x4d, 0x0b, 0x68, 0x6e, 0xae,
 	0xcc, 0xca, 0x43, 0x07, 0xd5, 0x96, 0xf5, 0x02
 };
-static const u8 enc_nonce089[] __initconst = {
+static const u8 enc_nonce089[] = {
 	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
 	0x08, 0x09, 0x0a, 0x0b
 };
-static const u8 enc_key089[] __initconst = {
+static const u8 enc_key089[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -4729,7 +4728,7 @@ static const u8 enc_key089[] __initconst = {
 };
 
 /* wycheproof - special case tag */
-static const u8 enc_input090[] __initconst = {
+static const u8 enc_input090[] = {
 	0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6,
 	0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd,
 	0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b,
@@ -4739,7 +4738,7 @@ static const u8 enc_input090[] __initconst = {
 	0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63,
 	0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d
 };
-static const u8 enc_output090[] __initconst = {
+static const u8 enc_output090[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4751,17 +4750,17 @@ static const u8 enc_output090[] __initconst = {
 	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
 	0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f
 };
-static const u8 enc_assoc090[] __initconst = {
+static const u8 enc_assoc090[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0x85, 0x40, 0xb4, 0x64, 0x35, 0x77, 0x07, 0xbe,
 	0x3a, 0x39, 0xd5, 0x5c, 0x34, 0xf8, 0xbc, 0xb3
 };
-static const u8 enc_nonce090[] __initconst = {
+static const u8 enc_nonce090[] = {
 	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
 	0x08, 0x09, 0x0a, 0x0b
 };
-static const u8 enc_key090[] __initconst = {
+static const u8 enc_key090[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -4769,7 +4768,7 @@ static const u8 enc_key090[] __initconst = {
 };
 
 /* wycheproof - special case tag */
-static const u8 enc_input091[] __initconst = {
+static const u8 enc_input091[] = {
 	0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6,
 	0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd,
 	0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b,
@@ -4779,7 +4778,7 @@ static const u8 enc_input091[] __initconst = {
 	0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63,
 	0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d
 };
-static const u8 enc_output091[] __initconst = {
+static const u8 enc_output091[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4791,17 +4790,17 @@ static const u8 enc_output091[] __initconst = {
 	0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
 	0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00
 };
-static const u8 enc_assoc091[] __initconst = {
+static const u8 enc_assoc091[] = {
 	0x4f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0x66, 0x23, 0xd9, 0x90, 0xb8, 0x98, 0xd8, 0x30,
 	0xd2, 0x12, 0xaf, 0x23, 0x83, 0x33, 0x07, 0x01
 };
-static const u8 enc_nonce091[] __initconst = {
+static const u8 enc_nonce091[] = {
 	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
 	0x08, 0x09, 0x0a, 0x0b
 };
-static const u8 enc_key091[] __initconst = {
+static const u8 enc_key091[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -4809,7 +4808,7 @@ static const u8 enc_key091[] __initconst = {
 };
 
 /* wycheproof - special case tag */
-static const u8 enc_input092[] __initconst = {
+static const u8 enc_input092[] = {
 	0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6,
 	0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd,
 	0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b,
@@ -4819,7 +4818,7 @@ static const u8 enc_input092[] __initconst = {
 	0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63,
 	0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d
 };
-static const u8 enc_output092[] __initconst = {
+static const u8 enc_output092[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4831,17 +4830,17 @@ static const u8 enc_output092[] __initconst = {
 	0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
 };
-static const u8 enc_assoc092[] __initconst = {
+static const u8 enc_assoc092[] = {
 	0x83, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0x5f, 0x16, 0xd0, 0x9f, 0x17, 0x78, 0x72, 0x11,
 	0xb7, 0xd4, 0x84, 0xe0, 0x24, 0xf8, 0x97, 0x01
 };
-static const u8 enc_nonce092[] __initconst = {
+static const u8 enc_nonce092[] = {
 	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
 	0x08, 0x09, 0x0a, 0x0b
 };
-static const u8 enc_key092[] __initconst = {
+static const u8 enc_key092[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -4849,7 +4848,7 @@ static const u8 enc_key092[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input093[] __initconst = {
+static const u8 enc_input093[] = {
 	0x00, 0x52, 0x35, 0xd2, 0xa9, 0x19, 0xf2, 0x8d,
 	0x3d, 0xb7, 0x66, 0x4a, 0x34, 0xae, 0x6b, 0x44,
 	0x4d, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
@@ -4861,7 +4860,7 @@ static const u8 enc_input093[] __initconst = {
 	0x83, 0xdc, 0xe9, 0xf3, 0x07, 0x3e, 0xfa, 0xdb,
 	0x7d, 0x23, 0xb8, 0x7a, 0xce, 0x35, 0x16, 0x8c
 };
-static const u8 enc_output093[] __initconst = {
+static const u8 enc_output093[] = {
 	0x00, 0x39, 0xe2, 0xfd, 0x2f, 0xd3, 0x12, 0x14,
 	0x9e, 0x98, 0x98, 0x80, 0x88, 0x48, 0x13, 0xe7,
 	0xca, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -4875,13 +4874,13 @@ static const u8 enc_output093[] __initconst = {
 	0xa5, 0x19, 0xac, 0x1a, 0x35, 0xb4, 0xa5, 0x77,
 	0x87, 0x51, 0x0a, 0xf7, 0x8d, 0x8d, 0x20, 0x0a
 };
-static const u8 enc_assoc093[] __initconst = {
+static const u8 enc_assoc093[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce093[] __initconst = {
+static const u8 enc_nonce093[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key093[] __initconst = {
+static const u8 enc_key093[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -4889,7 +4888,7 @@ static const u8 enc_key093[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input094[] __initconst = {
+static const u8 enc_input094[] = {
 	0xd3, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
 	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
 	0xe5, 0xda, 0x78, 0x76, 0x6f, 0xa1, 0x92, 0x90,
@@ -4903,7 +4902,7 @@ static const u8 enc_input094[] __initconst = {
 	0x01, 0x49, 0xef, 0x50, 0x4b, 0x71, 0xb1, 0x20,
 	0xca, 0x4f, 0xf3, 0x95, 0x19, 0xc2, 0xc2, 0x10
 };
-static const u8 enc_output094[] __initconst = {
+static const u8 enc_output094[] = {
 	0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0x62, 0x18, 0xb2, 0x7f, 0x83, 0xb8, 0xb4, 0x66,
@@ -4919,13 +4918,13 @@ static const u8 enc_output094[] __initconst = {
 	0x30, 0x2f, 0xe8, 0x2a, 0xb0, 0xa0, 0x9a, 0xf6,
 	0x44, 0x00, 0xd0, 0x15, 0xae, 0x83, 0xd9, 0xcc
 };
-static const u8 enc_assoc094[] __initconst = {
+static const u8 enc_assoc094[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce094[] __initconst = {
+static const u8 enc_nonce094[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key094[] __initconst = {
+static const u8 enc_key094[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -4933,7 +4932,7 @@ static const u8 enc_key094[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input095[] __initconst = {
+static const u8 enc_input095[] = {
 	0xe9, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
 	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
 	0x6d, 0xf1, 0x39, 0x4e, 0xdc, 0x53, 0x9b, 0x5b,
@@ -4947,7 +4946,7 @@ static const u8 enc_input095[] __initconst = {
 	0x99, 0x52, 0xae, 0x08, 0x18, 0xc3, 0x89, 0x79,
 	0xc0, 0x74, 0x13, 0x71, 0x1a, 0x9a, 0xf7, 0x13
 };
-static const u8 enc_output095[] __initconst = {
+static const u8 enc_output095[] = {
 	0xe9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xea, 0x33, 0xf3, 0x47, 0x30, 0x4a, 0xbd, 0xad,
@@ -4963,13 +4962,13 @@ static const u8 enc_output095[] __initconst = {
 	0x98, 0xa7, 0xe8, 0x36, 0xe0, 0xee, 0x4d, 0x02,
 	0x35, 0x00, 0xd0, 0x55, 0x7e, 0xc2, 0xcb, 0xe0
 };
-static const u8 enc_assoc095[] __initconst = {
+static const u8 enc_assoc095[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce095[] __initconst = {
+static const u8 enc_nonce095[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key095[] __initconst = {
+static const u8 enc_key095[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -4977,7 +4976,7 @@ static const u8 enc_key095[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input096[] __initconst = {
+static const u8 enc_input096[] = {
 	0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
 	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
 	0x64, 0xf9, 0x0f, 0x5b, 0x26, 0x92, 0xb8, 0x60,
@@ -4987,7 +4986,7 @@ static const u8 enc_input096[] __initconst = {
 	0x03, 0x35, 0x61, 0xe7, 0xca, 0xca, 0x6d, 0x94,
 	0x1d, 0xc3, 0xcd, 0x69, 0x14, 0xad, 0x69, 0x04
 };
-static const u8 enc_output096[] __initconst = {
+static const u8 enc_output096[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xe3, 0x3b, 0xc5, 0x52, 0xca, 0x8b, 0x9e, 0x96,
@@ -4999,13 +4998,13 @@ static const u8 enc_output096[] __initconst = {
 	0x6a, 0xb8, 0xdc, 0xe2, 0xc5, 0x9d, 0xa4, 0x73,
 	0x71, 0x30, 0xb0, 0x25, 0x2f, 0x68, 0xa8, 0xd8
 };
-static const u8 enc_assoc096[] __initconst = {
+static const u8 enc_assoc096[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce096[] __initconst = {
+static const u8 enc_nonce096[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key096[] __initconst = {
+static const u8 enc_key096[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5013,7 +5012,7 @@ static const u8 enc_key096[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input097[] __initconst = {
+static const u8 enc_input097[] = {
 	0x68, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
 	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
 	0xb0, 0x8f, 0x25, 0x67, 0x5b, 0x9b, 0xcb, 0xf6,
@@ -5027,7 +5026,7 @@ static const u8 enc_input097[] __initconst = {
 	0x65, 0x0e, 0xc6, 0x2d, 0x75, 0x70, 0x72, 0xce,
 	0xe6, 0xff, 0x23, 0x31, 0x86, 0xdd, 0x1c, 0x8f
 };
-static const u8 enc_output097[] __initconst = {
+static const u8 enc_output097[] = {
 	0x68, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0x37, 0x4d, 0xef, 0x6e, 0xb7, 0x82, 0xed, 0x00,
@@ -5043,13 +5042,13 @@ static const u8 enc_output097[] __initconst = {
 	0x04, 0x4d, 0xea, 0x60, 0x88, 0x80, 0x41, 0x2b,
 	0xfd, 0xff, 0xcf, 0x35, 0x57, 0x9e, 0x9b, 0x26
 };
-static const u8 enc_assoc097[] __initconst = {
+static const u8 enc_assoc097[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce097[] __initconst = {
+static const u8 enc_nonce097[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key097[] __initconst = {
+static const u8 enc_key097[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5057,7 +5056,7 @@ static const u8 enc_key097[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input098[] __initconst = {
+static const u8 enc_input098[] = {
 	0x6d, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
 	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
 	0xa1, 0x61, 0xb5, 0xab, 0x04, 0x09, 0x00, 0x62,
@@ -5071,7 +5070,7 @@ static const u8 enc_input098[] __initconst = {
 	0x8e, 0xdc, 0x36, 0x6c, 0xd6, 0x97, 0x65, 0x6f,
 	0xca, 0x81, 0xfb, 0x13, 0x3c, 0xed, 0x79, 0xa1
 };
-static const u8 enc_output098[] __initconst = {
+static const u8 enc_output098[] = {
 	0x6d, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0x26, 0xa3, 0x7f, 0xa2, 0xe8, 0x10, 0x26, 0x94,
@@ -5087,13 +5086,13 @@ static const u8 enc_output098[] __initconst = {
 	0x1e, 0x6b, 0xea, 0x63, 0x14, 0x54, 0x2e, 0x2e,
 	0xf9, 0xff, 0xcf, 0x45, 0x0b, 0x2e, 0x98, 0x2b
 };
-static const u8 enc_assoc098[] __initconst = {
+static const u8 enc_assoc098[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce098[] __initconst = {
+static const u8 enc_nonce098[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key098[] __initconst = {
+static const u8 enc_key098[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5101,7 +5100,7 @@ static const u8 enc_key098[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input099[] __initconst = {
+static const u8 enc_input099[] = {
 	0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
 	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
 	0xfc, 0x01, 0xb8, 0x91, 0xe5, 0xf0, 0xf9, 0x12,
@@ -5111,7 +5110,7 @@ static const u8 enc_input099[] __initconst = {
 	0x60, 0xcd, 0x9e, 0xa1, 0x0c, 0x29, 0xa3, 0x66,
 	0x54, 0xe7, 0xa2, 0x8e, 0x76, 0x1b, 0xec, 0xd8
 };
-static const u8 enc_output099[] __initconst = {
+static const u8 enc_output099[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0x7b, 0xc3, 0x72, 0x98, 0x09, 0xe9, 0xdf, 0xe4,
@@ -5123,13 +5122,13 @@ static const u8 enc_output099[] __initconst = {
 	0xed, 0x20, 0x17, 0xc8, 0xdb, 0xa4, 0x77, 0x56,
 	0x29, 0x04, 0x9d, 0x78, 0x6e, 0x3b, 0xce, 0xb1
 };
-static const u8 enc_assoc099[] __initconst = {
+static const u8 enc_assoc099[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce099[] __initconst = {
+static const u8 enc_nonce099[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key099[] __initconst = {
+static const u8 enc_key099[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5137,7 +5136,7 @@ static const u8 enc_key099[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input100[] __initconst = {
+static const u8 enc_input100[] = {
 	0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
 	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
 	0x6b, 0x6d, 0xc9, 0xd2, 0x1a, 0x81, 0x9e, 0x70,
@@ -5147,7 +5146,7 @@ static const u8 enc_input100[] __initconst = {
 	0x10, 0xb9, 0x2f, 0x5f, 0xfe, 0xf9, 0x8b, 0x84,
 	0x7c, 0xf1, 0x7a, 0x9c, 0x98, 0xd8, 0x83, 0xe5
 };
-static const u8 enc_output100[] __initconst = {
+static const u8 enc_output100[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xec, 0xaf, 0x03, 0xdb, 0xf6, 0x98, 0xb8, 0x86,
@@ -5159,13 +5158,13 @@ static const u8 enc_output100[] __initconst = {
 	0x07, 0x3f, 0x17, 0xcb, 0x67, 0x78, 0x64, 0x59,
 	0x25, 0x04, 0x9d, 0x88, 0x22, 0xcb, 0xca, 0xb6
 };
-static const u8 enc_assoc100[] __initconst = {
+static const u8 enc_assoc100[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce100[] __initconst = {
+static const u8 enc_nonce100[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key100[] __initconst = {
+static const u8 enc_key100[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5173,7 +5172,7 @@ static const u8 enc_key100[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input101[] __initconst = {
+static const u8 enc_input101[] = {
 	0xff, 0xcb, 0x2b, 0x11, 0x06, 0xf8, 0x23, 0x4c,
 	0x5e, 0x99, 0xd4, 0xdb, 0x4c, 0x70, 0x48, 0xde,
 	0x32, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
@@ -5185,7 +5184,7 @@ static const u8 enc_input101[] __initconst = {
 	0xce, 0xbe, 0xf5, 0xe9, 0x88, 0x5a, 0x80, 0xea,
 	0x76, 0xd9, 0x75, 0xc1, 0x44, 0xa4, 0x18, 0x88
 };
-static const u8 enc_output101[] __initconst = {
+static const u8 enc_output101[] = {
 	0xff, 0xa0, 0xfc, 0x3e, 0x80, 0x32, 0xc3, 0xd5,
 	0xfd, 0xb6, 0x2a, 0x11, 0xf0, 0x96, 0x30, 0x7d,
 	0xb5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -5199,13 +5198,13 @@ static const u8 enc_output101[] __initconst = {
 	0x8b, 0x9b, 0xb4, 0xb4, 0x86, 0x12, 0x89, 0x65,
 	0x8c, 0x69, 0x6a, 0x83, 0x40, 0x15, 0x04, 0x05
 };
-static const u8 enc_assoc101[] __initconst = {
+static const u8 enc_assoc101[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce101[] __initconst = {
+static const u8 enc_nonce101[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key101[] __initconst = {
+static const u8 enc_key101[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5213,7 +5212,7 @@ static const u8 enc_key101[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input102[] __initconst = {
+static const u8 enc_input102[] = {
 	0x6f, 0x9e, 0x70, 0xed, 0x3b, 0x8b, 0xac, 0xa0,
 	0x26, 0xe4, 0x6a, 0x5a, 0x09, 0x43, 0x15, 0x8d,
 	0x21, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
@@ -5225,7 +5224,7 @@ static const u8 enc_input102[] __initconst = {
 	0xd4, 0x36, 0x51, 0xfd, 0x14, 0x9c, 0x26, 0x0b,
 	0xcb, 0xdd, 0x7b, 0x12, 0x68, 0x01, 0x31, 0x8c
 };
-static const u8 enc_output102[] __initconst = {
+static const u8 enc_output102[] = {
 	0x6f, 0xf5, 0xa7, 0xc2, 0xbd, 0x41, 0x4c, 0x39,
 	0x85, 0xcb, 0x94, 0x90, 0xb5, 0xa5, 0x6d, 0x2e,
 	0xa6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -5239,13 +5238,13 @@ static const u8 enc_output102[] __initconst = {
 	0x8b, 0x3b, 0xbd, 0x51, 0x64, 0x44, 0x59, 0x56,
 	0x8d, 0x81, 0xca, 0x1f, 0xa7, 0x2c, 0xe4, 0x04
 };
-static const u8 enc_assoc102[] __initconst = {
+static const u8 enc_assoc102[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce102[] __initconst = {
+static const u8 enc_nonce102[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key102[] __initconst = {
+static const u8 enc_key102[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5253,7 +5252,7 @@ static const u8 enc_key102[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input103[] __initconst = {
+static const u8 enc_input103[] = {
 	0x41, 0x2b, 0x08, 0x0a, 0x3e, 0x19, 0xc1, 0x0d,
 	0x44, 0xa1, 0xaf, 0x1e, 0xab, 0xde, 0xb4, 0xce,
 	0x35, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
@@ -5265,7 +5264,7 @@ static const u8 enc_input103[] __initconst = {
 	0xb3, 0xd4, 0xe9, 0x90, 0x90, 0x34, 0xc6, 0x14,
 	0xb1, 0x0a, 0xff, 0x55, 0x25, 0xd0, 0x9d, 0x8d
 };
-static const u8 enc_output103[] __initconst = {
+static const u8 enc_output103[] = {
 	0x41, 0x40, 0xdf, 0x25, 0xb8, 0xd3, 0x21, 0x94,
 	0xe7, 0x8e, 0x51, 0xd4, 0x17, 0x38, 0xcc, 0x6d,
 	0xb2, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -5279,13 +5278,13 @@ static const u8 enc_output103[] __initconst = {
 	0x86, 0xfb, 0xab, 0x2b, 0x4a, 0x94, 0xf4, 0x7a,
 	0xa5, 0x6f, 0x0a, 0xea, 0x65, 0xd1, 0x10, 0x08
 };
-static const u8 enc_assoc103[] __initconst = {
+static const u8 enc_assoc103[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce103[] __initconst = {
+static const u8 enc_nonce103[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key103[] __initconst = {
+static const u8 enc_key103[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5293,7 +5292,7 @@ static const u8 enc_key103[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input104[] __initconst = {
+static const u8 enc_input104[] = {
 	0xb2, 0x47, 0xa7, 0x47, 0x23, 0x49, 0x1a, 0xac,
 	0xac, 0xaa, 0xd7, 0x09, 0xc9, 0x1e, 0x93, 0x2b,
 	0x31, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
@@ -5305,7 +5304,7 @@ static const u8 enc_input104[] __initconst = {
 	0x42, 0x89, 0x79, 0x44, 0xc2, 0xa2, 0x8f, 0xa1,
 	0x76, 0x11, 0xd7, 0xfa, 0x5c, 0x22, 0xad, 0x8f
 };
-static const u8 enc_output104[] __initconst = {
+static const u8 enc_output104[] = {
 	0xb2, 0x2c, 0x70, 0x68, 0xa5, 0x83, 0xfa, 0x35,
 	0x0f, 0x85, 0x29, 0xc3, 0x75, 0xf8, 0xeb, 0x88,
 	0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -5319,13 +5318,13 @@ static const u8 enc_output104[] __initconst = {
 	0xa0, 0x19, 0xac, 0x2e, 0xd6, 0x67, 0xe1, 0x7d,
 	0xa1, 0x6f, 0x0a, 0xfa, 0x19, 0x61, 0x0d, 0x0d
 };
-static const u8 enc_assoc104[] __initconst = {
+static const u8 enc_assoc104[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce104[] __initconst = {
+static const u8 enc_nonce104[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key104[] __initconst = {
+static const u8 enc_key104[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5333,7 +5332,7 @@ static const u8 enc_key104[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input105[] __initconst = {
+static const u8 enc_input105[] = {
 	0x74, 0x0f, 0x9e, 0x49, 0xf6, 0x10, 0xef, 0xa5,
 	0x85, 0xb6, 0x59, 0xca, 0x6e, 0xd8, 0xb4, 0x99,
 	0x2d, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
@@ -5345,7 +5344,7 @@ static const u8 enc_input105[] __initconst = {
 	0x99, 0x7a, 0xeb, 0x0c, 0x27, 0x95, 0x62, 0x46,
 	0x69, 0xc3, 0x87, 0xf9, 0x11, 0x6a, 0xc1, 0x8d
 };
-static const u8 enc_output105[] __initconst = {
+static const u8 enc_output105[] = {
 	0x74, 0x64, 0x49, 0x66, 0x70, 0xda, 0x0f, 0x3c,
 	0x26, 0x99, 0xa7, 0x00, 0xd2, 0x3e, 0xcc, 0x3a,
 	0xaa, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -5359,13 +5358,13 @@ static const u8 enc_output105[] __initconst = {
 	0x73, 0x6e, 0x18, 0x18, 0x16, 0x96, 0xa5, 0x88,
 	0x9c, 0x31, 0x59, 0xfa, 0xab, 0xab, 0x20, 0xfd
 };
-static const u8 enc_assoc105[] __initconst = {
+static const u8 enc_assoc105[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce105[] __initconst = {
+static const u8 enc_nonce105[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key105[] __initconst = {
+static const u8 enc_key105[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5373,7 +5372,7 @@ static const u8 enc_key105[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input106[] __initconst = {
+static const u8 enc_input106[] = {
 	0xad, 0xba, 0x5d, 0x10, 0x5b, 0xc8, 0xaa, 0x06,
 	0x2c, 0x23, 0x36, 0xcb, 0x88, 0x9d, 0xdb, 0xd5,
 	0x37, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
@@ -5385,7 +5384,7 @@ static const u8 enc_input106[] __initconst = {
 	0xcf, 0x2b, 0x22, 0x5d, 0xb1, 0x60, 0x7a, 0x10,
 	0xe6, 0xd5, 0x40, 0x1e, 0x53, 0xb4, 0x2a, 0x8d
 };
-static const u8 enc_output106[] __initconst = {
+static const u8 enc_output106[] = {
 	0xad, 0xd1, 0x8a, 0x3f, 0xdd, 0x02, 0x4a, 0x9f,
 	0x8f, 0x0c, 0xc8, 0x01, 0x34, 0x7b, 0xa3, 0x76,
 	0xb0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -5399,13 +5398,13 @@ static const u8 enc_output106[] __initconst = {
 	0xba, 0xd5, 0x8f, 0x10, 0xa9, 0x1e, 0x6a, 0x88,
 	0x9a, 0xba, 0x32, 0xfd, 0x17, 0xd8, 0x33, 0x1a
 };
-static const u8 enc_assoc106[] __initconst = {
+static const u8 enc_assoc106[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce106[] __initconst = {
+static const u8 enc_nonce106[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key106[] __initconst = {
+static const u8 enc_key106[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5413,7 +5412,7 @@ static const u8 enc_key106[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input107[] __initconst = {
+static const u8 enc_input107[] = {
 	0xfe, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
 	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
 	0xc0, 0x01, 0xed, 0xc5, 0xda, 0x44, 0x2e, 0x71,
@@ -5427,7 +5426,7 @@ static const u8 enc_input107[] __initconst = {
 	0x00, 0x26, 0x6e, 0xa1, 0xe4, 0x36, 0x44, 0xa3,
 	0x4d, 0x8d, 0xd1, 0xdc, 0x93, 0xf2, 0xfa, 0x13
 };
-static const u8 enc_output107[] __initconst = {
+static const u8 enc_output107[] = {
 	0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0x47, 0xc3, 0x27, 0xcc, 0x36, 0x5d, 0x08, 0x87,
@@ -5443,13 +5442,13 @@ static const u8 enc_output107[] __initconst = {
 	0xd6, 0x8c, 0xe1, 0x74, 0x07, 0x9a, 0xdd, 0x02,
 	0x8d, 0xd0, 0x5c, 0xf8, 0x14, 0x63, 0x04, 0x88
 };
-static const u8 enc_assoc107[] __initconst = {
+static const u8 enc_assoc107[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce107[] __initconst = {
+static const u8 enc_nonce107[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key107[] __initconst = {
+static const u8 enc_key107[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5457,7 +5456,7 @@ static const u8 enc_key107[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input108[] __initconst = {
+static const u8 enc_input108[] = {
 	0xb5, 0x13, 0xb0, 0x6a, 0xb9, 0xac, 0x14, 0x43,
 	0x5a, 0xcb, 0x8a, 0xa3, 0xa3, 0x7a, 0xfd, 0xb6,
 	0x54, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
@@ -5469,7 +5468,7 @@ static const u8 enc_input108[] __initconst = {
 	0xb9, 0xc2, 0x7c, 0x30, 0x28, 0xaa, 0x8d, 0x69,
 	0xef, 0x06, 0xaf, 0xc0, 0xb5, 0x9e, 0xda, 0x8e
 };
-static const u8 enc_output108[] __initconst = {
+static const u8 enc_output108[] = {
 	0xb5, 0x78, 0x67, 0x45, 0x3f, 0x66, 0xf4, 0xda,
 	0xf9, 0xe4, 0x74, 0x69, 0x1f, 0x9c, 0x85, 0x15,
 	0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -5483,13 +5482,13 @@ static const u8 enc_output108[] __initconst = {
 	0xaa, 0x48, 0xa3, 0x88, 0x7d, 0x4b, 0x05, 0x96,
 	0x99, 0xc2, 0xfd, 0xf9, 0xc6, 0x78, 0x7e, 0x0a
 };
-static const u8 enc_assoc108[] __initconst = {
+static const u8 enc_assoc108[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce108[] __initconst = {
+static const u8 enc_nonce108[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key108[] __initconst = {
+static const u8 enc_key108[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5497,7 +5496,7 @@ static const u8 enc_key108[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input109[] __initconst = {
+static const u8 enc_input109[] = {
 	0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
 	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
 	0xd4, 0xf1, 0x09, 0xe8, 0x14, 0xce, 0xa8, 0x5a,
@@ -5511,7 +5510,7 @@ static const u8 enc_input109[] __initconst = {
 	0x1b, 0x64, 0x89, 0xba, 0x84, 0xd8, 0xf5, 0x59,
 	0x82, 0x9e, 0xd9, 0xbd, 0xa2, 0x29, 0x0f, 0x16
 };
-static const u8 enc_output109[] __initconst = {
+static const u8 enc_output109[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0x53, 0x33, 0xc3, 0xe1, 0xf8, 0xd7, 0x8e, 0xac,
@@ -5527,13 +5526,13 @@ static const u8 enc_output109[] __initconst = {
 	0xb9, 0x36, 0xa8, 0x17, 0xf2, 0x21, 0x1a, 0xf1,
 	0x29, 0xe2, 0xcf, 0x16, 0x0f, 0xd4, 0x2b, 0xcb
 };
-static const u8 enc_assoc109[] __initconst = {
+static const u8 enc_assoc109[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce109[] __initconst = {
+static const u8 enc_nonce109[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key109[] __initconst = {
+static const u8 enc_key109[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5541,7 +5540,7 @@ static const u8 enc_key109[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input110[] __initconst = {
+static const u8 enc_input110[] = {
 	0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
 	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
 	0xdf, 0x4c, 0x62, 0x03, 0x2d, 0x41, 0x19, 0xb5,
@@ -5555,7 +5554,7 @@ static const u8 enc_input110[] __initconst = {
 	0xb2, 0xa0, 0xc1, 0x84, 0x4b, 0x4e, 0x35, 0xd4,
 	0x1e, 0x5d, 0xa2, 0x10, 0xf6, 0x2f, 0x84, 0x12
 };
-static const u8 enc_output110[] __initconst = {
+static const u8 enc_output110[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0x58, 0x8e, 0xa8, 0x0a, 0xc1, 0x58, 0x3f, 0x43,
@@ -5571,13 +5570,13 @@ static const u8 enc_output110[] __initconst = {
 	0x9f, 0x7a, 0xc4, 0x35, 0x1f, 0x6b, 0x91, 0xe6,
 	0x30, 0x97, 0xa7, 0x13, 0x11, 0x5d, 0x05, 0xbe
 };
-static const u8 enc_assoc110[] __initconst = {
+static const u8 enc_assoc110[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce110[] __initconst = {
+static const u8 enc_nonce110[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key110[] __initconst = {
+static const u8 enc_key110[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5585,7 +5584,7 @@ static const u8 enc_key110[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input111[] __initconst = {
+static const u8 enc_input111[] = {
 	0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
 	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
 	0x13, 0xf8, 0x0a, 0x00, 0x6d, 0xc1, 0xbb, 0xda,
@@ -5599,7 +5598,7 @@ static const u8 enc_input111[] __initconst = {
 	0x2b, 0x6c, 0x89, 0x1d, 0x37, 0xc7, 0xe1, 0x1a,
 	0x56, 0x41, 0x91, 0x9c, 0x49, 0x4d, 0x95, 0x16
 };
-static const u8 enc_output111[] __initconst = {
+static const u8 enc_output111[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0x94, 0x3a, 0xc0, 0x09, 0x81, 0xd8, 0x9d, 0x2c,
@@ -5615,13 +5614,13 @@ static const u8 enc_output111[] __initconst = {
 	0x9a, 0x18, 0xa8, 0x28, 0x07, 0x02, 0x69, 0xf4,
 	0x47, 0x00, 0xd0, 0x09, 0xe7, 0x17, 0x1c, 0xc9
 };
-static const u8 enc_assoc111[] __initconst = {
+static const u8 enc_assoc111[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce111[] __initconst = {
+static const u8 enc_nonce111[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key111[] __initconst = {
+static const u8 enc_key111[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5629,7 +5628,7 @@ static const u8 enc_key111[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input112[] __initconst = {
+static const u8 enc_input112[] = {
 	0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
 	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
 	0x82, 0xe5, 0x9b, 0x45, 0x82, 0x91, 0x50, 0x38,
@@ -5643,7 +5642,7 @@ static const u8 enc_input112[] __initconst = {
 	0xfe, 0x55, 0xf9, 0x2a, 0xdc, 0x08, 0xb5, 0xaa,
 	0x95, 0x48, 0xa9, 0x2d, 0x63, 0xaf, 0xe1, 0x13
 };
-static const u8 enc_output112[] __initconst = {
+static const u8 enc_output112[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0x05, 0x27, 0x51, 0x4c, 0x6e, 0x88, 0x76, 0xce,
@@ -5659,13 +5658,13 @@ static const u8 enc_output112[] __initconst = {
 	0xb4, 0x36, 0xa8, 0x2b, 0x93, 0xd5, 0x55, 0xf7,
 	0x43, 0x00, 0xd0, 0x19, 0x9b, 0xa7, 0x18, 0xce
 };
-static const u8 enc_assoc112[] __initconst = {
+static const u8 enc_assoc112[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce112[] __initconst = {
+static const u8 enc_nonce112[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key112[] __initconst = {
+static const u8 enc_key112[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5673,7 +5672,7 @@ static const u8 enc_key112[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input113[] __initconst = {
+static const u8 enc_input113[] = {
 	0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
 	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
 	0xf1, 0xd1, 0x28, 0x87, 0xb7, 0x21, 0x69, 0x86,
@@ -5687,7 +5686,7 @@ static const u8 enc_input113[] __initconst = {
 	0xef, 0xe3, 0x69, 0x79, 0xed, 0x9e, 0x7d, 0x3e,
 	0xc9, 0x52, 0x41, 0x4e, 0x49, 0xb1, 0x30, 0x16
 };
-static const u8 enc_output113[] __initconst = {
+static const u8 enc_output113[] = {
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0x76, 0x13, 0xe2, 0x8e, 0x5b, 0x38, 0x4f, 0x70,
@@ -5703,13 +5702,13 @@ static const u8 enc_output113[] __initconst = {
 	0xce, 0x54, 0xa8, 0x2e, 0x1f, 0xa9, 0x42, 0xfa,
 	0x3f, 0x00, 0xd0, 0x29, 0x4f, 0x37, 0x15, 0xd3
 };
-static const u8 enc_assoc113[] __initconst = {
+static const u8 enc_assoc113[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce113[] __initconst = {
+static const u8 enc_nonce113[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key113[] __initconst = {
+static const u8 enc_key113[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5717,7 +5716,7 @@ static const u8 enc_key113[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input114[] __initconst = {
+static const u8 enc_input114[] = {
 	0xcb, 0xf1, 0xda, 0x9e, 0x0b, 0xa9, 0x37, 0x73,
 	0x74, 0xe6, 0x9e, 0x1c, 0x0e, 0x60, 0x0c, 0xfc,
 	0x34, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
@@ -5729,7 +5728,7 @@ static const u8 enc_input114[] __initconst = {
 	0x66, 0x68, 0xdb, 0xc8, 0xf5, 0xf2, 0x0e, 0xf2,
 	0xb3, 0xf3, 0x8f, 0x00, 0xe2, 0x03, 0x17, 0x88
 };
-static const u8 enc_output114[] __initconst = {
+static const u8 enc_output114[] = {
 	0xcb, 0x9a, 0x0d, 0xb1, 0x8d, 0x63, 0xd7, 0xea,
 	0xd7, 0xc9, 0x60, 0xd6, 0xb2, 0x86, 0x74, 0x5f,
 	0xb3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -5743,13 +5742,13 @@ static const u8 enc_output114[] __initconst = {
 	0x23, 0x83, 0xab, 0x0b, 0x79, 0x92, 0x05, 0x69,
 	0x9b, 0x51, 0x0a, 0xa7, 0x09, 0xbf, 0x31, 0xf1
 };
-static const u8 enc_assoc114[] __initconst = {
+static const u8 enc_assoc114[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce114[] __initconst = {
+static const u8 enc_nonce114[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key114[] __initconst = {
+static const u8 enc_key114[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5757,7 +5756,7 @@ static const u8 enc_key114[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input115[] __initconst = {
+static const u8 enc_input115[] = {
 	0x8f, 0x27, 0x86, 0x94, 0xc4, 0xe9, 0xda, 0xeb,
 	0xd5, 0x8d, 0x3e, 0x5b, 0x96, 0x6e, 0x8b, 0x68,
 	0x42, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09,
@@ -5769,7 +5768,7 @@ static const u8 enc_input115[] __initconst = {
 	0xde, 0x04, 0x9a, 0x00, 0xa8, 0x64, 0x06, 0x4b,
 	0xbc, 0xd4, 0x6f, 0xe4, 0xe4, 0x5b, 0x42, 0x8f
 };
-static const u8 enc_output115[] __initconst = {
+static const u8 enc_output115[] = {
 	0x8f, 0x4c, 0x51, 0xbb, 0x42, 0x23, 0x3a, 0x72,
 	0x76, 0xa2, 0xc0, 0x91, 0x2a, 0x88, 0xf3, 0xcb,
 	0xc5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -5783,13 +5782,13 @@ static const u8 enc_output115[] __initconst = {
 	0x8b, 0xfb, 0xab, 0x17, 0xa9, 0xe0, 0xb8, 0x74,
 	0x8b, 0x51, 0x0a, 0xe7, 0xd9, 0xfd, 0x23, 0x05
 };
-static const u8 enc_assoc115[] __initconst = {
+static const u8 enc_assoc115[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce115[] __initconst = {
+static const u8 enc_nonce115[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key115[] __initconst = {
+static const u8 enc_key115[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5797,7 +5796,7 @@ static const u8 enc_key115[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input116[] __initconst = {
+static const u8 enc_input116[] = {
 	0xd5, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
 	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
 	0x9a, 0x22, 0xd7, 0x0a, 0x48, 0xe2, 0x4f, 0xdd,
@@ -5811,7 +5810,7 @@ static const u8 enc_input116[] __initconst = {
 	0x3f, 0x91, 0xf8, 0xe7, 0xc7, 0xb1, 0x96, 0x25,
 	0x64, 0x61, 0x9c, 0x5e, 0x7e, 0x9b, 0xf6, 0x13
 };
-static const u8 enc_output116[] __initconst = {
+static const u8 enc_output116[] = {
 	0xd5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0x1d, 0xe0, 0x1d, 0x03, 0xa4, 0xfb, 0x69, 0x2b,
@@ -5827,13 +5826,13 @@ static const u8 enc_output116[] __initconst = {
 	0x49, 0xbc, 0x6e, 0x9f, 0xc5, 0x1c, 0x4d, 0x50,
 	0x30, 0x36, 0x64, 0x4d, 0x84, 0x27, 0x73, 0xd2
 };
-static const u8 enc_assoc116[] __initconst = {
+static const u8 enc_assoc116[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce116[] __initconst = {
+static const u8 enc_nonce116[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key116[] __initconst = {
+static const u8 enc_key116[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5841,7 +5840,7 @@ static const u8 enc_key116[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input117[] __initconst = {
+static const u8 enc_input117[] = {
 	0xdb, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
 	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
 	0x75, 0xd5, 0x64, 0x3a, 0xa5, 0xaf, 0x93, 0x4d,
@@ -5855,7 +5854,7 @@ static const u8 enc_input117[] __initconst = {
 	0x28, 0x3f, 0x6b, 0x32, 0x18, 0x07, 0x5f, 0xc9,
 	0x5f, 0x6b, 0xb4, 0xff, 0x45, 0x6d, 0xc1, 0x11
 };
-static const u8 enc_output117[] __initconst = {
+static const u8 enc_output117[] = {
 	0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xf2, 0x17, 0xae, 0x33, 0x49, 0xb6, 0xb5, 0xbb,
@@ -5871,13 +5870,13 @@ static const u8 enc_output117[] __initconst = {
 	0x63, 0xda, 0x6e, 0xa2, 0x51, 0xf0, 0x39, 0x53,
 	0x2c, 0x36, 0x64, 0x5d, 0x38, 0xb7, 0x6f, 0xd7
 };
-static const u8 enc_assoc117[] __initconst = {
+static const u8 enc_assoc117[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce117[] __initconst = {
+static const u8 enc_nonce117[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key117[] __initconst = {
+static const u8 enc_key117[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5885,7 +5884,7 @@ static const u8 enc_key117[] __initconst = {
 };
 
 /* wycheproof - edge case intermediate sums in poly1305 */
-static const u8 enc_input118[] __initconst = {
+static const u8 enc_input118[] = {
 	0x93, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66,
 	0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c,
 	0x62, 0x48, 0x39, 0x60, 0x42, 0x16, 0xe4, 0x03,
@@ -5899,7 +5898,7 @@ static const u8 enc_input118[] __initconst = {
 	0x90, 0xec, 0xf2, 0x1a, 0x04, 0xe6, 0x30, 0x85,
 	0x8b, 0xb6, 0x56, 0x52, 0xb5, 0xb1, 0x80, 0x16
 };
-static const u8 enc_output118[] __initconst = {
+static const u8 enc_output118[] = {
 	0x93, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 	0xe5, 0x8a, 0xf3, 0x69, 0xae, 0x0f, 0xc2, 0xf5,
@@ -5915,13 +5914,13 @@ static const u8 enc_output118[] __initconst = {
 	0x73, 0xeb, 0x27, 0x24, 0xb5, 0xc4, 0x05, 0xf0,
 	0x4d, 0x00, 0xd0, 0xf1, 0x58, 0x40, 0xa1, 0xc1
 };
-static const u8 enc_assoc118[] __initconst = {
+static const u8 enc_assoc118[] = {
 	0xff, 0xff, 0xff, 0xff
 };
-static const u8 enc_nonce118[] __initconst = {
+static const u8 enc_nonce118[] = {
 	0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52
 };
-static const u8 enc_key118[] __initconst = {
+static const u8 enc_key118[] = {
 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
 	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
@@ -5929,7 +5928,7 @@ static const u8 enc_key118[] __initconst = {
 };
 
 static const struct chacha20poly1305_testvec
-chacha20poly1305_enc_vectors[] __initconst = {
+chacha20poly1305_enc_vectors[] = {
 	{ enc_input001, enc_output001, enc_assoc001, enc_nonce001, enc_key001,
 	  sizeof(enc_input001), sizeof(enc_assoc001), sizeof(enc_nonce001) },
 	{ enc_input002, enc_output002, enc_assoc002, enc_nonce002, enc_key002,
@@ -6168,7 +6167,7 @@ chacha20poly1305_enc_vectors[] __initconst = {
 	  sizeof(enc_input118), sizeof(enc_assoc118), sizeof(enc_nonce118) }
 };
 
-static const u8 dec_input001[] __initconst = {
+static const u8 dec_input001[] = {
 	0x64, 0xa0, 0x86, 0x15, 0x75, 0x86, 0x1a, 0xf4,
 	0x60, 0xf0, 0x62, 0xc7, 0x9b, 0xe6, 0x43, 0xbd,
 	0x5e, 0x80, 0x5c, 0xfd, 0x34, 0x5c, 0xf3, 0x89,
@@ -6206,7 +6205,7 @@ static const u8 dec_input001[] __initconst = {
 	0x22, 0x39, 0x23, 0x36, 0xfe, 0xa1, 0x85, 0x1f,
 	0x38
 };
-static const u8 dec_output001[] __initconst = {
+static const u8 dec_output001[] = {
 	0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74,
 	0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20,
 	0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66,
@@ -6242,95 +6241,95 @@ static const u8 dec_output001[] __initconst = {
 	0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80,
 	0x9d
 };
-static const u8 dec_assoc001[] __initconst = {
+static const u8 dec_assoc001[] = {
 	0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x4e, 0x91
 };
-static const u8 dec_nonce001[] __initconst = {
+static const u8 dec_nonce001[] = {
 	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08
 };
-static const u8 dec_key001[] __initconst = {
+static const u8 dec_key001[] = {
 	0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a,
 	0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0,
 	0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09,
 	0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0
 };
 
-static const u8 dec_input002[] __initconst = {
+static const u8 dec_input002[] = {
 	0xea, 0xe0, 0x1e, 0x9e, 0x2c, 0x91, 0xaa, 0xe1,
 	0xdb, 0x5d, 0x99, 0x3f, 0x8a, 0xf7, 0x69, 0x92
 };
-static const u8 dec_output002[] __initconst = { };
-static const u8 dec_assoc002[] __initconst = { };
-static const u8 dec_nonce002[] __initconst = {
+static const u8 dec_output002[] = { };
+static const u8 dec_assoc002[] = { };
+static const u8 dec_nonce002[] = {
 	0xca, 0xbf, 0x33, 0x71, 0x32, 0x45, 0x77, 0x8e
 };
-static const u8 dec_key002[] __initconst = {
+static const u8 dec_key002[] = {
 	0x4c, 0xf5, 0x96, 0x83, 0x38, 0xe6, 0xae, 0x7f,
 	0x2d, 0x29, 0x25, 0x76, 0xd5, 0x75, 0x27, 0x86,
 	0x91, 0x9a, 0x27, 0x7a, 0xfb, 0x46, 0xc5, 0xef,
 	0x94, 0x81, 0x79, 0x57, 0x14, 0x59, 0x40, 0x68
 };
 
-static const u8 dec_input003[] __initconst = {
+static const u8 dec_input003[] = {
 	0xdd, 0x6b, 0x3b, 0x82, 0xce, 0x5a, 0xbd, 0xd6,
 	0xa9, 0x35, 0x83, 0xd8, 0x8c, 0x3d, 0x85, 0x77
 };
-static const u8 dec_output003[] __initconst = { };
-static const u8 dec_assoc003[] __initconst = {
+static const u8 dec_output003[] = { };
+static const u8 dec_assoc003[] = {
 	0x33, 0x10, 0x41, 0x12, 0x1f, 0xf3, 0xd2, 0x6b
 };
-static const u8 dec_nonce003[] __initconst = {
+static const u8 dec_nonce003[] = {
 	0x3d, 0x86, 0xb5, 0x6b, 0xc8, 0xa3, 0x1f, 0x1d
 };
-static const u8 dec_key003[] __initconst = {
+static const u8 dec_key003[] = {
 	0x2d, 0xb0, 0x5d, 0x40, 0xc8, 0xed, 0x44, 0x88,
 	0x34, 0xd1, 0x13, 0xaf, 0x57, 0xa1, 0xeb, 0x3a,
 	0x2a, 0x80, 0x51, 0x36, 0xec, 0x5b, 0xbc, 0x08,
 	0x93, 0x84, 0x21, 0xb5, 0x13, 0x88, 0x3c, 0x0d
 };
 
-static const u8 dec_input004[] __initconst = {
+static const u8 dec_input004[] = {
 	0xb7, 0x1b, 0xb0, 0x73, 0x59, 0xb0, 0x84, 0xb2,
 	0x6d, 0x8e, 0xab, 0x94, 0x31, 0xa1, 0xae, 0xac,
 	0x89
 };
-static const u8 dec_output004[] __initconst = {
+static const u8 dec_output004[] = {
 	0xa4
 };
-static const u8 dec_assoc004[] __initconst = {
+static const u8 dec_assoc004[] = {
 	0x6a, 0xe2, 0xad, 0x3f, 0x88, 0x39, 0x5a, 0x40
 };
-static const u8 dec_nonce004[] __initconst = {
+static const u8 dec_nonce004[] = {
 	0xd2, 0x32, 0x1f, 0x29, 0x28, 0xc6, 0xc4, 0xc4
 };
-static const u8 dec_key004[] __initconst = {
+static const u8 dec_key004[] = {
 	0x4b, 0x28, 0x4b, 0xa3, 0x7b, 0xbe, 0xe9, 0xf8,
 	0x31, 0x80, 0x82, 0xd7, 0xd8, 0xe8, 0xb5, 0xa1,
 	0xe2, 0x18, 0x18, 0x8a, 0x9c, 0xfa, 0xa3, 0x3d,
 	0x25, 0x71, 0x3e, 0x40, 0xbc, 0x54, 0x7a, 0x3e
 };
 
-static const u8 dec_input005[] __initconst = {
+static const u8 dec_input005[] = {
 	0xbf, 0xe1, 0x5b, 0x0b, 0xdb, 0x6b, 0xf5, 0x5e,
 	0x6c, 0x5d, 0x84, 0x44, 0x39, 0x81, 0xc1, 0x9c,
 	0xac
 };
-static const u8 dec_output005[] __initconst = {
+static const u8 dec_output005[] = {
 	0x2d
 };
-static const u8 dec_assoc005[] __initconst = { };
-static const u8 dec_nonce005[] __initconst = {
+static const u8 dec_assoc005[] = { };
+static const u8 dec_nonce005[] = {
 	0x20, 0x1c, 0xaa, 0x5f, 0x9c, 0xbf, 0x92, 0x30
 };
-static const u8 dec_key005[] __initconst = {
+static const u8 dec_key005[] = {
 	0x66, 0xca, 0x9c, 0x23, 0x2a, 0x4b, 0x4b, 0x31,
 	0x0e, 0x92, 0x89, 0x8b, 0xf4, 0x93, 0xc7, 0x87,
 	0x98, 0xa3, 0xd8, 0x39, 0xf8, 0xf4, 0xa7, 0x01,
 	0xc0, 0x2e, 0x0a, 0xa6, 0x7e, 0x5a, 0x78, 0x87
 };
 
-static const u8 dec_input006[] __initconst = {
+static const u8 dec_input006[] = {
 	0x8b, 0x06, 0xd3, 0x31, 0xb0, 0x93, 0x45, 0xb1,
 	0x75, 0x6e, 0x26, 0xf9, 0x67, 0xbc, 0x90, 0x15,
 	0x81, 0x2c, 0xb5, 0xf0, 0xc6, 0x2b, 0xc7, 0x8c,
@@ -6351,7 +6350,7 @@ static const u8 dec_input006[] __initconst = {
 	0x8d, 0x31, 0xf7, 0x7a, 0x39, 0x4d, 0x8f, 0x9a,
 	0xeb
 };
-static const u8 dec_output006[] __initconst = {
+static const u8 dec_output006[] = {
 	0x33, 0x2f, 0x94, 0xc1, 0xa4, 0xef, 0xcc, 0x2a,
 	0x5b, 0xa6, 0xe5, 0x8f, 0x1d, 0x40, 0xf0, 0x92,
 	0x3c, 0xd9, 0x24, 0x11, 0xa9, 0x71, 0xf9, 0x37,
@@ -6370,20 +6369,20 @@ static const u8 dec_output006[] __initconst = {
 	0x69, 0x21, 0x70, 0xd8, 0xa4, 0x4b, 0xc8, 0xde,
 	0x8f
 };
-static const u8 dec_assoc006[] __initconst = {
+static const u8 dec_assoc006[] = {
 	0x70, 0xd3, 0x33, 0xf3, 0x8b, 0x18, 0x0b
 };
-static const u8 dec_nonce006[] __initconst = {
+static const u8 dec_nonce006[] = {
 	0xdf, 0x51, 0x84, 0x82, 0x42, 0x0c, 0x75, 0x9c
 };
-static const u8 dec_key006[] __initconst = {
+static const u8 dec_key006[] = {
 	0x68, 0x7b, 0x8d, 0x8e, 0xe3, 0xc4, 0xdd, 0xae,
 	0xdf, 0x72, 0x7f, 0x53, 0x72, 0x25, 0x1e, 0x78,
 	0x91, 0xcb, 0x69, 0x76, 0x1f, 0x49, 0x93, 0xf9,
 	0x6f, 0x21, 0xcc, 0x39, 0x9c, 0xad, 0xb1, 0x01
 };
 
-static const u8 dec_input007[] __initconst = {
+static const u8 dec_input007[] = {
 	0x85, 0x04, 0xc2, 0xed, 0x8d, 0xfd, 0x97, 0x5c,
 	0xd2, 0xb7, 0xe2, 0xc1, 0x6b, 0xa3, 0xba, 0xf8,
 	0xc9, 0x50, 0xc3, 0xc6, 0xa5, 0xe3, 0xa4, 0x7c,
@@ -6419,7 +6418,7 @@ static const u8 dec_input007[] __initconst = {
 	0x7a, 0x4b, 0xbc, 0xb0, 0x10, 0x5c, 0x96, 0x42,
 	0x3a, 0x00, 0x98, 0xcd, 0x15, 0xe8, 0xb7, 0x53
 };
-static const u8 dec_output007[] __initconst = {
+static const u8 dec_output007[] = {
 	0x9b, 0x18, 0xdb, 0xdd, 0x9a, 0x0f, 0x3e, 0xa5,
 	0x15, 0x17, 0xde, 0xdf, 0x08, 0x9d, 0x65, 0x0a,
 	0x67, 0x30, 0x12, 0xe2, 0x34, 0x77, 0x4b, 0xc1,
@@ -6453,18 +6452,18 @@ static const u8 dec_output007[] __initconst = {
 	0x40, 0x5f, 0x99, 0xb7, 0x73, 0xec, 0x9b, 0x2b,
 	0xf0, 0x65, 0x11, 0xc8, 0xd0, 0x0a, 0x9f, 0xd3
 };
-static const u8 dec_assoc007[] __initconst = { };
-static const u8 dec_nonce007[] __initconst = {
+static const u8 dec_assoc007[] = { };
+static const u8 dec_nonce007[] = {
 	0xde, 0x7b, 0xef, 0xc3, 0x65, 0x1b, 0x68, 0xb0
 };
-static const u8 dec_key007[] __initconst = {
+static const u8 dec_key007[] = {
 	0x8d, 0xb8, 0x91, 0x48, 0xf0, 0xe7, 0x0a, 0xbd,
 	0xf9, 0x3f, 0xcd, 0xd9, 0xa0, 0x1e, 0x42, 0x4c,
 	0xe7, 0xde, 0x25, 0x3d, 0xa3, 0xd7, 0x05, 0x80,
 	0x8d, 0xf2, 0x82, 0xac, 0x44, 0x16, 0x51, 0x01
 };
 
-static const u8 dec_input008[] __initconst = {
+static const u8 dec_input008[] = {
 	0x14, 0xf6, 0x41, 0x37, 0xa6, 0xd4, 0x27, 0xcd,
 	0xdb, 0x06, 0x3e, 0x9a, 0x4e, 0xab, 0xd5, 0xb1,
 	0x1e, 0x6b, 0xd2, 0xbc, 0x11, 0xf4, 0x28, 0x93,
@@ -6532,7 +6531,7 @@ static const u8 dec_input008[] __initconst = {
 	0x54, 0xfd, 0x0d, 0x98, 0x1c, 0x5a, 0x6f, 0x1f,
 	0x9a, 0x40, 0xcd, 0xa2, 0xff, 0x6a, 0xf1, 0x54
 };
-static const u8 dec_output008[] __initconst = {
+static const u8 dec_output008[] = {
 	0xc3, 0x09, 0x94, 0x62, 0xe6, 0x46, 0x2e, 0x10,
 	0xbe, 0x00, 0xe4, 0xfc, 0xf3, 0x40, 0xa3, 0xe2,
 	0x0f, 0xc2, 0x8b, 0x28, 0xdc, 0xba, 0xb4, 0x3c,
@@ -6598,18 +6597,18 @@ static const u8 dec_output008[] __initconst = {
 	0x2e, 0x79, 0xb5, 0xe2, 0xb8, 0xe8, 0xb9, 0x7b,
 	0xd5, 0x10, 0xcb, 0xff, 0x5d, 0x14, 0x73, 0xf3
 };
-static const u8 dec_assoc008[] __initconst = { };
-static const u8 dec_nonce008[] __initconst = {
+static const u8 dec_assoc008[] = { };
+static const u8 dec_nonce008[] = {
 	0x0e, 0x0d, 0x57, 0xbb, 0x7b, 0x40, 0x54, 0x02
 };
-static const u8 dec_key008[] __initconst = {
+static const u8 dec_key008[] = {
 	0xf2, 0xaa, 0x4f, 0x99, 0xfd, 0x3e, 0xa8, 0x53,
 	0xc1, 0x44, 0xe9, 0x81, 0x18, 0xdc, 0xf5, 0xf0,
 	0x3e, 0x44, 0x15, 0x59, 0xe0, 0xc5, 0x44, 0x86,
 	0xc3, 0x91, 0xa8, 0x75, 0xc0, 0x12, 0x46, 0xba
 };
 
-static const u8 dec_input009[] __initconst = {
+static const u8 dec_input009[] = {
 	0xfd, 0x81, 0x8d, 0xd0, 0x3d, 0xb4, 0xd5, 0xdf,
 	0xd3, 0x42, 0x47, 0x5a, 0x6d, 0x19, 0x27, 0x66,
 	0x4b, 0x2e, 0x0c, 0x27, 0x9c, 0x96, 0x4c, 0x72,
@@ -6678,7 +6677,7 @@ static const u8 dec_input009[] __initconst = {
 	0x43, 0xb5, 0xd9, 0x90, 0xe1, 0x85, 0xf5, 0xa8,
 	0xae
 };
-static const u8 dec_output009[] __initconst = {
+static const u8 dec_output009[] = {
 	0xe6, 0xc3, 0xdb, 0x63, 0x55, 0x15, 0xe3, 0x5b,
 	0xb7, 0x4b, 0x27, 0x8b, 0x5a, 0xdd, 0xc2, 0xe8,
 	0x3a, 0x6b, 0xd7, 0x81, 0x96, 0x35, 0x97, 0xca,
@@ -6745,21 +6744,21 @@ static const u8 dec_output009[] __initconst = {
 	0x74, 0xb9, 0xe2, 0xd6, 0x1c, 0x80, 0x2c, 0x52,
 	0x65
 };
-static const u8 dec_assoc009[] __initconst = {
+static const u8 dec_assoc009[] = {
 	0x5a, 0x27, 0xff, 0xeb, 0xdf, 0x84, 0xb2, 0x9e,
 	0xef
 };
-static const u8 dec_nonce009[] __initconst = {
+static const u8 dec_nonce009[] = {
 	0xef, 0x2d, 0x63, 0xee, 0x6b, 0x80, 0x8b, 0x78
 };
-static const u8 dec_key009[] __initconst = {
+static const u8 dec_key009[] = {
 	0xea, 0xbc, 0x56, 0x99, 0xe3, 0x50, 0xff, 0xc5,
 	0xcc, 0x1a, 0xd7, 0xc1, 0x57, 0x72, 0xea, 0x86,
 	0x5b, 0x89, 0x88, 0x61, 0x3d, 0x2f, 0x9b, 0xb2,
 	0xe7, 0x9c, 0xec, 0x74, 0x6e, 0x3e, 0xf4, 0x3b
 };
 
-static const u8 dec_input010[] __initconst = {
+static const u8 dec_input010[] = {
 	0xe5, 0x26, 0xa4, 0x3d, 0xbd, 0x33, 0xd0, 0x4b,
 	0x6f, 0x05, 0xa7, 0x6e, 0x12, 0x7a, 0xd2, 0x74,
 	0xa6, 0xdd, 0xbd, 0x95, 0xeb, 0xf9, 0xa4, 0xf1,
@@ -6891,7 +6890,7 @@ static const u8 dec_input010[] __initconst = {
 	0x80, 0x6b, 0xf6, 0x1f, 0xc3, 0xcc, 0x97, 0xc9,
 	0x24, 0x9f, 0xf3, 0xaf, 0x43, 0x14, 0xd5, 0xa0
 };
-static const u8 dec_output010[] __initconst = {
+static const u8 dec_output010[] = {
 	0x42, 0x93, 0xe4, 0xeb, 0x97, 0xb0, 0x57, 0xbf,
 	0x1a, 0x8b, 0x1f, 0xe4, 0x5f, 0x36, 0x20, 0x3c,
 	0xef, 0x0a, 0xa9, 0x48, 0x5f, 0x5f, 0x37, 0x22,
@@ -7021,21 +7020,21 @@ static const u8 dec_output010[] __initconst = {
 	0x5a, 0xc0, 0x33, 0x1f, 0xcb, 0x05, 0x6d, 0x5c,
 	0x06, 0x87, 0x52, 0xa2, 0x8f, 0x26, 0xd5, 0x4f
 };
-static const u8 dec_assoc010[] __initconst = {
+static const u8 dec_assoc010[] = {
 	0xd2, 0xa1, 0x70, 0xdb, 0x7a, 0xf8, 0xfa, 0x27,
 	0xba, 0x73, 0x0f, 0xbf, 0x3d, 0x1e, 0x82, 0xb2
 };
-static const u8 dec_nonce010[] __initconst = {
+static const u8 dec_nonce010[] = {
 	0xdb, 0x92, 0x0f, 0x7f, 0x17, 0x54, 0x0c, 0x30
 };
-static const u8 dec_key010[] __initconst = {
+static const u8 dec_key010[] = {
 	0x47, 0x11, 0xeb, 0x86, 0x2b, 0x2c, 0xab, 0x44,
 	0x34, 0xda, 0x7f, 0x57, 0x03, 0x39, 0x0c, 0xaf,
 	0x2c, 0x14, 0xfd, 0x65, 0x23, 0xe9, 0x8e, 0x74,
 	0xd5, 0x08, 0x68, 0x08, 0xe7, 0xb4, 0x72, 0xd7
 };
 
-static const u8 dec_input011[] __initconst = {
+static const u8 dec_input011[] = {
 	0x6a, 0xfc, 0x4b, 0x25, 0xdf, 0xc0, 0xe4, 0xe8,
 	0x17, 0x4d, 0x4c, 0xc9, 0x7e, 0xde, 0x3a, 0xcc,
 	0x3c, 0xba, 0x6a, 0x77, 0x47, 0xdb, 0xe3, 0x74,
@@ -7281,7 +7280,7 @@ static const u8 dec_input011[] __initconst = {
 	0x86, 0xc4, 0xfc, 0x97, 0xae, 0x3f, 0x8f, 0x1e,
 	0x2b, 0xdf, 0xcd, 0xf9, 0x3c
 };
-static const u8 dec_output011[] __initconst = {
+static const u8 dec_output011[] = {
 	0x7a, 0x57, 0xf2, 0xc7, 0x06, 0x3f, 0x50, 0x7b,
 	0x36, 0x1a, 0x66, 0x5c, 0xb9, 0x0e, 0x5e, 0x3b,
 	0x45, 0x60, 0xbe, 0x9a, 0x31, 0x9f, 0xff, 0x5d,
@@ -7525,20 +7524,20 @@ static const u8 dec_output011[] __initconst = {
 	0x5c, 0x42, 0x16, 0xb8, 0x28, 0xeb, 0x1b, 0x61,
 	0x10, 0x1e, 0xbf, 0xec, 0xa8
 };
-static const u8 dec_assoc011[] __initconst = {
+static const u8 dec_assoc011[] = {
 	0xd6, 0x31, 0xda, 0x5d, 0x42, 0x5e, 0xd7
 };
-static const u8 dec_nonce011[] __initconst = {
+static const u8 dec_nonce011[] = {
 	0xfd, 0x87, 0xd4, 0xd8, 0x62, 0xfd, 0xec, 0xaa
 };
-static const u8 dec_key011[] __initconst = {
+static const u8 dec_key011[] = {
 	0x35, 0x4e, 0xb5, 0x70, 0x50, 0x42, 0x8a, 0x85,
 	0xf2, 0xfb, 0xed, 0x7b, 0xd0, 0x9e, 0x97, 0xca,
 	0xfa, 0x98, 0x66, 0x63, 0xee, 0x37, 0xcc, 0x52,
 	0xfe, 0xd1, 0xdf, 0x95, 0x15, 0x34, 0x29, 0x38
 };
 
-static const u8 dec_input012[] __initconst = {
+static const u8 dec_input012[] = {
 	0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3,
 	0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf,
 	0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1,
@@ -7794,7 +7793,7 @@ static const u8 dec_input012[] __initconst = {
 	0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62,
 	0x70, 0xcf, 0xd6
 };
-static const u8 dec_output012[] __initconst = {
+static const u8 dec_output012[] = {
 	0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0,
 	0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5,
 	0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57,
@@ -8048,7 +8047,7 @@ static const u8 dec_output012[] __initconst = {
 	0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d,
 	0x78, 0xec, 0x00
 };
-static const u8 dec_assoc012[] __initconst = {
+static const u8 dec_assoc012[] = {
 	0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8,
 	0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce,
 	0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c,
@@ -8058,17 +8057,17 @@ static const u8 dec_assoc012[] __initconst = {
 	0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b,
 	0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9
 };
-static const u8 dec_nonce012[] __initconst = {
+static const u8 dec_nonce012[] = {
 	0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06
 };
-static const u8 dec_key012[] __initconst = {
+static const u8 dec_key012[] = {
 	0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e,
 	0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d,
 	0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e,
 	0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64
 };
 
-static const u8 dec_input013[] __initconst = {
+static const u8 dec_input013[] = {
 	0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3,
 	0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf,
 	0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1,
@@ -8324,7 +8323,7 @@ static const u8 dec_input013[] __initconst = {
 	0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62,
 	0x70, 0xcf, 0xd7
 };
-static const u8 dec_output013[] __initconst = {
+static const u8 dec_output013[] = {
 	0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0,
 	0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5,
 	0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57,
@@ -8578,7 +8577,7 @@ static const u8 dec_output013[] __initconst = {
 	0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d,
 	0x78, 0xec, 0x00
 };
-static const u8 dec_assoc013[] __initconst = {
+static const u8 dec_assoc013[] = {
 	0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8,
 	0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce,
 	0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c,
@@ -8588,10 +8587,10 @@ static const u8 dec_assoc013[] __initconst = {
 	0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b,
 	0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9
 };
-static const u8 dec_nonce013[] __initconst = {
+static const u8 dec_nonce013[] = {
 	0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06
 };
-static const u8 dec_key013[] __initconst = {
+static const u8 dec_key013[] = {
 	0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e,
 	0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d,
 	0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e,
@@ -8599,7 +8598,7 @@ static const u8 dec_key013[] __initconst = {
 };
 
 static const struct chacha20poly1305_testvec
-chacha20poly1305_dec_vectors[] __initconst = {
+chacha20poly1305_dec_vectors[] = {
 	{ dec_input001, dec_output001, dec_assoc001, dec_nonce001, dec_key001,
 	  sizeof(dec_input001), sizeof(dec_assoc001), sizeof(dec_nonce001) },
 	{ dec_input002, dec_output002, dec_assoc002, dec_nonce002, dec_key002,
@@ -8629,7 +8628,7 @@ chacha20poly1305_dec_vectors[] __initconst = {
 	  true }
 };
 
-static const u8 xenc_input001[] __initconst = {
+static const u8 xenc_input001[] = {
 	0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74,
 	0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20,
 	0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66,
@@ -8665,7 +8664,7 @@ static const u8 xenc_input001[] __initconst = {
 	0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80,
 	0x9d
 };
-static const u8 xenc_output001[] __initconst = {
+static const u8 xenc_output001[] = {
 	0x1a, 0x6e, 0x3a, 0xd9, 0xfd, 0x41, 0x3f, 0x77,
 	0x54, 0x72, 0x0a, 0x70, 0x9a, 0xa0, 0x29, 0x92,
 	0x2e, 0xed, 0x93, 0xcf, 0x0f, 0x71, 0x88, 0x18,
@@ -8703,16 +8702,16 @@ static const u8 xenc_output001[] __initconst = {
 	0x93, 0xa6, 0x85, 0x5b, 0x40, 0x39, 0x5c, 0xc5,
 	0x9c
 };
-static const u8 xenc_assoc001[] __initconst = {
+static const u8 xenc_assoc001[] = {
 	0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x4e, 0x91
 };
-static const u8 xenc_nonce001[] __initconst = {
+static const u8 xenc_nonce001[] = {
 	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
 	0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
 	0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17
 };
-static const u8 xenc_key001[] __initconst = {
+static const u8 xenc_key001[] = {
 	0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a,
 	0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0,
 	0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09,
@@ -8720,12 +8719,12 @@ static const u8 xenc_key001[] __initconst = {
 };
 
 static const struct chacha20poly1305_testvec
-xchacha20poly1305_enc_vectors[] __initconst = {
+xchacha20poly1305_enc_vectors[] = {
 	{ xenc_input001, xenc_output001, xenc_assoc001, xenc_nonce001, xenc_key001,
 	  sizeof(xenc_input001), sizeof(xenc_assoc001), sizeof(xenc_nonce001) }
 };
 
-static const u8 xdec_input001[] __initconst = {
+static const u8 xdec_input001[] = {
 	0x1a, 0x6e, 0x3a, 0xd9, 0xfd, 0x41, 0x3f, 0x77,
 	0x54, 0x72, 0x0a, 0x70, 0x9a, 0xa0, 0x29, 0x92,
 	0x2e, 0xed, 0x93, 0xcf, 0x0f, 0x71, 0x88, 0x18,
@@ -8763,7 +8762,7 @@ static const u8 xdec_input001[] __initconst = {
 	0x93, 0xa6, 0x85, 0x5b, 0x40, 0x39, 0x5c, 0xc5,
 	0x9c
 };
-static const u8 xdec_output001[] __initconst = {
+static const u8 xdec_output001[] = {
 	0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74,
 	0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20,
 	0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66,
@@ -8799,16 +8798,16 @@ static const u8 xdec_output001[] __initconst = {
 	0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80,
 	0x9d
 };
-static const u8 xdec_assoc001[] __initconst = {
+static const u8 xdec_assoc001[] = {
 	0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x4e, 0x91
 };
-static const u8 xdec_nonce001[] __initconst = {
+static const u8 xdec_nonce001[] = {
 	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
 	0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
 	0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17
 };
-static const u8 xdec_key001[] __initconst = {
+static const u8 xdec_key001[] = {
 	0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a,
 	0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0,
 	0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09,
@@ -8816,15 +8815,15 @@ static const u8 xdec_key001[] __initconst = {
 };
 
 static const struct chacha20poly1305_testvec
-xchacha20poly1305_dec_vectors[] __initconst = {
+xchacha20poly1305_dec_vectors[] = {
 	{ xdec_input001, xdec_output001, xdec_assoc001, xdec_nonce001, xdec_key001,
 	  sizeof(xdec_input001), sizeof(xdec_assoc001), sizeof(xdec_nonce001) }
 };
 
-/* This is for the selftests-only, since it is only useful for the purpose of
+/* This is for the tests only, since it is only useful for the purpose of
  * testing the underlying primitives and interactions.
  */
-static void __init
+static void
 chacha20poly1305_encrypt_bignonce(u8 *dst, const u8 *src, const size_t src_len,
 				  const u8 *ad, const size_t ad_len,
 				  const u8 nonce[12],
@@ -8858,11 +8857,12 @@ chacha20poly1305_encrypt_bignonce(u8 *dst, const u8 *src, const size_t src_len,
 	poly1305_final(&poly1305_state, dst + src_len);
 }
 
-static void __init
-chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len,
-				  const u8 *ad, const size_t ad_len,
-				  const u8 *nonce, const size_t nonce_len,
-				  const u8 key[CHACHA20POLY1305_KEY_SIZE])
+static void
+chacha20poly1305_test_encrypt(struct kunit *test, u8 *dst,
+			      const u8 *src, const size_t src_len,
+			      const u8 *ad, const size_t ad_len,
+			      const u8 *nonce, const size_t nonce_len,
+			      const u8 key[CHACHA20POLY1305_KEY_SIZE])
 {
 	if (nonce_len == 8)
 		chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len,
@@ -8871,10 +8871,10 @@ chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len,
 		chacha20poly1305_encrypt_bignonce(dst, src, src_len, ad,
 						  ad_len, nonce, key);
 	else
-		BUG();
+		KUNIT_FAIL(test, "bad nonce_len: %zu", nonce_len);
 }
 
-static bool __init
+static bool
 decryption_success(bool func_ret, bool expect_failure, int memcmp_result)
 {
 	if (expect_failure)
@@ -8882,25 +8882,23 @@ decryption_success(bool func_ret, bool expect_failure, int memcmp_result)
 	return func_ret && !memcmp_result;
 }
 
-bool __init chacha20poly1305_selftest(void)
+static void test_chacha20poly1305(struct kunit *test)
 {
 	enum { MAXIMUM_TEST_BUFFER_LEN = 1UL << 12 };
 	size_t i, j, k, total_len;
 	u8 *computed_output = NULL, *input = NULL;
-	bool success = true, ret;
+	bool ret;
 	struct scatterlist sg_src[3];
 
-	computed_output = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL);
-	input = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL);
-	if (!computed_output || !input) {
-		pr_err("chacha20poly1305 self-test malloc: FAIL\n");
-		success = false;
-		goto out;
-	}
+	computed_output = kunit_kmalloc(test, MAXIMUM_TEST_BUFFER_LEN,
+					GFP_KERNEL);
+	input = kunit_kmalloc(test, MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, computed_output);
+	KUNIT_ASSERT_NOT_NULL(test, input);
 
 	for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) {
 		memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN);
-		chacha20poly1305_selftest_encrypt(computed_output,
+		chacha20poly1305_test_encrypt(test, computed_output,
 					chacha20poly1305_enc_vectors[i].input,
 					chacha20poly1305_enc_vectors[i].ilen,
 					chacha20poly1305_enc_vectors[i].assoc,
@@ -8908,14 +8906,13 @@ bool __init chacha20poly1305_selftest(void)
 					chacha20poly1305_enc_vectors[i].nonce,
 					chacha20poly1305_enc_vectors[i].nlen,
 					chacha20poly1305_enc_vectors[i].key);
-		if (memcmp(computed_output,
-			   chacha20poly1305_enc_vectors[i].output,
-			   chacha20poly1305_enc_vectors[i].ilen +
-							POLY1305_DIGEST_SIZE)) {
-			pr_err("chacha20poly1305 encryption self-test %zu: FAIL\n",
-			       i + 1);
-			success = false;
-		}
+		KUNIT_EXPECT_TRUE_MSG(
+			test,
+			memcmp(computed_output,
+			       chacha20poly1305_enc_vectors[i].output,
+			       chacha20poly1305_enc_vectors[i].ilen +
+				       POLY1305_DIGEST_SIZE) == 0,
+			"chacha20poly1305 encryption test %zu: FAIL", i + 1);
 	}
 
 	for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) {
@@ -8931,14 +8928,13 @@ bool __init chacha20poly1305_selftest(void)
 			chacha20poly1305_enc_vectors[i].alen,
 			get_unaligned_le64(chacha20poly1305_enc_vectors[i].nonce),
 			chacha20poly1305_enc_vectors[i].key);
-		if (!ret || memcmp(computed_output,
-				   chacha20poly1305_enc_vectors[i].output,
-				   chacha20poly1305_enc_vectors[i].ilen +
-							POLY1305_DIGEST_SIZE)) {
-			pr_err("chacha20poly1305 sg encryption self-test %zu: FAIL\n",
-			       i + 1);
-			success = false;
-		}
+		KUNIT_EXPECT_TRUE_MSG(
+			test,
+			ret && memcmp(computed_output,
+				      chacha20poly1305_enc_vectors[i].output,
+				      chacha20poly1305_enc_vectors[i].ilen +
+					      POLY1305_DIGEST_SIZE) == 0,
+			"chacha20poly1305 sg encryption test %zu: FAIL", i + 1);
 	}
 
 	for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) {
@@ -8950,16 +8946,15 @@ bool __init chacha20poly1305_selftest(void)
 			chacha20poly1305_dec_vectors[i].alen,
 			get_unaligned_le64(chacha20poly1305_dec_vectors[i].nonce),
 			chacha20poly1305_dec_vectors[i].key);
-		if (!decryption_success(ret,
-				chacha20poly1305_dec_vectors[i].failure,
+		KUNIT_EXPECT_TRUE_MSG(
+			test,
+			decryption_success(
+				ret, chacha20poly1305_dec_vectors[i].failure,
 				memcmp(computed_output,
 				       chacha20poly1305_dec_vectors[i].output,
 				       chacha20poly1305_dec_vectors[i].ilen -
-							POLY1305_DIGEST_SIZE))) {
-			pr_err("chacha20poly1305 decryption self-test %zu: FAIL\n",
-			       i + 1);
-			success = false;
-		}
+					       POLY1305_DIGEST_SIZE)),
+			"chacha20poly1305 decryption test %zu: FAIL", i + 1);
 	}
 
 	for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) {
@@ -8973,15 +8968,15 @@ bool __init chacha20poly1305_selftest(void)
 			chacha20poly1305_dec_vectors[i].alen,
 			get_unaligned_le64(chacha20poly1305_dec_vectors[i].nonce),
 			chacha20poly1305_dec_vectors[i].key);
-		if (!decryption_success(ret,
-			chacha20poly1305_dec_vectors[i].failure,
-			memcmp(computed_output, chacha20poly1305_dec_vectors[i].output,
-			       chacha20poly1305_dec_vectors[i].ilen -
-							POLY1305_DIGEST_SIZE))) {
-			pr_err("chacha20poly1305 sg decryption self-test %zu: FAIL\n",
-			       i + 1);
-			success = false;
-		}
+		KUNIT_EXPECT_TRUE_MSG(
+			test,
+			decryption_success(
+				ret, chacha20poly1305_dec_vectors[i].failure,
+				memcmp(computed_output,
+				       chacha20poly1305_dec_vectors[i].output,
+				       chacha20poly1305_dec_vectors[i].ilen -
+					       POLY1305_DIGEST_SIZE)),
+			"chacha20poly1305 sg decryption test %zu: FAIL", i + 1);
 	}
 
 	for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_enc_vectors); ++i) {
@@ -8993,14 +8988,13 @@ bool __init chacha20poly1305_selftest(void)
 					xchacha20poly1305_enc_vectors[i].alen,
 					xchacha20poly1305_enc_vectors[i].nonce,
 					xchacha20poly1305_enc_vectors[i].key);
-		if (memcmp(computed_output,
-			   xchacha20poly1305_enc_vectors[i].output,
-			   xchacha20poly1305_enc_vectors[i].ilen +
-							POLY1305_DIGEST_SIZE)) {
-			pr_err("xchacha20poly1305 encryption self-test %zu: FAIL\n",
-			       i + 1);
-			success = false;
-		}
+		KUNIT_EXPECT_TRUE_MSG(
+			test,
+			memcmp(computed_output,
+			       xchacha20poly1305_enc_vectors[i].output,
+			       xchacha20poly1305_enc_vectors[i].ilen +
+				       POLY1305_DIGEST_SIZE) == 0,
+			"xchacha20poly1305 encryption test %zu: FAIL", i + 1);
 	}
 
 	for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_dec_vectors); ++i) {
@@ -9012,16 +9006,15 @@ bool __init chacha20poly1305_selftest(void)
 					xchacha20poly1305_dec_vectors[i].alen,
 					xchacha20poly1305_dec_vectors[i].nonce,
 					xchacha20poly1305_dec_vectors[i].key);
-		if (!decryption_success(ret,
-				xchacha20poly1305_dec_vectors[i].failure,
+		KUNIT_EXPECT_TRUE_MSG(
+			test,
+			decryption_success(
+				ret, xchacha20poly1305_dec_vectors[i].failure,
 				memcmp(computed_output,
 				       xchacha20poly1305_dec_vectors[i].output,
 				       xchacha20poly1305_dec_vectors[i].ilen -
-							POLY1305_DIGEST_SIZE))) {
-			pr_err("xchacha20poly1305 decryption self-test %zu: FAIL\n",
-			       i + 1);
-			success = false;
-		}
+					       POLY1305_DIGEST_SIZE)),
+			"xchacha20poly1305 decryption test %zu: FAIL", i + 1);
 	}
 
 	for (total_len = POLY1305_DIGEST_SIZE; IS_ENABLED(DEBUG_CHACHA20POLY1305_SLOW_CHUNK_TEST)
@@ -9067,16 +9060,26 @@ bool __init chacha20poly1305_selftest(void)
 				continue;
 
 			chunkfail:
-				pr_err("chacha20poly1305 chunked self-test %zu/%zu/%zu: FAIL\n",
-				       total_len, i, j);
-				success = false;
+				KUNIT_FAIL(
+					test,
+					"chacha20poly1305 chunked test %zu/%zu/%zu: FAIL\n",
+					total_len, i, j);
 			}
 
 		}
 	}
-
-out:
-	kfree(computed_output);
-	kfree(input);
-	return success;
 }
+
+static struct kunit_case chacha20poly1305_test_cases[] = {
+	KUNIT_CASE(test_chacha20poly1305),
+	{},
+};
+
+static struct kunit_suite chacha20poly1305_test_suite = {
+	.name = "chacha20poly1305",
+	.test_cases = chacha20poly1305_test_cases,
+};
+kunit_test_suite(chacha20poly1305_test_suite);
+
+MODULE_DESCRIPTION("KUnit tests for ChaCha20Poly1305");
+MODULE_LICENSE("GPL");
diff --git a/lib/crypto/tests/ghash-testvecs.h b/lib/crypto/tests/ghash-testvecs.h
new file mode 100644
index 000000000000..759eb4072336
--- /dev/null
+++ b/lib/crypto/tests/ghash-testvecs.h
@@ -0,0 +1,186 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* This file was generated by: ./scripts/crypto/gen-hash-testvecs.py ghash */
+
+static const struct {
+	size_t data_len;
+	u8 digest[GHASH_DIGEST_SIZE];
+} hash_testvecs[] = {
+	{
+		.data_len = 0,
+		.digest = {
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		},
+	},
+	{
+		.data_len = 1,
+		.digest = {
+			0x13, 0x91, 0xa1, 0x11, 0x08, 0xc3, 0x7e, 0xeb,
+			0x21, 0x42, 0x4a, 0xd6, 0x45, 0x0f, 0x41, 0xa7,
+		},
+	},
+	{
+		.data_len = 2,
+		.digest = {
+			0xde, 0x00, 0x63, 0x3f, 0x71, 0x0f, 0xc6, 0x29,
+			0x53, 0x2e, 0x49, 0xd9, 0xc2, 0xb7, 0x73, 0xce,
+		},
+	},
+	{
+		.data_len = 3,
+		.digest = {
+			0xcf, 0xc7, 0xa8, 0x20, 0x24, 0xe9, 0x7a, 0x6c,
+			0x2c, 0x2a, 0x34, 0x70, 0x26, 0xba, 0xd5, 0x9a,
+		},
+	},
+	{
+		.data_len = 16,
+		.digest = {
+			0xaa, 0xe0, 0xdc, 0x7f, 0xcf, 0x8b, 0xe6, 0x0c,
+			0x2e, 0x93, 0x89, 0x7d, 0x68, 0x4e, 0xc2, 0x63,
+		},
+	},
+	{
+		.data_len = 32,
+		.digest = {
+			0x4b, 0x8b, 0x93, 0x5c, 0x79, 0xad, 0x85, 0x08,
+			0xd3, 0x8a, 0xcd, 0xdd, 0x4c, 0x6e, 0x0e, 0x6f,
+		},
+	},
+	{
+		.data_len = 48,
+		.digest = {
+			0xfa, 0xa0, 0x25, 0xdd, 0x61, 0x9a, 0x52, 0x9a,
+			0xea, 0xee, 0xc6, 0x62, 0xb2, 0xba, 0x11, 0x49,
+		},
+	},
+	{
+		.data_len = 49,
+		.digest = {
+			0x23, 0xf1, 0x05, 0xeb, 0x30, 0x40, 0xb9, 0x1d,
+			0xe6, 0x35, 0x51, 0x4e, 0x0f, 0xc0, 0x1b, 0x9e,
+		},
+	},
+	{
+		.data_len = 63,
+		.digest = {
+			0x8d, 0xcf, 0xa0, 0xc8, 0x83, 0x21, 0x06, 0x81,
+			0xc6, 0x36, 0xd5, 0x62, 0xbf, 0xa0, 0xcd, 0x9c,
+		},
+	},
+	{
+		.data_len = 64,
+		.digest = {
+			0xe7, 0xca, 0xbe, 0xe7, 0x66, 0xc8, 0x85, 0xad,
+			0xbc, 0xaf, 0x58, 0x21, 0xd7, 0x67, 0x82, 0x15,
+		},
+	},
+	{
+		.data_len = 65,
+		.digest = {
+			0x9f, 0x48, 0x10, 0xd9, 0xa2, 0x6b, 0x9d, 0xe0,
+			0xb1, 0x87, 0xe1, 0x39, 0xc3, 0xd7, 0xee, 0x09,
+		},
+	},
+	{
+		.data_len = 127,
+		.digest = {
+			0xa4, 0x36, 0xb7, 0x82, 0xd2, 0x67, 0x7e, 0xaf,
+			0x5d, 0xfd, 0x67, 0x9c, 0x1d, 0x9f, 0xe4, 0xf7,
+		},
+	},
+	{
+		.data_len = 128,
+		.digest = {
+			0x57, 0xe7, 0x1d, 0x78, 0xf0, 0x8e, 0xc7, 0x0c,
+			0x15, 0xee, 0x18, 0xc4, 0xd1, 0x75, 0x90, 0xaa,
+		},
+	},
+	{
+		.data_len = 129,
+		.digest = {
+			0x9b, 0xad, 0x81, 0xa9, 0x22, 0xb2, 0x19, 0x53,
+			0x60, 0x30, 0xe7, 0xa0, 0x4f, 0xd6, 0x72, 0x42,
+		},
+	},
+	{
+		.data_len = 256,
+		.digest = {
+			0xf7, 0x33, 0x42, 0xbf, 0x58, 0xde, 0x88, 0x0f,
+			0x8d, 0x3d, 0xa6, 0x11, 0x14, 0xc3, 0xf1, 0xdc,
+		},
+	},
+	{
+		.data_len = 511,
+		.digest = {
+			0x59, 0xdc, 0xa9, 0xc0, 0x4e, 0xd6, 0x97, 0xb3,
+			0x60, 0xaf, 0xa8, 0xa0, 0xea, 0x54, 0x8e, 0xc3,
+		},
+	},
+	{
+		.data_len = 513,
+		.digest = {
+			0xa2, 0x23, 0x37, 0xcc, 0x97, 0xec, 0xea, 0xbe,
+			0xd6, 0xc7, 0x13, 0xf7, 0x93, 0x73, 0xc0, 0x64,
+		},
+	},
+	{
+		.data_len = 1000,
+		.digest = {
+			0x46, 0x8b, 0x43, 0x77, 0x9b, 0xc2, 0xfc, 0xa4,
+			0x68, 0x6a, 0x6c, 0x07, 0xa4, 0x6f, 0x47, 0x65,
+		},
+	},
+	{
+		.data_len = 3333,
+		.digest = {
+			0x69, 0x7f, 0x19, 0xc3, 0xb9, 0xa4, 0xff, 0x40,
+			0xe3, 0x03, 0x71, 0xa3, 0x88, 0x8a, 0xf1, 0xbd,
+		},
+	},
+	{
+		.data_len = 4096,
+		.digest = {
+			0x4d, 0x65, 0xe6, 0x9c, 0xeb, 0x6a, 0x46, 0x8d,
+			0xe9, 0x32, 0x96, 0x72, 0xb3, 0x0d, 0x08, 0xa9,
+		},
+	},
+	{
+		.data_len = 4128,
+		.digest = {
+			0xfc, 0xa1, 0x74, 0x46, 0x21, 0x64, 0xa7, 0x64,
+			0xbe, 0x47, 0x03, 0x1e, 0x05, 0xf7, 0xd8, 0x37,
+		},
+	},
+	{
+		.data_len = 4160,
+		.digest = {
+			0x70, 0x5b, 0xe9, 0x17, 0xab, 0xd5, 0xa2, 0xee,
+			0xcb, 0x39, 0xa4, 0x81, 0x2f, 0x41, 0x70, 0xae,
+		},
+	},
+	{
+		.data_len = 4224,
+		.digest = {
+			0x07, 0xbd, 0xb6, 0x52, 0xe2, 0x75, 0x2c, 0x33,
+			0x6d, 0x1b, 0x63, 0x56, 0x58, 0xda, 0x98, 0x55,
+		},
+	},
+	{
+		.data_len = 16384,
+		.digest = {
+			0x9c, 0xb5, 0xf4, 0x14, 0xe8, 0xa8, 0x4a, 0xde,
+			0xee, 0x7b, 0xbb, 0xd6, 0x21, 0x6d, 0x6a, 0x69,
+		},
+	},
+};
+
+static const u8 hash_testvec_consolidated[GHASH_DIGEST_SIZE] = {
+	0x08, 0xef, 0xf5, 0x27, 0xb1, 0xca, 0xd4, 0x1d,
+	0xad, 0x38, 0x69, 0x88, 0x6b, 0x16, 0xdf, 0xa8,
+};
+
+static const u8 ghash_allones_hashofhashes[GHASH_DIGEST_SIZE] = {
+	0xef, 0x85, 0x58, 0xf8, 0x54, 0x9c, 0x5e, 0x54,
+	0xd9, 0xbe, 0x04, 0x1f, 0xff, 0xff, 0xff, 0xff,
+};
diff --git a/lib/crypto/tests/ghash_kunit.c b/lib/crypto/tests/ghash_kunit.c
new file mode 100644
index 000000000000..68b3837a3607
--- /dev/null
+++ b/lib/crypto/tests/ghash_kunit.c
@@ -0,0 +1,194 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright 2026 Google LLC
+ */
+#include <crypto/gf128hash.h>
+#include "ghash-testvecs.h"
+
+/*
+ * A fixed key used when presenting GHASH as an unkeyed hash function in order
+ * to reuse hash-test-template.h.  At the beginning of the test suite, this is
+ * initialized to a key prepared from bytes generated from a fixed seed.
+ */
+static struct ghash_key test_key;
+
+static void ghash_init_withtestkey(struct ghash_ctx *ctx)
+{
+	ghash_init(ctx, &test_key);
+}
+
+static void ghash_withtestkey(const u8 *data, size_t len,
+			      u8 out[GHASH_BLOCK_SIZE])
+{
+	ghash(&test_key, data, len, out);
+}
+
+/* Generate the HASH_KUNIT_CASES using hash-test-template.h. */
+#define HASH ghash_withtestkey
+#define HASH_CTX ghash_ctx
+#define HASH_SIZE GHASH_BLOCK_SIZE
+#define HASH_INIT ghash_init_withtestkey
+#define HASH_UPDATE ghash_update
+#define HASH_FINAL ghash_final
+#include "hash-test-template.h"
+
+/*
+ * Test a key and messages containing all one bits.  This is useful to detect
+ * overflow bugs in implementations that emulate carryless multiplication using
+ * a series of standard multiplications with the bits spread out.
+ */
+static void test_ghash_allones_key_and_message(struct kunit *test)
+{
+	struct ghash_key key;
+	struct ghash_ctx hashofhashes_ctx;
+	u8 hash[GHASH_BLOCK_SIZE];
+
+	static_assert(TEST_BUF_LEN >= 4096);
+	memset(test_buf, 0xff, 4096);
+
+	ghash_preparekey(&key, test_buf);
+	ghash_init(&hashofhashes_ctx, &key);
+	for (size_t len = 0; len <= 4096; len += 16) {
+		ghash(&key, test_buf, len, hash);
+		ghash_update(&hashofhashes_ctx, hash, sizeof(hash));
+	}
+	ghash_final(&hashofhashes_ctx, hash);
+	KUNIT_ASSERT_MEMEQ(test, hash, ghash_allones_hashofhashes,
+			   sizeof(hash));
+}
+
+#define MAX_LEN_FOR_KEY_CHECK 1024
+
+/*
+ * Given two prepared keys which should be identical (but may differ in
+ * alignment and/or whether they are followed by a guard page or not), verify
+ * that they produce consistent results on various data lengths.
+ */
+static void check_key_consistency(struct kunit *test,
+				  const struct ghash_key *key1,
+				  const struct ghash_key *key2)
+{
+	u8 *data = test_buf;
+	u8 hash1[GHASH_BLOCK_SIZE];
+	u8 hash2[GHASH_BLOCK_SIZE];
+
+	rand_bytes(data, MAX_LEN_FOR_KEY_CHECK);
+	KUNIT_ASSERT_MEMEQ(test, key1, key2, sizeof(*key1));
+
+	for (int i = 0; i < 100; i++) {
+		size_t len = rand_length(MAX_LEN_FOR_KEY_CHECK);
+
+		ghash(key1, data, len, hash1);
+		ghash(key2, data, len, hash2);
+		KUNIT_ASSERT_MEMEQ(test, hash1, hash2, sizeof(hash1));
+	}
+}
+
+/* Test that no buffer overreads occur on either raw_key or ghash_key. */
+static void test_ghash_with_guarded_key(struct kunit *test)
+{
+	u8 raw_key[GHASH_BLOCK_SIZE];
+	u8 *guarded_raw_key = &test_buf[TEST_BUF_LEN - sizeof(raw_key)];
+	struct ghash_key key1, key2;
+	struct ghash_key *guarded_key =
+		(struct ghash_key *)&test_buf[TEST_BUF_LEN - sizeof(key1)];
+
+	/* Prepare with regular buffers. */
+	rand_bytes(raw_key, sizeof(raw_key));
+	ghash_preparekey(&key1, raw_key);
+
+	/* Prepare with guarded raw_key, then check that it works. */
+	memcpy(guarded_raw_key, raw_key, sizeof(raw_key));
+	ghash_preparekey(&key2, guarded_raw_key);
+	check_key_consistency(test, &key1, &key2);
+
+	/* Prepare guarded ghash_key, then check that it works. */
+	ghash_preparekey(guarded_key, raw_key);
+	check_key_consistency(test, &key1, guarded_key);
+}
+
+/*
+ * Test that ghash_key only needs to be aligned to
+ * __alignof__(struct ghash_key), i.e. 8 bytes.  The assembly code may prefer
+ * 16-byte or higher alignment, but it mustn't require it.
+ */
+static void test_ghash_with_minimally_aligned_key(struct kunit *test)
+{
+	u8 raw_key[GHASH_BLOCK_SIZE];
+	struct ghash_key key;
+	struct ghash_key *minaligned_key =
+		(struct ghash_key *)&test_buf[MAX_LEN_FOR_KEY_CHECK +
+					      __alignof__(struct ghash_key)];
+
+	KUNIT_ASSERT_TRUE(test, IS_ALIGNED((uintptr_t)minaligned_key,
+					   __alignof__(struct ghash_key)));
+	KUNIT_ASSERT_TRUE(test, !IS_ALIGNED((uintptr_t)minaligned_key,
+					    2 * __alignof__(struct ghash_key)));
+
+	rand_bytes(raw_key, sizeof(raw_key));
+	ghash_preparekey(&key, raw_key);
+	ghash_preparekey(minaligned_key, raw_key);
+	check_key_consistency(test, &key, minaligned_key);
+}
+
+struct ghash_irq_test_state {
+	struct ghash_key expected_key;
+	u8 raw_key[GHASH_BLOCK_SIZE];
+};
+
+static bool ghash_irq_test_func(void *state_)
+{
+	struct ghash_irq_test_state *state = state_;
+	struct ghash_key key;
+
+	ghash_preparekey(&key, state->raw_key);
+	return memcmp(&key, &state->expected_key, sizeof(key)) == 0;
+}
+
+/*
+ * Test that ghash_preparekey() produces the same output regardless of whether
+ * FPU or vector registers are usable when it is called.
+ */
+static void test_ghash_preparekey_in_irqs(struct kunit *test)
+{
+	struct ghash_irq_test_state state;
+
+	rand_bytes(state.raw_key, sizeof(state.raw_key));
+	ghash_preparekey(&state.expected_key, state.raw_key);
+	kunit_run_irq_test(test, ghash_irq_test_func, 200000, &state);
+}
+
+static int ghash_suite_init(struct kunit_suite *suite)
+{
+	u8 raw_key[GHASH_BLOCK_SIZE];
+
+	rand_bytes_seeded_from_len(raw_key, sizeof(raw_key));
+	ghash_preparekey(&test_key, raw_key);
+	return hash_suite_init(suite);
+}
+
+static void ghash_suite_exit(struct kunit_suite *suite)
+{
+	hash_suite_exit(suite);
+}
+
+static struct kunit_case ghash_test_cases[] = {
+	HASH_KUNIT_CASES,
+	KUNIT_CASE(test_ghash_allones_key_and_message),
+	KUNIT_CASE(test_ghash_with_guarded_key),
+	KUNIT_CASE(test_ghash_with_minimally_aligned_key),
+	KUNIT_CASE(test_ghash_preparekey_in_irqs),
+	KUNIT_CASE(benchmark_hash),
+	{},
+};
+
+static struct kunit_suite ghash_test_suite = {
+	.name = "ghash",
+	.test_cases = ghash_test_cases,
+	.suite_init = ghash_suite_init,
+	.suite_exit = ghash_suite_exit,
+};
+kunit_test_suite(ghash_test_suite);
+
+MODULE_DESCRIPTION("KUnit tests and benchmark for GHASH");
+MODULE_LICENSE("GPL");
diff --git a/lib/crypto/tests/polyval_kunit.c b/lib/crypto/tests/polyval_kunit.c
index f47f41a39a41..d1f53a690ab8 100644
--- a/lib/crypto/tests/polyval_kunit.c
+++ b/lib/crypto/tests/polyval_kunit.c
@@ -2,7 +2,7 @@
 /*
  * Copyright 2025 Google LLC
  */
-#include <crypto/polyval.h>
+#include <crypto/gf128hash.h>
 #include "polyval-testvecs.h"
 
 /*
diff --git a/lib/crypto/tests/sm3-testvecs.h b/lib/crypto/tests/sm3-testvecs.h
new file mode 100644
index 000000000000..5e788c29f487
--- /dev/null
+++ b/lib/crypto/tests/sm3-testvecs.h
@@ -0,0 +1,231 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* This file was generated by: ./scripts/crypto/gen-hash-testvecs.py sm3 */
+
+static const struct {
+	size_t data_len;
+	u8 digest[SM3_DIGEST_SIZE];
+} hash_testvecs[] = {
+	{
+		.data_len = 0,
+		.digest = {
+			0x1a, 0xb2, 0x1d, 0x83, 0x55, 0xcf, 0xa1, 0x7f,
+			0x8e, 0x61, 0x19, 0x48, 0x31, 0xe8, 0x1a, 0x8f,
+			0x22, 0xbe, 0xc8, 0xc7, 0x28, 0xfe, 0xfb, 0x74,
+			0x7e, 0xd0, 0x35, 0xeb, 0x50, 0x82, 0xaa, 0x2b,
+		},
+	},
+	{
+		.data_len = 1,
+		.digest = {
+			0xb6, 0x22, 0x2c, 0x39, 0xdc, 0x14, 0x9a, 0xee,
+			0x01, 0x9a, 0xcb, 0x0d, 0xe6, 0xc6, 0x75, 0x6e,
+			0x8f, 0x18, 0x7b, 0x0e, 0xe8, 0x98, 0x61, 0x71,
+			0x2b, 0xd8, 0x38, 0xa9, 0xee, 0x2c, 0x1e, 0x93,
+		},
+	},
+	{
+		.data_len = 2,
+		.digest = {
+			0x62, 0x0c, 0x66, 0x77, 0x67, 0x28, 0x74, 0x8a,
+			0xe3, 0x64, 0xea, 0x44, 0x6a, 0x3f, 0x34, 0x61,
+			0x55, 0xc5, 0xaa, 0xb2, 0x6c, 0x67, 0x97, 0x68,
+			0x68, 0xae, 0x4d, 0x64, 0xa8, 0xb6, 0x72, 0x3e,
+		},
+	},
+	{
+		.data_len = 3,
+		.digest = {
+			0x71, 0xd4, 0x63, 0xb1, 0xfa, 0x27, 0xc7, 0xae,
+			0x65, 0xed, 0x5c, 0x93, 0x70, 0xe0, 0x09, 0x34,
+			0x2f, 0x42, 0xe6, 0x71, 0x16, 0x8e, 0x90, 0x90,
+			0x9a, 0xdd, 0xa6, 0x44, 0x66, 0x71, 0x18, 0xf9,
+		},
+	},
+	{
+		.data_len = 16,
+		.digest = {
+			0x79, 0x0b, 0x68, 0xb5, 0x41, 0xc1, 0x97, 0xa0,
+			0x50, 0xe6, 0x93, 0x70, 0xf6, 0x98, 0x49, 0xea,
+			0x92, 0xc9, 0xd0, 0xb1, 0x46, 0xbd, 0x4a, 0x0c,
+			0x8e, 0xe8, 0xf3, 0xe4, 0x8f, 0x90, 0x33, 0x3c,
+		},
+	},
+	{
+		.data_len = 32,
+		.digest = {
+			0x32, 0x9f, 0xa3, 0x18, 0x18, 0x45, 0xe0, 0x28,
+			0xd3, 0xa4, 0x41, 0x3a, 0x25, 0x62, 0x9c, 0x95,
+			0xab, 0xfe, 0x02, 0xe0, 0x37, 0x7d, 0x3c, 0xc4,
+			0xce, 0x69, 0x57, 0x5a, 0x07, 0x0e, 0xb9, 0xf5,
+		},
+	},
+	{
+		.data_len = 48,
+		.digest = {
+			0x0c, 0xcf, 0x7c, 0x48, 0x44, 0xa0, 0xb0, 0x8d,
+			0xdf, 0xbe, 0x22, 0x14, 0x7e, 0xd4, 0xc3, 0x8d,
+			0x6a, 0x23, 0xfc, 0x44, 0x0e, 0x0f, 0xde, 0xa5,
+			0x7c, 0x8b, 0xc4, 0x8b, 0xab, 0x8c, 0x87, 0x41,
+		},
+	},
+	{
+		.data_len = 49,
+		.digest = {
+			0xb3, 0x76, 0x8b, 0x19, 0xf9, 0x10, 0xa9, 0x56,
+			0x4f, 0xce, 0x27, 0xaa, 0x65, 0x96, 0xe5, 0xdb,
+			0x90, 0x9b, 0x92, 0xcd, 0x32, 0x0d, 0x16, 0xac,
+			0xf8, 0xd0, 0x66, 0x62, 0x10, 0xf0, 0x44, 0xdf,
+		},
+	},
+	{
+		.data_len = 63,
+		.digest = {
+			0x07, 0xc9, 0x45, 0x65, 0x9f, 0x68, 0x75, 0xc3,
+			0x74, 0xb2, 0x3b, 0x0c, 0x97, 0x05, 0xd3, 0x13,
+			0xc0, 0xb6, 0x21, 0xed, 0xf6, 0x10, 0x7a, 0xed,
+			0xec, 0xd8, 0x10, 0x29, 0xbf, 0x7a, 0x78, 0x37,
+		},
+	},
+	{
+		.data_len = 64,
+		.digest = {
+			0x3e, 0x69, 0x18, 0x45, 0xd8, 0x25, 0x6f, 0x44,
+			0xc0, 0x02, 0xe5, 0xcf, 0xcd, 0x94, 0x42, 0xa9,
+			0xd5, 0x12, 0x62, 0x10, 0x15, 0xa0, 0xf9, 0x16,
+			0x19, 0x2d, 0x8d, 0x63, 0x31, 0xf2, 0x2f, 0x36,
+		},
+	},
+	{
+		.data_len = 65,
+		.digest = {
+			0x6b, 0x3e, 0xc0, 0x20, 0xb7, 0x74, 0x30, 0xa0,
+			0xc6, 0x5c, 0xee, 0xbe, 0xdc, 0xe6, 0xe5, 0x4f,
+			0x3c, 0x61, 0x8d, 0x91, 0xac, 0x31, 0x4b, 0x2a,
+			0xdf, 0x1c, 0xef, 0x24, 0xdc, 0x0a, 0x10, 0xe8,
+		},
+	},
+	{
+		.data_len = 127,
+		.digest = {
+			0xab, 0xd6, 0xa1, 0xbf, 0x39, 0x43, 0x75, 0xda,
+			0xbf, 0xc7, 0x22, 0xcc, 0x4e, 0xfc, 0xe4, 0x42,
+			0x6d, 0x1b, 0x87, 0x25, 0x64, 0x7f, 0x88, 0xf7,
+			0xc3, 0x0a, 0x0a, 0x4c, 0xd6, 0xa7, 0x68, 0xae,
+		},
+	},
+	{
+		.data_len = 128,
+		.digest = {
+			0x1b, 0x70, 0xd4, 0x5f, 0x6c, 0xe4, 0x2d, 0x58,
+			0x2d, 0x0f, 0x9a, 0x12, 0x34, 0xbb, 0x5e, 0x38,
+			0xd8, 0x1f, 0x6a, 0x46, 0x8a, 0xef, 0xdb, 0x68,
+			0x18, 0x62, 0xbb, 0x85, 0xfc, 0xc4, 0x6e, 0x2e,
+		},
+	},
+	{
+		.data_len = 129,
+		.digest = {
+			0x33, 0x62, 0xba, 0xa7, 0x4a, 0xbc, 0xd7, 0x7b,
+			0xd4, 0x67, 0x6d, 0x3e, 0xea, 0xe8, 0xb0, 0x64,
+			0x0d, 0xf3, 0xae, 0x1d, 0x52, 0x24, 0x11, 0x9f,
+			0xda, 0xa9, 0x7f, 0xd5, 0x22, 0x1a, 0xde, 0x8a,
+		},
+	},
+	{
+		.data_len = 256,
+		.digest = {
+			0x70, 0xa8, 0xa6, 0x2b, 0xfb, 0x1f, 0x3b, 0x5a,
+			0xcc, 0x71, 0x76, 0x9e, 0x25, 0x4c, 0xfa, 0x8f,
+			0x39, 0x4a, 0x21, 0x8a, 0x9d, 0x74, 0x8d, 0x2c,
+			0x31, 0xa5, 0xb5, 0xff, 0x30, 0xc1, 0x14, 0xc4,
+		},
+	},
+	{
+		.data_len = 511,
+		.digest = {
+			0x39, 0xd0, 0x8c, 0x5f, 0xfc, 0x36, 0xc2, 0x7c,
+			0xdb, 0x8b, 0x2e, 0xdc, 0x9d, 0x1b, 0xd1, 0xba,
+			0x9b, 0x52, 0x6b, 0x35, 0x46, 0x46, 0x75, 0x73,
+			0xe5, 0x62, 0x96, 0x6e, 0xf3, 0xba, 0xd9, 0x19,
+		},
+	},
+	{
+		.data_len = 513,
+		.digest = {
+			0x76, 0xa0, 0x3d, 0xa2, 0x5f, 0xd4, 0xa6, 0xbe,
+			0x6b, 0xdb, 0xed, 0x14, 0x9e, 0xa8, 0x15, 0x77,
+			0xa9, 0x38, 0x30, 0x6b, 0x68, 0xfa, 0xb6, 0xe2,
+			0x93, 0x19, 0x24, 0x72, 0x67, 0x20, 0x72, 0xc3,
+		},
+	},
+	{
+		.data_len = 1000,
+		.digest = {
+			0x16, 0xbc, 0x33, 0x77, 0x0b, 0xcf, 0x93, 0x5e,
+			0xec, 0x7d, 0x8d, 0x3c, 0xae, 0xd9, 0x75, 0xdf,
+			0x46, 0x24, 0x17, 0x7e, 0x03, 0x88, 0xf2, 0x75,
+			0xa9, 0x18, 0xa6, 0x1c, 0x7a, 0x74, 0x0d, 0xf3,
+		},
+	},
+	{
+		.data_len = 3333,
+		.digest = {
+			0xdb, 0x54, 0x89, 0xe7, 0x1c, 0x50, 0xf2, 0xbf,
+			0xde, 0x3a, 0xbf, 0x5b, 0xee, 0x5a, 0x46, 0x62,
+			0x20, 0xb1, 0x80, 0x48, 0xac, 0x56, 0x33, 0xb3,
+			0xbb, 0x3f, 0xfa, 0x02, 0xc6, 0x43, 0xb5, 0x8c,
+		},
+	},
+	{
+		.data_len = 4096,
+		.digest = {
+			0xdf, 0x0d, 0xed, 0x3b, 0x8f, 0xea, 0x81, 0xfd,
+			0xd6, 0x34, 0xae, 0x74, 0x24, 0x3a, 0x15, 0x38,
+			0xe7, 0xcf, 0x45, 0x7e, 0x8f, 0xf5, 0x50, 0x6c,
+			0xaa, 0x27, 0x23, 0x92, 0x6d, 0xab, 0x3b, 0xde,
+		},
+	},
+	{
+		.data_len = 4128,
+		.digest = {
+			0x6a, 0xbd, 0x56, 0x5a, 0xf1, 0xc6, 0x40, 0x4d,
+			0xf3, 0x50, 0x77, 0x87, 0x86, 0x63, 0x1b, 0x4d,
+			0x21, 0x99, 0x96, 0xad, 0x24, 0x62, 0xce, 0xc0,
+			0x3e, 0xb7, 0x35, 0x52, 0x56, 0x0e, 0x55, 0x85,
+		},
+	},
+	{
+		.data_len = 4160,
+		.digest = {
+			0x5b, 0xc1, 0x1f, 0x25, 0x43, 0xa3, 0x1c, 0xa0,
+			0x8c, 0xfc, 0x41, 0xf1, 0xcc, 0xb3, 0x95, 0x95,
+			0xe0, 0xb9, 0xd3, 0x29, 0xf4, 0x08, 0x31, 0x47,
+			0x6d, 0x09, 0xa8, 0x2e, 0xa5, 0xf4, 0xf1, 0x8d,
+		},
+	},
+	{
+		.data_len = 4224,
+		.digest = {
+			0xec, 0x56, 0x1e, 0xa6, 0x1f, 0xb2, 0x87, 0xb2,
+			0x7e, 0x15, 0xd6, 0x30, 0x08, 0x74, 0xb0, 0x48,
+			0x90, 0x2a, 0xbe, 0x2f, 0x80, 0x3a, 0x88, 0xcc,
+			0xd7, 0xc5, 0x87, 0x8c, 0x04, 0xef, 0x78, 0x71,
+		},
+	},
+	{
+		.data_len = 16384,
+		.digest = {
+			0xe7, 0xb8, 0x84, 0x20, 0xff, 0xd5, 0x53, 0xe6,
+			0x14, 0x31, 0x12, 0x75, 0xb7, 0x9a, 0x4f, 0x63,
+			0x63, 0x00, 0xfe, 0x2c, 0x54, 0xee, 0x06, 0xfc,
+			0x12, 0x16, 0xe5, 0xdc, 0xa4, 0x40, 0x62, 0x12,
+		},
+	},
+};
+
+static const u8 hash_testvec_consolidated[SM3_DIGEST_SIZE] = {
+	0xe6, 0x58, 0xd4, 0x8e, 0x74, 0x92, 0xdf, 0xfe,
+	0x58, 0x05, 0xe5, 0x29, 0x83, 0xfb, 0xb7, 0x51,
+	0x7e, 0x66, 0x0c, 0x49, 0x3e, 0x11, 0x7e, 0x9b,
+	0xb1, 0x83, 0x3a, 0xa6, 0xb0, 0x3c, 0xf5, 0xe0,
+};
diff --git a/lib/crypto/tests/sm3_kunit.c b/lib/crypto/tests/sm3_kunit.c
new file mode 100644
index 000000000000..dc8136acdff6
--- /dev/null
+++ b/lib/crypto/tests/sm3_kunit.c
@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright 2026 Google LLC
+ */
+#include <crypto/sm3.h>
+#include "sm3-testvecs.h"
+
+#define HASH sm3
+#define HASH_CTX sm3_ctx
+#define HASH_SIZE SM3_DIGEST_SIZE
+#define HASH_INIT sm3_init
+#define HASH_UPDATE sm3_update
+#define HASH_FINAL sm3_final
+#include "hash-test-template.h"
+
+static struct kunit_case sm3_test_cases[] = {
+	HASH_KUNIT_CASES,
+	KUNIT_CASE(benchmark_hash),
+	{},
+};
+
+static struct kunit_suite sm3_test_suite = {
+	.name = "sm3",
+	.test_cases = sm3_test_cases,
+	.suite_init = hash_suite_init,
+	.suite_exit = hash_suite_exit,
+};
+kunit_test_suite(sm3_test_suite);
+
+MODULE_DESCRIPTION("KUnit tests and benchmark for SM3");
+MODULE_LICENSE("GPL");
diff --git a/lib/crypto/x86/polyval.h b/lib/crypto/x86/gf128hash.h
index ef8797521420..6b79b06caab0 100644
--- a/lib/crypto/x86/polyval.h
+++ b/lib/crypto/x86/gf128hash.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-or-later */
 /*
- * POLYVAL library functions, x86_64 optimized
+ * GHASH and POLYVAL, x86_64 optimized
  *
  * Copyright 2025 Google LLC
  */
@@ -9,14 +9,22 @@
 
 #define NUM_H_POWERS 8
 
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul);
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul_avx);
 
+asmlinkage void polyval_mul_pclmul(struct polyval_elem *a,
+				   const struct polyval_elem *b);
 asmlinkage void polyval_mul_pclmul_avx(struct polyval_elem *a,
 				       const struct polyval_elem *b);
+
+asmlinkage void ghash_blocks_pclmul(struct polyval_elem *acc,
+				    const struct polyval_elem *key,
+				    const u8 *data, size_t nblocks);
 asmlinkage void polyval_blocks_pclmul_avx(struct polyval_elem *acc,
 					  const struct polyval_key *key,
 					  const u8 *data, size_t nblocks);
 
+#define polyval_preparekey_arch polyval_preparekey_arch
 static void polyval_preparekey_arch(struct polyval_key *key,
 				    const u8 raw_key[POLYVAL_BLOCK_SIZE])
 {
@@ -40,18 +48,58 @@ static void polyval_preparekey_arch(struct polyval_key *key,
 	}
 }
 
-static void polyval_mul_arch(struct polyval_elem *acc,
-			     const struct polyval_key *key)
+static void polyval_mul_x86(struct polyval_elem *a,
+			    const struct polyval_elem *b)
 {
-	if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) {
+	if (static_branch_likely(&have_pclmul) && irq_fpu_usable()) {
 		kernel_fpu_begin();
-		polyval_mul_pclmul_avx(acc, &key->h_powers[NUM_H_POWERS - 1]);
+		if (static_branch_likely(&have_pclmul_avx))
+			polyval_mul_pclmul_avx(a, b);
+		else
+			polyval_mul_pclmul(a, b);
 		kernel_fpu_end();
 	} else {
-		polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]);
+		polyval_mul_generic(a, b);
+	}
+}
+
+#define ghash_mul_arch ghash_mul_arch
+static void ghash_mul_arch(struct polyval_elem *acc,
+			   const struct ghash_key *key)
+{
+	polyval_mul_x86(acc, &key->h);
+}
+
+#define polyval_mul_arch polyval_mul_arch
+static void polyval_mul_arch(struct polyval_elem *acc,
+			     const struct polyval_key *key)
+{
+	polyval_mul_x86(acc, &key->h_powers[NUM_H_POWERS - 1]);
+}
+
+#define ghash_blocks_arch ghash_blocks_arch
+static void ghash_blocks_arch(struct polyval_elem *acc,
+			      const struct ghash_key *key,
+			      const u8 *data, size_t nblocks)
+{
+	if (static_branch_likely(&have_pclmul) && irq_fpu_usable()) {
+		do {
+			/* Allow rescheduling every 4 KiB. */
+			size_t n = min_t(size_t, nblocks,
+					 4096 / GHASH_BLOCK_SIZE);
+
+			kernel_fpu_begin();
+			ghash_blocks_pclmul(acc, &key->h, data, n);
+			kernel_fpu_end();
+			data += n * GHASH_BLOCK_SIZE;
+			nblocks -= n;
+		} while (nblocks);
+	} else {
+		ghash_blocks_generic(acc, &key->h, data, nblocks);
 	}
 }
 
+#define polyval_blocks_arch polyval_blocks_arch
 static void polyval_blocks_arch(struct polyval_elem *acc,
 				const struct polyval_key *key,
 				const u8 *data, size_t nblocks)
@@ -74,10 +122,12 @@ static void polyval_blocks_arch(struct polyval_elem *acc,
 	}
 }
 
-#define polyval_mod_init_arch polyval_mod_init_arch
-static void polyval_mod_init_arch(void)
+#define gf128hash_mod_init_arch gf128hash_mod_init_arch
+static void gf128hash_mod_init_arch(void)
 {
-	if (boot_cpu_has(X86_FEATURE_PCLMULQDQ) &&
-	    boot_cpu_has(X86_FEATURE_AVX))
-		static_branch_enable(&have_pclmul_avx);
+	if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) {
+		static_branch_enable(&have_pclmul);
+		if (boot_cpu_has(X86_FEATURE_AVX))
+			static_branch_enable(&have_pclmul_avx);
+	}
 }
diff --git a/lib/crypto/x86/ghash-pclmul.S b/lib/crypto/x86/ghash-pclmul.S
new file mode 100644
index 000000000000..6ffb5aea6063
--- /dev/null
+++ b/lib/crypto/x86/ghash-pclmul.S
@@ -0,0 +1,127 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Accelerated GHASH implementation with Intel PCLMULQDQ-NI
+ * instructions. This file contains accelerated part of ghash
+ * implementation. More information about PCLMULQDQ can be found at:
+ *
+ * https://www.intel.com/content/dam/develop/external/us/en/documents/clmul-wp-rev-2-02-2014-04-20.pdf
+ *
+ * Copyright (c) 2009 Intel Corp.
+ *   Author: Huang Ying <ying.huang@intel.com>
+ *	     Vinodh Gopal
+ *	     Erdinc Ozturk
+ *	     Deniz Karakoyunlu
+ */
+
+#include <linux/linkage.h>
+#include <asm/frame.h>
+
+.section	.rodata.cst16.bswap_mask, "aM", @progbits, 16
+.align 16
+.Lbswap_mask:
+	.octa 0x000102030405060708090a0b0c0d0e0f
+
+#define ACC	%xmm0
+#define KEY	%xmm1
+#define T1	%xmm2
+#define T2	%xmm3
+#define T3	%xmm4
+#define BSWAP	%xmm5
+#define IN1	%xmm6
+
+.text
+
+/*
+ * __clmul_gf128mul_ble:	internal ABI
+ * input:
+ *	ACC:			operand1
+ *	KEY:			operand2, hash_key << 1 mod poly
+ * output:
+ *	ACC:			operand1 * operand2 mod poly
+ * changed:
+ *	T1
+ *	T2
+ *	T3
+ */
+SYM_FUNC_START_LOCAL(__clmul_gf128mul_ble)
+	movaps ACC, T1
+	pshufd $0b01001110, ACC, T2
+	pshufd $0b01001110, KEY, T3
+	pxor ACC, T2
+	pxor KEY, T3
+
+	pclmulqdq $0x00, KEY, ACC	# ACC = a0 * b0
+	pclmulqdq $0x11, KEY, T1	# T1 = a1 * b1
+	pclmulqdq $0x00, T3, T2		# T2 = (a1 + a0) * (b1 + b0)
+	pxor ACC, T2
+	pxor T1, T2			# T2 = a0 * b1 + a1 * b0
+
+	movaps T2, T3
+	pslldq $8, T3
+	psrldq $8, T2
+	pxor T3, ACC
+	pxor T2, T1			# <T1:ACC> is result of
+					# carry-less multiplication
+
+	# first phase of the reduction
+	movaps ACC, T3
+	psllq $1, T3
+	pxor ACC, T3
+	psllq $5, T3
+	pxor ACC, T3
+	psllq $57, T3
+	movaps T3, T2
+	pslldq $8, T2
+	psrldq $8, T3
+	pxor T2, ACC
+	pxor T3, T1
+
+	# second phase of the reduction
+	movaps ACC, T2
+	psrlq $5, T2
+	pxor ACC, T2
+	psrlq $1, T2
+	pxor ACC, T2
+	psrlq $1, T2
+	pxor T2, T1
+	pxor T1, ACC
+	RET
+SYM_FUNC_END(__clmul_gf128mul_ble)
+
+/*
+ * void polyval_mul_pclmul(struct polyval_elem *a,
+ *			   const struct polyval_elem *b)
+ */
+SYM_FUNC_START(polyval_mul_pclmul)
+	FRAME_BEGIN
+	movups (%rdi), ACC
+	movups (%rsi), KEY
+	call __clmul_gf128mul_ble
+	movups ACC, (%rdi)
+	FRAME_END
+	RET
+SYM_FUNC_END(polyval_mul_pclmul)
+
+/*
+ * void ghash_blocks_pclmul(struct polyval_elem *acc,
+ *			    const struct polyval_elem *key,
+ *			    const u8 *data, size_t nblocks)
+ */
+SYM_FUNC_START(ghash_blocks_pclmul)
+	FRAME_BEGIN
+	movaps .Lbswap_mask(%rip), BSWAP
+	movups (%rdi), ACC
+	movups (%rsi), KEY
+.align 4
+.Lnext_block:
+	movups (%rdx), IN1
+	pshufb BSWAP, IN1
+	pxor IN1, ACC
+	call __clmul_gf128mul_ble
+	add $16, %rdx
+	dec %rcx
+	jnz .Lnext_block
+	movups ACC, (%rdi)
+	FRAME_END
+	RET
+SYM_FUNC_END(ghash_blocks_pclmul)
diff --git a/lib/crypto/x86/sha256.h b/lib/crypto/x86/sha256.h
index 38e33b22a092..0ee69d8e39fe 100644
--- a/lib/crypto/x86/sha256.h
+++ b/lib/crypto/x86/sha256.h
@@ -31,6 +31,27 @@ DEFINE_X86_SHA256_FN(sha256_blocks_avx, sha256_transform_avx);
 DEFINE_X86_SHA256_FN(sha256_blocks_avx2, sha256_transform_rorx);
 DEFINE_X86_SHA256_FN(sha256_blocks_ni, sha256_ni_transform);
 
+#define PHE_ALIGNMENT 16
+static void sha256_blocks_phe(struct sha256_block_state *state,
+			      const u8 *data, size_t nblocks)
+{
+	/*
+	 * On Zhaoxin processors, XSHA256 requires the %rdi register
+	 * in 64-bit mode (or %edi in 32-bit mode) to point to
+	 * a 32-byte, 16-byte-aligned buffer.
+	 */
+	u8 buf[32 + PHE_ALIGNMENT - 1];
+	u8 *dst = PTR_ALIGN(&buf[0], PHE_ALIGNMENT);
+	size_t padding = -1;
+
+	memcpy(dst, state, SHA256_DIGEST_SIZE);
+	asm volatile(".byte 0xf3,0x0f,0xa6,0xd0" /* REP XSHA256 */
+		     : "+a"(padding), "+c"(nblocks), "+S"(data)
+		     : "D"(dst)
+		     : "memory");
+	memcpy(state, dst, SHA256_DIGEST_SIZE);
+}
+
 static void sha256_blocks(struct sha256_block_state *state,
 			  const u8 *data, size_t nblocks)
 {
@@ -79,6 +100,10 @@ static void sha256_mod_init_arch(void)
 	if (boot_cpu_has(X86_FEATURE_SHA_NI)) {
 		static_call_update(sha256_blocks_x86, sha256_blocks_ni);
 		static_branch_enable(&have_sha_ni);
+	} else if (IS_ENABLED(CONFIG_CPU_SUP_ZHAOXIN) &&
+		   boot_cpu_has(X86_FEATURE_PHE_EN) &&
+		   boot_cpu_data.x86 >= 0x07) {
+		static_call_update(sha256_blocks_x86, sha256_blocks_phe);
 	} else if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
 				     NULL) &&
 		   boot_cpu_has(X86_FEATURE_AVX)) {
diff --git a/lib/crypto/x86/sm3-avx-asm_64.S b/lib/crypto/x86/sm3-avx-asm_64.S
new file mode 100644
index 000000000000..a1925b136010
--- /dev/null
+++ b/lib/crypto/x86/sm3-avx-asm_64.S
@@ -0,0 +1,516 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * SM3 AVX accelerated transform.
+ * specified in: https://datatracker.ietf.org/doc/html/draft-sca-cfrg-sm3-02
+ *
+ * Copyright (C) 2021 Jussi Kivilinna <jussi.kivilinna@iki.fi>
+ * Copyright (C) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
+ */
+
+/* Based on SM3 AES/BMI2 accelerated work by libgcrypt at:
+ *  https://gnupg.org/software/libgcrypt/index.html
+ */
+
+#include <linux/linkage.h>
+#include <asm/frame.h>
+
+/* State structure */
+
+#define state_h0 0
+#define state_h1 4
+#define state_h2 8
+#define state_h3 12
+#define state_h4 16
+#define state_h5 20
+#define state_h6 24
+#define state_h7 28
+
+/* Constants */
+
+/* Round constant macros */
+
+#define K0   2043430169  /* 0x79cc4519 */
+#define K1   -208106958  /* 0xf3988a32 */
+#define K2   -416213915  /* 0xe7311465 */
+#define K3   -832427829  /* 0xce6228cb */
+#define K4  -1664855657  /* 0x9cc45197 */
+#define K5    965255983  /* 0x3988a32f */
+#define K6   1930511966  /* 0x7311465e */
+#define K7   -433943364  /* 0xe6228cbc */
+#define K8   -867886727  /* 0xcc451979 */
+#define K9  -1735773453  /* 0x988a32f3 */
+#define K10   823420391  /* 0x311465e7 */
+#define K11  1646840782  /* 0x6228cbce */
+#define K12 -1001285732  /* 0xc451979c */
+#define K13 -2002571463  /* 0x88a32f39 */
+#define K14   289824371  /* 0x11465e73 */
+#define K15   579648742  /* 0x228cbce6 */
+#define K16 -1651869049  /* 0x9d8a7a87 */
+#define K17   991229199  /* 0x3b14f50f */
+#define K18  1982458398  /* 0x7629ea1e */
+#define K19  -330050500  /* 0xec53d43c */
+#define K20  -660100999  /* 0xd8a7a879 */
+#define K21 -1320201997  /* 0xb14f50f3 */
+#define K22  1654563303  /* 0x629ea1e7 */
+#define K23  -985840690  /* 0xc53d43ce */
+#define K24 -1971681379  /* 0x8a7a879d */
+#define K25   351604539  /* 0x14f50f3b */
+#define K26   703209078  /* 0x29ea1e76 */
+#define K27  1406418156  /* 0x53d43cec */
+#define K28 -1482130984  /* 0xa7a879d8 */
+#define K29  1330705329  /* 0x4f50f3b1 */
+#define K30 -1633556638  /* 0x9ea1e762 */
+#define K31  1027854021  /* 0x3d43cec5 */
+#define K32  2055708042  /* 0x7a879d8a */
+#define K33  -183551212  /* 0xf50f3b14 */
+#define K34  -367102423  /* 0xea1e7629 */
+#define K35  -734204845  /* 0xd43cec53 */
+#define K36 -1468409689  /* 0xa879d8a7 */
+#define K37  1358147919  /* 0x50f3b14f */
+#define K38 -1578671458  /* 0xa1e7629e */
+#define K39  1137624381  /* 0x43cec53d */
+#define K40 -2019718534  /* 0x879d8a7a */
+#define K41   255530229  /* 0x0f3b14f5 */
+#define K42   511060458  /* 0x1e7629ea */
+#define K43  1022120916  /* 0x3cec53d4 */
+#define K44  2044241832  /* 0x79d8a7a8 */
+#define K45  -206483632  /* 0xf3b14f50 */
+#define K46  -412967263  /* 0xe7629ea1 */
+#define K47  -825934525  /* 0xcec53d43 */
+#define K48 -1651869049  /* 0x9d8a7a87 */
+#define K49   991229199  /* 0x3b14f50f */
+#define K50  1982458398  /* 0x7629ea1e */
+#define K51  -330050500  /* 0xec53d43c */
+#define K52  -660100999  /* 0xd8a7a879 */
+#define K53 -1320201997  /* 0xb14f50f3 */
+#define K54  1654563303  /* 0x629ea1e7 */
+#define K55  -985840690  /* 0xc53d43ce */
+#define K56 -1971681379  /* 0x8a7a879d */
+#define K57   351604539  /* 0x14f50f3b */
+#define K58   703209078  /* 0x29ea1e76 */
+#define K59  1406418156  /* 0x53d43cec */
+#define K60 -1482130984  /* 0xa7a879d8 */
+#define K61  1330705329  /* 0x4f50f3b1 */
+#define K62 -1633556638  /* 0x9ea1e762 */
+#define K63  1027854021  /* 0x3d43cec5 */
+
+/* Register macros */
+
+#define RSTATE %rdi
+#define RDATA  %rsi
+#define RNBLKS %rdx
+
+#define t0 %eax
+#define t1 %ebx
+#define t2 %ecx
+
+#define a %r8d
+#define b %r9d
+#define c %r10d
+#define d %r11d
+#define e %r12d
+#define f %r13d
+#define g %r14d
+#define h %r15d
+
+#define W0 %xmm0
+#define W1 %xmm1
+#define W2 %xmm2
+#define W3 %xmm3
+#define W4 %xmm4
+#define W5 %xmm5
+
+#define XTMP0 %xmm6
+#define XTMP1 %xmm7
+#define XTMP2 %xmm8
+#define XTMP3 %xmm9
+#define XTMP4 %xmm10
+#define XTMP5 %xmm11
+#define XTMP6 %xmm12
+
+#define BSWAP_REG %xmm15
+
+/* Stack structure */
+
+#define STACK_W_SIZE        (32 * 2 * 3)
+#define STACK_REG_SAVE_SIZE (64)
+
+#define STACK_W             (0)
+#define STACK_REG_SAVE      (STACK_W + STACK_W_SIZE)
+#define STACK_SIZE          (STACK_REG_SAVE + STACK_REG_SAVE_SIZE)
+
+/* Instruction helpers. */
+
+#define roll2(v, reg)		\
+	roll $(v), reg;
+
+#define roll3mov(v, src, dst)	\
+	movl src, dst;		\
+	roll $(v), dst;
+
+#define roll3(v, src, dst)	\
+	rorxl $(32-(v)), src, dst;
+
+#define addl2(a, out)		\
+	leal (a, out), out;
+
+/* Round function macros. */
+
+#define GG1(x, y, z, o, t)	\
+	movl x, o;		\
+	xorl y, o;		\
+	xorl z, o;
+
+#define FF1(x, y, z, o, t) GG1(x, y, z, o, t)
+
+#define GG2(x, y, z, o, t)	\
+	andnl z, x, o;		\
+	movl y, t;		\
+	andl x, t;		\
+	addl2(t, o);
+
+#define FF2(x, y, z, o, t)	\
+	movl y, o;		\
+	xorl x, o;		\
+	movl y, t;		\
+	andl x, t;		\
+	andl z, o;		\
+	xorl t, o;
+
+#define R(i, a, b, c, d, e, f, g, h, round, widx, wtype)		\
+	/* rol(a, 12) => t0 */						\
+	roll3mov(12, a, t0); /* rorxl here would reduce perf by 6% on zen3 */ \
+	/* rol (t0 + e + t), 7) => t1 */				\
+	leal K##round(t0, e, 1), t1;					\
+	roll2(7, t1);							\
+	/* h + w1 => h */						\
+	addl wtype##_W1_ADDR(round, widx), h;				\
+	/* h + t1 => h */						\
+	addl2(t1, h);							\
+	/* t1 ^ t0 => t0 */						\
+	xorl t1, t0;							\
+	/* w1w2 + d => d */						\
+	addl wtype##_W1W2_ADDR(round, widx), d;				\
+	/* FF##i(a,b,c) => t1 */					\
+	FF##i(a, b, c, t1, t2);						\
+	/* d + t1 => d */						\
+	addl2(t1, d);							\
+	/* GG#i(e,f,g) => t2 */						\
+	GG##i(e, f, g, t2, t1);						\
+	/* h + t2 => h */						\
+	addl2(t2, h);							\
+	/* rol (f, 19) => f */						\
+	roll2(19, f);							\
+	/* d + t0 => d */						\
+	addl2(t0, d);							\
+	/* rol (b, 9) => b */						\
+	roll2(9, b);							\
+	/* P0(h) => h */						\
+	roll3(9, h, t2);						\
+	roll3(17, h, t1);						\
+	xorl t2, h;							\
+	xorl t1, h;
+
+#define R1(a, b, c, d, e, f, g, h, round, widx, wtype) \
+	R(1, a, b, c, d, e, f, g, h, round, widx, wtype)
+
+#define R2(a, b, c, d, e, f, g, h, round, widx, wtype) \
+	R(2, a, b, c, d, e, f, g, h, round, widx, wtype)
+
+/* Input expansion macros. */
+
+/* Byte-swapped input address. */
+#define IW_W_ADDR(round, widx, offs) \
+	(STACK_W + ((round) / 4) * 64 + (offs) + ((widx) * 4))(%rsp)
+
+/* Expanded input address. */
+#define XW_W_ADDR(round, widx, offs) \
+	(STACK_W + ((((round) / 3) - 4) % 2) * 64 + (offs) + ((widx) * 4))(%rsp)
+
+/* Rounds 1-12, byte-swapped input block addresses. */
+#define IW_W1_ADDR(round, widx)   IW_W_ADDR(round, widx, 0)
+#define IW_W1W2_ADDR(round, widx) IW_W_ADDR(round, widx, 32)
+
+/* Rounds 1-12, expanded input block addresses. */
+#define XW_W1_ADDR(round, widx)   XW_W_ADDR(round, widx, 0)
+#define XW_W1W2_ADDR(round, widx) XW_W_ADDR(round, widx, 32)
+
+/* Input block loading. */
+#define LOAD_W_XMM_1()							\
+	vmovdqu 0*16(RDATA), XTMP0; /* XTMP0: w3, w2, w1, w0 */		\
+	vmovdqu 1*16(RDATA), XTMP1; /* XTMP1: w7, w6, w5, w4 */		\
+	vmovdqu 2*16(RDATA), XTMP2; /* XTMP2: w11, w10, w9, w8 */	\
+	vmovdqu 3*16(RDATA), XTMP3; /* XTMP3: w15, w14, w13, w12 */	\
+	vpshufb BSWAP_REG, XTMP0, XTMP0;				\
+	vpshufb BSWAP_REG, XTMP1, XTMP1;				\
+	vpshufb BSWAP_REG, XTMP2, XTMP2;				\
+	vpshufb BSWAP_REG, XTMP3, XTMP3;				\
+	vpxor XTMP0, XTMP1, XTMP4;					\
+	vpxor XTMP1, XTMP2, XTMP5;					\
+	vpxor XTMP2, XTMP3, XTMP6;					\
+	leaq 64(RDATA), RDATA;						\
+	vmovdqa XTMP0, IW_W1_ADDR(0, 0);				\
+	vmovdqa XTMP4, IW_W1W2_ADDR(0, 0);				\
+	vmovdqa XTMP1, IW_W1_ADDR(4, 0);				\
+	vmovdqa XTMP5, IW_W1W2_ADDR(4, 0);
+
+#define LOAD_W_XMM_2()				\
+	vmovdqa XTMP2, IW_W1_ADDR(8, 0);	\
+	vmovdqa XTMP6, IW_W1W2_ADDR(8, 0);
+
+#define LOAD_W_XMM_3()							\
+	vpshufd $0b00000000, XTMP0, W0; /* W0: xx, w0, xx, xx */	\
+	vpshufd $0b11111001, XTMP0, W1; /* W1: xx, w3, w2, w1 */	\
+	vmovdqa XTMP1, W2;              /* W2: xx, w6, w5, w4 */	\
+	vpalignr $12, XTMP1, XTMP2, W3; /* W3: xx, w9, w8, w7 */	\
+	vpalignr $8, XTMP2, XTMP3, W4;  /* W4: xx, w12, w11, w10 */	\
+	vpshufd $0b11111001, XTMP3, W5; /* W5: xx, w15, w14, w13 */
+
+/* Message scheduling. Note: 3 words per XMM register. */
+#define SCHED_W_0(round, w0, w1, w2, w3, w4, w5)			\
+	/* Load (w[i - 16]) => XTMP0 */					\
+	vpshufd $0b10111111, w0, XTMP0;					\
+	vpalignr $12, XTMP0, w1, XTMP0; /* XTMP0: xx, w2, w1, w0 */	\
+	/* Load (w[i - 13]) => XTMP1 */					\
+	vpshufd $0b10111111, w1, XTMP1;					\
+	vpalignr $12, XTMP1, w2, XTMP1;					\
+	/* w[i - 9] == w3 */						\
+	/* XMM3 ^ XTMP0 => XTMP0 */					\
+	vpxor w3, XTMP0, XTMP0;
+
+#define SCHED_W_1(round, w0, w1, w2, w3, w4, w5)	\
+	/* w[i - 3] == w5 */				\
+	/* rol(XMM5, 15) ^ XTMP0 => XTMP0 */		\
+	vpslld $15, w5, XTMP2;				\
+	vpsrld $(32-15), w5, XTMP3;			\
+	vpxor XTMP2, XTMP3, XTMP3;			\
+	vpxor XTMP3, XTMP0, XTMP0;			\
+	/* rol(XTMP1, 7) => XTMP1 */			\
+	vpslld $7, XTMP1, XTMP5;			\
+	vpsrld $(32-7), XTMP1, XTMP1;			\
+	vpxor XTMP5, XTMP1, XTMP1;			\
+	/* XMM4 ^ XTMP1 => XTMP1 */			\
+	vpxor w4, XTMP1, XTMP1;				\
+	/* w[i - 6] == XMM4 */				\
+	/* P1(XTMP0) ^ XTMP1 => XMM0 */			\
+	vpslld $15, XTMP0, XTMP5;			\
+	vpsrld $(32-15), XTMP0, XTMP6;			\
+	vpslld $23, XTMP0, XTMP2;			\
+	vpsrld $(32-23), XTMP0, XTMP3;			\
+	vpxor XTMP0, XTMP1, XTMP1;			\
+	vpxor XTMP6, XTMP5, XTMP5;			\
+	vpxor XTMP3, XTMP2, XTMP2;			\
+	vpxor XTMP2, XTMP5, XTMP5;			\
+	vpxor XTMP5, XTMP1, w0;
+
+#define SCHED_W_2(round, w0, w1, w2, w3, w4, w5)	\
+	/* W1 in XMM12 */				\
+	vpshufd $0b10111111, w4, XTMP4;			\
+	vpalignr $12, XTMP4, w5, XTMP4;			\
+	vmovdqa XTMP4, XW_W1_ADDR((round), 0);		\
+	/* W1 ^ W2 => XTMP1 */				\
+	vpxor w0, XTMP4, XTMP1;				\
+	vmovdqa XTMP1, XW_W1W2_ADDR((round), 0);
+
+
+.section	.rodata.cst16, "aM", @progbits, 16
+.align 16
+
+.Lbe32mask:
+	.long 0x00010203, 0x04050607, 0x08090a0b, 0x0c0d0e0f
+
+.text
+
+/*
+ * Transform nblocks*64 bytes (nblocks*16 32-bit words) at DATA.
+ *
+ * void sm3_transform_avx(struct sm3_block_state *state,
+ *                        const u8 *data, size_t nblocks);
+ */
+SYM_FUNC_START(sm3_transform_avx)
+	/* input:
+	 *	%rdi: state
+	 *	%rsi: data
+	 *	%rdx: nblocks
+	 */
+	vzeroupper;
+
+	pushq %rbp;
+	movq %rsp, %rbp;
+
+	movq %rdx, RNBLKS;
+
+	subq $STACK_SIZE, %rsp;
+	andq $(~63), %rsp;
+
+	movq %rbx, (STACK_REG_SAVE + 0 * 8)(%rsp);
+	movq %r15, (STACK_REG_SAVE + 1 * 8)(%rsp);
+	movq %r14, (STACK_REG_SAVE + 2 * 8)(%rsp);
+	movq %r13, (STACK_REG_SAVE + 3 * 8)(%rsp);
+	movq %r12, (STACK_REG_SAVE + 4 * 8)(%rsp);
+
+	vmovdqa .Lbe32mask (%rip), BSWAP_REG;
+
+	/* Get the values of the chaining variables. */
+	movl state_h0(RSTATE), a;
+	movl state_h1(RSTATE), b;
+	movl state_h2(RSTATE), c;
+	movl state_h3(RSTATE), d;
+	movl state_h4(RSTATE), e;
+	movl state_h5(RSTATE), f;
+	movl state_h6(RSTATE), g;
+	movl state_h7(RSTATE), h;
+
+.align 16
+.Loop:
+	/* Load data part1. */
+	LOAD_W_XMM_1();
+
+	leaq -1(RNBLKS), RNBLKS;
+
+	/* Transform 0-3 + Load data part2. */
+	R1(a, b, c, d, e, f, g, h, 0, 0, IW); LOAD_W_XMM_2();
+	R1(d, a, b, c, h, e, f, g, 1, 1, IW);
+	R1(c, d, a, b, g, h, e, f, 2, 2, IW);
+	R1(b, c, d, a, f, g, h, e, 3, 3, IW); LOAD_W_XMM_3();
+
+	/* Transform 4-7 + Precalc 12-14. */
+	R1(a, b, c, d, e, f, g, h, 4, 0, IW);
+	R1(d, a, b, c, h, e, f, g, 5, 1, IW);
+	R1(c, d, a, b, g, h, e, f, 6, 2, IW); SCHED_W_0(12, W0, W1, W2, W3, W4, W5);
+	R1(b, c, d, a, f, g, h, e, 7, 3, IW); SCHED_W_1(12, W0, W1, W2, W3, W4, W5);
+
+	/* Transform 8-11 + Precalc 12-17. */
+	R1(a, b, c, d, e, f, g, h, 8, 0, IW); SCHED_W_2(12, W0, W1, W2, W3, W4, W5);
+	R1(d, a, b, c, h, e, f, g, 9, 1, IW); SCHED_W_0(15, W1, W2, W3, W4, W5, W0);
+	R1(c, d, a, b, g, h, e, f, 10, 2, IW); SCHED_W_1(15, W1, W2, W3, W4, W5, W0);
+	R1(b, c, d, a, f, g, h, e, 11, 3, IW); SCHED_W_2(15, W1, W2, W3, W4, W5, W0);
+
+	/* Transform 12-14 + Precalc 18-20 */
+	R1(a, b, c, d, e, f, g, h, 12, 0, XW); SCHED_W_0(18, W2, W3, W4, W5, W0, W1);
+	R1(d, a, b, c, h, e, f, g, 13, 1, XW); SCHED_W_1(18, W2, W3, W4, W5, W0, W1);
+	R1(c, d, a, b, g, h, e, f, 14, 2, XW); SCHED_W_2(18, W2, W3, W4, W5, W0, W1);
+
+	/* Transform 15-17 + Precalc 21-23 */
+	R1(b, c, d, a, f, g, h, e, 15, 0, XW); SCHED_W_0(21, W3, W4, W5, W0, W1, W2);
+	R2(a, b, c, d, e, f, g, h, 16, 1, XW); SCHED_W_1(21, W3, W4, W5, W0, W1, W2);
+	R2(d, a, b, c, h, e, f, g, 17, 2, XW); SCHED_W_2(21, W3, W4, W5, W0, W1, W2);
+
+	/* Transform 18-20 + Precalc 24-26 */
+	R2(c, d, a, b, g, h, e, f, 18, 0, XW); SCHED_W_0(24, W4, W5, W0, W1, W2, W3);
+	R2(b, c, d, a, f, g, h, e, 19, 1, XW); SCHED_W_1(24, W4, W5, W0, W1, W2, W3);
+	R2(a, b, c, d, e, f, g, h, 20, 2, XW); SCHED_W_2(24, W4, W5, W0, W1, W2, W3);
+
+	/* Transform 21-23 + Precalc 27-29 */
+	R2(d, a, b, c, h, e, f, g, 21, 0, XW); SCHED_W_0(27, W5, W0, W1, W2, W3, W4);
+	R2(c, d, a, b, g, h, e, f, 22, 1, XW); SCHED_W_1(27, W5, W0, W1, W2, W3, W4);
+	R2(b, c, d, a, f, g, h, e, 23, 2, XW); SCHED_W_2(27, W5, W0, W1, W2, W3, W4);
+
+	/* Transform 24-26 + Precalc 30-32 */
+	R2(a, b, c, d, e, f, g, h, 24, 0, XW); SCHED_W_0(30, W0, W1, W2, W3, W4, W5);
+	R2(d, a, b, c, h, e, f, g, 25, 1, XW); SCHED_W_1(30, W0, W1, W2, W3, W4, W5);
+	R2(c, d, a, b, g, h, e, f, 26, 2, XW); SCHED_W_2(30, W0, W1, W2, W3, W4, W5);
+
+	/* Transform 27-29 + Precalc 33-35 */
+	R2(b, c, d, a, f, g, h, e, 27, 0, XW); SCHED_W_0(33, W1, W2, W3, W4, W5, W0);
+	R2(a, b, c, d, e, f, g, h, 28, 1, XW); SCHED_W_1(33, W1, W2, W3, W4, W5, W0);
+	R2(d, a, b, c, h, e, f, g, 29, 2, XW); SCHED_W_2(33, W1, W2, W3, W4, W5, W0);
+
+	/* Transform 30-32 + Precalc 36-38 */
+	R2(c, d, a, b, g, h, e, f, 30, 0, XW); SCHED_W_0(36, W2, W3, W4, W5, W0, W1);
+	R2(b, c, d, a, f, g, h, e, 31, 1, XW); SCHED_W_1(36, W2, W3, W4, W5, W0, W1);
+	R2(a, b, c, d, e, f, g, h, 32, 2, XW); SCHED_W_2(36, W2, W3, W4, W5, W0, W1);
+
+	/* Transform 33-35 + Precalc 39-41 */
+	R2(d, a, b, c, h, e, f, g, 33, 0, XW); SCHED_W_0(39, W3, W4, W5, W0, W1, W2);
+	R2(c, d, a, b, g, h, e, f, 34, 1, XW); SCHED_W_1(39, W3, W4, W5, W0, W1, W2);
+	R2(b, c, d, a, f, g, h, e, 35, 2, XW); SCHED_W_2(39, W3, W4, W5, W0, W1, W2);
+
+	/* Transform 36-38 + Precalc 42-44 */
+	R2(a, b, c, d, e, f, g, h, 36, 0, XW); SCHED_W_0(42, W4, W5, W0, W1, W2, W3);
+	R2(d, a, b, c, h, e, f, g, 37, 1, XW); SCHED_W_1(42, W4, W5, W0, W1, W2, W3);
+	R2(c, d, a, b, g, h, e, f, 38, 2, XW); SCHED_W_2(42, W4, W5, W0, W1, W2, W3);
+
+	/* Transform 39-41 + Precalc 45-47 */
+	R2(b, c, d, a, f, g, h, e, 39, 0, XW); SCHED_W_0(45, W5, W0, W1, W2, W3, W4);
+	R2(a, b, c, d, e, f, g, h, 40, 1, XW); SCHED_W_1(45, W5, W0, W1, W2, W3, W4);
+	R2(d, a, b, c, h, e, f, g, 41, 2, XW); SCHED_W_2(45, W5, W0, W1, W2, W3, W4);
+
+	/* Transform 42-44 + Precalc 48-50 */
+	R2(c, d, a, b, g, h, e, f, 42, 0, XW); SCHED_W_0(48, W0, W1, W2, W3, W4, W5);
+	R2(b, c, d, a, f, g, h, e, 43, 1, XW); SCHED_W_1(48, W0, W1, W2, W3, W4, W5);
+	R2(a, b, c, d, e, f, g, h, 44, 2, XW); SCHED_W_2(48, W0, W1, W2, W3, W4, W5);
+
+	/* Transform 45-47 + Precalc 51-53 */
+	R2(d, a, b, c, h, e, f, g, 45, 0, XW); SCHED_W_0(51, W1, W2, W3, W4, W5, W0);
+	R2(c, d, a, b, g, h, e, f, 46, 1, XW); SCHED_W_1(51, W1, W2, W3, W4, W5, W0);
+	R2(b, c, d, a, f, g, h, e, 47, 2, XW); SCHED_W_2(51, W1, W2, W3, W4, W5, W0);
+
+	/* Transform 48-50 + Precalc 54-56 */
+	R2(a, b, c, d, e, f, g, h, 48, 0, XW); SCHED_W_0(54, W2, W3, W4, W5, W0, W1);
+	R2(d, a, b, c, h, e, f, g, 49, 1, XW); SCHED_W_1(54, W2, W3, W4, W5, W0, W1);
+	R2(c, d, a, b, g, h, e, f, 50, 2, XW); SCHED_W_2(54, W2, W3, W4, W5, W0, W1);
+
+	/* Transform 51-53 + Precalc 57-59 */
+	R2(b, c, d, a, f, g, h, e, 51, 0, XW); SCHED_W_0(57, W3, W4, W5, W0, W1, W2);
+	R2(a, b, c, d, e, f, g, h, 52, 1, XW); SCHED_W_1(57, W3, W4, W5, W0, W1, W2);
+	R2(d, a, b, c, h, e, f, g, 53, 2, XW); SCHED_W_2(57, W3, W4, W5, W0, W1, W2);
+
+	/* Transform 54-56 + Precalc 60-62 */
+	R2(c, d, a, b, g, h, e, f, 54, 0, XW); SCHED_W_0(60, W4, W5, W0, W1, W2, W3);
+	R2(b, c, d, a, f, g, h, e, 55, 1, XW); SCHED_W_1(60, W4, W5, W0, W1, W2, W3);
+	R2(a, b, c, d, e, f, g, h, 56, 2, XW); SCHED_W_2(60, W4, W5, W0, W1, W2, W3);
+
+	/* Transform 57-59 + Precalc 63 */
+	R2(d, a, b, c, h, e, f, g, 57, 0, XW); SCHED_W_0(63, W5, W0, W1, W2, W3, W4);
+	R2(c, d, a, b, g, h, e, f, 58, 1, XW);
+	R2(b, c, d, a, f, g, h, e, 59, 2, XW); SCHED_W_1(63, W5, W0, W1, W2, W3, W4);
+
+	/* Transform 60-62 + Precalc 63 */
+	R2(a, b, c, d, e, f, g, h, 60, 0, XW);
+	R2(d, a, b, c, h, e, f, g, 61, 1, XW); SCHED_W_2(63, W5, W0, W1, W2, W3, W4);
+	R2(c, d, a, b, g, h, e, f, 62, 2, XW);
+
+	/* Transform 63 */
+	R2(b, c, d, a, f, g, h, e, 63, 0, XW);
+
+	/* Update the chaining variables. */
+	xorl state_h0(RSTATE), a;
+	xorl state_h1(RSTATE), b;
+	xorl state_h2(RSTATE), c;
+	xorl state_h3(RSTATE), d;
+	movl a, state_h0(RSTATE);
+	movl b, state_h1(RSTATE);
+	movl c, state_h2(RSTATE);
+	movl d, state_h3(RSTATE);
+	xorl state_h4(RSTATE), e;
+	xorl state_h5(RSTATE), f;
+	xorl state_h6(RSTATE), g;
+	xorl state_h7(RSTATE), h;
+	movl e, state_h4(RSTATE);
+	movl f, state_h5(RSTATE);
+	movl g, state_h6(RSTATE);
+	movl h, state_h7(RSTATE);
+
+	cmpq $0, RNBLKS;
+	jne .Loop;
+
+	vzeroall;
+
+	movq (STACK_REG_SAVE + 0 * 8)(%rsp), %rbx;
+	movq (STACK_REG_SAVE + 1 * 8)(%rsp), %r15;
+	movq (STACK_REG_SAVE + 2 * 8)(%rsp), %r14;
+	movq (STACK_REG_SAVE + 3 * 8)(%rsp), %r13;
+	movq (STACK_REG_SAVE + 4 * 8)(%rsp), %r12;
+
+	vmovdqa %xmm0, IW_W1_ADDR(0, 0);
+	vmovdqa %xmm0, IW_W1W2_ADDR(0, 0);
+	vmovdqa %xmm0, IW_W1_ADDR(4, 0);
+	vmovdqa %xmm0, IW_W1W2_ADDR(4, 0);
+	vmovdqa %xmm0, IW_W1_ADDR(8, 0);
+	vmovdqa %xmm0, IW_W1W2_ADDR(8, 0);
+
+	movq %rbp, %rsp;
+	popq %rbp;
+	RET;
+SYM_FUNC_END(sm3_transform_avx)
diff --git a/lib/crypto/x86/sm3.h b/lib/crypto/x86/sm3.h
new file mode 100644
index 000000000000..3834780f2f6a
--- /dev/null
+++ b/lib/crypto/x86/sm3.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * SM3 optimized for x86_64
+ *
+ * Copyright 2026 Google LLC
+ */
+#include <asm/fpu/api.h>
+#include <linux/static_call.h>
+
+asmlinkage void sm3_transform_avx(struct sm3_block_state *state,
+				  const u8 *data, size_t nblocks);
+
+static void sm3_blocks_avx(struct sm3_block_state *state,
+			   const u8 *data, size_t nblocks)
+{
+	if (likely(irq_fpu_usable())) {
+		kernel_fpu_begin();
+		sm3_transform_avx(state, data, nblocks);
+		kernel_fpu_end();
+	} else {
+		sm3_blocks_generic(state, data, nblocks);
+	}
+}
+
+DEFINE_STATIC_CALL(sm3_blocks_x86, sm3_blocks_generic);
+
+static void sm3_blocks(struct sm3_block_state *state,
+		       const u8 *data, size_t nblocks)
+{
+	static_call(sm3_blocks_x86)(state, data, nblocks);
+}
+
+#define sm3_mod_init_arch sm3_mod_init_arch
+static void sm3_mod_init_arch(void)
+{
+	if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_BMI2) &&
+	    cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
+		static_call_update(sm3_blocks_x86, sm3_blocks_avx);
+}
author	Linus Torvalds <torvalds@linux-foundation.org>	2026-04-13 17:31:39 -0700
committer	Linus Torvalds <torvalds@linux-foundation.org>	2026-04-13 17:31:39 -0700
commit	370c3883195566ee3e7d79e0146c3d735a406573 (patch)
tree	5ef1663d7822a3094ce80d1379e7c0630200e5d9 /lib
parent	9932f00bf40d281151de5694bc0f097cb9b5616c (diff)
parent	12b11e47f126d097839fd2f077636e2139b0151b (diff)