diff options
Diffstat (limited to 'lib')
64 files changed, 6778 insertions, 1502 deletions
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 30d069676309..94eb2667b7e1 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -3058,6 +3058,8 @@ config HW_BREAKPOINT_KUNIT_TEST If unsure, say N. +source "lib/crypto/tests/Kconfig" + config SIPHASH_KUNIT_TEST tristate "Perform selftest on siphash functions" if !KUNIT_ALL_TESTS depends on KUNIT diff --git a/lib/crypto/.kunitconfig b/lib/crypto/.kunitconfig index 6b2ce28ae509..3efc854a2c08 100644 --- a/lib/crypto/.kunitconfig +++ b/lib/crypto/.kunitconfig @@ -1,28 +1,13 @@ CONFIG_KUNIT=y -# These kconfig options select all the CONFIG_CRYPTO_LIB_* symbols that have a -# corresponding KUnit test. Those symbols cannot be directly enabled here, -# since they are hidden symbols. -CONFIG_CRYPTO=y -CONFIG_CRYPTO_ADIANTUM=y -CONFIG_CRYPTO_BLAKE2B=y -CONFIG_CRYPTO_CHACHA20POLY1305=y -CONFIG_CRYPTO_HCTR2=y -CONFIG_CRYPTO_MD5=y -CONFIG_CRYPTO_MLDSA=y -CONFIG_CRYPTO_SHA1=y -CONFIG_CRYPTO_SHA256=y -CONFIG_CRYPTO_SHA512=y -CONFIG_CRYPTO_SHA3=y -CONFIG_INET=y -CONFIG_IPV6=y -CONFIG_NET=y -CONFIG_NETDEVICES=y -CONFIG_WIREGUARD=y +CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=y +CONFIG_CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST=y CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST=y CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_CHACHA20POLY1305_KUNIT_TEST=y CONFIG_CRYPTO_LIB_CURVE25519_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_GHASH_KUNIT_TEST=y CONFIG_CRYPTO_LIB_MD5_KUNIT_TEST=y CONFIG_CRYPTO_LIB_MLDSA_KUNIT_TEST=y CONFIG_CRYPTO_LIB_NH_KUNIT_TEST=y @@ -32,3 +17,4 @@ CONFIG_CRYPTO_LIB_SHA1_KUNIT_TEST=y CONFIG_CRYPTO_LIB_SHA256_KUNIT_TEST=y CONFIG_CRYPTO_LIB_SHA512_KUNIT_TEST=y CONFIG_CRYPTO_LIB_SHA3_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_SM3_KUNIT_TEST=y diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 032f9755f999..d3904b72dae7 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -1,7 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -menu "Crypto library routines" - config CRYPTO_HASH_INFO bool @@ -10,6 +8,8 @@ config CRYPTO_LIB_UTILS config CRYPTO_LIB_AES tristate + # Select dependencies of modes that are part of libaes. + select CRYPTO_LIB_UTILS if CRYPTO_LIB_AES_CBC_MACS config CRYPTO_LIB_AES_ARCH bool @@ -28,10 +28,18 @@ config CRYPTO_LIB_AESCFB select CRYPTO_LIB_AES select CRYPTO_LIB_UTILS +config CRYPTO_LIB_AES_CBC_MACS + tristate + select CRYPTO_LIB_AES + help + The AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC library functions. Select + this if your module uses any of the functions from + <crypto/aes-cbc-macs.h>. + config CRYPTO_LIB_AESGCM tristate select CRYPTO_LIB_AES - select CRYPTO_LIB_GF128MUL + select CRYPTO_LIB_GF128HASH select CRYPTO_LIB_UTILS config CRYPTO_LIB_ARC4 @@ -70,7 +78,7 @@ config CRYPTO_LIB_CHACHA_ARCH bool depends on CRYPTO_LIB_CHACHA && !UML && !KMSAN default y if ARM - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if MIPS && CPU_MIPS32_R2 default y if PPC64 && CPU_LITTLE_ENDIAN && VSX default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ @@ -100,6 +108,23 @@ config CRYPTO_LIB_CURVE25519_GENERIC config CRYPTO_LIB_DES tristate +config CRYPTO_LIB_GF128HASH + tristate + help + The GHASH and POLYVAL library functions. Select this if your module + uses any of the functions from <crypto/gf128hash.h>. + +config CRYPTO_LIB_GF128HASH_ARCH + bool + depends on CRYPTO_LIB_GF128HASH && !UML + default y if ARM && KERNEL_MODE_NEON + default y if ARM64 + default y if PPC64 && VSX + default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ + RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS + default y if S390 + default y if X86_64 + config CRYPTO_LIB_MD5 tristate help @@ -109,9 +134,7 @@ config CRYPTO_LIB_MD5 config CRYPTO_LIB_MD5_ARCH bool depends on CRYPTO_LIB_MD5 && !UML - default y if MIPS && CPU_CAVIUM_OCTEON default y if PPC - default y if SPARC64 config CRYPTO_LIB_MLDSA tristate @@ -130,7 +153,7 @@ config CRYPTO_LIB_NH_ARCH bool depends on CRYPTO_LIB_NH && !UML && !KMSAN default y if ARM && KERNEL_MODE_NEON - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if X86_64 config CRYPTO_LIB_POLY1305 @@ -143,7 +166,7 @@ config CRYPTO_LIB_POLY1305_ARCH bool depends on CRYPTO_LIB_POLY1305 && !UML && !KMSAN default y if ARM - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if MIPS # The PPC64 code needs to be fixed to work in softirq context. default y if PPC64 && CPU_LITTLE_ENDIAN && VSX && BROKEN @@ -168,18 +191,6 @@ config CRYPTO_LIB_POLY1305_RSIZE default 9 if ARM || ARM64 default 1 -config CRYPTO_LIB_POLYVAL - tristate - help - The POLYVAL library functions. Select this if your module uses any of - the functions from <crypto/polyval.h>. - -config CRYPTO_LIB_POLYVAL_ARCH - bool - depends on CRYPTO_LIB_POLYVAL && !UML - default y if ARM64 && KERNEL_MODE_NEON - default y if X86_64 - config CRYPTO_LIB_CHACHA20POLY1305 tristate select CRYPTO_LIB_CHACHA @@ -196,7 +207,7 @@ config CRYPTO_LIB_SHA1_ARCH bool depends on CRYPTO_LIB_SHA1 && !UML default y if ARM - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if MIPS && CPU_CAVIUM_OCTEON default y if PPC default y if S390 @@ -252,12 +263,19 @@ config CRYPTO_LIB_SHA3 config CRYPTO_LIB_SHA3_ARCH bool depends on CRYPTO_LIB_SHA3 && !UML - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if S390 config CRYPTO_LIB_SM3 tristate + help + The SM3 library functions. Select this if your module uses any of the + functions from <crypto/sm3.h>. -source "lib/crypto/tests/Kconfig" - -endmenu +config CRYPTO_LIB_SM3_ARCH + bool + depends on CRYPTO_LIB_SM3 && !UML + default y if ARM64 + default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ + RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS + default y if X86_64 diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index dc7a56f7287d..4ad91f390038 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -8,6 +8,10 @@ quiet_cmd_perlasm = PERLASM $@ quiet_cmd_perlasm_with_args = PERLASM $@ cmd_perlasm_with_args = $(PERL) $(<) void $(@) +ppc64-perlasm-flavour-y := linux-ppc64 +ppc64-perlasm-flavour-$(CONFIG_PPC64_ELF_ABI_V2) := linux-ppc64-elfv2 +ppc64-perlasm-flavour-$(CONFIG_CPU_LITTLE_ENDIAN) := linux-ppc64le + obj-$(CONFIG_KUNIT) += tests/ obj-$(CONFIG_CRYPTO_HASH_INFO) += hash_info.o @@ -23,11 +27,10 @@ ifeq ($(CONFIG_CRYPTO_LIB_AES_ARCH),y) CFLAGS_aes.o += -I$(src)/$(SRCARCH) libaes-$(CONFIG_ARM) += arm/aes-cipher-core.o - -ifeq ($(CONFIG_ARM64),y) -libaes-y += arm64/aes-cipher-core.o -libaes-$(CONFIG_KERNEL_MODE_NEON) += arm64/aes-ce-core.o -endif +libaes-$(CONFIG_ARM64) += arm64/aes-cipher-core.o \ + arm64/aes-ce-core.o \ + arm64/aes-ce.o \ + arm64/aes-neon.o ifeq ($(CONFIG_PPC),y) ifeq ($(CONFIG_SPE),y) @@ -37,11 +40,8 @@ libaes-y += powerpc/aes-spe-core.o \ powerpc/aes-tab-4k.o else libaes-y += powerpc/aesp8-ppc.o -aes-perlasm-flavour-y := linux-ppc64 -aes-perlasm-flavour-$(CONFIG_PPC64_ELF_ABI_V2) := linux-ppc64-elfv2 -aes-perlasm-flavour-$(CONFIG_CPU_LITTLE_ENDIAN) := linux-ppc64le quiet_cmd_perlasm_aes = PERLASM $@ - cmd_perlasm_aes = $(PERL) $< $(aes-perlasm-flavour-y) $@ + cmd_perlasm_aes = $(PERL) $< $(ppc64-perlasm-flavour-y) $@ # Use if_changed instead of cmd, in case the flavour changed. $(obj)/powerpc/aesp8-ppc.S: $(src)/powerpc/aesp8-ppc.pl FORCE $(call if_changed,perlasm_aes) @@ -125,7 +125,6 @@ endif # CONFIG_CRYPTO_LIB_CHACHA_ARCH obj-$(CONFIG_CRYPTO_LIB_CHACHA20POLY1305) += libchacha20poly1305.o libchacha20poly1305-y += chacha20poly1305.o -libchacha20poly1305-$(CONFIG_CRYPTO_SELFTESTS) += chacha20poly1305-selftest.o ################################################################################ @@ -158,12 +157,39 @@ libdes-y := des.o ################################################################################ +obj-$(CONFIG_CRYPTO_LIB_GF128HASH) += libgf128hash.o +libgf128hash-y := gf128hash.o +ifeq ($(CONFIG_CRYPTO_LIB_GF128HASH_ARCH),y) +CFLAGS_gf128hash.o += -I$(src)/$(SRCARCH) +libgf128hash-$(CONFIG_ARM) += arm/ghash-neon-core.o +libgf128hash-$(CONFIG_ARM64) += arm64/ghash-neon-core.o \ + arm64/polyval-ce-core.o + +ifeq ($(CONFIG_PPC),y) +libgf128hash-y += powerpc/ghashp8-ppc.o +quiet_cmd_perlasm_ghash = PERLASM $@ + cmd_perlasm_ghash = $(PERL) $< $(ppc64-perlasm-flavour-y) $@ +$(obj)/powerpc/ghashp8-ppc.S: $(src)/powerpc/ghashp8-ppc.pl FORCE + $(call if_changed,perlasm_ghash) +targets += powerpc/ghashp8-ppc.S +OBJECT_FILES_NON_STANDARD_powerpc/ghashp8-ppc.o := y +endif + +libgf128hash-$(CONFIG_RISCV) += riscv/ghash-riscv64-zvkg.o +libgf128hash-$(CONFIG_X86) += x86/ghash-pclmul.o \ + x86/polyval-pclmul-avx.o +endif # CONFIG_CRYPTO_LIB_GF128HASH_ARCH + +# clean-files must be defined unconditionally +clean-files += powerpc/ghashp8-ppc.S + +################################################################################ + obj-$(CONFIG_CRYPTO_LIB_MD5) += libmd5.o libmd5-y := md5.o ifeq ($(CONFIG_CRYPTO_LIB_MD5_ARCH),y) CFLAGS_md5.o += -I$(src)/$(SRCARCH) libmd5-$(CONFIG_PPC) += powerpc/md5-asm.o -libmd5-$(CONFIG_SPARC) += sparc/md5_asm.o endif # CONFIG_CRYPTO_LIB_MD5_ARCH ################################################################################ @@ -255,16 +281,6 @@ clean-files += arm/poly1305-core.S \ ################################################################################ -obj-$(CONFIG_CRYPTO_LIB_POLYVAL) += libpolyval.o -libpolyval-y := polyval.o -ifeq ($(CONFIG_CRYPTO_LIB_POLYVAL_ARCH),y) -CFLAGS_polyval.o += -I$(src)/$(SRCARCH) -libpolyval-$(CONFIG_ARM64) += arm64/polyval-ce-core.o -libpolyval-$(CONFIG_X86) += x86/polyval-pclmul-avx.o -endif - -################################################################################ - obj-$(CONFIG_CRYPTO_LIB_SHA1) += libsha1.o libsha1-y := sha1.o ifeq ($(CONFIG_CRYPTO_LIB_SHA1_ARCH),y) @@ -300,10 +316,9 @@ AFLAGS_arm/sha256-core.o += $(aflags-thumb2-y) endif ifeq ($(CONFIG_ARM64),y) -libsha256-y += arm64/sha256-core.o +libsha256-y += arm64/sha256-ce.o arm64/sha256-core.o $(obj)/arm64/sha256-core.S: $(src)/arm64/sha2-armv8.pl $(call cmd,perlasm_with_args) -libsha256-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha256-ce.o endif libsha256-$(CONFIG_PPC) += powerpc/sha256-spe-asm.o @@ -330,10 +345,9 @@ AFLAGS_arm/sha512-core.o += $(aflags-thumb2-y) endif ifeq ($(CONFIG_ARM64),y) -libsha512-y += arm64/sha512-core.o +libsha512-y += arm64/sha512-ce-core.o arm64/sha512-core.o $(obj)/arm64/sha512-core.S: $(src)/arm64/sha2-armv8.pl $(call cmd,perlasm_with_args) -libsha512-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha512-ce-core.o endif libsha512-$(CONFIG_RISCV) += riscv/sha512-riscv64-zvknhb-zvkb.o @@ -355,13 +369,22 @@ endif # CONFIG_CRYPTO_LIB_SHA3_ARCH ################################################################################ +obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o +libsm3-y := sm3.o +ifeq ($(CONFIG_CRYPTO_LIB_SM3_ARCH),y) +CFLAGS_sm3.o += -I$(src)/$(SRCARCH) +libsm3-$(CONFIG_ARM64) += arm64/sm3-ce-core.o \ + arm64/sm3-neon-core.o +libsm3-$(CONFIG_RISCV) += riscv/sm3-riscv64-zvksh-zvkb.o +libsm3-$(CONFIG_X86) += x86/sm3-avx-asm_64.o +endif # CONFIG_CRYPTO_LIB_SM3_ARCH + +################################################################################ + obj-$(CONFIG_MPILIB) += mpi/ obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) += simd.o -obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o -libsm3-y := sm3.o - # clean-files must be defined unconditionally clean-files += arm/sha256-core.S arm/sha512-core.S clean-files += arm64/sha256-core.S arm64/sha512-core.S diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c index b73e19f1bb95..ca733f15b2a8 100644 --- a/lib/crypto/aes.c +++ b/lib/crypto/aes.c @@ -4,12 +4,15 @@ * Copyright 2026 Google LLC */ +#include <crypto/aes-cbc-macs.h> #include <crypto/aes.h> +#include <crypto/utils.h> #include <linux/cache.h> #include <linux/crypto.h> #include <linux/export.h> #include <linux/module.h> #include <linux/unaligned.h> +#include "fips.h" static const u8 ____cacheline_aligned aes_sbox[] = { 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, @@ -512,10 +515,235 @@ void aes_decrypt(const struct aes_key *key, u8 out[AES_BLOCK_SIZE], } EXPORT_SYMBOL(aes_decrypt); -#ifdef aes_mod_init_arch +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) + +#ifndef aes_cbcmac_blocks_arch +static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE], + const struct aes_enckey *key, const u8 *data, + size_t nblocks, bool enc_before, + bool enc_after) +{ + return false; +} +#endif + +/* This assumes nblocks >= 1. */ +static void aes_cbcmac_blocks(u8 h[AES_BLOCK_SIZE], + const struct aes_enckey *key, const u8 *data, + size_t nblocks, bool enc_before, bool enc_after) +{ + if (aes_cbcmac_blocks_arch(h, key, data, nblocks, enc_before, + enc_after)) + return; + + if (enc_before) + aes_encrypt(key, h, h); + for (; nblocks > 1; nblocks--) { + crypto_xor(h, data, AES_BLOCK_SIZE); + data += AES_BLOCK_SIZE; + aes_encrypt(key, h, h); + } + crypto_xor(h, data, AES_BLOCK_SIZE); + if (enc_after) + aes_encrypt(key, h, h); +} + +int aes_cmac_preparekey(struct aes_cmac_key *key, const u8 *in_key, + size_t key_len) +{ + u64 hi, lo, mask; + int err; + + /* Prepare the AES key. */ + err = aes_prepareenckey(&key->aes, in_key, key_len); + if (err) + return err; + + /* + * Prepare the subkeys K1 and K2 by encrypting the all-zeroes block, + * then multiplying by 'x' and 'x^2' (respectively) in GF(2^128). + * Reference: NIST SP 800-38B, Section 6.1 "Subkey Generation". + */ + memset(key->k_final[0].b, 0, AES_BLOCK_SIZE); + aes_encrypt(&key->aes, key->k_final[0].b, key->k_final[0].b); + hi = be64_to_cpu(key->k_final[0].w[0]); + lo = be64_to_cpu(key->k_final[0].w[1]); + for (int i = 0; i < 2; i++) { + mask = ((s64)hi >> 63) & 0x87; + hi = (hi << 1) ^ (lo >> 63); + lo = (lo << 1) ^ mask; + key->k_final[i].w[0] = cpu_to_be64(hi); + key->k_final[i].w[1] = cpu_to_be64(lo); + } + return 0; +} +EXPORT_SYMBOL_GPL(aes_cmac_preparekey); + +void aes_xcbcmac_preparekey(struct aes_cmac_key *key, + const u8 in_key[AES_KEYSIZE_128]) +{ + static const u8 constants[3][AES_BLOCK_SIZE] = { + { [0 ... AES_BLOCK_SIZE - 1] = 0x1 }, + { [0 ... AES_BLOCK_SIZE - 1] = 0x2 }, + { [0 ... AES_BLOCK_SIZE - 1] = 0x3 }, + }; + u8 new_aes_key[AES_BLOCK_SIZE]; + + static_assert(AES_BLOCK_SIZE == AES_KEYSIZE_128); + aes_prepareenckey(&key->aes, in_key, AES_BLOCK_SIZE); + aes_encrypt(&key->aes, new_aes_key, constants[0]); + aes_encrypt(&key->aes, key->k_final[0].b, constants[1]); + aes_encrypt(&key->aes, key->k_final[1].b, constants[2]); + aes_prepareenckey(&key->aes, new_aes_key, AES_BLOCK_SIZE); + memzero_explicit(new_aes_key, AES_BLOCK_SIZE); +} +EXPORT_SYMBOL_GPL(aes_xcbcmac_preparekey); + +void aes_cmac_update(struct aes_cmac_ctx *ctx, const u8 *data, size_t data_len) +{ + bool enc_before = false; + size_t nblocks; + + if (ctx->partial_len) { + /* XOR data into a pending block. */ + size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); + + crypto_xor(&ctx->h[ctx->partial_len], data, l); + data += l; + data_len -= l; + ctx->partial_len += l; + if (data_len == 0) { + /* + * Either the pending block hasn't been filled yet, or + * no more data was given so it's not yet known whether + * the block is the final block. + */ + return; + } + /* Pending block has been filled and isn't the final block. */ + enc_before = true; + } + + nblocks = data_len / AES_BLOCK_SIZE; + data_len %= AES_BLOCK_SIZE; + if (nblocks == 0) { + /* 0 additional full blocks, then optionally a partial block */ + if (enc_before) + aes_encrypt(&ctx->key->aes, ctx->h, ctx->h); + crypto_xor(ctx->h, data, data_len); + ctx->partial_len = data_len; + } else if (data_len != 0) { + /* 1 or more additional full blocks, then a partial block */ + aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, + enc_before, /* enc_after= */ true); + data += nblocks * AES_BLOCK_SIZE; + crypto_xor(ctx->h, data, data_len); + ctx->partial_len = data_len; + } else { + /* + * 1 or more additional full blocks only. Encryption of the + * last block is delayed until it's known whether it's the final + * block in the message or not. + */ + aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, + enc_before, /* enc_after= */ false); + ctx->partial_len = AES_BLOCK_SIZE; + } +} +EXPORT_SYMBOL_GPL(aes_cmac_update); + +void aes_cmac_final(struct aes_cmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) +{ + if (ctx->partial_len == AES_BLOCK_SIZE) { + /* Final block is a full block. Use k_final[0]. */ + crypto_xor(ctx->h, ctx->key->k_final[0].b, AES_BLOCK_SIZE); + } else { + /* Final block is a partial block. Pad, and use k_final[1]. */ + ctx->h[ctx->partial_len] ^= 0x80; + crypto_xor(ctx->h, ctx->key->k_final[1].b, AES_BLOCK_SIZE); + } + aes_encrypt(&ctx->key->aes, out, ctx->h); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(aes_cmac_final); + +void aes_cbcmac_update(struct aes_cbcmac_ctx *ctx, const u8 *data, + size_t data_len) +{ + bool enc_before = false; + size_t nblocks; + + if (ctx->partial_len) { + size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); + + crypto_xor(&ctx->h[ctx->partial_len], data, l); + data += l; + data_len -= l; + ctx->partial_len += l; + if (ctx->partial_len < AES_BLOCK_SIZE) + return; + enc_before = true; + } + + nblocks = data_len / AES_BLOCK_SIZE; + data_len %= AES_BLOCK_SIZE; + if (nblocks == 0) { + if (enc_before) + aes_encrypt(ctx->key, ctx->h, ctx->h); + } else { + aes_cbcmac_blocks(ctx->h, ctx->key, data, nblocks, enc_before, + /* enc_after= */ true); + data += nblocks * AES_BLOCK_SIZE; + } + crypto_xor(ctx->h, data, data_len); + ctx->partial_len = data_len; +} +EXPORT_SYMBOL_NS_GPL(aes_cbcmac_update, "CRYPTO_INTERNAL"); + +void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) +{ + if (ctx->partial_len) + aes_encrypt(ctx->key, out, ctx->h); + else + memcpy(out, ctx->h, AES_BLOCK_SIZE); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_NS_GPL(aes_cbcmac_final, "CRYPTO_INTERNAL"); + +/* + * FIPS cryptographic algorithm self-test for AES-CMAC. As per the FIPS 140-3 + * Implementation Guidance, a cryptographic algorithm self-test for at least one + * of AES-GCM, AES-CCM, AES-CMAC, or AES-GMAC is required if any of those modes + * is implemented. This fulfills that requirement via AES-CMAC. + * + * This is just for FIPS. The full tests are in the KUnit test suite. + */ +static void __init aes_cmac_fips_test(void) +{ + struct aes_cmac_key key; + u8 mac[AES_BLOCK_SIZE]; + + if (aes_cmac_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != + 0) + panic("aes: CMAC FIPS self-test failed (preparekey)\n"); + aes_cmac(&key, fips_test_data, sizeof(fips_test_data), mac); + if (memcmp(fips_test_aes_cmac_value, mac, sizeof(mac)) != 0) + panic("aes: CMAC FIPS self-test failed (wrong MAC)\n"); + memzero_explicit(&key, sizeof(key)); +} +#else /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ +static inline void aes_cmac_fips_test(void) +{ +} +#endif /* !CONFIG_CRYPTO_LIB_AES_CBC_MACS */ + static int __init aes_mod_init(void) { +#ifdef aes_mod_init_arch aes_mod_init_arch(); +#endif + if (fips_enabled) + aes_cmac_fips_test(); return 0; } subsys_initcall(aes_mod_init); @@ -524,7 +752,6 @@ static void __exit aes_mod_exit(void) { } module_exit(aes_mod_exit); -#endif MODULE_DESCRIPTION("AES block cipher"); MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); diff --git a/lib/crypto/aescfb.c b/lib/crypto/aescfb.c index 147e5211728f..82cd55436055 100644 --- a/lib/crypto/aescfb.c +++ b/lib/crypto/aescfb.c @@ -6,28 +6,9 @@ */ #include <crypto/aes.h> -#include <crypto/algapi.h> +#include <crypto/utils.h> #include <linux/export.h> #include <linux/module.h> -#include <asm/irqflags.h> - -static void aescfb_encrypt_block(const struct aes_enckey *key, void *dst, - const void *src) -{ - unsigned long flags; - - /* - * In AES-CFB, the AES encryption operates on known 'plaintext' (the IV - * and ciphertext), making it susceptible to timing attacks on the - * encryption key. The AES library already mitigates this risk to some - * extent by pulling the entire S-box into the caches before doing any - * substitutions, but this strategy is more effective when running with - * interrupts disabled. - */ - local_irq_save(flags); - aes_encrypt(key, dst, src); - local_irq_restore(flags); -} /** * aescfb_encrypt - Perform AES-CFB encryption on a block of data @@ -45,7 +26,7 @@ void aescfb_encrypt(const struct aes_enckey *key, u8 *dst, const u8 *src, const u8 *v = iv; while (len > 0) { - aescfb_encrypt_block(key, ks, v); + aes_encrypt(key, ks, v); crypto_xor_cpy(dst, src, ks, min(len, AES_BLOCK_SIZE)); v = dst; @@ -72,7 +53,7 @@ void aescfb_decrypt(const struct aes_enckey *key, u8 *dst, const u8 *src, { u8 ks[2][AES_BLOCK_SIZE]; - aescfb_encrypt_block(key, ks[0], iv); + aes_encrypt(key, ks[0], iv); for (int i = 0; len > 0; i ^= 1) { if (len > AES_BLOCK_SIZE) @@ -81,7 +62,7 @@ void aescfb_decrypt(const struct aes_enckey *key, u8 *dst, const u8 *src, * performing the XOR, as that may update in place and * overwrite the ciphertext. */ - aescfb_encrypt_block(key, ks[!i], src); + aes_encrypt(key, ks[!i], src); crypto_xor_cpy(dst, src, ks[i], min(len, AES_BLOCK_SIZE)); diff --git a/lib/crypto/aesgcm.c b/lib/crypto/aesgcm.c index 02f5b5f32c76..1da31e1f747d 100644 --- a/lib/crypto/aesgcm.c +++ b/lib/crypto/aesgcm.c @@ -5,30 +5,10 @@ * Copyright 2022 Google LLC */ -#include <crypto/algapi.h> #include <crypto/gcm.h> -#include <crypto/ghash.h> +#include <crypto/utils.h> #include <linux/export.h> #include <linux/module.h> -#include <asm/irqflags.h> - -static void aesgcm_encrypt_block(const struct aes_enckey *key, void *dst, - const void *src) -{ - unsigned long flags; - - /* - * In AES-GCM, both the GHASH key derivation and the CTR mode - * encryption operate on known plaintext, making them susceptible to - * timing attacks on the encryption key. The AES library already - * mitigates this risk to some extent by pulling the entire S-box into - * the caches before doing any substitutions, but this strategy is more - * effective when running with interrupts disabled. - */ - local_irq_save(flags); - aes_encrypt(key, dst, src); - local_irq_restore(flags); -} /** * aesgcm_expandkey - Expands the AES and GHASH keys for the AES-GCM key @@ -45,7 +25,7 @@ static void aesgcm_encrypt_block(const struct aes_enckey *key, void *dst, int aesgcm_expandkey(struct aesgcm_ctx *ctx, const u8 *key, unsigned int keysize, unsigned int authsize) { - u8 kin[AES_BLOCK_SIZE] = {}; + u8 h[AES_BLOCK_SIZE] = {}; int ret; ret = crypto_gcm_check_authsize(authsize) ?: @@ -54,24 +34,13 @@ int aesgcm_expandkey(struct aesgcm_ctx *ctx, const u8 *key, return ret; ctx->authsize = authsize; - aesgcm_encrypt_block(&ctx->aes_key, &ctx->ghash_key, kin); - + aes_encrypt(&ctx->aes_key, h, h); + ghash_preparekey(&ctx->ghash_key, h); + memzero_explicit(h, sizeof(h)); return 0; } EXPORT_SYMBOL(aesgcm_expandkey); -static void aesgcm_ghash(be128 *ghash, const be128 *key, const void *src, - int len) -{ - while (len > 0) { - crypto_xor((u8 *)ghash, src, min(len, GHASH_BLOCK_SIZE)); - gf128mul_lle(ghash, key); - - src += GHASH_BLOCK_SIZE; - len -= GHASH_BLOCK_SIZE; - } -} - /** * aesgcm_mac - Generates the authentication tag using AES-GCM algorithm. * @ctx: The data structure that will hold the AES-GCM key schedule @@ -88,20 +57,33 @@ static void aesgcm_ghash(be128 *ghash, const be128 *key, const void *src, static void aesgcm_mac(const struct aesgcm_ctx *ctx, const u8 *src, int src_len, const u8 *assoc, int assoc_len, __be32 *ctr, u8 *authtag) { - be128 tail = { cpu_to_be64(assoc_len * 8), cpu_to_be64(src_len * 8) }; - u8 buf[AES_BLOCK_SIZE]; - be128 ghash = {}; + static const u8 zeroes[GHASH_BLOCK_SIZE]; + __be64 tail[2] = { + cpu_to_be64((u64)assoc_len * 8), + cpu_to_be64((u64)src_len * 8), + }; + struct ghash_ctx ghash; + u8 ghash_out[AES_BLOCK_SIZE]; + u8 enc_ctr[AES_BLOCK_SIZE]; + + ghash_init(&ghash, &ctx->ghash_key); - aesgcm_ghash(&ghash, &ctx->ghash_key, assoc, assoc_len); - aesgcm_ghash(&ghash, &ctx->ghash_key, src, src_len); - aesgcm_ghash(&ghash, &ctx->ghash_key, &tail, sizeof(tail)); + ghash_update(&ghash, assoc, assoc_len); + ghash_update(&ghash, zeroes, -assoc_len & (GHASH_BLOCK_SIZE - 1)); + + ghash_update(&ghash, src, src_len); + ghash_update(&ghash, zeroes, -src_len & (GHASH_BLOCK_SIZE - 1)); + + ghash_update(&ghash, (const u8 *)&tail, sizeof(tail)); + + ghash_final(&ghash, ghash_out); ctr[3] = cpu_to_be32(1); - aesgcm_encrypt_block(&ctx->aes_key, buf, ctr); - crypto_xor_cpy(authtag, buf, (u8 *)&ghash, ctx->authsize); + aes_encrypt(&ctx->aes_key, enc_ctr, (const u8 *)ctr); + crypto_xor_cpy(authtag, ghash_out, enc_ctr, ctx->authsize); - memzero_explicit(&ghash, sizeof(ghash)); - memzero_explicit(buf, sizeof(buf)); + memzero_explicit(ghash_out, sizeof(ghash_out)); + memzero_explicit(enc_ctr, sizeof(enc_ctr)); } static void aesgcm_crypt(const struct aesgcm_ctx *ctx, u8 *dst, const u8 *src, @@ -119,7 +101,7 @@ static void aesgcm_crypt(const struct aesgcm_ctx *ctx, u8 *dst, const u8 *src, * len', this cannot happen, so no explicit test is necessary. */ ctr[3] = cpu_to_be32(n++); - aesgcm_encrypt_block(&ctx->aes_key, buf, ctr); + aes_encrypt(&ctx->aes_key, buf, (const u8 *)ctr); crypto_xor_cpy(dst, src, buf, min(len, AES_BLOCK_SIZE)); dst += AES_BLOCK_SIZE; diff --git a/lib/crypto/arm/gf128hash.h b/lib/crypto/arm/gf128hash.h new file mode 100644 index 000000000000..c33c8cbe51fe --- /dev/null +++ b/lib/crypto/arm/gf128hash.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * GHASH, arm optimized + * + * Copyright 2026 Google LLC + */ + +#include <asm/hwcap.h> +#include <asm/neon.h> +#include <asm/simd.h> + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); + +void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg, + const u8 *src, const struct polyval_elem *h); + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_neon) && may_use_simd()) { + do { + /* Allow rescheduling every 4 KiB. */ + size_t n = + min_t(size_t, nblocks, 4096 / GHASH_BLOCK_SIZE); + + scoped_ksimd() + pmull_ghash_update_p8(n, acc, data, &key->h); + data += n * GHASH_BLOCK_SIZE; + nblocks -= n; + } while (nblocks); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (elf_hwcap & HWCAP_NEON) + static_branch_enable(&have_neon); +} diff --git a/lib/crypto/arm/ghash-neon-core.S b/lib/crypto/arm/ghash-neon-core.S new file mode 100644 index 000000000000..eeffd12504a9 --- /dev/null +++ b/lib/crypto/arm/ghash-neon-core.S @@ -0,0 +1,209 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Accelerated GHASH implementation with NEON vmull.p8 instructions. + * + * Copyright (C) 2015 - 2017 Linaro Ltd. + * Copyright (C) 2023 Google LLC. <ardb@google.com> + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> + + .fpu neon + + SHASH .req q0 + T1 .req q1 + XL .req q2 + XM .req q3 + XH .req q4 + IN1 .req q4 + + SHASH_L .req d0 + SHASH_H .req d1 + T1_L .req d2 + T1_H .req d3 + XL_L .req d4 + XL_H .req d5 + XM_L .req d6 + XM_H .req d7 + XH_L .req d8 + + t0l .req d10 + t0h .req d11 + t1l .req d12 + t1h .req d13 + t2l .req d14 + t2h .req d15 + t3l .req d16 + t3h .req d17 + t4l .req d18 + t4h .req d19 + + t0q .req q5 + t1q .req q6 + t2q .req q7 + t3q .req q8 + t4q .req q9 + + s1l .req d20 + s1h .req d21 + s2l .req d22 + s2h .req d23 + s3l .req d24 + s3h .req d25 + s4l .req d26 + s4h .req d27 + + SHASH2_p8 .req d28 + + k16 .req d29 + k32 .req d30 + k48 .req d31 + + T2 .req q7 + + .text + + /* + * This implementation of 64x64 -> 128 bit polynomial multiplication + * using vmull.p8 instructions (8x8 -> 16) is taken from the paper + * "Fast Software Polynomial Multiplication on ARM Processors Using + * the NEON Engine" by Danilo Camara, Conrado Gouvea, Julio Lopez and + * Ricardo Dahab (https://hal.inria.fr/hal-01506572) + * + * It has been slightly tweaked for in-order performance, and to allow + * 'rq' to overlap with 'ad' or 'bd'. + */ + .macro __pmull_p8, rq, ad, bd, b1=t4l, b2=t3l, b3=t4l, b4=t3l + vext.8 t0l, \ad, \ad, #1 @ A1 + .ifc \b1, t4l + vext.8 t4l, \bd, \bd, #1 @ B1 + .endif + vmull.p8 t0q, t0l, \bd @ F = A1*B + vext.8 t1l, \ad, \ad, #2 @ A2 + vmull.p8 t4q, \ad, \b1 @ E = A*B1 + .ifc \b2, t3l + vext.8 t3l, \bd, \bd, #2 @ B2 + .endif + vmull.p8 t1q, t1l, \bd @ H = A2*B + vext.8 t2l, \ad, \ad, #3 @ A3 + vmull.p8 t3q, \ad, \b2 @ G = A*B2 + veor t0q, t0q, t4q @ L = E + F + .ifc \b3, t4l + vext.8 t4l, \bd, \bd, #3 @ B3 + .endif + vmull.p8 t2q, t2l, \bd @ J = A3*B + veor t0l, t0l, t0h @ t0 = (L) (P0 + P1) << 8 + veor t1q, t1q, t3q @ M = G + H + .ifc \b4, t3l + vext.8 t3l, \bd, \bd, #4 @ B4 + .endif + vmull.p8 t4q, \ad, \b3 @ I = A*B3 + veor t1l, t1l, t1h @ t1 = (M) (P2 + P3) << 16 + vmull.p8 t3q, \ad, \b4 @ K = A*B4 + vand t0h, t0h, k48 + vand t1h, t1h, k32 + veor t2q, t2q, t4q @ N = I + J + veor t0l, t0l, t0h + veor t1l, t1l, t1h + veor t2l, t2l, t2h @ t2 = (N) (P4 + P5) << 24 + vand t2h, t2h, k16 + veor t3l, t3l, t3h @ t3 = (K) (P6 + P7) << 32 + vmov.i64 t3h, #0 + vext.8 t0q, t0q, t0q, #15 + veor t2l, t2l, t2h + vext.8 t1q, t1q, t1q, #14 + vmull.p8 \rq, \ad, \bd @ D = A*B + vext.8 t2q, t2q, t2q, #13 + vext.8 t3q, t3q, t3q, #12 + veor t0q, t0q, t1q + veor t2q, t2q, t3q + veor \rq, \rq, t0q + veor \rq, \rq, t2q + .endm + + .macro __pmull_reduce_p8 + veor XL_H, XL_H, XM_L + veor XH_L, XH_L, XM_H + + vshl.i64 T1, XL, #57 + vshl.i64 T2, XL, #62 + veor T1, T1, T2 + vshl.i64 T2, XL, #63 + veor T1, T1, T2 + veor XL_H, XL_H, T1_L + veor XH_L, XH_L, T1_H + + vshr.u64 T1, XL, #1 + veor XH, XH, XL + veor XL, XL, T1 + vshr.u64 T1, T1, #6 + vshr.u64 XL, XL, #1 + .endm + + .macro vrev64_if_be a +#ifdef CONFIG_CPU_BIG_ENDIAN + vrev64.8 \a, \a +#endif + .endm + + .macro ghash_update + vld1.64 {XL}, [r1] + vrev64_if_be XL + +0: + vld1.8 {T1}, [r2]! + subs r0, r0, #1 + + /* multiply XL by SHASH in GF(2^128) */ + vrev64.8 T1, T1 + + vext.8 IN1, T1, T1, #8 + veor T1_L, T1_L, XL_H + veor XL, XL, IN1 + + __pmull_p8 XH, XL_H, SHASH_H, s1h, s2h, s3h, s4h @ a1 * b1 + veor T1, T1, XL + __pmull_p8 XL, XL_L, SHASH_L, s1l, s2l, s3l, s4l @ a0 * b0 + __pmull_p8 XM, T1_L, SHASH2_p8 @ (a1+a0)(b1+b0) + + veor T1, XL, XH + veor XM, XM, T1 + + __pmull_reduce_p8 + + veor T1, T1, XH + veor XL, XL, T1 + + bne 0b + .endm + + /* + * void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg, + * const u8 *src, + * const struct polyval_elem *h) + */ +ENTRY(pmull_ghash_update_p8) + vld1.64 {SHASH}, [r3] + vrev64_if_be SHASH + veor SHASH2_p8, SHASH_L, SHASH_H + + vext.8 s1l, SHASH_L, SHASH_L, #1 + vext.8 s2l, SHASH_L, SHASH_L, #2 + vext.8 s3l, SHASH_L, SHASH_L, #3 + vext.8 s4l, SHASH_L, SHASH_L, #4 + vext.8 s1h, SHASH_H, SHASH_H, #1 + vext.8 s2h, SHASH_H, SHASH_H, #2 + vext.8 s3h, SHASH_H, SHASH_H, #3 + vext.8 s4h, SHASH_H, SHASH_H, #4 + + vmov.i64 k16, #0xffff + vmov.i64 k32, #0xffffffff + vmov.i64 k48, #0xffffffffffff + + ghash_update + vrev64_if_be XL + vst1.64 {XL}, [r1] + + bx lr +ENDPROC(pmull_ghash_update_p8) diff --git a/lib/crypto/arm64/aes-ce.S b/lib/crypto/arm64/aes-ce.S new file mode 100644 index 000000000000..b853e02f7b1e --- /dev/null +++ b/lib/crypto/arm64/aes-ce.S @@ -0,0 +1,145 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * AES cipher for ARMv8 with Crypto Extensions + * + * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> + +#define AES_FUNC_START(func) SYM_FUNC_START(ce_ ## func) +#define AES_FUNC_END(func) SYM_FUNC_END(ce_ ## func) + + .arch armv8-a+crypto + + xtsmask .req v16 + cbciv .req v16 + vctr .req v16 + + .macro xts_reload_mask, tmp + .endm + + .macro xts_cts_skip_tw, reg, lbl + .endm + + /* preload all round keys */ + .macro load_round_keys, rk, nr, tmp + add \tmp, \rk, \nr, sxtw #4 + sub \tmp, \tmp, #160 + ld1 {v17.4s-v20.4s}, [\rk] + ld1 {v21.4s-v24.4s}, [\tmp], #64 + ld1 {v25.4s-v28.4s}, [\tmp], #64 + ld1 {v29.4s-v31.4s}, [\tmp] + .endm + + /* prepare for encryption with key in rk[] */ + .macro enc_prepare, rounds, rk, temp + load_round_keys \rk, \rounds, \temp + .endm + + /* prepare for encryption (again) but with new key in rk[] */ + .macro enc_switch_key, rounds, rk, temp + load_round_keys \rk, \rounds, \temp + .endm + + /* prepare for decryption with key in rk[] */ + .macro dec_prepare, rounds, rk, temp + load_round_keys \rk, \rounds, \temp + .endm + + .macro do_enc_Nx, de, mc, k, i0, i1, i2, i3, i4 + aes\de \i0\().16b, \k\().16b + aes\mc \i0\().16b, \i0\().16b + .ifnb \i1 + aes\de \i1\().16b, \k\().16b + aes\mc \i1\().16b, \i1\().16b + .ifnb \i3 + aes\de \i2\().16b, \k\().16b + aes\mc \i2\().16b, \i2\().16b + aes\de \i3\().16b, \k\().16b + aes\mc \i3\().16b, \i3\().16b + .ifnb \i4 + aes\de \i4\().16b, \k\().16b + aes\mc \i4\().16b, \i4\().16b + .endif + .endif + .endif + .endm + + /* up to 5 interleaved encryption rounds with the same round key */ + .macro round_Nx, enc, k, i0, i1, i2, i3, i4 + .ifc \enc, e + do_enc_Nx e, mc, \k, \i0, \i1, \i2, \i3, \i4 + .else + do_enc_Nx d, imc, \k, \i0, \i1, \i2, \i3, \i4 + .endif + .endm + + /* up to 5 interleaved final rounds */ + .macro fin_round_Nx, de, k, k2, i0, i1, i2, i3, i4 + aes\de \i0\().16b, \k\().16b + .ifnb \i1 + aes\de \i1\().16b, \k\().16b + .ifnb \i3 + aes\de \i2\().16b, \k\().16b + aes\de \i3\().16b, \k\().16b + .ifnb \i4 + aes\de \i4\().16b, \k\().16b + .endif + .endif + .endif + eor \i0\().16b, \i0\().16b, \k2\().16b + .ifnb \i1 + eor \i1\().16b, \i1\().16b, \k2\().16b + .ifnb \i3 + eor \i2\().16b, \i2\().16b, \k2\().16b + eor \i3\().16b, \i3\().16b, \k2\().16b + .ifnb \i4 + eor \i4\().16b, \i4\().16b, \k2\().16b + .endif + .endif + .endif + .endm + + /* up to 5 interleaved blocks */ + .macro do_block_Nx, enc, rounds, i0, i1, i2, i3, i4 + tbz \rounds, #2, .L\@ /* 128 bits */ + round_Nx \enc, v17, \i0, \i1, \i2, \i3, \i4 + round_Nx \enc, v18, \i0, \i1, \i2, \i3, \i4 + tbz \rounds, #1, .L\@ /* 192 bits */ + round_Nx \enc, v19, \i0, \i1, \i2, \i3, \i4 + round_Nx \enc, v20, \i0, \i1, \i2, \i3, \i4 +.L\@: .irp key, v21, v22, v23, v24, v25, v26, v27, v28, v29 + round_Nx \enc, \key, \i0, \i1, \i2, \i3, \i4 + .endr + fin_round_Nx \enc, v30, v31, \i0, \i1, \i2, \i3, \i4 + .endm + + .macro encrypt_block, in, rounds, t0, t1, t2 + do_block_Nx e, \rounds, \in + .endm + + .macro encrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2 + do_block_Nx e, \rounds, \i0, \i1, \i2, \i3 + .endm + + .macro encrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2 + do_block_Nx e, \rounds, \i0, \i1, \i2, \i3, \i4 + .endm + + .macro decrypt_block, in, rounds, t0, t1, t2 + do_block_Nx d, \rounds, \in + .endm + + .macro decrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2 + do_block_Nx d, \rounds, \i0, \i1, \i2, \i3 + .endm + + .macro decrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2 + do_block_Nx d, \rounds, \i0, \i1, \i2, \i3, \i4 + .endm + +#define MAX_STRIDE 5 + +#include "aes-modes.S" diff --git a/lib/crypto/arm64/aes-cipher-core.S b/lib/crypto/arm64/aes-cipher-core.S index 651f701c56a8..0b05ec4be65f 100644 --- a/lib/crypto/arm64/aes-cipher-core.S +++ b/lib/crypto/arm64/aes-cipher-core.S @@ -87,11 +87,6 @@ ldp w8, w9, [rk], #16 ldp w10, w11, [rk, #-8] -CPU_BE( rev w4, w4 ) -CPU_BE( rev w5, w5 ) -CPU_BE( rev w6, w6 ) -CPU_BE( rev w7, w7 ) - eor w4, w4, w8 eor w5, w5, w9 eor w6, w6, w10 @@ -112,11 +107,6 @@ CPU_BE( rev w7, w7 ) 3: adr_l tt, \ltab \round w4, w5, w6, w7, w8, w9, w10, w11, \bsz, b -CPU_BE( rev w4, w4 ) -CPU_BE( rev w5, w5 ) -CPU_BE( rev w6, w6 ) -CPU_BE( rev w7, w7 ) - stp w4, w5, [out] stp w6, w7, [out, #8] ret diff --git a/lib/crypto/arm64/aes-modes.S b/lib/crypto/arm64/aes-modes.S new file mode 100644 index 000000000000..10e537317eaf --- /dev/null +++ b/lib/crypto/arm64/aes-modes.S @@ -0,0 +1,867 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Chaining mode wrappers for AES + * + * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> + */ + +/* included by aes-ce.S and aes-neon.S */ + + .text + .align 4 + +#ifndef MAX_STRIDE +#define MAX_STRIDE 4 +#endif + +#if MAX_STRIDE == 4 +#define ST4(x...) x +#define ST5(x...) +#else +#define ST4(x...) +#define ST5(x...) x +#endif + +SYM_FUNC_START_LOCAL(aes_encrypt_block4x) + encrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7 + ret +SYM_FUNC_END(aes_encrypt_block4x) + +SYM_FUNC_START_LOCAL(aes_decrypt_block4x) + decrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7 + ret +SYM_FUNC_END(aes_decrypt_block4x) + +#if MAX_STRIDE == 5 +SYM_FUNC_START_LOCAL(aes_encrypt_block5x) + encrypt_block5x v0, v1, v2, v3, v4, w3, x2, x8, w7 + ret +SYM_FUNC_END(aes_encrypt_block5x) + +SYM_FUNC_START_LOCAL(aes_decrypt_block5x) + decrypt_block5x v0, v1, v2, v3, v4, w3, x2, x8, w7 + ret +SYM_FUNC_END(aes_decrypt_block5x) +#endif + + /* + * aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, + * int blocks) + * aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, + * int blocks) + */ + +AES_FUNC_START(aes_ecb_encrypt) + frame_push 0 + + enc_prepare w3, x2, x5 + +.LecbencloopNx: + subs w4, w4, #MAX_STRIDE + bmi .Lecbenc1x + ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */ +ST4( bl aes_encrypt_block4x ) +ST5( ld1 {v4.16b}, [x1], #16 ) +ST5( bl aes_encrypt_block5x ) + st1 {v0.16b-v3.16b}, [x0], #64 +ST5( st1 {v4.16b}, [x0], #16 ) + b .LecbencloopNx +.Lecbenc1x: + adds w4, w4, #MAX_STRIDE + beq .Lecbencout +.Lecbencloop: + ld1 {v0.16b}, [x1], #16 /* get next pt block */ + encrypt_block v0, w3, x2, x5, w6 + st1 {v0.16b}, [x0], #16 + subs w4, w4, #1 + bne .Lecbencloop +.Lecbencout: + frame_pop + ret +AES_FUNC_END(aes_ecb_encrypt) + + +AES_FUNC_START(aes_ecb_decrypt) + frame_push 0 + + dec_prepare w3, x2, x5 + +.LecbdecloopNx: + subs w4, w4, #MAX_STRIDE + bmi .Lecbdec1x + ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */ +ST4( bl aes_decrypt_block4x ) +ST5( ld1 {v4.16b}, [x1], #16 ) +ST5( bl aes_decrypt_block5x ) + st1 {v0.16b-v3.16b}, [x0], #64 +ST5( st1 {v4.16b}, [x0], #16 ) + b .LecbdecloopNx +.Lecbdec1x: + adds w4, w4, #MAX_STRIDE + beq .Lecbdecout +.Lecbdecloop: + ld1 {v0.16b}, [x1], #16 /* get next ct block */ + decrypt_block v0, w3, x2, x5, w6 + st1 {v0.16b}, [x0], #16 + subs w4, w4, #1 + bne .Lecbdecloop +.Lecbdecout: + frame_pop + ret +AES_FUNC_END(aes_ecb_decrypt) + + + /* + * aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, + * int blocks, u8 iv[]) + * aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, + * int blocks, u8 iv[]) + * aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[], + * int rounds, int blocks, u8 iv[], + * u32 const rk2[]); + * aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[], + * int rounds, int blocks, u8 iv[], + * u32 const rk2[]); + */ + +AES_FUNC_START(aes_essiv_cbc_encrypt) + ld1 {v4.16b}, [x5] /* get iv */ + + mov w8, #14 /* AES-256: 14 rounds */ + enc_prepare w8, x6, x7 + encrypt_block v4, w8, x6, x7, w9 + enc_switch_key w3, x2, x6 + b .Lcbcencloop4x + +AES_FUNC_START(aes_cbc_encrypt) + ld1 {v4.16b}, [x5] /* get iv */ + enc_prepare w3, x2, x6 + +.Lcbcencloop4x: + subs w4, w4, #4 + bmi .Lcbcenc1x + ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */ + eor v0.16b, v0.16b, v4.16b /* ..and xor with iv */ + encrypt_block v0, w3, x2, x6, w7 + eor v1.16b, v1.16b, v0.16b + encrypt_block v1, w3, x2, x6, w7 + eor v2.16b, v2.16b, v1.16b + encrypt_block v2, w3, x2, x6, w7 + eor v3.16b, v3.16b, v2.16b + encrypt_block v3, w3, x2, x6, w7 + st1 {v0.16b-v3.16b}, [x0], #64 + mov v4.16b, v3.16b + b .Lcbcencloop4x +.Lcbcenc1x: + adds w4, w4, #4 + beq .Lcbcencout +.Lcbcencloop: + ld1 {v0.16b}, [x1], #16 /* get next pt block */ + eor v4.16b, v4.16b, v0.16b /* ..and xor with iv */ + encrypt_block v4, w3, x2, x6, w7 + st1 {v4.16b}, [x0], #16 + subs w4, w4, #1 + bne .Lcbcencloop +.Lcbcencout: + st1 {v4.16b}, [x5] /* return iv */ + ret +AES_FUNC_END(aes_cbc_encrypt) +AES_FUNC_END(aes_essiv_cbc_encrypt) + +AES_FUNC_START(aes_essiv_cbc_decrypt) + ld1 {cbciv.16b}, [x5] /* get iv */ + + mov w8, #14 /* AES-256: 14 rounds */ + enc_prepare w8, x6, x7 + encrypt_block cbciv, w8, x6, x7, w9 + b .Lessivcbcdecstart + +AES_FUNC_START(aes_cbc_decrypt) + ld1 {cbciv.16b}, [x5] /* get iv */ +.Lessivcbcdecstart: + frame_push 0 + dec_prepare w3, x2, x6 + +.LcbcdecloopNx: + subs w4, w4, #MAX_STRIDE + bmi .Lcbcdec1x + ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */ +#if MAX_STRIDE == 5 + ld1 {v4.16b}, [x1], #16 /* get 1 ct block */ + mov v5.16b, v0.16b + mov v6.16b, v1.16b + mov v7.16b, v2.16b + bl aes_decrypt_block5x + sub x1, x1, #32 + eor v0.16b, v0.16b, cbciv.16b + eor v1.16b, v1.16b, v5.16b + ld1 {v5.16b}, [x1], #16 /* reload 1 ct block */ + ld1 {cbciv.16b}, [x1], #16 /* reload 1 ct block */ + eor v2.16b, v2.16b, v6.16b + eor v3.16b, v3.16b, v7.16b + eor v4.16b, v4.16b, v5.16b +#else + mov v4.16b, v0.16b + mov v5.16b, v1.16b + mov v6.16b, v2.16b + bl aes_decrypt_block4x + sub x1, x1, #16 + eor v0.16b, v0.16b, cbciv.16b + eor v1.16b, v1.16b, v4.16b + ld1 {cbciv.16b}, [x1], #16 /* reload 1 ct block */ + eor v2.16b, v2.16b, v5.16b + eor v3.16b, v3.16b, v6.16b +#endif + st1 {v0.16b-v3.16b}, [x0], #64 +ST5( st1 {v4.16b}, [x0], #16 ) + b .LcbcdecloopNx +.Lcbcdec1x: + adds w4, w4, #MAX_STRIDE + beq .Lcbcdecout +.Lcbcdecloop: + ld1 {v1.16b}, [x1], #16 /* get next ct block */ + mov v0.16b, v1.16b /* ...and copy to v0 */ + decrypt_block v0, w3, x2, x6, w7 + eor v0.16b, v0.16b, cbciv.16b /* xor with iv => pt */ + mov cbciv.16b, v1.16b /* ct is next iv */ + st1 {v0.16b}, [x0], #16 + subs w4, w4, #1 + bne .Lcbcdecloop +.Lcbcdecout: + st1 {cbciv.16b}, [x5] /* return iv */ + frame_pop + ret +AES_FUNC_END(aes_cbc_decrypt) +AES_FUNC_END(aes_essiv_cbc_decrypt) + + + /* + * aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[], + * int rounds, int bytes, u8 const iv[]) + * aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[], + * int rounds, int bytes, u8 const iv[]) + */ + +AES_FUNC_START(aes_cbc_cts_encrypt) + adr_l x8, .Lcts_permute_table + sub x4, x4, #16 + add x9, x8, #32 + add x8, x8, x4 + sub x9, x9, x4 + ld1 {v3.16b}, [x8] + ld1 {v4.16b}, [x9] + + ld1 {v0.16b}, [x1], x4 /* overlapping loads */ + ld1 {v1.16b}, [x1] + + ld1 {v5.16b}, [x5] /* get iv */ + enc_prepare w3, x2, x6 + + eor v0.16b, v0.16b, v5.16b /* xor with iv */ + tbl v1.16b, {v1.16b}, v4.16b + encrypt_block v0, w3, x2, x6, w7 + + eor v1.16b, v1.16b, v0.16b + tbl v0.16b, {v0.16b}, v3.16b + encrypt_block v1, w3, x2, x6, w7 + + add x4, x0, x4 + st1 {v0.16b}, [x4] /* overlapping stores */ + st1 {v1.16b}, [x0] + ret +AES_FUNC_END(aes_cbc_cts_encrypt) + +AES_FUNC_START(aes_cbc_cts_decrypt) + adr_l x8, .Lcts_permute_table + sub x4, x4, #16 + add x9, x8, #32 + add x8, x8, x4 + sub x9, x9, x4 + ld1 {v3.16b}, [x8] + ld1 {v4.16b}, [x9] + + ld1 {v0.16b}, [x1], x4 /* overlapping loads */ + ld1 {v1.16b}, [x1] + + ld1 {v5.16b}, [x5] /* get iv */ + dec_prepare w3, x2, x6 + + decrypt_block v0, w3, x2, x6, w7 + tbl v2.16b, {v0.16b}, v3.16b + eor v2.16b, v2.16b, v1.16b + + tbx v0.16b, {v1.16b}, v4.16b + decrypt_block v0, w3, x2, x6, w7 + eor v0.16b, v0.16b, v5.16b /* xor with iv */ + + add x4, x0, x4 + st1 {v2.16b}, [x4] /* overlapping stores */ + st1 {v0.16b}, [x0] + ret +AES_FUNC_END(aes_cbc_cts_decrypt) + + .section ".rodata", "a" + .align 6 +.Lcts_permute_table: + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7 + .byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .previous + + /* + * This macro generates the code for CTR and XCTR mode. + */ +.macro ctr_encrypt xctr + // Arguments + OUT .req x0 + IN .req x1 + KEY .req x2 + ROUNDS_W .req w3 + BYTES_W .req w4 + IV .req x5 + BYTE_CTR_W .req w6 // XCTR only + // Intermediate values + CTR_W .req w11 // XCTR only + CTR .req x11 // XCTR only + IV_PART .req x12 + BLOCKS .req x13 + BLOCKS_W .req w13 + + frame_push 0 + + enc_prepare ROUNDS_W, KEY, IV_PART + ld1 {vctr.16b}, [IV] + + /* + * Keep 64 bits of the IV in a register. For CTR mode this lets us + * easily increment the IV. For XCTR mode this lets us efficiently XOR + * the 64-bit counter with the IV. + */ + .if \xctr + umov IV_PART, vctr.d[0] + lsr CTR_W, BYTE_CTR_W, #4 + .else + umov IV_PART, vctr.d[1] + rev IV_PART, IV_PART + .endif + +.LctrloopNx\xctr: + add BLOCKS_W, BYTES_W, #15 + sub BYTES_W, BYTES_W, #MAX_STRIDE << 4 + lsr BLOCKS_W, BLOCKS_W, #4 + mov w8, #MAX_STRIDE + cmp BLOCKS_W, w8 + csel BLOCKS_W, BLOCKS_W, w8, lt + + /* + * Set up the counter values in v0-v{MAX_STRIDE-1}. + * + * If we are encrypting less than MAX_STRIDE blocks, the tail block + * handling code expects the last keystream block to be in + * v{MAX_STRIDE-1}. For example: if encrypting two blocks with + * MAX_STRIDE=5, then v3 and v4 should have the next two counter blocks. + */ + .if \xctr + add CTR, CTR, BLOCKS + .else + adds IV_PART, IV_PART, BLOCKS + .endif + mov v0.16b, vctr.16b + mov v1.16b, vctr.16b + mov v2.16b, vctr.16b + mov v3.16b, vctr.16b +ST5( mov v4.16b, vctr.16b ) + .if \xctr + sub x6, CTR, #MAX_STRIDE - 1 + sub x7, CTR, #MAX_STRIDE - 2 + sub x8, CTR, #MAX_STRIDE - 3 + sub x9, CTR, #MAX_STRIDE - 4 +ST5( sub x10, CTR, #MAX_STRIDE - 5 ) + eor x6, x6, IV_PART + eor x7, x7, IV_PART + eor x8, x8, IV_PART + eor x9, x9, IV_PART +ST5( eor x10, x10, IV_PART ) + mov v0.d[0], x6 + mov v1.d[0], x7 + mov v2.d[0], x8 + mov v3.d[0], x9 +ST5( mov v4.d[0], x10 ) + .else + bcs 0f + .subsection 1 + /* + * This subsection handles carries. + * + * Conditional branching here is allowed with respect to time + * invariance since the branches are dependent on the IV instead + * of the plaintext or key. This code is rarely executed in + * practice anyway. + */ + + /* Apply carry to outgoing counter. */ +0: umov x8, vctr.d[0] + rev x8, x8 + add x8, x8, #1 + rev x8, x8 + ins vctr.d[0], x8 + + /* + * Apply carry to counter blocks if needed. + * + * Since the carry flag was set, we know 0 <= IV_PART < + * MAX_STRIDE. Using the value of IV_PART we can determine how + * many counter blocks need to be updated. + */ + cbz IV_PART, 2f + adr x16, 1f + sub x16, x16, IV_PART, lsl #3 + br x16 + bti c + mov v0.d[0], vctr.d[0] + bti c + mov v1.d[0], vctr.d[0] + bti c + mov v2.d[0], vctr.d[0] + bti c + mov v3.d[0], vctr.d[0] +ST5( bti c ) +ST5( mov v4.d[0], vctr.d[0] ) +1: b 2f + .previous + +2: rev x7, IV_PART + ins vctr.d[1], x7 + sub x7, IV_PART, #MAX_STRIDE - 1 + sub x8, IV_PART, #MAX_STRIDE - 2 + sub x9, IV_PART, #MAX_STRIDE - 3 + rev x7, x7 + rev x8, x8 + mov v1.d[1], x7 + rev x9, x9 +ST5( sub x10, IV_PART, #MAX_STRIDE - 4 ) + mov v2.d[1], x8 +ST5( rev x10, x10 ) + mov v3.d[1], x9 +ST5( mov v4.d[1], x10 ) + .endif + + /* + * If there are at least MAX_STRIDE blocks left, XOR the data with + * keystream and store. Otherwise jump to tail handling. + */ + tbnz BYTES_W, #31, .Lctrtail\xctr + ld1 {v5.16b-v7.16b}, [IN], #48 +ST4( bl aes_encrypt_block4x ) +ST5( bl aes_encrypt_block5x ) + eor v0.16b, v5.16b, v0.16b +ST4( ld1 {v5.16b}, [IN], #16 ) + eor v1.16b, v6.16b, v1.16b +ST5( ld1 {v5.16b-v6.16b}, [IN], #32 ) + eor v2.16b, v7.16b, v2.16b + eor v3.16b, v5.16b, v3.16b +ST5( eor v4.16b, v6.16b, v4.16b ) + st1 {v0.16b-v3.16b}, [OUT], #64 +ST5( st1 {v4.16b}, [OUT], #16 ) + cbz BYTES_W, .Lctrout\xctr + b .LctrloopNx\xctr + +.Lctrout\xctr: + .if !\xctr + st1 {vctr.16b}, [IV] /* return next CTR value */ + .endif + frame_pop + ret + +.Lctrtail\xctr: + /* + * Handle up to MAX_STRIDE * 16 - 1 bytes of plaintext + * + * This code expects the last keystream block to be in v{MAX_STRIDE-1}. + * For example: if encrypting two blocks with MAX_STRIDE=5, then v3 and + * v4 should have the next two counter blocks. + * + * This allows us to store the ciphertext by writing to overlapping + * regions of memory. Any invalid ciphertext blocks get overwritten by + * correctly computed blocks. This approach greatly simplifies the + * logic for storing the ciphertext. + */ + mov x16, #16 + ands w7, BYTES_W, #0xf + csel x13, x7, x16, ne + +ST5( cmp BYTES_W, #64 - (MAX_STRIDE << 4)) +ST5( csel x14, x16, xzr, gt ) + cmp BYTES_W, #48 - (MAX_STRIDE << 4) + csel x15, x16, xzr, gt + cmp BYTES_W, #32 - (MAX_STRIDE << 4) + csel x16, x16, xzr, gt + cmp BYTES_W, #16 - (MAX_STRIDE << 4) + + adr_l x9, .Lcts_permute_table + add x9, x9, x13 + ble .Lctrtail1x\xctr + +ST5( ld1 {v5.16b}, [IN], x14 ) + ld1 {v6.16b}, [IN], x15 + ld1 {v7.16b}, [IN], x16 + +ST4( bl aes_encrypt_block4x ) +ST5( bl aes_encrypt_block5x ) + + ld1 {v8.16b}, [IN], x13 + ld1 {v9.16b}, [IN] + ld1 {v10.16b}, [x9] + +ST4( eor v6.16b, v6.16b, v0.16b ) +ST4( eor v7.16b, v7.16b, v1.16b ) +ST4( tbl v3.16b, {v3.16b}, v10.16b ) +ST4( eor v8.16b, v8.16b, v2.16b ) +ST4( eor v9.16b, v9.16b, v3.16b ) + +ST5( eor v5.16b, v5.16b, v0.16b ) +ST5( eor v6.16b, v6.16b, v1.16b ) +ST5( tbl v4.16b, {v4.16b}, v10.16b ) +ST5( eor v7.16b, v7.16b, v2.16b ) +ST5( eor v8.16b, v8.16b, v3.16b ) +ST5( eor v9.16b, v9.16b, v4.16b ) + +ST5( st1 {v5.16b}, [OUT], x14 ) + st1 {v6.16b}, [OUT], x15 + st1 {v7.16b}, [OUT], x16 + add x13, x13, OUT + st1 {v9.16b}, [x13] // overlapping stores + st1 {v8.16b}, [OUT] + b .Lctrout\xctr + +.Lctrtail1x\xctr: + /* + * Handle <= 16 bytes of plaintext + * + * This code always reads and writes 16 bytes. To avoid out of bounds + * accesses, XCTR and CTR modes must use a temporary buffer when + * encrypting/decrypting less than 16 bytes. + * + * This code is unusual in that it loads the input and stores the output + * relative to the end of the buffers rather than relative to the start. + * This causes unusual behaviour when encrypting/decrypting less than 16 + * bytes; the end of the data is expected to be at the end of the + * temporary buffer rather than the start of the data being at the start + * of the temporary buffer. + */ + sub x8, x7, #16 + csel x7, x7, x8, eq + add IN, IN, x7 + add OUT, OUT, x7 + ld1 {v5.16b}, [IN] + ld1 {v6.16b}, [OUT] +ST5( mov v3.16b, v4.16b ) + encrypt_block v3, ROUNDS_W, KEY, x8, w7 + ld1 {v10.16b-v11.16b}, [x9] + tbl v3.16b, {v3.16b}, v10.16b + sshr v11.16b, v11.16b, #7 + eor v5.16b, v5.16b, v3.16b + bif v5.16b, v6.16b, v11.16b + st1 {v5.16b}, [OUT] + b .Lctrout\xctr + + // Arguments + .unreq OUT + .unreq IN + .unreq KEY + .unreq ROUNDS_W + .unreq BYTES_W + .unreq IV + .unreq BYTE_CTR_W // XCTR only + // Intermediate values + .unreq CTR_W // XCTR only + .unreq CTR // XCTR only + .unreq IV_PART + .unreq BLOCKS + .unreq BLOCKS_W +.endm + + /* + * aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, + * int bytes, u8 ctr[]) + * + * The input and output buffers must always be at least 16 bytes even if + * encrypting/decrypting less than 16 bytes. Otherwise out of bounds + * accesses will occur. The data to be encrypted/decrypted is expected + * to be at the end of this 16-byte temporary buffer rather than the + * start. + */ + +AES_FUNC_START(aes_ctr_encrypt) + ctr_encrypt 0 +AES_FUNC_END(aes_ctr_encrypt) + + /* + * aes_xctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, + * int bytes, u8 const iv[], int byte_ctr) + * + * The input and output buffers must always be at least 16 bytes even if + * encrypting/decrypting less than 16 bytes. Otherwise out of bounds + * accesses will occur. The data to be encrypted/decrypted is expected + * to be at the end of this 16-byte temporary buffer rather than the + * start. + */ + +AES_FUNC_START(aes_xctr_encrypt) + ctr_encrypt 1 +AES_FUNC_END(aes_xctr_encrypt) + + + /* + * aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds, + * int bytes, u8 const rk2[], u8 iv[], int first) + * aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds, + * int bytes, u8 const rk2[], u8 iv[], int first) + */ + + .macro next_tweak, out, in, tmp + sshr \tmp\().2d, \in\().2d, #63 + and \tmp\().16b, \tmp\().16b, xtsmask.16b + add \out\().2d, \in\().2d, \in\().2d + ext \tmp\().16b, \tmp\().16b, \tmp\().16b, #8 + eor \out\().16b, \out\().16b, \tmp\().16b + .endm + + .macro xts_load_mask, tmp + movi xtsmask.2s, #0x1 + movi \tmp\().2s, #0x87 + uzp1 xtsmask.4s, xtsmask.4s, \tmp\().4s + .endm + +AES_FUNC_START(aes_xts_encrypt) + frame_push 0 + + ld1 {v4.16b}, [x6] + xts_load_mask v8 + cbz w7, .Lxtsencnotfirst + + enc_prepare w3, x5, x8 + xts_cts_skip_tw w7, .LxtsencNx + encrypt_block v4, w3, x5, x8, w7 /* first tweak */ + enc_switch_key w3, x2, x8 + b .LxtsencNx + +.Lxtsencnotfirst: + enc_prepare w3, x2, x8 +.LxtsencloopNx: + next_tweak v4, v4, v8 +.LxtsencNx: + subs w4, w4, #64 + bmi .Lxtsenc1x + ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */ + next_tweak v5, v4, v8 + eor v0.16b, v0.16b, v4.16b + next_tweak v6, v5, v8 + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + next_tweak v7, v6, v8 + eor v3.16b, v3.16b, v7.16b + bl aes_encrypt_block4x + eor v3.16b, v3.16b, v7.16b + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + st1 {v0.16b-v3.16b}, [x0], #64 + mov v4.16b, v7.16b + cbz w4, .Lxtsencret + xts_reload_mask v8 + b .LxtsencloopNx +.Lxtsenc1x: + adds w4, w4, #64 + beq .Lxtsencout + subs w4, w4, #16 + bmi .LxtsencctsNx +.Lxtsencloop: + ld1 {v0.16b}, [x1], #16 +.Lxtsencctsout: + eor v0.16b, v0.16b, v4.16b + encrypt_block v0, w3, x2, x8, w7 + eor v0.16b, v0.16b, v4.16b + cbz w4, .Lxtsencout + subs w4, w4, #16 + next_tweak v4, v4, v8 + bmi .Lxtsenccts + st1 {v0.16b}, [x0], #16 + b .Lxtsencloop +.Lxtsencout: + st1 {v0.16b}, [x0] +.Lxtsencret: + st1 {v4.16b}, [x6] + frame_pop + ret + +.LxtsencctsNx: + mov v0.16b, v3.16b + sub x0, x0, #16 +.Lxtsenccts: + adr_l x8, .Lcts_permute_table + + add x1, x1, w4, sxtw /* rewind input pointer */ + add w4, w4, #16 /* # bytes in final block */ + add x9, x8, #32 + add x8, x8, x4 + sub x9, x9, x4 + add x4, x0, x4 /* output address of final block */ + + ld1 {v1.16b}, [x1] /* load final block */ + ld1 {v2.16b}, [x8] + ld1 {v3.16b}, [x9] + + tbl v2.16b, {v0.16b}, v2.16b + tbx v0.16b, {v1.16b}, v3.16b + st1 {v2.16b}, [x4] /* overlapping stores */ + mov w4, wzr + b .Lxtsencctsout +AES_FUNC_END(aes_xts_encrypt) + +AES_FUNC_START(aes_xts_decrypt) + frame_push 0 + + /* subtract 16 bytes if we are doing CTS */ + sub w8, w4, #0x10 + tst w4, #0xf + csel w4, w4, w8, eq + + ld1 {v4.16b}, [x6] + xts_load_mask v8 + xts_cts_skip_tw w7, .Lxtsdecskiptw + cbz w7, .Lxtsdecnotfirst + + enc_prepare w3, x5, x8 + encrypt_block v4, w3, x5, x8, w7 /* first tweak */ +.Lxtsdecskiptw: + dec_prepare w3, x2, x8 + b .LxtsdecNx + +.Lxtsdecnotfirst: + dec_prepare w3, x2, x8 +.LxtsdecloopNx: + next_tweak v4, v4, v8 +.LxtsdecNx: + subs w4, w4, #64 + bmi .Lxtsdec1x + ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */ + next_tweak v5, v4, v8 + eor v0.16b, v0.16b, v4.16b + next_tweak v6, v5, v8 + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + next_tweak v7, v6, v8 + eor v3.16b, v3.16b, v7.16b + bl aes_decrypt_block4x + eor v3.16b, v3.16b, v7.16b + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + st1 {v0.16b-v3.16b}, [x0], #64 + mov v4.16b, v7.16b + cbz w4, .Lxtsdecout + xts_reload_mask v8 + b .LxtsdecloopNx +.Lxtsdec1x: + adds w4, w4, #64 + beq .Lxtsdecout + subs w4, w4, #16 +.Lxtsdecloop: + ld1 {v0.16b}, [x1], #16 + bmi .Lxtsdeccts +.Lxtsdecctsout: + eor v0.16b, v0.16b, v4.16b + decrypt_block v0, w3, x2, x8, w7 + eor v0.16b, v0.16b, v4.16b + st1 {v0.16b}, [x0], #16 + cbz w4, .Lxtsdecout + subs w4, w4, #16 + next_tweak v4, v4, v8 + b .Lxtsdecloop +.Lxtsdecout: + st1 {v4.16b}, [x6] + frame_pop + ret + +.Lxtsdeccts: + adr_l x8, .Lcts_permute_table + + add x1, x1, w4, sxtw /* rewind input pointer */ + add w4, w4, #16 /* # bytes in final block */ + add x9, x8, #32 + add x8, x8, x4 + sub x9, x9, x4 + add x4, x0, x4 /* output address of final block */ + + next_tweak v5, v4, v8 + + ld1 {v1.16b}, [x1] /* load final block */ + ld1 {v2.16b}, [x8] + ld1 {v3.16b}, [x9] + + eor v0.16b, v0.16b, v5.16b + decrypt_block v0, w3, x2, x8, w7 + eor v0.16b, v0.16b, v5.16b + + tbl v2.16b, {v0.16b}, v2.16b + tbx v0.16b, {v1.16b}, v3.16b + + st1 {v2.16b}, [x4] /* overlapping stores */ + mov w4, wzr + b .Lxtsdecctsout +AES_FUNC_END(aes_xts_decrypt) + +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) + /* + * void aes_mac_update(u8 const in[], u32 const rk[], int rounds, + * size_t blocks, u8 dg[], int enc_before, + * int enc_after); + */ +AES_FUNC_START(aes_mac_update) + ld1 {v0.16b}, [x4] /* get dg */ + enc_prepare w2, x1, x7 + cbz w5, .Lmacloop4x + + encrypt_block v0, w2, x1, x7, w8 + +.Lmacloop4x: + subs x3, x3, #4 + bmi .Lmac1x + ld1 {v1.16b-v4.16b}, [x0], #64 /* get next pt block */ + eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */ + encrypt_block v0, w2, x1, x7, w8 + eor v0.16b, v0.16b, v2.16b + encrypt_block v0, w2, x1, x7, w8 + eor v0.16b, v0.16b, v3.16b + encrypt_block v0, w2, x1, x7, w8 + eor v0.16b, v0.16b, v4.16b + cmp x3, xzr + csinv w5, w6, wzr, eq + cbz w5, .Lmacout + encrypt_block v0, w2, x1, x7, w8 + st1 {v0.16b}, [x4] /* return dg */ + b .Lmacloop4x +.Lmac1x: + add x3, x3, #4 +.Lmacloop: + cbz x3, .Lmacout + ld1 {v1.16b}, [x0], #16 /* get next pt block */ + eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */ + + subs x3, x3, #1 + csinv w5, w6, wzr, eq + cbz w5, .Lmacout + +.Lmacenc: + encrypt_block v0, w2, x1, x7, w8 + b .Lmacloop + +.Lmacout: + st1 {v0.16b}, [x4] /* return dg */ + ret +AES_FUNC_END(aes_mac_update) +#endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ diff --git a/lib/crypto/arm64/aes-neon.S b/lib/crypto/arm64/aes-neon.S new file mode 100644 index 000000000000..f37b1dbd887f --- /dev/null +++ b/lib/crypto/arm64/aes-neon.S @@ -0,0 +1,250 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * AES cipher for ARMv8 NEON + * + * Copyright (C) 2013 - 2017 Linaro Ltd. <ard.biesheuvel@linaro.org> + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> + +#define AES_FUNC_START(func) SYM_FUNC_START(neon_ ## func) +#define AES_FUNC_END(func) SYM_FUNC_END(neon_ ## func) + + xtsmask .req v7 + cbciv .req v7 + vctr .req v4 + + .macro xts_reload_mask, tmp + xts_load_mask \tmp + .endm + + /* special case for the neon-bs driver calling into this one for CTS */ + .macro xts_cts_skip_tw, reg, lbl + tbnz \reg, #1, \lbl + .endm + + /* multiply by polynomial 'x' in GF(2^8) */ + .macro mul_by_x, out, in, temp, const + sshr \temp, \in, #7 + shl \out, \in, #1 + and \temp, \temp, \const + eor \out, \out, \temp + .endm + + /* multiply by polynomial 'x^2' in GF(2^8) */ + .macro mul_by_x2, out, in, temp, const + ushr \temp, \in, #6 + shl \out, \in, #2 + pmul \temp, \temp, \const + eor \out, \out, \temp + .endm + + /* preload the entire Sbox */ + .macro prepare, sbox, shiftrows, temp + movi v12.16b, #0x1b + ldr_l q13, \shiftrows, \temp + ldr_l q14, .Lror32by8, \temp + adr_l \temp, \sbox + ld1 {v16.16b-v19.16b}, [\temp], #64 + ld1 {v20.16b-v23.16b}, [\temp], #64 + ld1 {v24.16b-v27.16b}, [\temp], #64 + ld1 {v28.16b-v31.16b}, [\temp] + .endm + + /* do preload for encryption */ + .macro enc_prepare, ignore0, ignore1, temp + prepare crypto_aes_sbox, .LForward_ShiftRows, \temp + .endm + + .macro enc_switch_key, ignore0, ignore1, temp + /* do nothing */ + .endm + + /* do preload for decryption */ + .macro dec_prepare, ignore0, ignore1, temp + prepare crypto_aes_inv_sbox, .LReverse_ShiftRows, \temp + .endm + + /* apply SubBytes transformation using the preloaded Sbox */ + .macro sub_bytes, in + sub v9.16b, \in\().16b, v15.16b + tbl \in\().16b, {v16.16b-v19.16b}, \in\().16b + sub v10.16b, v9.16b, v15.16b + tbx \in\().16b, {v20.16b-v23.16b}, v9.16b + sub v11.16b, v10.16b, v15.16b + tbx \in\().16b, {v24.16b-v27.16b}, v10.16b + tbx \in\().16b, {v28.16b-v31.16b}, v11.16b + .endm + + /* apply MixColumns transformation */ + .macro mix_columns, in, enc + .if \enc == 0 + /* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */ + mul_by_x2 v8.16b, \in\().16b, v9.16b, v12.16b + eor \in\().16b, \in\().16b, v8.16b + rev32 v8.8h, v8.8h + eor \in\().16b, \in\().16b, v8.16b + .endif + + mul_by_x v9.16b, \in\().16b, v8.16b, v12.16b + rev32 v8.8h, \in\().8h + eor v8.16b, v8.16b, v9.16b + eor \in\().16b, \in\().16b, v8.16b + tbl \in\().16b, {\in\().16b}, v14.16b + eor \in\().16b, \in\().16b, v8.16b + .endm + + .macro do_block, enc, in, rounds, rk, rkp, i + ld1 {v15.4s}, [\rk] + add \rkp, \rk, #16 + mov \i, \rounds +.La\@: eor \in\().16b, \in\().16b, v15.16b /* ^round key */ + movi v15.16b, #0x40 + tbl \in\().16b, {\in\().16b}, v13.16b /* ShiftRows */ + sub_bytes \in + sub \i, \i, #1 + ld1 {v15.4s}, [\rkp], #16 + cbz \i, .Lb\@ + mix_columns \in, \enc + b .La\@ +.Lb\@: eor \in\().16b, \in\().16b, v15.16b /* ^round key */ + .endm + + .macro encrypt_block, in, rounds, rk, rkp, i + do_block 1, \in, \rounds, \rk, \rkp, \i + .endm + + .macro decrypt_block, in, rounds, rk, rkp, i + do_block 0, \in, \rounds, \rk, \rkp, \i + .endm + + /* + * Interleaved versions: functionally equivalent to the + * ones above, but applied to AES states in parallel. + */ + + .macro sub_bytes_4x, in0, in1, in2, in3 + sub v8.16b, \in0\().16b, v15.16b + tbl \in0\().16b, {v16.16b-v19.16b}, \in0\().16b + sub v9.16b, \in1\().16b, v15.16b + tbl \in1\().16b, {v16.16b-v19.16b}, \in1\().16b + sub v10.16b, \in2\().16b, v15.16b + tbl \in2\().16b, {v16.16b-v19.16b}, \in2\().16b + sub v11.16b, \in3\().16b, v15.16b + tbl \in3\().16b, {v16.16b-v19.16b}, \in3\().16b + tbx \in0\().16b, {v20.16b-v23.16b}, v8.16b + tbx \in1\().16b, {v20.16b-v23.16b}, v9.16b + sub v8.16b, v8.16b, v15.16b + tbx \in2\().16b, {v20.16b-v23.16b}, v10.16b + sub v9.16b, v9.16b, v15.16b + tbx \in3\().16b, {v20.16b-v23.16b}, v11.16b + sub v10.16b, v10.16b, v15.16b + tbx \in0\().16b, {v24.16b-v27.16b}, v8.16b + sub v11.16b, v11.16b, v15.16b + tbx \in1\().16b, {v24.16b-v27.16b}, v9.16b + sub v8.16b, v8.16b, v15.16b + tbx \in2\().16b, {v24.16b-v27.16b}, v10.16b + sub v9.16b, v9.16b, v15.16b + tbx \in3\().16b, {v24.16b-v27.16b}, v11.16b + sub v10.16b, v10.16b, v15.16b + tbx \in0\().16b, {v28.16b-v31.16b}, v8.16b + sub v11.16b, v11.16b, v15.16b + tbx \in1\().16b, {v28.16b-v31.16b}, v9.16b + tbx \in2\().16b, {v28.16b-v31.16b}, v10.16b + tbx \in3\().16b, {v28.16b-v31.16b}, v11.16b + .endm + + .macro mul_by_x_2x, out0, out1, in0, in1, tmp0, tmp1, const + sshr \tmp0\().16b, \in0\().16b, #7 + shl \out0\().16b, \in0\().16b, #1 + sshr \tmp1\().16b, \in1\().16b, #7 + and \tmp0\().16b, \tmp0\().16b, \const\().16b + shl \out1\().16b, \in1\().16b, #1 + and \tmp1\().16b, \tmp1\().16b, \const\().16b + eor \out0\().16b, \out0\().16b, \tmp0\().16b + eor \out1\().16b, \out1\().16b, \tmp1\().16b + .endm + + .macro mul_by_x2_2x, out0, out1, in0, in1, tmp0, tmp1, const + ushr \tmp0\().16b, \in0\().16b, #6 + shl \out0\().16b, \in0\().16b, #2 + ushr \tmp1\().16b, \in1\().16b, #6 + pmul \tmp0\().16b, \tmp0\().16b, \const\().16b + shl \out1\().16b, \in1\().16b, #2 + pmul \tmp1\().16b, \tmp1\().16b, \const\().16b + eor \out0\().16b, \out0\().16b, \tmp0\().16b + eor \out1\().16b, \out1\().16b, \tmp1\().16b + .endm + + .macro mix_columns_2x, in0, in1, enc + .if \enc == 0 + /* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */ + mul_by_x2_2x v8, v9, \in0, \in1, v10, v11, v12 + eor \in0\().16b, \in0\().16b, v8.16b + rev32 v8.8h, v8.8h + eor \in1\().16b, \in1\().16b, v9.16b + rev32 v9.8h, v9.8h + eor \in0\().16b, \in0\().16b, v8.16b + eor \in1\().16b, \in1\().16b, v9.16b + .endif + + mul_by_x_2x v8, v9, \in0, \in1, v10, v11, v12 + rev32 v10.8h, \in0\().8h + rev32 v11.8h, \in1\().8h + eor v10.16b, v10.16b, v8.16b + eor v11.16b, v11.16b, v9.16b + eor \in0\().16b, \in0\().16b, v10.16b + eor \in1\().16b, \in1\().16b, v11.16b + tbl \in0\().16b, {\in0\().16b}, v14.16b + tbl \in1\().16b, {\in1\().16b}, v14.16b + eor \in0\().16b, \in0\().16b, v10.16b + eor \in1\().16b, \in1\().16b, v11.16b + .endm + + .macro do_block_4x, enc, in0, in1, in2, in3, rounds, rk, rkp, i + ld1 {v15.4s}, [\rk] + add \rkp, \rk, #16 + mov \i, \rounds +.La\@: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */ + eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */ + eor \in2\().16b, \in2\().16b, v15.16b /* ^round key */ + eor \in3\().16b, \in3\().16b, v15.16b /* ^round key */ + movi v15.16b, #0x40 + tbl \in0\().16b, {\in0\().16b}, v13.16b /* ShiftRows */ + tbl \in1\().16b, {\in1\().16b}, v13.16b /* ShiftRows */ + tbl \in2\().16b, {\in2\().16b}, v13.16b /* ShiftRows */ + tbl \in3\().16b, {\in3\().16b}, v13.16b /* ShiftRows */ + sub_bytes_4x \in0, \in1, \in2, \in3 + sub \i, \i, #1 + ld1 {v15.4s}, [\rkp], #16 + cbz \i, .Lb\@ + mix_columns_2x \in0, \in1, \enc + mix_columns_2x \in2, \in3, \enc + b .La\@ +.Lb\@: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */ + eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */ + eor \in2\().16b, \in2\().16b, v15.16b /* ^round key */ + eor \in3\().16b, \in3\().16b, v15.16b /* ^round key */ + .endm + + .macro encrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i + do_block_4x 1, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i + .endm + + .macro decrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i + do_block_4x 0, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i + .endm + +#include "aes-modes.S" + + .section ".rodata", "a" + .align 4 +.LForward_ShiftRows: + .octa 0x0b06010c07020d08030e09040f0a0500 + +.LReverse_ShiftRows: + .octa 0x0306090c0f0205080b0e0104070a0d00 + +.Lror32by8: + .octa 0x0c0f0e0d080b0a090407060500030201 diff --git a/lib/crypto/arm64/aes.h b/lib/crypto/arm64/aes.h index 63eea6271ef9..9e9e45a6f787 100644 --- a/lib/crypto/arm64/aes.h +++ b/lib/crypto/arm64/aes.h @@ -11,6 +11,7 @@ #include <linux/unaligned.h> #include <linux/cpufeature.h> +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aes); struct aes_block { @@ -28,6 +29,9 @@ asmlinkage void __aes_ce_decrypt(const u32 inv_rk[], u8 out[AES_BLOCK_SIZE], asmlinkage u32 __aes_ce_sub(u32 l); asmlinkage void __aes_ce_invert(struct aes_block *out, const struct aes_block *in); +asmlinkage void neon_aes_mac_update(u8 const in[], u32 const rk[], int rounds, + size_t blocks, u8 dg[], int enc_before, + int enc_after); /* * Expand an AES key using the crypto extensions if supported and usable or @@ -48,8 +52,7 @@ static void aes_expandkey_arm64(u32 rndkeys[], u32 *inv_rndkeys, struct aes_block *key_enc, *key_dec; int i, j; - if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || - !static_branch_likely(&have_aes) || unlikely(!may_use_simd())) { + if (!static_branch_likely(&have_aes) || unlikely(!may_use_simd())) { aes_expandkey_generic(rndkeys, inv_rndkeys, in_key, key_len); return; } @@ -126,12 +129,40 @@ int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, } EXPORT_SYMBOL(ce_aes_expandkey); +EXPORT_SYMBOL_NS_GPL(neon_aes_ecb_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_ecb_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_cts_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_cts_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_ctr_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_xctr_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_xts_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_xts_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL"); + +EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_cts_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_cts_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_ctr_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_xctr_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_xts_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_xts_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL"); +#if IS_MODULE(CONFIG_CRYPTO_AES_ARM64_CE_CCM) +EXPORT_SYMBOL_NS_GPL(ce_aes_mac_update, "CRYPTO_INTERNAL"); +#endif + static void aes_encrypt_arch(const struct aes_enckey *key, u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_aes) && likely(may_use_simd())) { + if (static_branch_likely(&have_aes) && likely(may_use_simd())) { scoped_ksimd() __aes_ce_encrypt(key->k.rndkeys, out, in, key->nrounds); } else { @@ -143,8 +174,7 @@ static void aes_decrypt_arch(const struct aes_key *key, u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_aes) && likely(may_use_simd())) { + if (static_branch_likely(&have_aes) && likely(may_use_simd())) { scoped_ksimd() __aes_ce_decrypt(key->inv_k.inv_rndkeys, out, in, key->nrounds); @@ -154,11 +184,36 @@ static void aes_decrypt_arch(const struct aes_key *key, } } -#ifdef CONFIG_KERNEL_MODE_NEON +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) +#define aes_cbcmac_blocks_arch aes_cbcmac_blocks_arch +static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE], + const struct aes_enckey *key, const u8 *data, + size_t nblocks, bool enc_before, + bool enc_after) +{ + if (static_branch_likely(&have_neon) && likely(may_use_simd())) { + scoped_ksimd() { + if (static_branch_likely(&have_aes)) + ce_aes_mac_update(data, key->k.rndkeys, + key->nrounds, nblocks, h, + enc_before, enc_after); + else + neon_aes_mac_update(data, key->k.rndkeys, + key->nrounds, nblocks, h, + enc_before, enc_after); + } + return true; + } + return false; +} +#endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ + #define aes_mod_init_arch aes_mod_init_arch static void aes_mod_init_arch(void) { - if (cpu_have_named_feature(AES)) - static_branch_enable(&have_aes); + if (cpu_have_named_feature(ASIMD)) { + static_branch_enable(&have_neon); + if (cpu_have_named_feature(AES)) + static_branch_enable(&have_aes); + } } -#endif /* CONFIG_KERNEL_MODE_NEON */ diff --git a/lib/crypto/arm64/chacha-neon-core.S b/lib/crypto/arm64/chacha-neon-core.S index 80079586ecc7..cb18eec968bd 100644 --- a/lib/crypto/arm64/chacha-neon-core.S +++ b/lib/crypto/arm64/chacha-neon-core.S @@ -531,10 +531,6 @@ SYM_FUNC_START(chacha_4block_xor_neon) add v3.4s, v3.4s, v19.4s add a2, a2, w8 add a3, a3, w9 -CPU_BE( rev a0, a0 ) -CPU_BE( rev a1, a1 ) -CPU_BE( rev a2, a2 ) -CPU_BE( rev a3, a3 ) ld4r {v24.4s-v27.4s}, [x0], #16 ld4r {v28.4s-v31.4s}, [x0] @@ -555,10 +551,6 @@ CPU_BE( rev a3, a3 ) add v7.4s, v7.4s, v23.4s add a6, a6, w8 add a7, a7, w9 -CPU_BE( rev a4, a4 ) -CPU_BE( rev a5, a5 ) -CPU_BE( rev a6, a6 ) -CPU_BE( rev a7, a7 ) // x8[0-3] += s2[0] // x9[0-3] += s2[1] @@ -576,10 +568,6 @@ CPU_BE( rev a7, a7 ) add v11.4s, v11.4s, v27.4s add a10, a10, w8 add a11, a11, w9 -CPU_BE( rev a8, a8 ) -CPU_BE( rev a9, a9 ) -CPU_BE( rev a10, a10 ) -CPU_BE( rev a11, a11 ) // x12[0-3] += s3[0] // x13[0-3] += s3[1] @@ -597,10 +585,6 @@ CPU_BE( rev a11, a11 ) add v15.4s, v15.4s, v31.4s add a14, a14, w8 add a15, a15, w9 -CPU_BE( rev a12, a12 ) -CPU_BE( rev a13, a13 ) -CPU_BE( rev a14, a14 ) -CPU_BE( rev a15, a15 ) // interleave 32-bit words in state n, n+1 ldp w6, w7, [x2], #64 diff --git a/lib/crypto/arm64/chacha.h b/lib/crypto/arm64/chacha.h index ca8c6a8b0578..c6f8ddf98e2d 100644 --- a/lib/crypto/arm64/chacha.h +++ b/lib/crypto/arm64/chacha.h @@ -36,9 +36,9 @@ asmlinkage void hchacha_block_neon(const struct chacha_state *state, static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); static void chacha_doneon(struct chacha_state *state, u8 *dst, const u8 *src, - int bytes, int nrounds) + unsigned int bytes, int nrounds) { - while (bytes > 0) { + while (bytes) { int l = min(bytes, CHACHA_BLOCK_SIZE * 5); if (l <= CHACHA_BLOCK_SIZE) { @@ -76,16 +76,8 @@ static void chacha_crypt_arch(struct chacha_state *state, u8 *dst, !crypto_simd_usable()) return chacha_crypt_generic(state, dst, src, bytes, nrounds); - do { - unsigned int todo = min_t(unsigned int, bytes, SZ_4K); - - scoped_ksimd() - chacha_doneon(state, dst, src, todo, nrounds); - - bytes -= todo; - src += todo; - dst += todo; - } while (bytes); + scoped_ksimd() + chacha_doneon(state, dst, src, bytes, nrounds); } #define chacha_mod_init_arch chacha_mod_init_arch diff --git a/lib/crypto/arm64/gf128hash.h b/lib/crypto/arm64/gf128hash.h new file mode 100644 index 000000000000..1d1179f87f8d --- /dev/null +++ b/lib/crypto/arm64/gf128hash.h @@ -0,0 +1,121 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * GHASH and POLYVAL, arm64 optimized + * + * Copyright 2025 Google LLC + */ +#include <asm/simd.h> +#include <linux/cpufeature.h> + +#define NUM_H_POWERS 8 + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_asimd); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); + +asmlinkage void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg, + const u8 *src, + const struct polyval_elem *h); +asmlinkage void polyval_mul_pmull(struct polyval_elem *a, + const struct polyval_elem *b); +asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc, + const struct polyval_key *key, + const u8 *data, size_t nblocks); + +#define polyval_preparekey_arch polyval_preparekey_arch +static void polyval_preparekey_arch(struct polyval_key *key, + const u8 raw_key[POLYVAL_BLOCK_SIZE]) +{ + static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS); + memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE); + if (static_branch_likely(&have_pmull) && may_use_simd()) { + scoped_ksimd() { + for (int i = NUM_H_POWERS - 2; i >= 0; i--) { + key->h_powers[i] = key->h_powers[i + 1]; + polyval_mul_pmull( + &key->h_powers[i], + &key->h_powers[NUM_H_POWERS - 1]); + } + } + } else { + for (int i = NUM_H_POWERS - 2; i >= 0; i--) { + key->h_powers[i] = key->h_powers[i + 1]; + polyval_mul_generic(&key->h_powers[i], + &key->h_powers[NUM_H_POWERS - 1]); + } + } +} + +static void polyval_mul_arm64(struct polyval_elem *a, + const struct polyval_elem *b) +{ + if (static_branch_likely(&have_asimd) && may_use_simd()) { + static const u8 zeroes[GHASH_BLOCK_SIZE]; + + scoped_ksimd() { + if (static_branch_likely(&have_pmull)) { + polyval_mul_pmull(a, b); + } else { + /* + * Note that this is indeed equivalent to a + * POLYVAL multiplication, since it takes the + * accumulator and key in POLYVAL format, and + * byte-swapping a block of zeroes is a no-op. + */ + pmull_ghash_update_p8(1, a, zeroes, b); + } + } + } else { + polyval_mul_generic(a, b); + } +} + +#define ghash_mul_arch ghash_mul_arch +static void ghash_mul_arch(struct polyval_elem *acc, + const struct ghash_key *key) +{ + polyval_mul_arm64(acc, &key->h); +} + +#define polyval_mul_arch polyval_mul_arch +static void polyval_mul_arch(struct polyval_elem *acc, + const struct polyval_key *key) +{ + polyval_mul_arm64(acc, &key->h_powers[NUM_H_POWERS - 1]); +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_asimd) && may_use_simd()) { + scoped_ksimd() + pmull_ghash_update_p8(nblocks, acc, data, &key->h); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define polyval_blocks_arch polyval_blocks_arch +static void polyval_blocks_arch(struct polyval_elem *acc, + const struct polyval_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_pmull) && may_use_simd()) { + scoped_ksimd() + polyval_blocks_pmull(acc, key, data, nblocks); + } else { + polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1], + data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (cpu_have_named_feature(ASIMD)) { + static_branch_enable(&have_asimd); + if (cpu_have_named_feature(PMULL)) + static_branch_enable(&have_pmull); + } +} diff --git a/lib/crypto/arm64/ghash-neon-core.S b/lib/crypto/arm64/ghash-neon-core.S new file mode 100644 index 000000000000..4c5799172b49 --- /dev/null +++ b/lib/crypto/arm64/ghash-neon-core.S @@ -0,0 +1,220 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Accelerated GHASH implementation with ARMv8 ASIMD instructions. + * + * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org> + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> + + SHASH .req v0 + SHASH2 .req v1 + T1 .req v2 + T2 .req v3 + XM .req v5 + XL .req v6 + XH .req v7 + IN1 .req v7 + + k00_16 .req v8 + k32_48 .req v9 + + t3 .req v10 + t4 .req v11 + t5 .req v12 + t6 .req v13 + t7 .req v14 + t8 .req v15 + t9 .req v16 + + perm1 .req v17 + perm2 .req v18 + perm3 .req v19 + + sh1 .req v20 + sh2 .req v21 + sh3 .req v22 + sh4 .req v23 + + ss1 .req v24 + ss2 .req v25 + ss3 .req v26 + ss4 .req v27 + + .text + + .macro __pmull_p8, rq, ad, bd + ext t3.8b, \ad\().8b, \ad\().8b, #1 // A1 + ext t5.8b, \ad\().8b, \ad\().8b, #2 // A2 + ext t7.8b, \ad\().8b, \ad\().8b, #3 // A3 + + __pmull_p8_\bd \rq, \ad + .endm + + .macro __pmull2_p8, rq, ad, bd + tbl t3.16b, {\ad\().16b}, perm1.16b // A1 + tbl t5.16b, {\ad\().16b}, perm2.16b // A2 + tbl t7.16b, {\ad\().16b}, perm3.16b // A3 + + __pmull2_p8_\bd \rq, \ad + .endm + + .macro __pmull_p8_SHASH, rq, ad + __pmull_p8_tail \rq, \ad\().8b, SHASH.8b, 8b,, sh1, sh2, sh3, sh4 + .endm + + .macro __pmull_p8_SHASH2, rq, ad + __pmull_p8_tail \rq, \ad\().8b, SHASH2.8b, 8b,, ss1, ss2, ss3, ss4 + .endm + + .macro __pmull2_p8_SHASH, rq, ad + __pmull_p8_tail \rq, \ad\().16b, SHASH.16b, 16b, 2, sh1, sh2, sh3, sh4 + .endm + + .macro __pmull_p8_tail, rq, ad, bd, nb, t, b1, b2, b3, b4 + pmull\t t3.8h, t3.\nb, \bd // F = A1*B + pmull\t t4.8h, \ad, \b1\().\nb // E = A*B1 + pmull\t t5.8h, t5.\nb, \bd // H = A2*B + pmull\t t6.8h, \ad, \b2\().\nb // G = A*B2 + pmull\t t7.8h, t7.\nb, \bd // J = A3*B + pmull\t t8.8h, \ad, \b3\().\nb // I = A*B3 + pmull\t t9.8h, \ad, \b4\().\nb // K = A*B4 + pmull\t \rq\().8h, \ad, \bd // D = A*B + + eor t3.16b, t3.16b, t4.16b // L = E + F + eor t5.16b, t5.16b, t6.16b // M = G + H + eor t7.16b, t7.16b, t8.16b // N = I + J + + uzp1 t4.2d, t3.2d, t5.2d + uzp2 t3.2d, t3.2d, t5.2d + uzp1 t6.2d, t7.2d, t9.2d + uzp2 t7.2d, t7.2d, t9.2d + + // t3 = (L) (P0 + P1) << 8 + // t5 = (M) (P2 + P3) << 16 + eor t4.16b, t4.16b, t3.16b + and t3.16b, t3.16b, k32_48.16b + + // t7 = (N) (P4 + P5) << 24 + // t9 = (K) (P6 + P7) << 32 + eor t6.16b, t6.16b, t7.16b + and t7.16b, t7.16b, k00_16.16b + + eor t4.16b, t4.16b, t3.16b + eor t6.16b, t6.16b, t7.16b + + zip2 t5.2d, t4.2d, t3.2d + zip1 t3.2d, t4.2d, t3.2d + zip2 t9.2d, t6.2d, t7.2d + zip1 t7.2d, t6.2d, t7.2d + + ext t3.16b, t3.16b, t3.16b, #15 + ext t5.16b, t5.16b, t5.16b, #14 + ext t7.16b, t7.16b, t7.16b, #13 + ext t9.16b, t9.16b, t9.16b, #12 + + eor t3.16b, t3.16b, t5.16b + eor t7.16b, t7.16b, t9.16b + eor \rq\().16b, \rq\().16b, t3.16b + eor \rq\().16b, \rq\().16b, t7.16b + .endm + + .macro __pmull_pre_p8 + ext SHASH2.16b, SHASH.16b, SHASH.16b, #8 + eor SHASH2.16b, SHASH2.16b, SHASH.16b + + // k00_16 := 0x0000000000000000_000000000000ffff + // k32_48 := 0x00000000ffffffff_0000ffffffffffff + movi k32_48.2d, #0xffffffff + mov k32_48.h[2], k32_48.h[0] + ushr k00_16.2d, k32_48.2d, #32 + + // prepare the permutation vectors + mov_q x5, 0x080f0e0d0c0b0a09 + movi T1.8b, #8 + dup perm1.2d, x5 + eor perm1.16b, perm1.16b, T1.16b + ushr perm2.2d, perm1.2d, #8 + ushr perm3.2d, perm1.2d, #16 + ushr T1.2d, perm1.2d, #24 + sli perm2.2d, perm1.2d, #56 + sli perm3.2d, perm1.2d, #48 + sli T1.2d, perm1.2d, #40 + + // precompute loop invariants + tbl sh1.16b, {SHASH.16b}, perm1.16b + tbl sh2.16b, {SHASH.16b}, perm2.16b + tbl sh3.16b, {SHASH.16b}, perm3.16b + tbl sh4.16b, {SHASH.16b}, T1.16b + ext ss1.8b, SHASH2.8b, SHASH2.8b, #1 + ext ss2.8b, SHASH2.8b, SHASH2.8b, #2 + ext ss3.8b, SHASH2.8b, SHASH2.8b, #3 + ext ss4.8b, SHASH2.8b, SHASH2.8b, #4 + .endm + + .macro __pmull_reduce_p8 + eor XM.16b, XM.16b, T1.16b + + mov XL.d[1], XM.d[0] + mov XH.d[0], XM.d[1] + + shl T1.2d, XL.2d, #57 + shl T2.2d, XL.2d, #62 + eor T2.16b, T2.16b, T1.16b + shl T1.2d, XL.2d, #63 + eor T2.16b, T2.16b, T1.16b + ext T1.16b, XL.16b, XH.16b, #8 + eor T2.16b, T2.16b, T1.16b + + mov XL.d[1], T2.d[0] + mov XH.d[0], T2.d[1] + + ushr T2.2d, XL.2d, #1 + eor XH.16b, XH.16b, XL.16b + eor XL.16b, XL.16b, T2.16b + ushr T2.2d, T2.2d, #6 + ushr XL.2d, XL.2d, #1 + .endm + + /* + * void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg, + * const u8 *src, + * const struct polyval_elem *h) + */ +SYM_FUNC_START(pmull_ghash_update_p8) + ld1 {SHASH.2d}, [x3] + ld1 {XL.2d}, [x1] + + __pmull_pre_p8 + +0: ld1 {T1.2d}, [x2], #16 + sub x0, x0, #1 + + /* multiply XL by SHASH in GF(2^128) */ + rev64 T1.16b, T1.16b + + ext T2.16b, XL.16b, XL.16b, #8 + ext IN1.16b, T1.16b, T1.16b, #8 + eor T1.16b, T1.16b, T2.16b + eor XL.16b, XL.16b, IN1.16b + + __pmull2_p8 XH, XL, SHASH // a1 * b1 + eor T1.16b, T1.16b, XL.16b + __pmull_p8 XL, XL, SHASH // a0 * b0 + __pmull_p8 XM, T1, SHASH2 // (a1 + a0)(b1 + b0) + + eor T2.16b, XL.16b, XH.16b + ext T1.16b, XL.16b, XH.16b, #8 + eor XM.16b, XM.16b, T2.16b + + __pmull_reduce_p8 + + eor T2.16b, T2.16b, XH.16b + eor XL.16b, XL.16b, T2.16b + + cbnz x0, 0b + + st1 {XL.2d}, [x1] + ret +SYM_FUNC_END(pmull_ghash_update_p8) diff --git a/lib/crypto/arm64/poly1305.h b/lib/crypto/arm64/poly1305.h index b77669767cd6..3d4bde857699 100644 --- a/lib/crypto/arm64/poly1305.h +++ b/lib/crypto/arm64/poly1305.h @@ -27,17 +27,11 @@ static void poly1305_blocks(struct poly1305_block_state *state, const u8 *src, unsigned int len, u32 padbit) { if (static_branch_likely(&have_neon) && likely(may_use_simd())) { - do { - unsigned int todo = min_t(unsigned int, len, SZ_4K); - - scoped_ksimd() - poly1305_blocks_neon(state, src, todo, padbit); - - len -= todo; - src += todo; - } while (len); - } else + scoped_ksimd() + poly1305_blocks_neon(state, src, len, padbit); + } else { poly1305_blocks_arm64(state, src, len, padbit); + } } #define poly1305_mod_init_arch poly1305_mod_init_arch diff --git a/lib/crypto/arm64/polyval.h b/lib/crypto/arm64/polyval.h deleted file mode 100644 index a39763395e9b..000000000000 --- a/lib/crypto/arm64/polyval.h +++ /dev/null @@ -1,80 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * POLYVAL library functions, arm64 optimized - * - * Copyright 2025 Google LLC - */ -#include <asm/simd.h> -#include <linux/cpufeature.h> - -#define NUM_H_POWERS 8 - -static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); - -asmlinkage void polyval_mul_pmull(struct polyval_elem *a, - const struct polyval_elem *b); -asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc, - const struct polyval_key *key, - const u8 *data, size_t nblocks); - -static void polyval_preparekey_arch(struct polyval_key *key, - const u8 raw_key[POLYVAL_BLOCK_SIZE]) -{ - static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS); - memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE); - if (static_branch_likely(&have_pmull) && may_use_simd()) { - scoped_ksimd() { - for (int i = NUM_H_POWERS - 2; i >= 0; i--) { - key->h_powers[i] = key->h_powers[i + 1]; - polyval_mul_pmull( - &key->h_powers[i], - &key->h_powers[NUM_H_POWERS - 1]); - } - } - } else { - for (int i = NUM_H_POWERS - 2; i >= 0; i--) { - key->h_powers[i] = key->h_powers[i + 1]; - polyval_mul_generic(&key->h_powers[i], - &key->h_powers[NUM_H_POWERS - 1]); - } - } -} - -static void polyval_mul_arch(struct polyval_elem *acc, - const struct polyval_key *key) -{ - if (static_branch_likely(&have_pmull) && may_use_simd()) { - scoped_ksimd() - polyval_mul_pmull(acc, &key->h_powers[NUM_H_POWERS - 1]); - } else { - polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]); - } -} - -static void polyval_blocks_arch(struct polyval_elem *acc, - const struct polyval_key *key, - const u8 *data, size_t nblocks) -{ - if (static_branch_likely(&have_pmull) && may_use_simd()) { - do { - /* Allow rescheduling every 4 KiB. */ - size_t n = min_t(size_t, nblocks, - 4096 / POLYVAL_BLOCK_SIZE); - - scoped_ksimd() - polyval_blocks_pmull(acc, key, data, n); - data += n * POLYVAL_BLOCK_SIZE; - nblocks -= n; - } while (nblocks); - } else { - polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1], - data, nblocks); - } -} - -#define polyval_mod_init_arch polyval_mod_init_arch -static void polyval_mod_init_arch(void) -{ - if (cpu_have_named_feature(PMULL)) - static_branch_enable(&have_pmull); -} diff --git a/lib/crypto/arm64/sha1-ce-core.S b/lib/crypto/arm64/sha1-ce-core.S index 8fbd4767f0f0..40f2a6c8d0c5 100644 --- a/lib/crypto/arm64/sha1-ce-core.S +++ b/lib/crypto/arm64/sha1-ce-core.S @@ -62,10 +62,10 @@ .endm /* - * size_t __sha1_ce_transform(struct sha1_block_state *state, - * const u8 *data, size_t nblocks); + * void sha1_ce_transform(struct sha1_block_state *state, + * const u8 *data, size_t nblocks); */ -SYM_FUNC_START(__sha1_ce_transform) +SYM_FUNC_START(sha1_ce_transform) /* load round constants */ loadrc k0.4s, 0x5a827999, w6 loadrc k1.4s, 0x6ed9eba1, w6 @@ -80,10 +80,10 @@ SYM_FUNC_START(__sha1_ce_transform) 0: ld1 {v8.4s-v11.4s}, [x1], #64 sub x2, x2, #1 -CPU_LE( rev32 v8.16b, v8.16b ) -CPU_LE( rev32 v9.16b, v9.16b ) -CPU_LE( rev32 v10.16b, v10.16b ) -CPU_LE( rev32 v11.16b, v11.16b ) + rev32 v8.16b, v8.16b + rev32 v9.16b, v9.16b + rev32 v10.16b, v10.16b + rev32 v11.16b, v11.16b add t0.4s, v8.4s, k0.4s mov dg0v.16b, dgav.16b @@ -116,15 +116,11 @@ CPU_LE( rev32 v11.16b, v11.16b ) add dgbv.2s, dgbv.2s, dg1v.2s add dgav.4s, dgav.4s, dg0v.4s - /* return early if voluntary preemption is needed */ - cond_yield 1f, x5, x6 - /* handled all input blocks? */ cbnz x2, 0b /* store new state */ -1: st1 {dgav.4s}, [x0] + st1 {dgav.4s}, [x0] str dgb, [x0, #16] - mov x0, x2 ret -SYM_FUNC_END(__sha1_ce_transform) +SYM_FUNC_END(sha1_ce_transform) diff --git a/lib/crypto/arm64/sha1.h b/lib/crypto/arm64/sha1.h index bc7071f1be09..112c5d443c56 100644 --- a/lib/crypto/arm64/sha1.h +++ b/lib/crypto/arm64/sha1.h @@ -9,22 +9,15 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); -asmlinkage size_t __sha1_ce_transform(struct sha1_block_state *state, - const u8 *data, size_t nblocks); +asmlinkage void sha1_ce_transform(struct sha1_block_state *state, + const u8 *data, size_t nblocks); static void sha1_blocks(struct sha1_block_state *state, const u8 *data, size_t nblocks) { if (static_branch_likely(&have_ce) && likely(may_use_simd())) { - do { - size_t rem; - - scoped_ksimd() - rem = __sha1_ce_transform(state, data, nblocks); - - data += (nblocks - rem) * SHA1_BLOCK_SIZE; - nblocks = rem; - } while (nblocks); + scoped_ksimd() + sha1_ce_transform(state, data, nblocks); } else { sha1_blocks_generic(state, data, nblocks); } diff --git a/lib/crypto/arm64/sha256-ce.S b/lib/crypto/arm64/sha256-ce.S index e4bfe42a61a9..8fdbf0a9ff89 100644 --- a/lib/crypto/arm64/sha256-ce.S +++ b/lib/crypto/arm64/sha256-ce.S @@ -79,11 +79,11 @@ .endm /* - * size_t __sha256_ce_transform(struct sha256_block_state *state, - * const u8 *data, size_t nblocks); + * void sha256_ce_transform(struct sha256_block_state *state, + * const u8 *data, size_t nblocks); */ .text -SYM_FUNC_START(__sha256_ce_transform) +SYM_FUNC_START(sha256_ce_transform) load_round_constants x8 @@ -94,10 +94,10 @@ SYM_FUNC_START(__sha256_ce_transform) 0: ld1 {v16.4s-v19.4s}, [x1], #64 sub x2, x2, #1 -CPU_LE( rev32 v16.16b, v16.16b ) -CPU_LE( rev32 v17.16b, v17.16b ) -CPU_LE( rev32 v18.16b, v18.16b ) -CPU_LE( rev32 v19.16b, v19.16b ) + rev32 v16.16b, v16.16b + rev32 v17.16b, v17.16b + rev32 v18.16b, v18.16b + rev32 v19.16b, v19.16b add t0.4s, v16.4s, v0.4s mov dg0v.16b, dgav.16b @@ -127,17 +127,13 @@ CPU_LE( rev32 v19.16b, v19.16b ) add dgav.4s, dgav.4s, dg0v.4s add dgbv.4s, dgbv.4s, dg1v.4s - /* return early if voluntary preemption is needed */ - cond_yield 1f, x5, x6 - /* handled all input blocks? */ cbnz x2, 0b /* store new state */ -1: st1 {dgav.4s, dgbv.4s}, [x0] - mov x0, x2 + st1 {dgav.4s, dgbv.4s}, [x0] ret -SYM_FUNC_END(__sha256_ce_transform) +SYM_FUNC_END(sha256_ce_transform) .unreq dga .unreq dgav @@ -293,14 +289,14 @@ SYM_FUNC_START(sha256_ce_finup2x) ld1 {v20.4s-v23.4s}, [data2], #64 .Lfinup2x_loop_have_data: // Convert the words of the data blocks from big endian. -CPU_LE( rev32 v16.16b, v16.16b ) -CPU_LE( rev32 v17.16b, v17.16b ) -CPU_LE( rev32 v18.16b, v18.16b ) -CPU_LE( rev32 v19.16b, v19.16b ) -CPU_LE( rev32 v20.16b, v20.16b ) -CPU_LE( rev32 v21.16b, v21.16b ) -CPU_LE( rev32 v22.16b, v22.16b ) -CPU_LE( rev32 v23.16b, v23.16b ) + rev32 v16.16b, v16.16b + rev32 v17.16b, v17.16b + rev32 v18.16b, v18.16b + rev32 v19.16b, v19.16b + rev32 v20.16b, v20.16b + rev32 v21.16b, v21.16b + rev32 v22.16b, v22.16b + rev32 v23.16b, v23.16b .Lfinup2x_loop_have_bswapped_data: // Save the original state for each block. @@ -340,11 +336,8 @@ CPU_LE( rev32 v23.16b, v23.16b ) sub w8, len, #64 // w8 = len - 64 add data1, data1, w8, sxtw // data1 += len - 64 add data2, data2, w8, sxtw // data2 += len - 64 -CPU_LE( mov x9, #0x80 ) -CPU_LE( fmov d16, x9 ) -CPU_BE( movi v16.16b, #0 ) -CPU_BE( mov x9, #0x8000000000000000 ) -CPU_BE( mov v16.d[1], x9 ) + mov x9, #0x80 + fmov d16, x9 movi v17.16b, #0 stp q16, q17, [sp, #64] stp q17, q17, [sp, #96] @@ -352,7 +345,7 @@ CPU_BE( mov v16.d[1], x9 ) cmp len, #56 b.ge 1f // will count spill into its own block? lsl count, count, #3 -CPU_LE( rev count, count ) + rev count, count str count, [x9, #56] mov final_step, #2 // won't need count-only block b 2f @@ -397,10 +390,10 @@ CPU_LE( rev count, count ) .Lfinup2x_done: // Write the two digests with all bytes in the correct order. -CPU_LE( rev32 state0_a.16b, state0_a.16b ) -CPU_LE( rev32 state1_a.16b, state1_a.16b ) -CPU_LE( rev32 state0_b.16b, state0_b.16b ) -CPU_LE( rev32 state1_b.16b, state1_b.16b ) + rev32 state0_a.16b, state0_a.16b + rev32 state1_a.16b, state1_a.16b + rev32 state0_b.16b, state0_b.16b + rev32 state1_b.16b, state1_b.16b st1 {state0_a.4s-state1_a.4s}, [out1] st1 {state0_b.4s-state1_b.4s}, [out2] add sp, sp, #128 diff --git a/lib/crypto/arm64/sha256.h b/lib/crypto/arm64/sha256.h index 568dff0f276a..b4353d3c4dd0 100644 --- a/lib/crypto/arm64/sha256.h +++ b/lib/crypto/arm64/sha256.h @@ -14,27 +14,17 @@ asmlinkage void sha256_block_data_order(struct sha256_block_state *state, const u8 *data, size_t nblocks); asmlinkage void sha256_block_neon(struct sha256_block_state *state, const u8 *data, size_t nblocks); -asmlinkage size_t __sha256_ce_transform(struct sha256_block_state *state, - const u8 *data, size_t nblocks); +asmlinkage void sha256_ce_transform(struct sha256_block_state *state, + const u8 *data, size_t nblocks); static void sha256_blocks(struct sha256_block_state *state, const u8 *data, size_t nblocks) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_neon) && likely(may_use_simd())) { - if (static_branch_likely(&have_ce)) { - do { - size_t rem; - - scoped_ksimd() - rem = __sha256_ce_transform(state, data, - nblocks); - - data += (nblocks - rem) * SHA256_BLOCK_SIZE; - nblocks = rem; - } while (nblocks); - } else { - scoped_ksimd() + if (static_branch_likely(&have_neon) && likely(may_use_simd())) { + scoped_ksimd() { + if (static_branch_likely(&have_ce)) + sha256_ce_transform(state, data, nblocks); + else sha256_block_neon(state, data, nblocks); } } else { @@ -56,14 +46,9 @@ static bool sha256_finup_2x_arch(const struct __sha256_ctx *ctx, u8 out1[SHA256_DIGEST_SIZE], u8 out2[SHA256_DIGEST_SIZE]) { - /* - * The assembly requires len >= SHA256_BLOCK_SIZE && len <= INT_MAX. - * Further limit len to 65536 to avoid spending too long with preemption - * disabled. (Of course, in practice len is nearly always 4096 anyway.) - */ - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_ce) && len >= SHA256_BLOCK_SIZE && - len <= 65536 && likely(may_use_simd())) { + /* The assembly requires len >= SHA256_BLOCK_SIZE && len <= INT_MAX. */ + if (static_branch_likely(&have_ce) && len >= SHA256_BLOCK_SIZE && + len <= INT_MAX && likely(may_use_simd())) { scoped_ksimd() sha256_ce_finup2x(ctx, data1, data2, len, out1, out2); kmsan_unpoison_memory(out1, SHA256_DIGEST_SIZE); @@ -78,7 +63,6 @@ static bool sha256_finup_2x_is_optimized_arch(void) return static_key_enabled(&have_ce); } -#ifdef CONFIG_KERNEL_MODE_NEON #define sha256_mod_init_arch sha256_mod_init_arch static void sha256_mod_init_arch(void) { @@ -88,4 +72,3 @@ static void sha256_mod_init_arch(void) static_branch_enable(&have_ce); } } -#endif /* CONFIG_KERNEL_MODE_NEON */ diff --git a/lib/crypto/arm64/sha3-ce-core.S b/lib/crypto/arm64/sha3-ce-core.S index ace90b506490..b8ab01987ae0 100644 --- a/lib/crypto/arm64/sha3-ce-core.S +++ b/lib/crypto/arm64/sha3-ce-core.S @@ -37,8 +37,8 @@ .endm /* - * size_t sha3_ce_transform(struct sha3_state *state, const u8 *data, - * size_t nblocks, size_t block_size) + * void sha3_ce_transform(struct sha3_state *state, const u8 *data, + * size_t nblocks, size_t block_size) * * block_size is assumed to be one of 72 (SHA3-512), 104 (SHA3-384), 136 * (SHA3-256 and SHAKE256), 144 (SHA3-224), or 168 (SHAKE128). @@ -185,18 +185,16 @@ SYM_FUNC_START(sha3_ce_transform) eor v0.16b, v0.16b, v31.16b cbnz w8, 3b - cond_yield 4f, x8, x9 cbnz x2, 0b /* save state */ -4: st1 { v0.1d- v3.1d}, [x0], #32 + st1 { v0.1d- v3.1d}, [x0], #32 st1 { v4.1d- v7.1d}, [x0], #32 st1 { v8.1d-v11.1d}, [x0], #32 st1 {v12.1d-v15.1d}, [x0], #32 st1 {v16.1d-v19.1d}, [x0], #32 st1 {v20.1d-v23.1d}, [x0], #32 st1 {v24.1d}, [x0] - mov x0, x2 ret SYM_FUNC_END(sha3_ce_transform) diff --git a/lib/crypto/arm64/sha3.h b/lib/crypto/arm64/sha3.h index b602f1b3b282..eaaba3224acc 100644 --- a/lib/crypto/arm64/sha3.h +++ b/lib/crypto/arm64/sha3.h @@ -12,22 +12,15 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3); -asmlinkage size_t sha3_ce_transform(struct sha3_state *state, const u8 *data, - size_t nblocks, size_t block_size); +asmlinkage void sha3_ce_transform(struct sha3_state *state, const u8 *data, + size_t nblocks, size_t block_size); static void sha3_absorb_blocks(struct sha3_state *state, const u8 *data, size_t nblocks, size_t block_size) { if (static_branch_likely(&have_sha3) && likely(may_use_simd())) { - do { - size_t rem; - - scoped_ksimd() - rem = sha3_ce_transform(state, data, nblocks, - block_size); - data += (nblocks - rem) * block_size; - nblocks = rem; - } while (nblocks); + scoped_ksimd() + sha3_ce_transform(state, data, nblocks, block_size); } else { sha3_absorb_blocks_generic(state, data, nblocks, block_size); } diff --git a/lib/crypto/arm64/sha512-ce-core.S b/lib/crypto/arm64/sha512-ce-core.S index ffd51acfd1ee..daa2c63a20c0 100644 --- a/lib/crypto/arm64/sha512-ce-core.S +++ b/lib/crypto/arm64/sha512-ce-core.S @@ -93,11 +93,11 @@ .endm /* - * size_t __sha512_ce_transform(struct sha512_block_state *state, - * const u8 *data, size_t nblocks); + * void sha512_ce_transform(struct sha512_block_state *state, + * const u8 *data, size_t nblocks); */ .text -SYM_FUNC_START(__sha512_ce_transform) +SYM_FUNC_START(sha512_ce_transform) /* load state */ ld1 {v8.2d-v11.2d}, [x0] @@ -110,14 +110,14 @@ SYM_FUNC_START(__sha512_ce_transform) ld1 {v16.2d-v19.2d}, [x1], #64 sub x2, x2, #1 -CPU_LE( rev64 v12.16b, v12.16b ) -CPU_LE( rev64 v13.16b, v13.16b ) -CPU_LE( rev64 v14.16b, v14.16b ) -CPU_LE( rev64 v15.16b, v15.16b ) -CPU_LE( rev64 v16.16b, v16.16b ) -CPU_LE( rev64 v17.16b, v17.16b ) -CPU_LE( rev64 v18.16b, v18.16b ) -CPU_LE( rev64 v19.16b, v19.16b ) + rev64 v12.16b, v12.16b + rev64 v13.16b, v13.16b + rev64 v14.16b, v14.16b + rev64 v15.16b, v15.16b + rev64 v16.16b, v16.16b + rev64 v17.16b, v17.16b + rev64 v18.16b, v18.16b + rev64 v19.16b, v19.16b mov x4, x3 // rc pointer @@ -186,12 +186,10 @@ CPU_LE( rev64 v19.16b, v19.16b ) add v10.2d, v10.2d, v2.2d add v11.2d, v11.2d, v3.2d - cond_yield 3f, x4, x5 /* handled all input blocks? */ cbnz x2, 0b /* store new state */ -3: st1 {v8.2d-v11.2d}, [x0] - mov x0, x2 + st1 {v8.2d-v11.2d}, [x0] ret -SYM_FUNC_END(__sha512_ce_transform) +SYM_FUNC_END(sha512_ce_transform) diff --git a/lib/crypto/arm64/sha512.h b/lib/crypto/arm64/sha512.h index 7eb7ef04d268..5da27e6e23ea 100644 --- a/lib/crypto/arm64/sha512.h +++ b/lib/crypto/arm64/sha512.h @@ -12,34 +12,24 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha512_insns); asmlinkage void sha512_block_data_order(struct sha512_block_state *state, const u8 *data, size_t nblocks); -asmlinkage size_t __sha512_ce_transform(struct sha512_block_state *state, - const u8 *data, size_t nblocks); +asmlinkage void sha512_ce_transform(struct sha512_block_state *state, + const u8 *data, size_t nblocks); static void sha512_blocks(struct sha512_block_state *state, const u8 *data, size_t nblocks) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_sha512_insns) && + if (static_branch_likely(&have_sha512_insns) && likely(may_use_simd())) { - do { - size_t rem; - - scoped_ksimd() - rem = __sha512_ce_transform(state, data, nblocks); - - data += (nblocks - rem) * SHA512_BLOCK_SIZE; - nblocks = rem; - } while (nblocks); + scoped_ksimd() + sha512_ce_transform(state, data, nblocks); } else { sha512_block_data_order(state, data, nblocks); } } -#ifdef CONFIG_KERNEL_MODE_NEON #define sha512_mod_init_arch sha512_mod_init_arch static void sha512_mod_init_arch(void) { if (cpu_have_named_feature(SHA512)) static_branch_enable(&have_sha512_insns); } -#endif /* CONFIG_KERNEL_MODE_NEON */ diff --git a/lib/crypto/arm64/sm3-ce-core.S b/lib/crypto/arm64/sm3-ce-core.S new file mode 100644 index 000000000000..ee7f900d7cff --- /dev/null +++ b/lib/crypto/arm64/sm3-ce-core.S @@ -0,0 +1,138 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * sm3-ce-core.S - SM3 secure hash using ARMv8.2 Crypto Extensions + * + * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org> + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> + + .irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 + .set .Lv\b\().4s, \b + .endr + + .macro sm3partw1, rd, rn, rm + .inst 0xce60c000 | .L\rd | (.L\rn << 5) | (.L\rm << 16) + .endm + + .macro sm3partw2, rd, rn, rm + .inst 0xce60c400 | .L\rd | (.L\rn << 5) | (.L\rm << 16) + .endm + + .macro sm3ss1, rd, rn, rm, ra + .inst 0xce400000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16) + .endm + + .macro sm3tt1a, rd, rn, rm, imm2 + .inst 0xce408000 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16) + .endm + + .macro sm3tt1b, rd, rn, rm, imm2 + .inst 0xce408400 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16) + .endm + + .macro sm3tt2a, rd, rn, rm, imm2 + .inst 0xce408800 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16) + .endm + + .macro sm3tt2b, rd, rn, rm, imm2 + .inst 0xce408c00 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16) + .endm + + .macro round, ab, s0, t0, t1, i + sm3ss1 v5.4s, v8.4s, \t0\().4s, v9.4s + shl \t1\().4s, \t0\().4s, #1 + sri \t1\().4s, \t0\().4s, #31 + sm3tt1\ab v8.4s, v5.4s, v10.4s, \i + sm3tt2\ab v9.4s, v5.4s, \s0\().4s, \i + .endm + + .macro qround, ab, s0, s1, s2, s3, s4 + .ifnb \s4 + ext \s4\().16b, \s1\().16b, \s2\().16b, #12 + ext v6.16b, \s0\().16b, \s1\().16b, #12 + ext v7.16b, \s2\().16b, \s3\().16b, #8 + sm3partw1 \s4\().4s, \s0\().4s, \s3\().4s + .endif + + eor v10.16b, \s0\().16b, \s1\().16b + + round \ab, \s0, v11, v12, 0 + round \ab, \s0, v12, v11, 1 + round \ab, \s0, v11, v12, 2 + round \ab, \s0, v12, v11, 3 + + .ifnb \s4 + sm3partw2 \s4\().4s, v7.4s, v6.4s + .endif + .endm + + /* + * void sm3_ce_transform(struct sm3_block_state *state, + * const u8 *data, size_t nblocks) + */ + .text +SYM_FUNC_START(sm3_ce_transform) + /* load state */ + ld1 {v8.4s-v9.4s}, [x0] + rev64 v8.4s, v8.4s + rev64 v9.4s, v9.4s + ext v8.16b, v8.16b, v8.16b, #8 + ext v9.16b, v9.16b, v9.16b, #8 + + adr_l x8, .Lt + ldp s13, s14, [x8] + + /* load input */ +0: ld1 {v0.16b-v3.16b}, [x1], #64 + sub x2, x2, #1 + + mov v15.16b, v8.16b + mov v16.16b, v9.16b + + rev32 v0.16b, v0.16b + rev32 v1.16b, v1.16b + rev32 v2.16b, v2.16b + rev32 v3.16b, v3.16b + + ext v11.16b, v13.16b, v13.16b, #4 + + qround a, v0, v1, v2, v3, v4 + qround a, v1, v2, v3, v4, v0 + qround a, v2, v3, v4, v0, v1 + qround a, v3, v4, v0, v1, v2 + + ext v11.16b, v14.16b, v14.16b, #4 + + qround b, v4, v0, v1, v2, v3 + qround b, v0, v1, v2, v3, v4 + qround b, v1, v2, v3, v4, v0 + qround b, v2, v3, v4, v0, v1 + qround b, v3, v4, v0, v1, v2 + qround b, v4, v0, v1, v2, v3 + qround b, v0, v1, v2, v3, v4 + qround b, v1, v2, v3, v4, v0 + qround b, v2, v3, v4, v0, v1 + qround b, v3, v4 + qround b, v4, v0 + qround b, v0, v1 + + eor v8.16b, v8.16b, v15.16b + eor v9.16b, v9.16b, v16.16b + + /* handled all input blocks? */ + cbnz x2, 0b + + /* save state */ + rev64 v8.4s, v8.4s + rev64 v9.4s, v9.4s + ext v8.16b, v8.16b, v8.16b, #8 + ext v9.16b, v9.16b, v9.16b, #8 + st1 {v8.4s-v9.4s}, [x0] + ret +SYM_FUNC_END(sm3_ce_transform) + + .section ".rodata", "a" + .align 3 +.Lt: .word 0x79cc4519, 0x9d8a7a87 diff --git a/lib/crypto/arm64/sm3-neon-core.S b/lib/crypto/arm64/sm3-neon-core.S new file mode 100644 index 000000000000..ad874af13802 --- /dev/null +++ b/lib/crypto/arm64/sm3-neon-core.S @@ -0,0 +1,600 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * sm3-neon-core.S - SM3 secure hash using NEON instructions + * + * Linux/arm64 port of the libgcrypt SM3 implementation for AArch64 + * + * Copyright (C) 2021 Jussi Kivilinna <jussi.kivilinna@iki.fi> + * Copyright (c) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> + +/* Context structure */ + +#define state_h0 0 +#define state_h1 4 +#define state_h2 8 +#define state_h3 12 +#define state_h4 16 +#define state_h5 20 +#define state_h6 24 +#define state_h7 28 + +/* Stack structure */ + +#define STACK_W_SIZE (32 * 2 * 3) + +#define STACK_W (0) +#define STACK_SIZE (STACK_W + STACK_W_SIZE) + +/* Register macros */ + +#define RSTATE x0 +#define RDATA x1 +#define RNBLKS x2 +#define RKPTR x28 +#define RFRAME x29 + +#define ra w3 +#define rb w4 +#define rc w5 +#define rd w6 +#define re w7 +#define rf w8 +#define rg w9 +#define rh w10 + +#define t0 w11 +#define t1 w12 +#define t2 w13 +#define t3 w14 +#define t4 w15 +#define t5 w16 +#define t6 w17 + +#define k_even w19 +#define k_odd w20 + +#define addr0 x21 +#define addr1 x22 + +#define s0 w23 +#define s1 w24 +#define s2 w25 +#define s3 w26 + +#define W0 v0 +#define W1 v1 +#define W2 v2 +#define W3 v3 +#define W4 v4 +#define W5 v5 + +#define XTMP0 v6 +#define XTMP1 v7 +#define XTMP2 v16 +#define XTMP3 v17 +#define XTMP4 v18 +#define XTMP5 v19 +#define XTMP6 v20 + +/* Helper macros. */ + +#define _(...) /*_*/ + +#define clear_vec(x) \ + movi x.8h, #0; + +#define rolw(o, a, n) \ + ror o, a, #(32 - n); + +/* Round function macros. */ + +#define GG1_1(x, y, z, o, t) \ + eor o, x, y; +#define GG1_2(x, y, z, o, t) \ + eor o, o, z; +#define GG1_3(x, y, z, o, t) + +#define FF1_1(x, y, z, o, t) GG1_1(x, y, z, o, t) +#define FF1_2(x, y, z, o, t) +#define FF1_3(x, y, z, o, t) GG1_2(x, y, z, o, t) + +#define GG2_1(x, y, z, o, t) \ + bic o, z, x; +#define GG2_2(x, y, z, o, t) \ + and t, y, x; +#define GG2_3(x, y, z, o, t) \ + eor o, o, t; + +#define FF2_1(x, y, z, o, t) \ + eor o, x, y; +#define FF2_2(x, y, z, o, t) \ + and t, x, y; \ + and o, o, z; +#define FF2_3(x, y, z, o, t) \ + eor o, o, t; + +#define R(i, a, b, c, d, e, f, g, h, k, K_LOAD, round, widx, wtype, IOP, iop_param) \ + K_LOAD(round); \ + ldr t5, [sp, #(wtype##_W1_ADDR(round, widx))]; \ + rolw(t0, a, 12); /* rol(a, 12) => t0 */ \ + IOP(1, iop_param); \ + FF##i##_1(a, b, c, t1, t2); \ + ldr t6, [sp, #(wtype##_W1W2_ADDR(round, widx))]; \ + add k, k, e; \ + IOP(2, iop_param); \ + GG##i##_1(e, f, g, t3, t4); \ + FF##i##_2(a, b, c, t1, t2); \ + IOP(3, iop_param); \ + add k, k, t0; \ + add h, h, t5; \ + add d, d, t6; /* w1w2 + d => d */ \ + IOP(4, iop_param); \ + rolw(k, k, 7); /* rol (t0 + e + t), 7) => k */ \ + GG##i##_2(e, f, g, t3, t4); \ + add h, h, k; /* h + w1 + k => h */ \ + IOP(5, iop_param); \ + FF##i##_3(a, b, c, t1, t2); \ + eor t0, t0, k; /* k ^ t0 => t0 */ \ + GG##i##_3(e, f, g, t3, t4); \ + add d, d, t1; /* FF(a,b,c) + d => d */ \ + IOP(6, iop_param); \ + add t3, t3, h; /* GG(e,f,g) + h => t3 */ \ + rolw(b, b, 9); /* rol(b, 9) => b */ \ + eor h, t3, t3, ror #(32-9); \ + IOP(7, iop_param); \ + add d, d, t0; /* t0 + d => d */ \ + rolw(f, f, 19); /* rol(f, 19) => f */ \ + IOP(8, iop_param); \ + eor h, h, t3, ror #(32-17); /* P0(t3) => h */ + +#define R1(a, b, c, d, e, f, g, h, k, K_LOAD, round, widx, wtype, IOP, iop_param) \ + R(1, ##a, ##b, ##c, ##d, ##e, ##f, ##g, ##h, ##k, K_LOAD, round, widx, wtype, IOP, iop_param) + +#define R2(a, b, c, d, e, f, g, h, k, K_LOAD, round, widx, wtype, IOP, iop_param) \ + R(2, ##a, ##b, ##c, ##d, ##e, ##f, ##g, ##h, ##k, K_LOAD, round, widx, wtype, IOP, iop_param) + +#define KL(round) \ + ldp k_even, k_odd, [RKPTR, #(4*(round))]; + +/* Input expansion macros. */ + +/* Byte-swapped input address. */ +#define IW_W_ADDR(round, widx, offs) \ + (STACK_W + ((round) / 4) * 64 + (offs) + ((widx) * 4)) + +/* Expanded input address. */ +#define XW_W_ADDR(round, widx, offs) \ + (STACK_W + ((((round) / 3) - 4) % 2) * 64 + (offs) + ((widx) * 4)) + +/* Rounds 1-12, byte-swapped input block addresses. */ +#define IW_W1_ADDR(round, widx) IW_W_ADDR(round, widx, 32) +#define IW_W1W2_ADDR(round, widx) IW_W_ADDR(round, widx, 48) + +/* Rounds 1-12, expanded input block addresses. */ +#define XW_W1_ADDR(round, widx) XW_W_ADDR(round, widx, 0) +#define XW_W1W2_ADDR(round, widx) XW_W_ADDR(round, widx, 16) + +/* Input block loading. + * Interleaving within round function needed for in-order CPUs. */ +#define LOAD_W_VEC_1_1() \ + add addr0, sp, #IW_W1_ADDR(0, 0); +#define LOAD_W_VEC_1_2() \ + add addr1, sp, #IW_W1_ADDR(4, 0); +#define LOAD_W_VEC_1_3() \ + ld1 {W0.16b}, [RDATA], #16; +#define LOAD_W_VEC_1_4() \ + ld1 {W1.16b}, [RDATA], #16; +#define LOAD_W_VEC_1_5() \ + ld1 {W2.16b}, [RDATA], #16; +#define LOAD_W_VEC_1_6() \ + ld1 {W3.16b}, [RDATA], #16; +#define LOAD_W_VEC_1_7() \ + rev32 XTMP0.16b, W0.16b; +#define LOAD_W_VEC_1_8() \ + rev32 XTMP1.16b, W1.16b; +#define LOAD_W_VEC_2_1() \ + rev32 XTMP2.16b, W2.16b; +#define LOAD_W_VEC_2_2() \ + rev32 XTMP3.16b, W3.16b; +#define LOAD_W_VEC_2_3() \ + eor XTMP4.16b, XTMP1.16b, XTMP0.16b; +#define LOAD_W_VEC_2_4() \ + eor XTMP5.16b, XTMP2.16b, XTMP1.16b; +#define LOAD_W_VEC_2_5() \ + st1 {XTMP0.16b}, [addr0], #16; +#define LOAD_W_VEC_2_6() \ + st1 {XTMP4.16b}, [addr0]; \ + add addr0, sp, #IW_W1_ADDR(8, 0); +#define LOAD_W_VEC_2_7() \ + eor XTMP6.16b, XTMP3.16b, XTMP2.16b; +#define LOAD_W_VEC_2_8() \ + ext W0.16b, XTMP0.16b, XTMP0.16b, #8; /* W0: xx, w0, xx, xx */ +#define LOAD_W_VEC_3_1() \ + mov W2.16b, XTMP1.16b; /* W2: xx, w6, w5, w4 */ +#define LOAD_W_VEC_3_2() \ + st1 {XTMP1.16b}, [addr1], #16; +#define LOAD_W_VEC_3_3() \ + st1 {XTMP5.16b}, [addr1]; \ + ext W1.16b, XTMP0.16b, XTMP0.16b, #4; /* W1: xx, w3, w2, w1 */ +#define LOAD_W_VEC_3_4() \ + ext W3.16b, XTMP1.16b, XTMP2.16b, #12; /* W3: xx, w9, w8, w7 */ +#define LOAD_W_VEC_3_5() \ + ext W4.16b, XTMP2.16b, XTMP3.16b, #8; /* W4: xx, w12, w11, w10 */ +#define LOAD_W_VEC_3_6() \ + st1 {XTMP2.16b}, [addr0], #16; +#define LOAD_W_VEC_3_7() \ + st1 {XTMP6.16b}, [addr0]; +#define LOAD_W_VEC_3_8() \ + ext W5.16b, XTMP3.16b, XTMP3.16b, #4; /* W5: xx, w15, w14, w13 */ + +#define LOAD_W_VEC_1(iop_num, ...) \ + LOAD_W_VEC_1_##iop_num() +#define LOAD_W_VEC_2(iop_num, ...) \ + LOAD_W_VEC_2_##iop_num() +#define LOAD_W_VEC_3(iop_num, ...) \ + LOAD_W_VEC_3_##iop_num() + +/* Message scheduling. Note: 3 words per vector register. + * Interleaving within round function needed for in-order CPUs. */ +#define SCHED_W_1_1(round, w0, w1, w2, w3, w4, w5) \ + /* Load (w[i - 16]) => XTMP0 */ \ + /* Load (w[i - 13]) => XTMP5 */ \ + ext XTMP0.16b, w0.16b, w0.16b, #12; /* XTMP0: w0, xx, xx, xx */ +#define SCHED_W_1_2(round, w0, w1, w2, w3, w4, w5) \ + ext XTMP5.16b, w1.16b, w1.16b, #12; +#define SCHED_W_1_3(round, w0, w1, w2, w3, w4, w5) \ + ext XTMP0.16b, XTMP0.16b, w1.16b, #12; /* XTMP0: xx, w2, w1, w0 */ +#define SCHED_W_1_4(round, w0, w1, w2, w3, w4, w5) \ + ext XTMP5.16b, XTMP5.16b, w2.16b, #12; +#define SCHED_W_1_5(round, w0, w1, w2, w3, w4, w5) \ + /* w[i - 9] == w3 */ \ + /* W3 ^ XTMP0 => XTMP0 */ \ + eor XTMP0.16b, XTMP0.16b, w3.16b; +#define SCHED_W_1_6(round, w0, w1, w2, w3, w4, w5) \ + /* w[i - 3] == w5 */ \ + /* rol(XMM5, 15) ^ XTMP0 => XTMP0 */ \ + /* rol(XTMP5, 7) => XTMP1 */ \ + add addr0, sp, #XW_W1_ADDR((round), 0); \ + shl XTMP2.4s, w5.4s, #15; +#define SCHED_W_1_7(round, w0, w1, w2, w3, w4, w5) \ + shl XTMP1.4s, XTMP5.4s, #7; +#define SCHED_W_1_8(round, w0, w1, w2, w3, w4, w5) \ + sri XTMP2.4s, w5.4s, #(32-15); +#define SCHED_W_2_1(round, w0, w1, w2, w3, w4, w5) \ + sri XTMP1.4s, XTMP5.4s, #(32-7); +#define SCHED_W_2_2(round, w0, w1, w2, w3, w4, w5) \ + eor XTMP0.16b, XTMP0.16b, XTMP2.16b; +#define SCHED_W_2_3(round, w0, w1, w2, w3, w4, w5) \ + /* w[i - 6] == W4 */ \ + /* W4 ^ XTMP1 => XTMP1 */ \ + eor XTMP1.16b, XTMP1.16b, w4.16b; +#define SCHED_W_2_4(round, w0, w1, w2, w3, w4, w5) \ + /* P1(XTMP0) ^ XTMP1 => W0 */ \ + shl XTMP3.4s, XTMP0.4s, #15; +#define SCHED_W_2_5(round, w0, w1, w2, w3, w4, w5) \ + shl XTMP4.4s, XTMP0.4s, #23; +#define SCHED_W_2_6(round, w0, w1, w2, w3, w4, w5) \ + eor w0.16b, XTMP1.16b, XTMP0.16b; +#define SCHED_W_2_7(round, w0, w1, w2, w3, w4, w5) \ + sri XTMP3.4s, XTMP0.4s, #(32-15); +#define SCHED_W_2_8(round, w0, w1, w2, w3, w4, w5) \ + sri XTMP4.4s, XTMP0.4s, #(32-23); +#define SCHED_W_3_1(round, w0, w1, w2, w3, w4, w5) \ + eor w0.16b, w0.16b, XTMP3.16b; +#define SCHED_W_3_2(round, w0, w1, w2, w3, w4, w5) \ + /* Load (w[i - 3]) => XTMP2 */ \ + ext XTMP2.16b, w4.16b, w4.16b, #12; +#define SCHED_W_3_3(round, w0, w1, w2, w3, w4, w5) \ + eor w0.16b, w0.16b, XTMP4.16b; +#define SCHED_W_3_4(round, w0, w1, w2, w3, w4, w5) \ + ext XTMP2.16b, XTMP2.16b, w5.16b, #12; +#define SCHED_W_3_5(round, w0, w1, w2, w3, w4, w5) \ + /* W1 ^ W2 => XTMP3 */ \ + eor XTMP3.16b, XTMP2.16b, w0.16b; +#define SCHED_W_3_6(round, w0, w1, w2, w3, w4, w5) +#define SCHED_W_3_7(round, w0, w1, w2, w3, w4, w5) \ + st1 {XTMP2.16b-XTMP3.16b}, [addr0]; +#define SCHED_W_3_8(round, w0, w1, w2, w3, w4, w5) + +#define SCHED_W_W0W1W2W3W4W5_1(iop_num, round) \ + SCHED_W_1_##iop_num(round, W0, W1, W2, W3, W4, W5) +#define SCHED_W_W0W1W2W3W4W5_2(iop_num, round) \ + SCHED_W_2_##iop_num(round, W0, W1, W2, W3, W4, W5) +#define SCHED_W_W0W1W2W3W4W5_3(iop_num, round) \ + SCHED_W_3_##iop_num(round, W0, W1, W2, W3, W4, W5) + +#define SCHED_W_W1W2W3W4W5W0_1(iop_num, round) \ + SCHED_W_1_##iop_num(round, W1, W2, W3, W4, W5, W0) +#define SCHED_W_W1W2W3W4W5W0_2(iop_num, round) \ + SCHED_W_2_##iop_num(round, W1, W2, W3, W4, W5, W0) +#define SCHED_W_W1W2W3W4W5W0_3(iop_num, round) \ + SCHED_W_3_##iop_num(round, W1, W2, W3, W4, W5, W0) + +#define SCHED_W_W2W3W4W5W0W1_1(iop_num, round) \ + SCHED_W_1_##iop_num(round, W2, W3, W4, W5, W0, W1) +#define SCHED_W_W2W3W4W5W0W1_2(iop_num, round) \ + SCHED_W_2_##iop_num(round, W2, W3, W4, W5, W0, W1) +#define SCHED_W_W2W3W4W5W0W1_3(iop_num, round) \ + SCHED_W_3_##iop_num(round, W2, W3, W4, W5, W0, W1) + +#define SCHED_W_W3W4W5W0W1W2_1(iop_num, round) \ + SCHED_W_1_##iop_num(round, W3, W4, W5, W0, W1, W2) +#define SCHED_W_W3W4W5W0W1W2_2(iop_num, round) \ + SCHED_W_2_##iop_num(round, W3, W4, W5, W0, W1, W2) +#define SCHED_W_W3W4W5W0W1W2_3(iop_num, round) \ + SCHED_W_3_##iop_num(round, W3, W4, W5, W0, W1, W2) + +#define SCHED_W_W4W5W0W1W2W3_1(iop_num, round) \ + SCHED_W_1_##iop_num(round, W4, W5, W0, W1, W2, W3) +#define SCHED_W_W4W5W0W1W2W3_2(iop_num, round) \ + SCHED_W_2_##iop_num(round, W4, W5, W0, W1, W2, W3) +#define SCHED_W_W4W5W0W1W2W3_3(iop_num, round) \ + SCHED_W_3_##iop_num(round, W4, W5, W0, W1, W2, W3) + +#define SCHED_W_W5W0W1W2W3W4_1(iop_num, round) \ + SCHED_W_1_##iop_num(round, W5, W0, W1, W2, W3, W4) +#define SCHED_W_W5W0W1W2W3W4_2(iop_num, round) \ + SCHED_W_2_##iop_num(round, W5, W0, W1, W2, W3, W4) +#define SCHED_W_W5W0W1W2W3W4_3(iop_num, round) \ + SCHED_W_3_##iop_num(round, W5, W0, W1, W2, W3, W4) + + + /* + * Transform nblocks*64 bytes (nblocks*16 32-bit words) at 'data'. + * + * void sm3_neon_transform(struct sm3_block_state *state, + * const u8 *data, size_t nblocks) + */ + .text +.align 3 +SYM_FUNC_START(sm3_neon_transform) + ldp ra, rb, [RSTATE, #0] + ldp rc, rd, [RSTATE, #8] + ldp re, rf, [RSTATE, #16] + ldp rg, rh, [RSTATE, #24] + + stp x28, x29, [sp, #-16]! + stp x19, x20, [sp, #-16]! + stp x21, x22, [sp, #-16]! + stp x23, x24, [sp, #-16]! + stp x25, x26, [sp, #-16]! + mov RFRAME, sp + + sub addr0, sp, #STACK_SIZE + adr_l RKPTR, .LKtable + and sp, addr0, #(~63) + + /* Preload first block. */ + LOAD_W_VEC_1(1, 0) + LOAD_W_VEC_1(2, 0) + LOAD_W_VEC_1(3, 0) + LOAD_W_VEC_1(4, 0) + LOAD_W_VEC_1(5, 0) + LOAD_W_VEC_1(6, 0) + LOAD_W_VEC_1(7, 0) + LOAD_W_VEC_1(8, 0) + LOAD_W_VEC_2(1, 0) + LOAD_W_VEC_2(2, 0) + LOAD_W_VEC_2(3, 0) + LOAD_W_VEC_2(4, 0) + LOAD_W_VEC_2(5, 0) + LOAD_W_VEC_2(6, 0) + LOAD_W_VEC_2(7, 0) + LOAD_W_VEC_2(8, 0) + LOAD_W_VEC_3(1, 0) + LOAD_W_VEC_3(2, 0) + LOAD_W_VEC_3(3, 0) + LOAD_W_VEC_3(4, 0) + LOAD_W_VEC_3(5, 0) + LOAD_W_VEC_3(6, 0) + LOAD_W_VEC_3(7, 0) + LOAD_W_VEC_3(8, 0) + +.balign 16 +.Loop: + /* Transform 0-3 */ + R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 0, 0, IW, _, 0) + R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 1, 1, IW, _, 0) + R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 2, 2, IW, _, 0) + R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 3, 3, IW, _, 0) + + /* Transform 4-7 + Precalc 12-14 */ + R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 4, 0, IW, _, 0) + R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 5, 1, IW, _, 0) + R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 6, 2, IW, SCHED_W_W0W1W2W3W4W5_1, 12) + R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 7, 3, IW, SCHED_W_W0W1W2W3W4W5_2, 12) + + /* Transform 8-11 + Precalc 12-17 */ + R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 8, 0, IW, SCHED_W_W0W1W2W3W4W5_3, 12) + R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 9, 1, IW, SCHED_W_W1W2W3W4W5W0_1, 15) + R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 10, 2, IW, SCHED_W_W1W2W3W4W5W0_2, 15) + R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 11, 3, IW, SCHED_W_W1W2W3W4W5W0_3, 15) + + /* Transform 12-14 + Precalc 18-20 */ + R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 12, 0, XW, SCHED_W_W2W3W4W5W0W1_1, 18) + R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 13, 1, XW, SCHED_W_W2W3W4W5W0W1_2, 18) + R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 14, 2, XW, SCHED_W_W2W3W4W5W0W1_3, 18) + + /* Transform 15-17 + Precalc 21-23 */ + R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 15, 0, XW, SCHED_W_W3W4W5W0W1W2_1, 21) + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 16, 1, XW, SCHED_W_W3W4W5W0W1W2_2, 21) + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 17, 2, XW, SCHED_W_W3W4W5W0W1W2_3, 21) + + /* Transform 18-20 + Precalc 24-26 */ + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 18, 0, XW, SCHED_W_W4W5W0W1W2W3_1, 24) + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 19, 1, XW, SCHED_W_W4W5W0W1W2W3_2, 24) + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 20, 2, XW, SCHED_W_W4W5W0W1W2W3_3, 24) + + /* Transform 21-23 + Precalc 27-29 */ + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 21, 0, XW, SCHED_W_W5W0W1W2W3W4_1, 27) + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 22, 1, XW, SCHED_W_W5W0W1W2W3W4_2, 27) + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 23, 2, XW, SCHED_W_W5W0W1W2W3W4_3, 27) + + /* Transform 24-26 + Precalc 30-32 */ + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 24, 0, XW, SCHED_W_W0W1W2W3W4W5_1, 30) + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 25, 1, XW, SCHED_W_W0W1W2W3W4W5_2, 30) + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 26, 2, XW, SCHED_W_W0W1W2W3W4W5_3, 30) + + /* Transform 27-29 + Precalc 33-35 */ + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 27, 0, XW, SCHED_W_W1W2W3W4W5W0_1, 33) + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 28, 1, XW, SCHED_W_W1W2W3W4W5W0_2, 33) + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 29, 2, XW, SCHED_W_W1W2W3W4W5W0_3, 33) + + /* Transform 30-32 + Precalc 36-38 */ + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 30, 0, XW, SCHED_W_W2W3W4W5W0W1_1, 36) + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 31, 1, XW, SCHED_W_W2W3W4W5W0W1_2, 36) + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 32, 2, XW, SCHED_W_W2W3W4W5W0W1_3, 36) + + /* Transform 33-35 + Precalc 39-41 */ + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 33, 0, XW, SCHED_W_W3W4W5W0W1W2_1, 39) + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 34, 1, XW, SCHED_W_W3W4W5W0W1W2_2, 39) + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 35, 2, XW, SCHED_W_W3W4W5W0W1W2_3, 39) + + /* Transform 36-38 + Precalc 42-44 */ + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 36, 0, XW, SCHED_W_W4W5W0W1W2W3_1, 42) + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 37, 1, XW, SCHED_W_W4W5W0W1W2W3_2, 42) + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 38, 2, XW, SCHED_W_W4W5W0W1W2W3_3, 42) + + /* Transform 39-41 + Precalc 45-47 */ + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 39, 0, XW, SCHED_W_W5W0W1W2W3W4_1, 45) + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 40, 1, XW, SCHED_W_W5W0W1W2W3W4_2, 45) + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 41, 2, XW, SCHED_W_W5W0W1W2W3W4_3, 45) + + /* Transform 42-44 + Precalc 48-50 */ + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 42, 0, XW, SCHED_W_W0W1W2W3W4W5_1, 48) + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 43, 1, XW, SCHED_W_W0W1W2W3W4W5_2, 48) + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 44, 2, XW, SCHED_W_W0W1W2W3W4W5_3, 48) + + /* Transform 45-47 + Precalc 51-53 */ + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 45, 0, XW, SCHED_W_W1W2W3W4W5W0_1, 51) + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 46, 1, XW, SCHED_W_W1W2W3W4W5W0_2, 51) + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 47, 2, XW, SCHED_W_W1W2W3W4W5W0_3, 51) + + /* Transform 48-50 + Precalc 54-56 */ + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 48, 0, XW, SCHED_W_W2W3W4W5W0W1_1, 54) + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 49, 1, XW, SCHED_W_W2W3W4W5W0W1_2, 54) + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 50, 2, XW, SCHED_W_W2W3W4W5W0W1_3, 54) + + /* Transform 51-53 + Precalc 57-59 */ + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 51, 0, XW, SCHED_W_W3W4W5W0W1W2_1, 57) + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 52, 1, XW, SCHED_W_W3W4W5W0W1W2_2, 57) + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 53, 2, XW, SCHED_W_W3W4W5W0W1W2_3, 57) + + /* Transform 54-56 + Precalc 60-62 */ + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 54, 0, XW, SCHED_W_W4W5W0W1W2W3_1, 60) + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 55, 1, XW, SCHED_W_W4W5W0W1W2W3_2, 60) + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 56, 2, XW, SCHED_W_W4W5W0W1W2W3_3, 60) + + /* Transform 57-59 + Precalc 63 */ + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 57, 0, XW, SCHED_W_W5W0W1W2W3W4_1, 63) + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 58, 1, XW, SCHED_W_W5W0W1W2W3W4_2, 63) + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 59, 2, XW, SCHED_W_W5W0W1W2W3W4_3, 63) + + /* Transform 60 */ + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 60, 0, XW, _, _) + subs RNBLKS, RNBLKS, #1 + b.eq .Lend + + /* Transform 61-63 + Preload next block */ + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 61, 1, XW, LOAD_W_VEC_1, _) + ldp s0, s1, [RSTATE, #0] + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 62, 2, XW, LOAD_W_VEC_2, _) + ldp s2, s3, [RSTATE, #8] + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 63, 0, XW, LOAD_W_VEC_3, _) + + /* Update the chaining variables. */ + eor ra, ra, s0 + eor rb, rb, s1 + ldp s0, s1, [RSTATE, #16] + eor rc, rc, s2 + ldp k_even, k_odd, [RSTATE, #24] + eor rd, rd, s3 + eor re, re, s0 + stp ra, rb, [RSTATE, #0] + eor rf, rf, s1 + stp rc, rd, [RSTATE, #8] + eor rg, rg, k_even + stp re, rf, [RSTATE, #16] + eor rh, rh, k_odd + stp rg, rh, [RSTATE, #24] + b .Loop + +.Lend: + /* Transform 61-63 */ + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 61, 1, XW, _, _) + ldp s0, s1, [RSTATE, #0] + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 62, 2, XW, _, _) + ldp s2, s3, [RSTATE, #8] + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 63, 0, XW, _, _) + + /* Update the chaining variables. */ + eor ra, ra, s0 + clear_vec(W0) + eor rb, rb, s1 + clear_vec(W1) + ldp s0, s1, [RSTATE, #16] + clear_vec(W2) + eor rc, rc, s2 + clear_vec(W3) + ldp k_even, k_odd, [RSTATE, #24] + clear_vec(W4) + eor rd, rd, s3 + clear_vec(W5) + eor re, re, s0 + clear_vec(XTMP0) + stp ra, rb, [RSTATE, #0] + clear_vec(XTMP1) + eor rf, rf, s1 + clear_vec(XTMP2) + stp rc, rd, [RSTATE, #8] + clear_vec(XTMP3) + eor rg, rg, k_even + clear_vec(XTMP4) + stp re, rf, [RSTATE, #16] + clear_vec(XTMP5) + eor rh, rh, k_odd + clear_vec(XTMP6) + stp rg, rh, [RSTATE, #24] + + /* Clear message expansion area */ + add addr0, sp, #STACK_W + st1 {W0.16b-W3.16b}, [addr0], #64 + st1 {W0.16b-W3.16b}, [addr0], #64 + st1 {W0.16b-W3.16b}, [addr0] + + mov sp, RFRAME + + ldp x25, x26, [sp], #16 + ldp x23, x24, [sp], #16 + ldp x21, x22, [sp], #16 + ldp x19, x20, [sp], #16 + ldp x28, x29, [sp], #16 + + ret +SYM_FUNC_END(sm3_neon_transform) + + + .section ".rodata", "a" + + .align 4 +.LKtable: + .long 0x79cc4519, 0xf3988a32, 0xe7311465, 0xce6228cb + .long 0x9cc45197, 0x3988a32f, 0x7311465e, 0xe6228cbc + .long 0xcc451979, 0x988a32f3, 0x311465e7, 0x6228cbce + .long 0xc451979c, 0x88a32f39, 0x11465e73, 0x228cbce6 + .long 0x9d8a7a87, 0x3b14f50f, 0x7629ea1e, 0xec53d43c + .long 0xd8a7a879, 0xb14f50f3, 0x629ea1e7, 0xc53d43ce + .long 0x8a7a879d, 0x14f50f3b, 0x29ea1e76, 0x53d43cec + .long 0xa7a879d8, 0x4f50f3b1, 0x9ea1e762, 0x3d43cec5 + .long 0x7a879d8a, 0xf50f3b14, 0xea1e7629, 0xd43cec53 + .long 0xa879d8a7, 0x50f3b14f, 0xa1e7629e, 0x43cec53d + .long 0x879d8a7a, 0x0f3b14f5, 0x1e7629ea, 0x3cec53d4 + .long 0x79d8a7a8, 0xf3b14f50, 0xe7629ea1, 0xcec53d43 + .long 0x9d8a7a87, 0x3b14f50f, 0x7629ea1e, 0xec53d43c + .long 0xd8a7a879, 0xb14f50f3, 0x629ea1e7, 0xc53d43ce + .long 0x8a7a879d, 0x14f50f3b, 0x29ea1e76, 0x53d43cec + .long 0xa7a879d8, 0x4f50f3b1, 0x9ea1e762, 0x3d43cec5 diff --git a/lib/crypto/arm64/sm3.h b/lib/crypto/arm64/sm3.h new file mode 100644 index 000000000000..beb9cd82bb7d --- /dev/null +++ b/lib/crypto/arm64/sm3.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SM3 optimized for ARM64 + * + * Copyright 2026 Google LLC + */ +#include <asm/simd.h> +#include <linux/cpufeature.h> + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); + +asmlinkage void sm3_neon_transform(struct sm3_block_state *state, + const u8 *data, size_t nblocks); +asmlinkage void sm3_ce_transform(struct sm3_block_state *state, + const u8 *data, size_t nblocks); + +static void sm3_blocks(struct sm3_block_state *state, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_neon) && likely(may_use_simd())) { + scoped_ksimd() { + if (static_branch_likely(&have_ce)) + sm3_ce_transform(state, data, nblocks); + else + sm3_neon_transform(state, data, nblocks); + } + } else { + sm3_blocks_generic(state, data, nblocks); + } +} + +#define sm3_mod_init_arch sm3_mod_init_arch +static void sm3_mod_init_arch(void) +{ + if (cpu_have_named_feature(ASIMD)) { + static_branch_enable(&have_neon); + if (cpu_have_named_feature(SM3)) + static_branch_enable(&have_ce); + } +} diff --git a/lib/crypto/chacha.c b/lib/crypto/chacha.c index e0c7cb4af318..86e5d382a4e0 100644 --- a/lib/crypto/chacha.c +++ b/lib/crypto/chacha.c @@ -5,8 +5,8 @@ * Copyright (C) 2015 Martin Willi */ -#include <crypto/algapi.h> // for crypto_xor_cpy #include <crypto/chacha.h> +#include <crypto/utils.h> #include <linux/export.h> #include <linux/kernel.h> #include <linux/module.h> diff --git a/lib/crypto/chacha20poly1305.c b/lib/crypto/chacha20poly1305.c index 212ce33562af..ea42a28f4ff7 100644 --- a/lib/crypto/chacha20poly1305.c +++ b/lib/crypto/chacha20poly1305.c @@ -356,20 +356,6 @@ bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len } EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace); -static int __init chacha20poly1305_init(void) -{ - if (IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) && - WARN_ON(!chacha20poly1305_selftest())) - return -ENODEV; - return 0; -} - -static void __exit chacha20poly1305_exit(void) -{ -} - -module_init(chacha20poly1305_init); -module_exit(chacha20poly1305_exit); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction"); MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); diff --git a/lib/crypto/fips.h b/lib/crypto/fips.h index 023410c2e0db..9fc49747db64 100644 --- a/lib/crypto/fips.h +++ b/lib/crypto/fips.h @@ -43,3 +43,8 @@ static const u8 fips_test_sha3_256_value[] __initconst __maybe_unused = { 0xba, 0x9b, 0xb6, 0xaa, 0x32, 0xa7, 0x97, 0x00, 0x98, 0xdb, 0xff, 0xe7, 0xc6, 0xde, 0xb5, 0x82, }; + +static const u8 fips_test_aes_cmac_value[] __initconst __maybe_unused = { + 0xc5, 0x88, 0x28, 0x55, 0xd7, 0x2c, 0x00, 0xb6, + 0x6a, 0xa7, 0xfc, 0x82, 0x90, 0x81, 0xcf, 0x18, +}; diff --git a/lib/crypto/polyval.c b/lib/crypto/gf128hash.c index 5796275f574a..2650603d8ba8 100644 --- a/lib/crypto/polyval.c +++ b/lib/crypto/gf128hash.c @@ -1,34 +1,41 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * POLYVAL library functions + * GF(2^128) polynomial hashing: GHASH and POLYVAL * * Copyright 2025 Google LLC */ -#include <crypto/polyval.h> +#include <crypto/gf128hash.h> #include <linux/export.h> #include <linux/module.h> #include <linux/string.h> #include <linux/unaligned.h> /* - * POLYVAL is an almost-XOR-universal hash function. Similar to GHASH, POLYVAL - * interprets the message as the coefficients of a polynomial in GF(2^128) and - * evaluates that polynomial at a secret point. POLYVAL has a simple - * mathematical relationship with GHASH, but it uses a better field convention - * which makes it easier and faster to implement. + * GHASH and POLYVAL are almost-XOR-universal hash functions. They interpret + * the message as the coefficients of a polynomial in the finite field GF(2^128) + * and evaluate that polynomial at a secret point. * - * POLYVAL is not a cryptographic hash function, and it should be used only by - * algorithms that are specifically designed to use it. + * Neither GHASH nor POLYVAL is a cryptographic hash function. They should be + * used only by algorithms that are specifically designed to use them. * - * POLYVAL is specified by "AES-GCM-SIV: Nonce Misuse-Resistant Authenticated - * Encryption" (https://datatracker.ietf.org/doc/html/rfc8452) + * GHASH is the older variant, defined as part of GCM in NIST SP 800-38D + * (https://nvlpubs.nist.gov/nistpubs/legacy/sp/nistspecialpublication800-38d.pdf). + * GHASH is hard to implement directly, due to its backwards mapping between + * bits and polynomial coefficients. GHASH implementations typically pre and + * post-process the inputs and outputs (mainly by byte-swapping) to convert the + * GHASH computation into an equivalent computation over a different, + * easier-to-use representation of GF(2^128). * - * POLYVAL is also used by HCTR2. See "Length-preserving encryption with HCTR2" - * (https://eprint.iacr.org/2021/1441.pdf). + * POLYVAL is a newer GF(2^128) polynomial hash, originally defined as part of + * AES-GCM-SIV (https://datatracker.ietf.org/doc/html/rfc8452) and also used by + * HCTR2 (https://eprint.iacr.org/2021/1441.pdf). It uses that easier-to-use + * field representation directly, eliminating the data conversion steps. * - * This file provides a library API for POLYVAL. This API can delegate to - * either a generic implementation or an architecture-optimized implementation. + * This file provides library APIs for GHASH and POLYVAL. These APIs can + * delegate to either a generic implementation or an architecture-optimized + * implementation. Due to the mathematical relationship between GHASH and + * POLYVAL, in some cases code for one is reused with the other. * * For the generic implementation, we don't use the traditional table approach * to GF(2^128) multiplication. That approach is not constant-time and requires @@ -205,6 +212,19 @@ polyval_mul_generic(struct polyval_elem *a, const struct polyval_elem *b) a->hi = cpu_to_le64(c3); } +static void __maybe_unused ghash_blocks_generic(struct polyval_elem *acc, + const struct polyval_elem *key, + const u8 *data, size_t nblocks) +{ + do { + acc->lo ^= + cpu_to_le64(get_unaligned_be64((__be64 *)(data + 8))); + acc->hi ^= cpu_to_le64(get_unaligned_be64((__be64 *)data)); + polyval_mul_generic(acc, key); + data += GHASH_BLOCK_SIZE; + } while (--nblocks); +} + static void __maybe_unused polyval_blocks_generic(struct polyval_elem *acc, const struct polyval_elem *key, const u8 *data, size_t nblocks) @@ -217,16 +237,118 @@ polyval_blocks_generic(struct polyval_elem *acc, const struct polyval_elem *key, } while (--nblocks); } -/* Include the arch-optimized implementation of POLYVAL, if one is available. */ -#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH -#include "polyval.h" /* $(SRCARCH)/polyval.h */ +/* Convert the key from GHASH format to POLYVAL format. */ +static void __maybe_unused ghash_key_to_polyval(const u8 in[GHASH_BLOCK_SIZE], + struct polyval_elem *out) +{ + u64 hi = get_unaligned_be64(&in[0]); + u64 lo = get_unaligned_be64(&in[8]); + u64 mask = (s64)hi >> 63; + + hi = (hi << 1) ^ (lo >> 63) ^ (mask & ((u64)0xc2 << 56)); + lo = (lo << 1) ^ (mask & 1); + out->lo = cpu_to_le64(lo); + out->hi = cpu_to_le64(hi); +} + +/* Convert the accumulator from POLYVAL format to GHASH format. */ +static void polyval_acc_to_ghash(const struct polyval_elem *in, + u8 out[GHASH_BLOCK_SIZE]) +{ + put_unaligned_be64(le64_to_cpu(in->hi), &out[0]); + put_unaligned_be64(le64_to_cpu(in->lo), &out[8]); +} + +/* Convert the accumulator from GHASH format to POLYVAL format. */ +static void __maybe_unused ghash_acc_to_polyval(const u8 in[GHASH_BLOCK_SIZE], + struct polyval_elem *out) +{ + out->lo = cpu_to_le64(get_unaligned_be64(&in[8])); + out->hi = cpu_to_le64(get_unaligned_be64(&in[0])); +} + +#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH +#include "gf128hash.h" /* $(SRCARCH)/gf128hash.h */ +#endif + +void ghash_preparekey(struct ghash_key *key, const u8 raw_key[GHASH_BLOCK_SIZE]) +{ +#ifdef ghash_preparekey_arch + ghash_preparekey_arch(key, raw_key); +#else + ghash_key_to_polyval(raw_key, &key->h); +#endif +} +EXPORT_SYMBOL_GPL(ghash_preparekey); + +static void ghash_mul(struct ghash_ctx *ctx) +{ +#ifdef ghash_mul_arch + ghash_mul_arch(&ctx->acc, ctx->key); +#elif defined(ghash_blocks_arch) + static const u8 zeroes[GHASH_BLOCK_SIZE]; + + ghash_blocks_arch(&ctx->acc, ctx->key, zeroes, 1); +#else + polyval_mul_generic(&ctx->acc, &ctx->key->h); +#endif +} + +/* nblocks is always >= 1. */ +static void ghash_blocks(struct ghash_ctx *ctx, const u8 *data, size_t nblocks) +{ +#ifdef ghash_blocks_arch + ghash_blocks_arch(&ctx->acc, ctx->key, data, nblocks); +#else + ghash_blocks_generic(&ctx->acc, &ctx->key->h, data, nblocks); +#endif +} + +void ghash_update(struct ghash_ctx *ctx, const u8 *data, size_t len) +{ + if (unlikely(ctx->partial)) { + size_t n = min(len, GHASH_BLOCK_SIZE - ctx->partial); + + len -= n; + while (n--) + ctx->acc.bytes[GHASH_BLOCK_SIZE - 1 - ctx->partial++] ^= + *data++; + if (ctx->partial < GHASH_BLOCK_SIZE) + return; + ghash_mul(ctx); + } + if (len >= GHASH_BLOCK_SIZE) { + size_t nblocks = len / GHASH_BLOCK_SIZE; + + ghash_blocks(ctx, data, nblocks); + data += len & ~(GHASH_BLOCK_SIZE - 1); + len &= GHASH_BLOCK_SIZE - 1; + } + for (size_t i = 0; i < len; i++) + ctx->acc.bytes[GHASH_BLOCK_SIZE - 1 - i] ^= data[i]; + ctx->partial = len; +} +EXPORT_SYMBOL_GPL(ghash_update); + +void ghash_final(struct ghash_ctx *ctx, u8 out[GHASH_BLOCK_SIZE]) +{ + if (unlikely(ctx->partial)) + ghash_mul(ctx); + polyval_acc_to_ghash(&ctx->acc, out); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(ghash_final); + void polyval_preparekey(struct polyval_key *key, const u8 raw_key[POLYVAL_BLOCK_SIZE]) { +#ifdef polyval_preparekey_arch polyval_preparekey_arch(key, raw_key); +#else + memcpy(key->h.bytes, raw_key, POLYVAL_BLOCK_SIZE); +#endif } EXPORT_SYMBOL_GPL(polyval_preparekey); -#endif /* Else, polyval_preparekey() is an inline function. */ /* * polyval_mul_generic() and polyval_blocks_generic() take the key as a @@ -238,17 +360,22 @@ EXPORT_SYMBOL_GPL(polyval_preparekey); static void polyval_mul(struct polyval_ctx *ctx) { -#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH +#ifdef polyval_mul_arch polyval_mul_arch(&ctx->acc, ctx->key); +#elif defined(polyval_blocks_arch) + static const u8 zeroes[POLYVAL_BLOCK_SIZE]; + + polyval_blocks_arch(&ctx->acc, ctx->key, zeroes, 1); #else polyval_mul_generic(&ctx->acc, &ctx->key->h); #endif } +/* nblocks is always >= 1. */ static void polyval_blocks(struct polyval_ctx *ctx, const u8 *data, size_t nblocks) { -#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH +#ifdef polyval_blocks_arch polyval_blocks_arch(&ctx->acc, ctx->key, data, nblocks); #else polyval_blocks_generic(&ctx->acc, &ctx->key->h, data, nblocks); @@ -289,19 +416,19 @@ void polyval_final(struct polyval_ctx *ctx, u8 out[POLYVAL_BLOCK_SIZE]) } EXPORT_SYMBOL_GPL(polyval_final); -#ifdef polyval_mod_init_arch -static int __init polyval_mod_init(void) +#ifdef gf128hash_mod_init_arch +static int __init gf128hash_mod_init(void) { - polyval_mod_init_arch(); + gf128hash_mod_init_arch(); return 0; } -subsys_initcall(polyval_mod_init); +subsys_initcall(gf128hash_mod_init); -static void __exit polyval_mod_exit(void) +static void __exit gf128hash_mod_exit(void) { } -module_exit(polyval_mod_exit); +module_exit(gf128hash_mod_exit); #endif -MODULE_DESCRIPTION("POLYVAL almost-XOR-universal hash function"); +MODULE_DESCRIPTION("GF(2^128) polynomial hashing: GHASH and POLYVAL"); MODULE_LICENSE("GPL"); diff --git a/lib/crypto/gf128mul.c b/lib/crypto/gf128mul.c index e5a727b15f07..7ebf07ce1168 100644 --- a/lib/crypto/gf128mul.c +++ b/lib/crypto/gf128mul.c @@ -127,27 +127,9 @@ (i & 0x02 ? 0x0384 : 0) ^ (i & 0x01 ? 0x01c2 : 0) \ ) -static const u16 gf128mul_table_le[256] = gf128mul_dat(xda_le); static const u16 gf128mul_table_be[256] = gf128mul_dat(xda_be); -/* - * The following functions multiply a field element by x^8 in - * the polynomial field representation. They use 64-bit word operations - * to gain speed but compensate for machine endianness and hence work - * correctly on both styles of machine. - */ - -static void gf128mul_x8_lle(be128 *x) -{ - u64 a = be64_to_cpu(x->a); - u64 b = be64_to_cpu(x->b); - u64 _tt = gf128mul_table_le[b & 0xff]; - - x->b = cpu_to_be64((b >> 8) | (a << 56)); - x->a = cpu_to_be64((a >> 8) ^ (_tt << 48)); -} - -/* time invariant version of gf128mul_x8_lle */ +/* A table-less implementation of multiplying by x^8 */ static void gf128mul_x8_lle_ti(be128 *x) { u64 a = be64_to_cpu(x->a); @@ -305,58 +287,5 @@ void gf128mul_64k_bbe(be128 *a, const struct gf128mul_64k *t) } EXPORT_SYMBOL(gf128mul_64k_bbe); -/* This version uses 4k bytes of table space. - A 16 byte buffer has to be multiplied by a 16 byte key - value in GF(2^128). If we consider a GF(2^128) value in a - single byte, we can construct a table of the 256 16 byte - values that result from the 256 values of this byte. - This requires 4096 bytes. If we take the highest byte in - the buffer and use this table to get the result, we then - have to multiply by x^120 to get the final value. For the - next highest byte the result has to be multiplied by x^112 - and so on. But we can do this by accumulating the result - in an accumulator starting with the result for the top - byte. We repeatedly multiply the accumulator value by - x^8 and then add in (i.e. xor) the 16 bytes of the next - lower byte in the buffer, stopping when we reach the - lowest byte. This requires a 4096 byte table. -*/ -struct gf128mul_4k *gf128mul_init_4k_lle(const be128 *g) -{ - struct gf128mul_4k *t; - int j, k; - - t = kzalloc_obj(*t); - if (!t) - goto out; - - t->t[128] = *g; - for (j = 64; j > 0; j >>= 1) - gf128mul_x_lle(&t->t[j], &t->t[j+j]); - - for (j = 2; j < 256; j += j) - for (k = 1; k < j; ++k) - be128_xor(&t->t[j + k], &t->t[j], &t->t[k]); - -out: - return t; -} -EXPORT_SYMBOL(gf128mul_init_4k_lle); - -void gf128mul_4k_lle(be128 *a, const struct gf128mul_4k *t) -{ - u8 *ap = (u8 *)a; - be128 r[1]; - int i = 15; - - *r = t->t[ap[15]]; - while (i--) { - gf128mul_x8_lle(r); - be128_xor(r, r, &t->t[ap[i]]); - } - *a = *r; -} -EXPORT_SYMBOL(gf128mul_4k_lle); - MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Functions for multiplying elements of GF(2^128)"); diff --git a/lib/crypto/memneq.c b/lib/crypto/memneq.c index 44daacb8cb51..08924acd44bc 100644 --- a/lib/crypto/memneq.c +++ b/lib/crypto/memneq.c @@ -59,7 +59,7 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include <crypto/algapi.h> +#include <crypto/utils.h> #include <linux/export.h> #include <linux/module.h> #include <linux/unaligned.h> @@ -159,7 +159,7 @@ static inline unsigned long __crypto_memneq_16(const void *a, const void *b) /* Compare two areas of memory without leaking timing information, * and with special optimizations for common sizes. Users should * not call this function directly, but should instead use - * crypto_memneq defined in crypto/algapi.h. + * crypto_memneq defined in crypto/utils.h. */ noinline unsigned long __crypto_memneq(const void *a, const void *b, size_t size) diff --git a/lib/crypto/mips/md5.h b/lib/crypto/mips/md5.h deleted file mode 100644 index e08e28aeffa4..000000000000 --- a/lib/crypto/mips/md5.h +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Cryptographic API. - * - * MD5 Message Digest Algorithm (RFC1321). - * - * Adapted for OCTEON by Aaro Koskinen <aaro.koskinen@iki.fi>. - * - * Based on crypto/md5.c, which is: - * - * Derived from cryptoapi implementation, originally based on the - * public domain implementation written by Colin Plumb in 1993. - * - * Copyright (c) Cryptoapi developers. - * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) - * any later version. - */ - -#include <asm/octeon/crypto.h> -#include <asm/octeon/octeon.h> - -/* - * We pass everything as 64-bit. OCTEON can handle misaligned data. - */ - -static void md5_blocks(struct md5_block_state *state, - const u8 *data, size_t nblocks) -{ - struct octeon_cop2_state cop2_state; - u64 *state64 = (u64 *)state; - unsigned long flags; - - if (!octeon_has_crypto()) - return md5_blocks_generic(state, data, nblocks); - - cpu_to_le32_array(state->h, ARRAY_SIZE(state->h)); - - flags = octeon_crypto_enable(&cop2_state); - write_octeon_64bit_hash_dword(state64[0], 0); - write_octeon_64bit_hash_dword(state64[1], 1); - - do { - const u64 *block = (const u64 *)data; - - write_octeon_64bit_block_dword(block[0], 0); - write_octeon_64bit_block_dword(block[1], 1); - write_octeon_64bit_block_dword(block[2], 2); - write_octeon_64bit_block_dword(block[3], 3); - write_octeon_64bit_block_dword(block[4], 4); - write_octeon_64bit_block_dword(block[5], 5); - write_octeon_64bit_block_dword(block[6], 6); - octeon_md5_start(block[7]); - - data += MD5_BLOCK_SIZE; - } while (--nblocks); - - state64[0] = read_octeon_64bit_hash_dword(0); - state64[1] = read_octeon_64bit_hash_dword(1); - octeon_crypto_disable(&cop2_state, flags); - - le32_to_cpu_array(state->h, ARRAY_SIZE(state->h)); -} diff --git a/lib/crypto/powerpc/.gitignore b/lib/crypto/powerpc/.gitignore index 598ca7aff6b1..7aa71d83f739 100644 --- a/lib/crypto/powerpc/.gitignore +++ b/lib/crypto/powerpc/.gitignore @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only aesp8-ppc.S +ghashp8-ppc.S diff --git a/lib/crypto/powerpc/gf128hash.h b/lib/crypto/powerpc/gf128hash.h new file mode 100644 index 000000000000..629cd325d0c7 --- /dev/null +++ b/lib/crypto/powerpc/gf128hash.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * GHASH routines supporting VMX instructions on the Power 8 + * + * Copyright (C) 2015, 2019 International Business Machines Inc. + * Copyright (C) 2014 - 2018 Linaro Ltd. + * Copyright 2026 Google LLC + */ + +#include <asm/simd.h> +#include <asm/switch_to.h> +#include <linux/cpufeature.h> +#include <linux/jump_label.h> +#include <linux/preempt.h> +#include <linux/uaccess.h> + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto); + +void gcm_init_p8(u64 htable[4][2], const u8 h[16]); +void gcm_gmult_p8(u8 Xi[16], const u64 htable[4][2]); +void gcm_ghash_p8(u8 Xi[16], const u64 htable[4][2], const u8 *in, size_t len); + +#define ghash_preparekey_arch ghash_preparekey_arch +static void ghash_preparekey_arch(struct ghash_key *key, + const u8 raw_key[GHASH_BLOCK_SIZE]) +{ + ghash_key_to_polyval(raw_key, &key->h); + + if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) { + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + gcm_init_p8(key->htable, raw_key); + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + } else { + /* This reproduces gcm_init_p8() on both LE and BE systems. */ + key->htable[0][0] = 0; + key->htable[0][1] = 0xc200000000000000; + + key->htable[1][0] = 0; + key->htable[1][1] = le64_to_cpu(key->h.lo); + + key->htable[2][0] = le64_to_cpu(key->h.lo); + key->htable[2][1] = le64_to_cpu(key->h.hi); + + key->htable[3][0] = le64_to_cpu(key->h.hi); + key->htable[3][1] = 0; + } +} + +#define ghash_mul_arch ghash_mul_arch +static void ghash_mul_arch(struct polyval_elem *acc, + const struct ghash_key *key) +{ + if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) { + u8 ghash_acc[GHASH_BLOCK_SIZE]; + + polyval_acc_to_ghash(acc, ghash_acc); + + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + gcm_gmult_p8(ghash_acc, key->htable); + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + + ghash_acc_to_polyval(ghash_acc, acc); + memzero_explicit(ghash_acc, sizeof(ghash_acc)); + } else { + polyval_mul_generic(acc, &key->h); + } +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) { + u8 ghash_acc[GHASH_BLOCK_SIZE]; + + polyval_acc_to_ghash(acc, ghash_acc); + + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + gcm_ghash_p8(ghash_acc, key->htable, data, + nblocks * GHASH_BLOCK_SIZE); + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + + ghash_acc_to_polyval(ghash_acc, acc); + memzero_explicit(ghash_acc, sizeof(ghash_acc)); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (cpu_has_feature(CPU_FTR_ARCH_207S) && + (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO)) + static_branch_enable(&have_vec_crypto); +} diff --git a/lib/crypto/powerpc/ghashp8-ppc.pl b/lib/crypto/powerpc/ghashp8-ppc.pl new file mode 100644 index 000000000000..7c38eedc02cc --- /dev/null +++ b/lib/crypto/powerpc/ghashp8-ppc.pl @@ -0,0 +1,244 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 + +# This code is taken from the OpenSSL project but the author (Andy Polyakov) +# has relicensed it under the GPLv2. Therefore this program is free software; +# you can redistribute it and/or modify it under the terms of the GNU General +# Public License version 2 as published by the Free Software Foundation. +# +# The original headers, including the original license headers, are +# included below for completeness. + +# ==================================================================== +# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see https://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# GHASH for PowerISA v2.07. +# +# July 2014 +# +# Accurate performance measurements are problematic, because it's +# always virtualized setup with possibly throttled processor. +# Relative comparison is therefore more informative. This initial +# version is ~2.1x slower than hardware-assisted AES-128-CTR, ~12x +# faster than "4-bit" integer-only compiler-generated 64-bit code. +# "Initial version" means that there is room for futher improvement. + +$flavour=shift; +$output =shift; + +if ($flavour =~ /64/) { + $SIZE_T=8; + $LRSAVE=2*$SIZE_T; + $STU="stdu"; + $POP="ld"; + $PUSH="std"; +} elsif ($flavour =~ /32/) { + $SIZE_T=4; + $LRSAVE=$SIZE_T; + $STU="stwu"; + $POP="lwz"; + $PUSH="stw"; +} else { die "nonsense $flavour"; } + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or +( $xlate="${dir}../../../arch/powerpc/crypto/ppc-xlate.pl" and -f $xlate) or +die "can't locate ppc-xlate.pl"; + +open STDOUT,"| $^X $xlate $flavour $output" || die "can't call $xlate: $!"; + +my ($Xip,$Htbl,$inp,$len)=map("r$_",(3..6)); # argument block + +my ($Xl,$Xm,$Xh,$IN)=map("v$_",(0..3)); +my ($zero,$t0,$t1,$t2,$xC2,$H,$Hh,$Hl,$lemask)=map("v$_",(4..12)); +my $vrsave="r12"; + +$code=<<___; +.machine "any" + +.text + +.globl .gcm_init_p8 + lis r0,0xfff0 + li r8,0x10 + mfspr $vrsave,256 + li r9,0x20 + mtspr 256,r0 + li r10,0x30 + lvx_u $H,0,r4 # load H + le?xor r7,r7,r7 + le?addi r7,r7,0x8 # need a vperm start with 08 + le?lvsr 5,0,r7 + le?vspltisb 6,0x0f + le?vxor 5,5,6 # set a b-endian mask + le?vperm $H,$H,$H,5 + + vspltisb $xC2,-16 # 0xf0 + vspltisb $t0,1 # one + vaddubm $xC2,$xC2,$xC2 # 0xe0 + vxor $zero,$zero,$zero + vor $xC2,$xC2,$t0 # 0xe1 + vsldoi $xC2,$xC2,$zero,15 # 0xe1... + vsldoi $t1,$zero,$t0,1 # ...1 + vaddubm $xC2,$xC2,$xC2 # 0xc2... + vspltisb $t2,7 + vor $xC2,$xC2,$t1 # 0xc2....01 + vspltb $t1,$H,0 # most significant byte + vsl $H,$H,$t0 # H<<=1 + vsrab $t1,$t1,$t2 # broadcast carry bit + vand $t1,$t1,$xC2 + vxor $H,$H,$t1 # twisted H + + vsldoi $H,$H,$H,8 # twist even more ... + vsldoi $xC2,$zero,$xC2,8 # 0xc2.0 + vsldoi $Hl,$zero,$H,8 # ... and split + vsldoi $Hh,$H,$zero,8 + + stvx_u $xC2,0,r3 # save pre-computed table + stvx_u $Hl,r8,r3 + stvx_u $H, r9,r3 + stvx_u $Hh,r10,r3 + + mtspr 256,$vrsave + blr + .long 0 + .byte 0,12,0x14,0,0,0,2,0 + .long 0 +.size .gcm_init_p8,.-.gcm_init_p8 + +.globl .gcm_gmult_p8 + lis r0,0xfff8 + li r8,0x10 + mfspr $vrsave,256 + li r9,0x20 + mtspr 256,r0 + li r10,0x30 + lvx_u $IN,0,$Xip # load Xi + + lvx_u $Hl,r8,$Htbl # load pre-computed table + le?lvsl $lemask,r0,r0 + lvx_u $H, r9,$Htbl + le?vspltisb $t0,0x07 + lvx_u $Hh,r10,$Htbl + le?vxor $lemask,$lemask,$t0 + lvx_u $xC2,0,$Htbl + le?vperm $IN,$IN,$IN,$lemask + vxor $zero,$zero,$zero + + vpmsumd $Xl,$IN,$Hl # H.lo·Xi.lo + vpmsumd $Xm,$IN,$H # H.hi·Xi.lo+H.lo·Xi.hi + vpmsumd $Xh,$IN,$Hh # H.hi·Xi.hi + + vpmsumd $t2,$Xl,$xC2 # 1st phase + + vsldoi $t0,$Xm,$zero,8 + vsldoi $t1,$zero,$Xm,8 + vxor $Xl,$Xl,$t0 + vxor $Xh,$Xh,$t1 + + vsldoi $Xl,$Xl,$Xl,8 + vxor $Xl,$Xl,$t2 + + vsldoi $t1,$Xl,$Xl,8 # 2nd phase + vpmsumd $Xl,$Xl,$xC2 + vxor $t1,$t1,$Xh + vxor $Xl,$Xl,$t1 + + le?vperm $Xl,$Xl,$Xl,$lemask + stvx_u $Xl,0,$Xip # write out Xi + + mtspr 256,$vrsave + blr + .long 0 + .byte 0,12,0x14,0,0,0,2,0 + .long 0 +.size .gcm_gmult_p8,.-.gcm_gmult_p8 + +.globl .gcm_ghash_p8 + lis r0,0xfff8 + li r8,0x10 + mfspr $vrsave,256 + li r9,0x20 + mtspr 256,r0 + li r10,0x30 + lvx_u $Xl,0,$Xip # load Xi + + lvx_u $Hl,r8,$Htbl # load pre-computed table + le?lvsl $lemask,r0,r0 + lvx_u $H, r9,$Htbl + le?vspltisb $t0,0x07 + lvx_u $Hh,r10,$Htbl + le?vxor $lemask,$lemask,$t0 + lvx_u $xC2,0,$Htbl + le?vperm $Xl,$Xl,$Xl,$lemask + vxor $zero,$zero,$zero + + lvx_u $IN,0,$inp + addi $inp,$inp,16 + subi $len,$len,16 + le?vperm $IN,$IN,$IN,$lemask + vxor $IN,$IN,$Xl + b Loop + +.align 5 +Loop: + subic $len,$len,16 + vpmsumd $Xl,$IN,$Hl # H.lo·Xi.lo + subfe. r0,r0,r0 # borrow?-1:0 + vpmsumd $Xm,$IN,$H # H.hi·Xi.lo+H.lo·Xi.hi + and r0,r0,$len + vpmsumd $Xh,$IN,$Hh # H.hi·Xi.hi + add $inp,$inp,r0 + + vpmsumd $t2,$Xl,$xC2 # 1st phase + + vsldoi $t0,$Xm,$zero,8 + vsldoi $t1,$zero,$Xm,8 + vxor $Xl,$Xl,$t0 + vxor $Xh,$Xh,$t1 + + vsldoi $Xl,$Xl,$Xl,8 + vxor $Xl,$Xl,$t2 + lvx_u $IN,0,$inp + addi $inp,$inp,16 + + vsldoi $t1,$Xl,$Xl,8 # 2nd phase + vpmsumd $Xl,$Xl,$xC2 + le?vperm $IN,$IN,$IN,$lemask + vxor $t1,$t1,$Xh + vxor $IN,$IN,$t1 + vxor $IN,$IN,$Xl + beq Loop # did $len-=16 borrow? + + vxor $Xl,$Xl,$t1 + le?vperm $Xl,$Xl,$Xl,$lemask + stvx_u $Xl,0,$Xip # write out Xi + + mtspr 256,$vrsave + blr + .long 0 + .byte 0,12,0x14,0,0,0,4,0 + .long 0 +.size .gcm_ghash_p8,.-.gcm_ghash_p8 + +.asciz "GHASH for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>" +.align 2 +___ + +foreach (split("\n",$code)) { + if ($flavour =~ /le$/o) { # little-endian + s/le\?//o or + s/be\?/#be#/o; + } else { + s/le\?/#le#/o or + s/be\?//o; + } + print $_,"\n"; +} + +close STDOUT; # enforce flush diff --git a/lib/crypto/riscv/gf128hash.h b/lib/crypto/riscv/gf128hash.h new file mode 100644 index 000000000000..4301a0384f60 --- /dev/null +++ b/lib/crypto/riscv/gf128hash.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * GHASH, RISC-V optimized + * + * Copyright (C) 2023 VRULL GmbH + * Copyright (C) 2023 SiFive, Inc. + * Copyright 2026 Google LLC + */ + +#include <asm/simd.h> +#include <asm/vector.h> + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkg); + +asmlinkage void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE], + const u8 key[GHASH_BLOCK_SIZE], + const u8 *data, size_t nblocks); + +#define ghash_preparekey_arch ghash_preparekey_arch +static void ghash_preparekey_arch(struct ghash_key *key, + const u8 raw_key[GHASH_BLOCK_SIZE]) +{ + /* Save key in POLYVAL format for fallback */ + ghash_key_to_polyval(raw_key, &key->h); + + /* Save key in GHASH format for zvkg */ + memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE); +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_zvkg) && likely(may_use_simd())) { + u8 ghash_acc[GHASH_BLOCK_SIZE]; + + polyval_acc_to_ghash(acc, ghash_acc); + + kernel_vector_begin(); + ghash_zvkg(ghash_acc, key->h_raw, data, nblocks); + kernel_vector_end(); + + ghash_acc_to_polyval(ghash_acc, acc); + memzero_explicit(ghash_acc, sizeof(ghash_acc)); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (riscv_isa_extension_available(NULL, ZVKG) && + riscv_vector_vlen() >= 128) + static_branch_enable(&have_zvkg); +} diff --git a/lib/crypto/riscv/ghash-riscv64-zvkg.S b/lib/crypto/riscv/ghash-riscv64-zvkg.S new file mode 100644 index 000000000000..6a2a2f2bc7c8 --- /dev/null +++ b/lib/crypto/riscv/ghash-riscv64-zvkg.S @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */ +// +// This file is dual-licensed, meaning that you can use it under your +// choice of either of the following two licenses: +// +// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. +// +// Licensed under the Apache License 2.0 (the "License"). You can obtain +// a copy in the file LICENSE in the source distribution or at +// https://www.openssl.org/source/license.html +// +// or +// +// Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu> +// Copyright (c) 2023, Jerry Shih <jerry.shih@sifive.com> +// Copyright 2024 Google LLC +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// The generated code of this file depends on the following RISC-V extensions: +// - RV64I +// - RISC-V Vector ('V') with VLEN >= 128 +// - RISC-V Vector GCM/GMAC extension ('Zvkg') + +#include <linux/linkage.h> + +.text +.option arch, +zvkg + +#define ACCUMULATOR a0 +#define KEY a1 +#define DATA a2 +#define NBLOCKS a3 + +// void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE], +// const u8 key[GHASH_BLOCK_SIZE], +// const u8 *data, size_t nblocks); +// +// |nblocks| must be nonzero. +SYM_FUNC_START(ghash_zvkg) + vsetivli zero, 4, e32, m1, ta, ma + vle32.v v1, (ACCUMULATOR) + vle32.v v2, (KEY) +.Lnext_block: + vle32.v v3, (DATA) + vghsh.vv v1, v2, v3 + addi DATA, DATA, 16 + addi NBLOCKS, NBLOCKS, -1 + bnez NBLOCKS, .Lnext_block + + vse32.v v1, (ACCUMULATOR) + ret +SYM_FUNC_END(ghash_zvkg) diff --git a/lib/crypto/riscv/sm3-riscv64-zvksh-zvkb.S b/lib/crypto/riscv/sm3-riscv64-zvksh-zvkb.S new file mode 100644 index 000000000000..a1d4468b0485 --- /dev/null +++ b/lib/crypto/riscv/sm3-riscv64-zvksh-zvkb.S @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */ +// +// This file is dual-licensed, meaning that you can use it under your +// choice of either of the following two licenses: +// +// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. +// +// Licensed under the Apache License 2.0 (the "License"). You can obtain +// a copy in the file LICENSE in the source distribution or at +// https://www.openssl.org/source/license.html +// +// or +// +// Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu> +// Copyright (c) 2023, Jerry Shih <jerry.shih@sifive.com> +// Copyright 2024 Google LLC +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// The generated code of this file depends on the following RISC-V extensions: +// - RV64I +// - RISC-V Vector ('V') with VLEN >= 128 +// - RISC-V Vector SM3 Secure Hash extension ('Zvksh') +// - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb') + +#include <linux/linkage.h> + +.text +.option arch, +zvksh, +zvkb + +#define STATEP a0 +#define DATA a1 +#define NUM_BLOCKS a2 + +#define STATE v0 // LMUL=2 +#define PREV_STATE v2 // LMUL=2 +#define W0 v4 // LMUL=2 +#define W1 v6 // LMUL=2 +#define VTMP v8 // LMUL=2 + +.macro sm3_8rounds i, w0, w1 + // Do 4 rounds using W_{0+i}..W_{7+i}. + vsm3c.vi STATE, \w0, \i + 0 + vslidedown.vi VTMP, \w0, 2 + vsm3c.vi STATE, VTMP, \i + 1 + + // Compute W_{4+i}..W_{11+i}. + vslidedown.vi VTMP, \w0, 4 + vslideup.vi VTMP, \w1, 4 + + // Do 4 rounds using W_{4+i}..W_{11+i}. + vsm3c.vi STATE, VTMP, \i + 2 + vslidedown.vi VTMP, VTMP, 2 + vsm3c.vi STATE, VTMP, \i + 3 + +.if \i < 28 + // Compute W_{16+i}..W_{23+i}. + vsm3me.vv \w0, \w1, \w0 +.endif + // For the next 8 rounds, w0 and w1 are swapped. +.endm + +// void sm3_transform_zvksh_zvkb(struct sm3_block_state *state, +// const u8 *data, size_t nblocks); +SYM_FUNC_START(sm3_transform_zvksh_zvkb) + + // Load the state and endian-swap each 32-bit word. + vsetivli zero, 8, e32, m2, ta, ma + vle32.v STATE, (STATEP) + vrev8.v STATE, STATE + +.Lnext_block: + addi NUM_BLOCKS, NUM_BLOCKS, -1 + + // Save the previous state, as it's needed later. + vmv.v.v PREV_STATE, STATE + + // Load the next 512-bit message block into W0-W1. + vle32.v W0, (DATA) + addi DATA, DATA, 32 + vle32.v W1, (DATA) + addi DATA, DATA, 32 + + // Do the 64 rounds of SM3. + sm3_8rounds 0, W0, W1 + sm3_8rounds 4, W1, W0 + sm3_8rounds 8, W0, W1 + sm3_8rounds 12, W1, W0 + sm3_8rounds 16, W0, W1 + sm3_8rounds 20, W1, W0 + sm3_8rounds 24, W0, W1 + sm3_8rounds 28, W1, W0 + + // XOR in the previous state. + vxor.vv STATE, STATE, PREV_STATE + + // Repeat if more blocks remain. + bnez NUM_BLOCKS, .Lnext_block + + // Store the new state and return. + vrev8.v STATE, STATE + vse32.v STATE, (STATEP) + ret +SYM_FUNC_END(sm3_transform_zvksh_zvkb) diff --git a/lib/crypto/riscv/sm3.h b/lib/crypto/riscv/sm3.h new file mode 100644 index 000000000000..c1fbee7094e6 --- /dev/null +++ b/lib/crypto/riscv/sm3.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SM3 using the RISC-V vector crypto extensions + * + * Copyright (C) 2023 VRULL GmbH + * Author: Heiko Stuebner <heiko.stuebner@vrull.eu> + * + * Copyright (C) 2023 SiFive, Inc. + * Author: Jerry Shih <jerry.shih@sifive.com> + */ + +#include <asm/simd.h> +#include <asm/vector.h> + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions); + +asmlinkage void sm3_transform_zvksh_zvkb(struct sm3_block_state *state, + const u8 *data, size_t nblocks); + +static void sm3_blocks(struct sm3_block_state *state, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_extensions) && likely(may_use_simd())) { + kernel_vector_begin(); + sm3_transform_zvksh_zvkb(state, data, nblocks); + kernel_vector_end(); + } else { + sm3_blocks_generic(state, data, nblocks); + } +} + +#define sm3_mod_init_arch sm3_mod_init_arch +static void sm3_mod_init_arch(void) +{ + if (riscv_isa_extension_available(NULL, ZVKSH) && + riscv_isa_extension_available(NULL, ZVKB) && + riscv_vector_vlen() >= 128) + static_branch_enable(&have_extensions); +} diff --git a/lib/crypto/s390/gf128hash.h b/lib/crypto/s390/gf128hash.h new file mode 100644 index 000000000000..1e46ce4bca40 --- /dev/null +++ b/lib/crypto/s390/gf128hash.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * GHASH optimized using the CP Assist for Cryptographic Functions (CPACF) + * + * Copyright 2026 Google LLC + */ +#include <asm/cpacf.h> +#include <linux/cpufeature.h> + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_ghash); + +#define ghash_preparekey_arch ghash_preparekey_arch +static void ghash_preparekey_arch(struct ghash_key *key, + const u8 raw_key[GHASH_BLOCK_SIZE]) +{ + /* Save key in POLYVAL format for fallback */ + ghash_key_to_polyval(raw_key, &key->h); + + /* Save key in GHASH format for CPACF_KIMD_GHASH */ + memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE); +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_cpacf_ghash)) { + /* + * CPACF_KIMD_GHASH requires the accumulator and key in a single + * buffer, each using the GHASH convention. + */ + u8 ctx[2][GHASH_BLOCK_SIZE] __aligned(8); + + polyval_acc_to_ghash(acc, ctx[0]); + memcpy(ctx[1], key->h_raw, GHASH_BLOCK_SIZE); + + cpacf_kimd(CPACF_KIMD_GHASH, ctx, data, + nblocks * GHASH_BLOCK_SIZE); + + ghash_acc_to_polyval(ctx[0], acc); + memzero_explicit(ctx, sizeof(ctx)); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (cpu_have_feature(S390_CPU_FEATURE_MSA) && + cpacf_query_func(CPACF_KIMD, CPACF_KIMD_GHASH)) + static_branch_enable(&have_cpacf_ghash); +} diff --git a/lib/crypto/sm3.c b/lib/crypto/sm3.c index c6b9ad8a3ac6..b02b8a247adf 100644 --- a/lib/crypto/sm3.c +++ b/lib/crypto/sm3.c @@ -15,6 +15,13 @@ #include <linux/string.h> #include <linux/unaligned.h> +static const struct sm3_block_state sm3_iv = { + .h = { + SM3_IVA, SM3_IVB, SM3_IVC, SM3_IVD, + SM3_IVE, SM3_IVF, SM3_IVG, SM3_IVH, + }, +}; + static const u32 ____cacheline_aligned K[64] = { 0x79cc4519, 0xf3988a32, 0xe7311465, 0xce6228cb, 0x9cc45197, 0x3988a32f, 0x7311465e, 0xe6228cbc, @@ -72,18 +79,19 @@ static const u32 ____cacheline_aligned K[64] = { ^ rol32(W[(i-13) & 0x0f], 7) \ ^ W[(i-6) & 0x0f]) -static void sm3_transform(struct sm3_state *sctx, u8 const *data, u32 W[16]) +static void sm3_block_generic(struct sm3_block_state *state, + const u8 data[SM3_BLOCK_SIZE], u32 W[16]) { u32 a, b, c, d, e, f, g, h, ss1, ss2; - a = sctx->state[0]; - b = sctx->state[1]; - c = sctx->state[2]; - d = sctx->state[3]; - e = sctx->state[4]; - f = sctx->state[5]; - g = sctx->state[6]; - h = sctx->state[7]; + a = state->h[0]; + b = state->h[1]; + c = state->h[2]; + d = state->h[3]; + e = state->h[4]; + f = state->h[5]; + g = state->h[6]; + h = state->h[7]; R1(a, b, c, d, e, f, g, h, K[0], I(0), I(4)); R1(d, a, b, c, h, e, f, g, K[1], I(1), I(5)); @@ -153,14 +161,14 @@ static void sm3_transform(struct sm3_state *sctx, u8 const *data, u32 W[16]) R2(c, d, a, b, g, h, e, f, K[62], W1(62), W2(66)); R2(b, c, d, a, f, g, h, e, K[63], W1(63), W2(67)); - sctx->state[0] ^= a; - sctx->state[1] ^= b; - sctx->state[2] ^= c; - sctx->state[3] ^= d; - sctx->state[4] ^= e; - sctx->state[5] ^= f; - sctx->state[6] ^= g; - sctx->state[7] ^= h; + state->h[0] ^= a; + state->h[1] ^= b; + state->h[2] ^= c; + state->h[3] ^= d; + state->h[4] ^= e; + state->h[5] ^= f; + state->h[6] ^= g; + state->h[7] ^= h; } #undef R #undef R1 @@ -169,18 +177,114 @@ static void sm3_transform(struct sm3_state *sctx, u8 const *data, u32 W[16]) #undef W1 #undef W2 -void sm3_block_generic(struct sm3_state *sctx, u8 const *data, int blocks) +static void __maybe_unused sm3_blocks_generic(struct sm3_block_state *state, + const u8 *data, size_t nblocks) { u32 W[16]; do { - sm3_transform(sctx, data, W); + sm3_block_generic(state, data, W); data += SM3_BLOCK_SIZE; - } while (--blocks); + } while (--nblocks); memzero_explicit(W, sizeof(W)); } -EXPORT_SYMBOL_GPL(sm3_block_generic); -MODULE_DESCRIPTION("Generic SM3 library"); +#ifdef CONFIG_CRYPTO_LIB_SM3_ARCH +#include "sm3.h" /* $(SRCARCH)/sm3.h */ +#else +#define sm3_blocks sm3_blocks_generic +#endif + +void sm3_init(struct sm3_ctx *ctx) +{ + ctx->state = sm3_iv; + ctx->bytecount = 0; +} +EXPORT_SYMBOL_GPL(sm3_init); + +void sm3_update(struct sm3_ctx *ctx, const u8 *data, size_t len) +{ + size_t partial = ctx->bytecount % SM3_BLOCK_SIZE; + + ctx->bytecount += len; + + if (partial + len >= SM3_BLOCK_SIZE) { + size_t nblocks; + + if (partial) { + size_t l = SM3_BLOCK_SIZE - partial; + + memcpy(&ctx->buf[partial], data, l); + data += l; + len -= l; + + sm3_blocks(&ctx->state, ctx->buf, 1); + } + + nblocks = len / SM3_BLOCK_SIZE; + len %= SM3_BLOCK_SIZE; + + if (nblocks) { + sm3_blocks(&ctx->state, data, nblocks); + data += nblocks * SM3_BLOCK_SIZE; + } + partial = 0; + } + if (len) + memcpy(&ctx->buf[partial], data, len); +} +EXPORT_SYMBOL_GPL(sm3_update); + +static void __sm3_final(struct sm3_ctx *ctx, u8 out[SM3_DIGEST_SIZE]) +{ + u64 bitcount = ctx->bytecount << 3; + size_t partial = ctx->bytecount % SM3_BLOCK_SIZE; + + ctx->buf[partial++] = 0x80; + if (partial > SM3_BLOCK_SIZE - 8) { + memset(&ctx->buf[partial], 0, SM3_BLOCK_SIZE - partial); + sm3_blocks(&ctx->state, ctx->buf, 1); + partial = 0; + } + memset(&ctx->buf[partial], 0, SM3_BLOCK_SIZE - 8 - partial); + *(__be64 *)&ctx->buf[SM3_BLOCK_SIZE - 8] = cpu_to_be64(bitcount); + sm3_blocks(&ctx->state, ctx->buf, 1); + + for (size_t i = 0; i < SM3_DIGEST_SIZE; i += 4) + put_unaligned_be32(ctx->state.h[i / 4], out + i); +} + +void sm3_final(struct sm3_ctx *ctx, u8 out[SM3_DIGEST_SIZE]) +{ + __sm3_final(ctx, out); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(sm3_final); + +void sm3(const u8 *data, size_t len, u8 out[SM3_DIGEST_SIZE]) +{ + struct sm3_ctx ctx; + + sm3_init(&ctx); + sm3_update(&ctx, data, len); + sm3_final(&ctx, out); +} +EXPORT_SYMBOL_GPL(sm3); + +#ifdef sm3_mod_init_arch +static int __init sm3_mod_init(void) +{ + sm3_mod_init_arch(); + return 0; +} +subsys_initcall(sm3_mod_init); + +static void __exit sm3_mod_exit(void) +{ +} +module_exit(sm3_mod_exit); +#endif + +MODULE_DESCRIPTION("SM3 library functions"); MODULE_LICENSE("GPL v2"); diff --git a/lib/crypto/sparc/md5.h b/lib/crypto/sparc/md5.h deleted file mode 100644 index 3995f3e075eb..000000000000 --- a/lib/crypto/sparc/md5.h +++ /dev/null @@ -1,48 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * MD5 accelerated using the sparc64 crypto opcodes - * - * Copyright (c) Alan Smithee. - * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> - * Copyright (c) Jean-Francois Dive <jef@linuxbe.org> - * Copyright (c) Mathias Krause <minipli@googlemail.com> - * Copyright (c) Cryptoapi developers. - * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> - */ - -#include <asm/elf.h> -#include <asm/opcodes.h> -#include <asm/pstate.h> - -static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_md5_opcodes); - -asmlinkage void md5_sparc64_transform(struct md5_block_state *state, - const u8 *data, size_t nblocks); - -static void md5_blocks(struct md5_block_state *state, - const u8 *data, size_t nblocks) -{ - if (static_branch_likely(&have_md5_opcodes)) { - cpu_to_le32_array(state->h, ARRAY_SIZE(state->h)); - md5_sparc64_transform(state, data, nblocks); - le32_to_cpu_array(state->h, ARRAY_SIZE(state->h)); - } else { - md5_blocks_generic(state, data, nblocks); - } -} - -#define md5_mod_init_arch md5_mod_init_arch -static void md5_mod_init_arch(void) -{ - unsigned long cfr; - - if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) - return; - - __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); - if (!(cfr & CFR_MD5)) - return; - - static_branch_enable(&have_md5_opcodes); - pr_info("Using sparc64 md5 opcode optimized MD5 implementation\n"); -} diff --git a/lib/crypto/sparc/md5_asm.S b/lib/crypto/sparc/md5_asm.S deleted file mode 100644 index 60b544e4d205..000000000000 --- a/lib/crypto/sparc/md5_asm.S +++ /dev/null @@ -1,70 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#include <linux/linkage.h> -#include <asm/opcodes.h> -#include <asm/visasm.h> - -ENTRY(md5_sparc64_transform) - /* %o0 = digest, %o1 = data, %o2 = rounds */ - VISEntryHalf - ld [%o0 + 0x00], %f0 - ld [%o0 + 0x04], %f1 - andcc %o1, 0x7, %g0 - ld [%o0 + 0x08], %f2 - bne,pn %xcc, 10f - ld [%o0 + 0x0c], %f3 - -1: - ldd [%o1 + 0x00], %f8 - ldd [%o1 + 0x08], %f10 - ldd [%o1 + 0x10], %f12 - ldd [%o1 + 0x18], %f14 - ldd [%o1 + 0x20], %f16 - ldd [%o1 + 0x28], %f18 - ldd [%o1 + 0x30], %f20 - ldd [%o1 + 0x38], %f22 - - MD5 - - subcc %o2, 1, %o2 - bne,pt %xcc, 1b - add %o1, 0x40, %o1 - -5: - st %f0, [%o0 + 0x00] - st %f1, [%o0 + 0x04] - st %f2, [%o0 + 0x08] - st %f3, [%o0 + 0x0c] - retl - VISExitHalf -10: - alignaddr %o1, %g0, %o1 - - ldd [%o1 + 0x00], %f10 -1: - ldd [%o1 + 0x08], %f12 - ldd [%o1 + 0x10], %f14 - ldd [%o1 + 0x18], %f16 - ldd [%o1 + 0x20], %f18 - ldd [%o1 + 0x28], %f20 - ldd [%o1 + 0x30], %f22 - ldd [%o1 + 0x38], %f24 - ldd [%o1 + 0x40], %f26 - - faligndata %f10, %f12, %f8 - faligndata %f12, %f14, %f10 - faligndata %f14, %f16, %f12 - faligndata %f16, %f18, %f14 - faligndata %f18, %f20, %f16 - faligndata %f20, %f22, %f18 - faligndata %f22, %f24, %f20 - faligndata %f24, %f26, %f22 - - MD5 - - subcc %o2, 1, %o2 - fsrc2 %f26, %f10 - bne,pt %xcc, 1b - add %o1, 0x40, %o1 - - ba,a,pt %xcc, 5b -ENDPROC(md5_sparc64_transform) diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig index 0de289b429a9..9409c1a935c3 100644 --- a/lib/crypto/tests/Kconfig +++ b/lib/crypto/tests/Kconfig @@ -1,9 +1,18 @@ # SPDX-License-Identifier: GPL-2.0-or-later +config CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST + tristate "KUnit tests for AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC" if !KUNIT_ALL_TESTS + depends on KUNIT && CRYPTO_LIB_AES_CBC_MACS + default KUNIT_ALL_TESTS + select CRYPTO_LIB_BENCHMARK_VISIBLE + help + KUnit tests for the AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC message + authentication codes. + config CRYPTO_LIB_BLAKE2B_KUNIT_TEST tristate "KUnit tests for BLAKE2b" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_BLAKE2B - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the BLAKE2b cryptographic hash function. @@ -11,25 +20,42 @@ config CRYPTO_LIB_BLAKE2B_KUNIT_TEST config CRYPTO_LIB_BLAKE2S_KUNIT_TEST tristate "KUnit tests for BLAKE2s" if !KUNIT_ALL_TESTS depends on KUNIT - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE # No need to depend on CRYPTO_LIB_BLAKE2S here, as that option doesn't # exist; the BLAKE2s code is always built-in for the /dev/random driver. help KUnit tests for the BLAKE2s cryptographic hash function. +config CRYPTO_LIB_CHACHA20POLY1305_KUNIT_TEST + tristate "KUnit tests for ChaCha20Poly1305" if !KUNIT_ALL_TESTS + depends on KUNIT && CRYPTO_LIB_CHACHA20POLY1305 + default KUNIT_ALL_TESTS + select CRYPTO_LIB_BENCHMARK_VISIBLE + help + KUnit tests for the ChaCha20Poly1305 authenticated encryption + algorithm. + config CRYPTO_LIB_CURVE25519_KUNIT_TEST tristate "KUnit tests for Curve25519" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_CURVE25519 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the Curve25519 Diffie-Hellman function. +config CRYPTO_LIB_GHASH_KUNIT_TEST + tristate "KUnit tests for GHASH" if !KUNIT_ALL_TESTS + depends on KUNIT && CRYPTO_LIB_GF128HASH + default KUNIT_ALL_TESTS + select CRYPTO_LIB_BENCHMARK_VISIBLE + help + KUnit tests for the GHASH library functions. + config CRYPTO_LIB_MD5_KUNIT_TEST tristate "KUnit tests for MD5" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_MD5 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the MD5 cryptographic hash function and its @@ -38,7 +64,7 @@ config CRYPTO_LIB_MD5_KUNIT_TEST config CRYPTO_LIB_MLDSA_KUNIT_TEST tristate "KUnit tests for ML-DSA" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_MLDSA - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the ML-DSA digital signature algorithm. @@ -46,22 +72,22 @@ config CRYPTO_LIB_MLDSA_KUNIT_TEST config CRYPTO_LIB_NH_KUNIT_TEST tristate "KUnit tests for NH" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_NH - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS help KUnit tests for the NH almost-universal hash function. config CRYPTO_LIB_POLY1305_KUNIT_TEST tristate "KUnit tests for Poly1305" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_POLY1305 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the Poly1305 library functions. config CRYPTO_LIB_POLYVAL_KUNIT_TEST tristate "KUnit tests for POLYVAL" if !KUNIT_ALL_TESTS - depends on KUNIT && CRYPTO_LIB_POLYVAL - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + depends on KUNIT && CRYPTO_LIB_GF128HASH + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the POLYVAL library functions. @@ -69,7 +95,7 @@ config CRYPTO_LIB_POLYVAL_KUNIT_TEST config CRYPTO_LIB_SHA1_KUNIT_TEST tristate "KUnit tests for SHA-1" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_SHA1 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the SHA-1 cryptographic hash function and its @@ -80,7 +106,7 @@ config CRYPTO_LIB_SHA1_KUNIT_TEST config CRYPTO_LIB_SHA256_KUNIT_TEST tristate "KUnit tests for SHA-224 and SHA-256" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_SHA256 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the SHA-224 and SHA-256 cryptographic hash functions @@ -91,7 +117,7 @@ config CRYPTO_LIB_SHA256_KUNIT_TEST config CRYPTO_LIB_SHA512_KUNIT_TEST tristate "KUnit tests for SHA-384 and SHA-512" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_SHA512 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the SHA-384 and SHA-512 cryptographic hash functions @@ -100,13 +126,47 @@ config CRYPTO_LIB_SHA512_KUNIT_TEST config CRYPTO_LIB_SHA3_KUNIT_TEST tristate "KUnit tests for SHA-3" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_SHA3 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the SHA3 cryptographic hash and XOF functions, including SHA3-224, SHA3-256, SHA3-384, SHA3-512, SHAKE128 and SHAKE256. +config CRYPTO_LIB_SM3_KUNIT_TEST + tristate "KUnit tests for SM3" if !KUNIT_ALL_TESTS + depends on KUNIT && CRYPTO_LIB_SM3 + default KUNIT_ALL_TESTS + select CRYPTO_LIB_BENCHMARK_VISIBLE + help + KUnit tests for the SM3 cryptographic hash function. + +config CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT + tristate "Enable all crypto library code for KUnit tests" + depends on KUNIT + select CRYPTO_LIB_AES_CBC_MACS + select CRYPTO_LIB_BLAKE2B + select CRYPTO_LIB_CHACHA20POLY1305 + select CRYPTO_LIB_CURVE25519 + select CRYPTO_LIB_GF128HASH + select CRYPTO_LIB_MD5 + select CRYPTO_LIB_MLDSA + select CRYPTO_LIB_NH + select CRYPTO_LIB_POLY1305 + select CRYPTO_LIB_SHA1 + select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_SHA512 + select CRYPTO_LIB_SHA3 + select CRYPTO_LIB_SM3 + help + Enable all the crypto library code that has KUnit tests. + + Enable this only if you'd like to test all the crypto library code, + even code that wouldn't otherwise need to be built. + + You'll still need to enable the tests themselves, either individually + or using KUNIT_ALL_TESTS. + config CRYPTO_LIB_BENCHMARK_VISIBLE bool diff --git a/lib/crypto/tests/Makefile b/lib/crypto/tests/Makefile index f4262379f56c..a739413500b6 100644 --- a/lib/crypto/tests/Makefile +++ b/lib/crypto/tests/Makefile @@ -1,8 +1,11 @@ # SPDX-License-Identifier: GPL-2.0-or-later +obj-$(CONFIG_CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST) += aes_cbc_macs_kunit.o obj-$(CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST) += blake2b_kunit.o obj-$(CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST) += blake2s_kunit.o +obj-$(CONFIG_CRYPTO_LIB_CHACHA20POLY1305_KUNIT_TEST) += chacha20poly1305_kunit.o obj-$(CONFIG_CRYPTO_LIB_CURVE25519_KUNIT_TEST) += curve25519_kunit.o +obj-$(CONFIG_CRYPTO_LIB_GHASH_KUNIT_TEST) += ghash_kunit.o obj-$(CONFIG_CRYPTO_LIB_MD5_KUNIT_TEST) += md5_kunit.o obj-$(CONFIG_CRYPTO_LIB_MLDSA_KUNIT_TEST) += mldsa_kunit.o obj-$(CONFIG_CRYPTO_LIB_NH_KUNIT_TEST) += nh_kunit.o @@ -12,3 +15,4 @@ obj-$(CONFIG_CRYPTO_LIB_SHA1_KUNIT_TEST) += sha1_kunit.o obj-$(CONFIG_CRYPTO_LIB_SHA256_KUNIT_TEST) += sha224_kunit.o sha256_kunit.o obj-$(CONFIG_CRYPTO_LIB_SHA512_KUNIT_TEST) += sha384_kunit.o sha512_kunit.o obj-$(CONFIG_CRYPTO_LIB_SHA3_KUNIT_TEST) += sha3_kunit.o +obj-$(CONFIG_CRYPTO_LIB_SM3_KUNIT_TEST) += sm3_kunit.o diff --git a/lib/crypto/tests/aes-cmac-testvecs.h b/lib/crypto/tests/aes-cmac-testvecs.h new file mode 100644 index 000000000000..8177862ba5a3 --- /dev/null +++ b/lib/crypto/tests/aes-cmac-testvecs.h @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* This file was generated by: ./scripts/crypto/gen-hash-testvecs.py aes-cmac */ + +static const struct { + size_t data_len; + u8 digest[AES_BLOCK_SIZE]; +} hash_testvecs[] = { + { + .data_len = 0, + .digest = { + 0x9a, 0xeb, 0x94, 0xc1, 0xe9, 0xc1, 0x57, 0x49, + 0x7e, 0xba, 0x66, 0x47, 0x9f, 0x03, 0x2c, 0x5b, + }, + }, + { + .data_len = 1, + .digest = { + 0x52, 0xef, 0x65, 0xda, 0x7b, 0x92, 0x0c, 0x0f, + 0xdd, 0xd6, 0xb9, 0x68, 0x3f, 0xcc, 0x5f, 0xea, + }, + }, + { + .data_len = 2, + .digest = { + 0xc3, 0x95, 0x15, 0xea, 0x16, 0x33, 0xbe, 0x5a, + 0xd4, 0x2c, 0x25, 0x06, 0x15, 0xc6, 0x10, 0x19, + }, + }, + { + .data_len = 3, + .digest = { + 0x82, 0x41, 0x41, 0xd5, 0x33, 0x26, 0x0b, 0xb6, + 0xc8, 0xf7, 0x8d, 0x76, 0x8a, 0xfc, 0x0e, 0xe4, + }, + }, + { + .data_len = 16, + .digest = { + 0x94, 0x09, 0x80, 0x07, 0xba, 0x7c, 0xed, 0xd2, + 0x74, 0x72, 0x30, 0x26, 0xb5, 0x11, 0x64, 0xc1, + }, + }, + { + .data_len = 32, + .digest = { + 0xeb, 0xcf, 0x1e, 0x67, 0x21, 0x64, 0x93, 0xa0, + 0xea, 0xc4, 0xb9, 0x2d, 0x55, 0xc8, 0xac, 0x99, + }, + }, + { + .data_len = 48, + .digest = { + 0xd0, 0xd6, 0xdb, 0xe2, 0x45, 0x98, 0x6a, 0x7a, + 0x5f, 0xd6, 0xcd, 0x9d, 0x12, 0x26, 0x20, 0x87, + }, + }, + { + .data_len = 49, + .digest = { + 0x63, 0x25, 0x3c, 0xe2, 0x2a, 0xfa, 0xe3, 0x1e, + 0x54, 0x10, 0x18, 0x28, 0xc6, 0xb8, 0xcb, 0x58, + }, + }, + { + .data_len = 63, + .digest = { + 0x4d, 0xab, 0xae, 0x99, 0x90, 0x13, 0x3f, 0x4f, + 0x42, 0x0f, 0x19, 0x94, 0xa2, 0x52, 0xfd, 0xaf, + }, + }, + { + .data_len = 64, + .digest = { + 0xf7, 0x49, 0xb9, 0xa7, 0xf9, 0x3e, 0xa0, 0xca, + 0xb2, 0x6c, 0xd7, 0x87, 0x7d, 0x1e, 0xd2, 0xcb, + }, + }, + { + .data_len = 65, + .digest = { + 0x27, 0x2c, 0xb7, 0xc8, 0xdd, 0x26, 0xa9, 0xfe, + 0x37, 0x64, 0x84, 0x38, 0xa5, 0x7e, 0xbc, 0x04, + }, + }, + { + .data_len = 127, + .digest = { + 0xfd, 0x1f, 0x01, 0xa4, 0xea, 0x9b, 0xbd, 0xef, + 0x09, 0x97, 0x57, 0x60, 0x95, 0x23, 0xcc, 0x71, + }, + }, + { + .data_len = 128, + .digest = { + 0x28, 0x21, 0xee, 0x56, 0x9f, 0x38, 0xd6, 0x0e, + 0xe3, 0x22, 0x06, 0x20, 0xad, 0xd8, 0x33, 0x74, + }, + }, + { + .data_len = 129, + .digest = { + 0x07, 0x28, 0x4a, 0x2a, 0xd3, 0x85, 0xa6, 0x87, + 0x5c, 0x01, 0x8c, 0xb9, 0xd3, 0x4b, 0xce, 0x20, + }, + }, + { + .data_len = 256, + .digest = { + 0xe6, 0x12, 0x25, 0x6b, 0xf9, 0x69, 0x4d, 0x5a, + 0x1a, 0xb0, 0xe6, 0x11, 0x46, 0x24, 0x08, 0xdf, + }, + }, + { + .data_len = 511, + .digest = { + 0xce, 0x28, 0x1f, 0x14, 0xb9, 0xcc, 0x7e, 0x1f, + 0xb5, 0x13, 0x2b, 0x45, 0x04, 0x54, 0xe9, 0x5f, + }, + }, + { + .data_len = 513, + .digest = { + 0x63, 0x12, 0xbd, 0x85, 0x60, 0x1b, 0x99, 0x7e, + 0x0a, 0xf7, 0x0f, 0xc1, 0xb5, 0x66, 0xf8, 0x9a, + }, + }, + { + .data_len = 1000, + .digest = { + 0xbd, 0x49, 0x5e, 0x21, 0xc6, 0x58, 0x74, 0x6b, + 0x21, 0xc2, 0x62, 0x6a, 0x15, 0xca, 0x1d, 0x8a, + }, + }, + { + .data_len = 3333, + .digest = { + 0xfe, 0x6b, 0xfa, 0xfc, 0x4c, 0x0b, 0x63, 0x0d, + 0x41, 0x7f, 0xa9, 0xd8, 0xba, 0xe3, 0xce, 0xce, + }, + }, + { + .data_len = 4096, + .digest = { + 0x41, 0x7c, 0xbc, 0x2e, 0x2f, 0xff, 0xdf, 0x09, + 0x31, 0xc5, 0x79, 0x0a, 0x1d, 0x6e, 0x46, 0xec, + }, + }, + { + .data_len = 4128, + .digest = { + 0x6a, 0x9d, 0x86, 0xa8, 0xab, 0xa5, 0xc1, 0xc5, + 0x0d, 0x54, 0xf3, 0x51, 0x10, 0x46, 0x25, 0x5a, + }, + }, + { + .data_len = 4160, + .digest = { + 0x4c, 0x50, 0xff, 0x2a, 0xe9, 0xca, 0x9e, 0x07, + 0x8a, 0x86, 0x67, 0x5e, 0xe5, 0x0a, 0xfd, 0x69, + }, + }, + { + .data_len = 4224, + .digest = { + 0x3a, 0xfa, 0x80, 0x9d, 0x80, 0xe3, 0x1e, 0x95, + 0x53, 0x93, 0x39, 0x17, 0xd3, 0xda, 0x49, 0x15, + }, + }, + { + .data_len = 16384, + .digest = { + 0x48, 0xf4, 0x4f, 0x2d, 0x5d, 0xf2, 0x02, 0xcf, + 0x51, 0x3c, 0x1b, 0x12, 0x80, 0x8f, 0xb0, 0xd6, + }, + }, +}; + +static const u8 hash_testvec_consolidated[AES_BLOCK_SIZE] = { + 0x41, 0xad, 0x25, 0xa1, 0xeb, 0xce, 0x6b, 0x9c, + 0x06, 0xdf, 0x47, 0xc4, 0x3a, 0x59, 0x50, 0x07, +}; diff --git a/lib/crypto/tests/aes_cbc_macs_kunit.c b/lib/crypto/tests/aes_cbc_macs_kunit.c new file mode 100644 index 000000000000..ae3745212f03 --- /dev/null +++ b/lib/crypto/tests/aes_cbc_macs_kunit.c @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2026 Google LLC + */ +#include <crypto/aes-cbc-macs.h> +#include "aes-cmac-testvecs.h" + +/* + * A fixed key used when presenting AES-CMAC as an unkeyed hash function in + * order to reuse hash-test-template.h. At the beginning of the test suite, + * this is initialized to a key prepared from bytes generated from a fixed seed. + */ +static struct aes_cmac_key test_key; + +static void aes_cmac_init_withtestkey(struct aes_cmac_ctx *ctx) +{ + aes_cmac_init(ctx, &test_key); +} + +static void aes_cmac_withtestkey(const u8 *data, size_t data_len, + u8 out[AES_BLOCK_SIZE]) +{ + aes_cmac(&test_key, data, data_len, out); +} + +#define HASH aes_cmac_withtestkey +#define HASH_CTX aes_cmac_ctx +#define HASH_SIZE AES_BLOCK_SIZE +#define HASH_INIT aes_cmac_init_withtestkey +#define HASH_UPDATE aes_cmac_update +#define HASH_FINAL aes_cmac_final +#include "hash-test-template.h" + +static int aes_cbc_macs_suite_init(struct kunit_suite *suite) +{ + u8 raw_key[AES_KEYSIZE_256]; + int err; + + rand_bytes_seeded_from_len(raw_key, sizeof(raw_key)); + err = aes_cmac_preparekey(&test_key, raw_key, sizeof(raw_key)); + if (err) + return err; + return hash_suite_init(suite); +} + +static void aes_cbc_macs_suite_exit(struct kunit_suite *suite) +{ + hash_suite_exit(suite); +} + +/* Verify compatibility of the AES-CMAC implementation with RFC 4493. */ +static void test_aes_cmac_rfc4493(struct kunit *test) +{ + static const u8 raw_key[AES_KEYSIZE_128] = { + 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, + 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c, + }; + static const struct { + size_t data_len; + const u8 data[40]; + const u8 mac[AES_BLOCK_SIZE]; + } testvecs[] = { + { + /* Example 1 from RFC 4493 */ + .data_len = 0, + .mac = { + 0xbb, 0x1d, 0x69, 0x29, 0xe9, 0x59, 0x37, 0x28, + 0x7f, 0xa3, 0x7d, 0x12, 0x9b, 0x75, 0x67, 0x46, + }, + + }, + { + /* Example 2 from RFC 4493 */ + .data = { + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, + 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + }, + .data_len = 16, + .mac = { + 0x07, 0x0a, 0x16, 0xb4, 0x6b, 0x4d, 0x41, 0x44, + 0xf7, 0x9b, 0xdd, 0x9d, 0xd0, 0x4a, 0x28, 0x7c, + }, + }, + { + /* Example 3 from RFC 4493 */ + .data = { + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, + 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, + 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, + }, + .data_len = 40, + .mac = { + 0xdf, 0xa6, 0x67, 0x47, 0xde, 0x9a, 0xe6, 0x30, + 0x30, 0xca, 0x32, 0x61, 0x14, 0x97, 0xc8, 0x27, + }, + }, + }; + struct aes_cmac_key key; + int err; + + err = aes_cmac_preparekey(&key, raw_key, sizeof(raw_key)); + KUNIT_ASSERT_EQ(test, err, 0); + + for (size_t i = 0; i < ARRAY_SIZE(testvecs); i++) { + u8 mac[AES_BLOCK_SIZE]; + + aes_cmac(&key, testvecs[i].data, testvecs[i].data_len, mac); + KUNIT_ASSERT_MEMEQ(test, mac, testvecs[i].mac, AES_BLOCK_SIZE); + } +} + +/* + * Verify compatibility of the AES-XCBC-MAC implementation with RFC 3566. + * + * Additional AES-XCBC-MAC tests are not necessary, since the AES-XCBC-MAC + * implementation is well covered by the AES-CMAC tests already. Only the key + * preparation function differs; the rest of the code is shared. + */ +static void test_aes_xcbcmac_rfc3566(struct kunit *test) +{ + struct aes_cmac_key key; + /* AES-XCBC-MAC Test Case #4 from RFC 3566 */ + static const u8 raw_key[AES_KEYSIZE_128] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + }; + static const u8 message[20] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, + 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, + }; + static const u8 expected_mac[AES_BLOCK_SIZE] = { + 0x47, 0xf5, 0x1b, 0x45, 0x64, 0x96, 0x62, 0x15, + 0xb8, 0x98, 0x5c, 0x63, 0x05, 0x5e, 0xd3, 0x08, + }; + u8 actual_mac[AES_BLOCK_SIZE]; + + aes_xcbcmac_preparekey(&key, raw_key); + aes_cmac(&key, message, sizeof(message), actual_mac); + KUNIT_ASSERT_MEMEQ(test, actual_mac, expected_mac, AES_BLOCK_SIZE); +} + +static void test_aes_cbcmac_rfc3610(struct kunit *test) +{ + /* + * The following AES-CBC-MAC test vector is extracted from RFC 3610 + * Packet Vector #11. It required some rearrangement to get the actual + * input to AES-CBC-MAC from the values given. + */ + static const u8 raw_key[AES_KEYSIZE_128] = { + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, + }; + const size_t unpadded_data_len = 52; + static const u8 data[64] = { + /* clang-format off */ + /* CCM header */ + 0x61, 0x00, 0x00, 0x00, 0x0d, 0x0c, 0x0b, 0x0a, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0x00, 0x14, + /* CCM additional authentication blocks */ + 0x00, 0x0c, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, + 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x00, 0x00, + /* CCM message blocks */ + 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, + 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, + 0x1c, 0x1d, 0x1e, 0x1f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* clang-format on */ + }; + static const u8 expected_mac[AES_BLOCK_SIZE] = { + 0x6b, 0x5e, 0x24, 0x34, 0x12, 0xcc, 0xc2, 0xad, + 0x6f, 0x1b, 0x11, 0xc3, 0xa1, 0xa9, 0xd8, 0xbc, + }; + struct aes_enckey key; + struct aes_cbcmac_ctx ctx; + u8 actual_mac[AES_BLOCK_SIZE]; + int err; + + err = aes_prepareenckey(&key, raw_key, sizeof(raw_key)); + KUNIT_ASSERT_EQ(test, err, 0); + + /* + * Trailing zeroes should not affect the CBC-MAC value, up to the next + * AES block boundary. + */ + for (size_t data_len = unpadded_data_len; data_len <= sizeof(data); + data_len++) { + aes_cbcmac_init(&ctx, &key); + aes_cbcmac_update(&ctx, data, data_len); + aes_cbcmac_final(&ctx, actual_mac); + KUNIT_ASSERT_MEMEQ(test, actual_mac, expected_mac, + AES_BLOCK_SIZE); + + /* Incremental computations should produce the same result. */ + for (size_t part1_len = 0; part1_len <= data_len; part1_len++) { + aes_cbcmac_init(&ctx, &key); + aes_cbcmac_update(&ctx, data, part1_len); + aes_cbcmac_update(&ctx, &data[part1_len], + data_len - part1_len); + aes_cbcmac_final(&ctx, actual_mac); + KUNIT_ASSERT_MEMEQ(test, actual_mac, expected_mac, + AES_BLOCK_SIZE); + } + } +} + +static struct kunit_case aes_cbc_macs_test_cases[] = { + HASH_KUNIT_CASES, + KUNIT_CASE(test_aes_cmac_rfc4493), + KUNIT_CASE(test_aes_xcbcmac_rfc3566), + KUNIT_CASE(test_aes_cbcmac_rfc3610), + KUNIT_CASE(benchmark_hash), + {}, +}; + +static struct kunit_suite aes_cbc_macs_test_suite = { + .name = "aes_cbc_macs", + .test_cases = aes_cbc_macs_test_cases, + .suite_init = aes_cbc_macs_suite_init, + .suite_exit = aes_cbc_macs_suite_exit, +}; +kunit_test_suite(aes_cbc_macs_test_suite); + +MODULE_DESCRIPTION( + "KUnit tests and benchmark for AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC"); +MODULE_IMPORT_NS("CRYPTO_INTERNAL"); +MODULE_LICENSE("GPL"); diff --git a/lib/crypto/chacha20poly1305-selftest.c b/lib/crypto/tests/chacha20poly1305_kunit.c index e4c85bc5a6d7..97a68fab88a7 100644 --- a/lib/crypto/chacha20poly1305-selftest.c +++ b/lib/crypto/tests/chacha20poly1305_kunit.c @@ -6,9 +6,8 @@ #include <crypto/chacha20poly1305.h> #include <crypto/chacha.h> #include <crypto/poly1305.h> - +#include <kunit/test.h> #include <linux/unaligned.h> -#include <linux/bug.h> #include <linux/init.h> #include <linux/mm.h> #include <linux/kernel.h> @@ -27,7 +26,7 @@ struct chacha20poly1305_testvec { * chapoly construction. */ -static const u8 enc_input001[] __initconst = { +static const u8 enc_input001[] = { 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, @@ -63,7 +62,7 @@ static const u8 enc_input001[] __initconst = { 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, 0x9d }; -static const u8 enc_output001[] __initconst = { +static const u8 enc_output001[] = { 0x64, 0xa0, 0x86, 0x15, 0x75, 0x86, 0x1a, 0xf4, 0x60, 0xf0, 0x62, 0xc7, 0x9b, 0xe6, 0x43, 0xbd, 0x5e, 0x80, 0x5c, 0xfd, 0x34, 0x5c, 0xf3, 0x89, @@ -101,95 +100,95 @@ static const u8 enc_output001[] __initconst = { 0x22, 0x39, 0x23, 0x36, 0xfe, 0xa1, 0x85, 0x1f, 0x38 }; -static const u8 enc_assoc001[] __initconst = { +static const u8 enc_assoc001[] = { 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x91 }; -static const u8 enc_nonce001[] __initconst = { +static const u8 enc_nonce001[] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 }; -static const u8 enc_key001[] __initconst = { +static const u8 enc_key001[] = { 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 }; -static const u8 enc_input002[] __initconst = { }; -static const u8 enc_output002[] __initconst = { +static const u8 enc_input002[] = { }; +static const u8 enc_output002[] = { 0xea, 0xe0, 0x1e, 0x9e, 0x2c, 0x91, 0xaa, 0xe1, 0xdb, 0x5d, 0x99, 0x3f, 0x8a, 0xf7, 0x69, 0x92 }; -static const u8 enc_assoc002[] __initconst = { }; -static const u8 enc_nonce002[] __initconst = { +static const u8 enc_assoc002[] = { }; +static const u8 enc_nonce002[] = { 0xca, 0xbf, 0x33, 0x71, 0x32, 0x45, 0x77, 0x8e }; -static const u8 enc_key002[] __initconst = { +static const u8 enc_key002[] = { 0x4c, 0xf5, 0x96, 0x83, 0x38, 0xe6, 0xae, 0x7f, 0x2d, 0x29, 0x25, 0x76, 0xd5, 0x75, 0x27, 0x86, 0x91, 0x9a, 0x27, 0x7a, 0xfb, 0x46, 0xc5, 0xef, 0x94, 0x81, 0x79, 0x57, 0x14, 0x59, 0x40, 0x68 }; -static const u8 enc_input003[] __initconst = { }; -static const u8 enc_output003[] __initconst = { +static const u8 enc_input003[] = { }; +static const u8 enc_output003[] = { 0xdd, 0x6b, 0x3b, 0x82, 0xce, 0x5a, 0xbd, 0xd6, 0xa9, 0x35, 0x83, 0xd8, 0x8c, 0x3d, 0x85, 0x77 }; -static const u8 enc_assoc003[] __initconst = { +static const u8 enc_assoc003[] = { 0x33, 0x10, 0x41, 0x12, 0x1f, 0xf3, 0xd2, 0x6b }; -static const u8 enc_nonce003[] __initconst = { +static const u8 enc_nonce003[] = { 0x3d, 0x86, 0xb5, 0x6b, 0xc8, 0xa3, 0x1f, 0x1d }; -static const u8 enc_key003[] __initconst = { +static const u8 enc_key003[] = { 0x2d, 0xb0, 0x5d, 0x40, 0xc8, 0xed, 0x44, 0x88, 0x34, 0xd1, 0x13, 0xaf, 0x57, 0xa1, 0xeb, 0x3a, 0x2a, 0x80, 0x51, 0x36, 0xec, 0x5b, 0xbc, 0x08, 0x93, 0x84, 0x21, 0xb5, 0x13, 0x88, 0x3c, 0x0d }; -static const u8 enc_input004[] __initconst = { +static const u8 enc_input004[] = { 0xa4 }; -static const u8 enc_output004[] __initconst = { +static const u8 enc_output004[] = { 0xb7, 0x1b, 0xb0, 0x73, 0x59, 0xb0, 0x84, 0xb2, 0x6d, 0x8e, 0xab, 0x94, 0x31, 0xa1, 0xae, 0xac, 0x89 }; -static const u8 enc_assoc004[] __initconst = { +static const u8 enc_assoc004[] = { 0x6a, 0xe2, 0xad, 0x3f, 0x88, 0x39, 0x5a, 0x40 }; -static const u8 enc_nonce004[] __initconst = { +static const u8 enc_nonce004[] = { 0xd2, 0x32, 0x1f, 0x29, 0x28, 0xc6, 0xc4, 0xc4 }; -static const u8 enc_key004[] __initconst = { +static const u8 enc_key004[] = { 0x4b, 0x28, 0x4b, 0xa3, 0x7b, 0xbe, 0xe9, 0xf8, 0x31, 0x80, 0x82, 0xd7, 0xd8, 0xe8, 0xb5, 0xa1, 0xe2, 0x18, 0x18, 0x8a, 0x9c, 0xfa, 0xa3, 0x3d, 0x25, 0x71, 0x3e, 0x40, 0xbc, 0x54, 0x7a, 0x3e }; -static const u8 enc_input005[] __initconst = { +static const u8 enc_input005[] = { 0x2d }; -static const u8 enc_output005[] __initconst = { +static const u8 enc_output005[] = { 0xbf, 0xe1, 0x5b, 0x0b, 0xdb, 0x6b, 0xf5, 0x5e, 0x6c, 0x5d, 0x84, 0x44, 0x39, 0x81, 0xc1, 0x9c, 0xac }; -static const u8 enc_assoc005[] __initconst = { }; -static const u8 enc_nonce005[] __initconst = { +static const u8 enc_assoc005[] = { }; +static const u8 enc_nonce005[] = { 0x20, 0x1c, 0xaa, 0x5f, 0x9c, 0xbf, 0x92, 0x30 }; -static const u8 enc_key005[] __initconst = { +static const u8 enc_key005[] = { 0x66, 0xca, 0x9c, 0x23, 0x2a, 0x4b, 0x4b, 0x31, 0x0e, 0x92, 0x89, 0x8b, 0xf4, 0x93, 0xc7, 0x87, 0x98, 0xa3, 0xd8, 0x39, 0xf8, 0xf4, 0xa7, 0x01, 0xc0, 0x2e, 0x0a, 0xa6, 0x7e, 0x5a, 0x78, 0x87 }; -static const u8 enc_input006[] __initconst = { +static const u8 enc_input006[] = { 0x33, 0x2f, 0x94, 0xc1, 0xa4, 0xef, 0xcc, 0x2a, 0x5b, 0xa6, 0xe5, 0x8f, 0x1d, 0x40, 0xf0, 0x92, 0x3c, 0xd9, 0x24, 0x11, 0xa9, 0x71, 0xf9, 0x37, @@ -208,7 +207,7 @@ static const u8 enc_input006[] __initconst = { 0x69, 0x21, 0x70, 0xd8, 0xa4, 0x4b, 0xc8, 0xde, 0x8f }; -static const u8 enc_output006[] __initconst = { +static const u8 enc_output006[] = { 0x8b, 0x06, 0xd3, 0x31, 0xb0, 0x93, 0x45, 0xb1, 0x75, 0x6e, 0x26, 0xf9, 0x67, 0xbc, 0x90, 0x15, 0x81, 0x2c, 0xb5, 0xf0, 0xc6, 0x2b, 0xc7, 0x8c, @@ -229,20 +228,20 @@ static const u8 enc_output006[] __initconst = { 0x8d, 0x31, 0xf7, 0x7a, 0x39, 0x4d, 0x8f, 0x9a, 0xeb }; -static const u8 enc_assoc006[] __initconst = { +static const u8 enc_assoc006[] = { 0x70, 0xd3, 0x33, 0xf3, 0x8b, 0x18, 0x0b }; -static const u8 enc_nonce006[] __initconst = { +static const u8 enc_nonce006[] = { 0xdf, 0x51, 0x84, 0x82, 0x42, 0x0c, 0x75, 0x9c }; -static const u8 enc_key006[] __initconst = { +static const u8 enc_key006[] = { 0x68, 0x7b, 0x8d, 0x8e, 0xe3, 0xc4, 0xdd, 0xae, 0xdf, 0x72, 0x7f, 0x53, 0x72, 0x25, 0x1e, 0x78, 0x91, 0xcb, 0x69, 0x76, 0x1f, 0x49, 0x93, 0xf9, 0x6f, 0x21, 0xcc, 0x39, 0x9c, 0xad, 0xb1, 0x01 }; -static const u8 enc_input007[] __initconst = { +static const u8 enc_input007[] = { 0x9b, 0x18, 0xdb, 0xdd, 0x9a, 0x0f, 0x3e, 0xa5, 0x15, 0x17, 0xde, 0xdf, 0x08, 0x9d, 0x65, 0x0a, 0x67, 0x30, 0x12, 0xe2, 0x34, 0x77, 0x4b, 0xc1, @@ -276,7 +275,7 @@ static const u8 enc_input007[] __initconst = { 0x40, 0x5f, 0x99, 0xb7, 0x73, 0xec, 0x9b, 0x2b, 0xf0, 0x65, 0x11, 0xc8, 0xd0, 0x0a, 0x9f, 0xd3 }; -static const u8 enc_output007[] __initconst = { +static const u8 enc_output007[] = { 0x85, 0x04, 0xc2, 0xed, 0x8d, 0xfd, 0x97, 0x5c, 0xd2, 0xb7, 0xe2, 0xc1, 0x6b, 0xa3, 0xba, 0xf8, 0xc9, 0x50, 0xc3, 0xc6, 0xa5, 0xe3, 0xa4, 0x7c, @@ -312,18 +311,18 @@ static const u8 enc_output007[] __initconst = { 0x7a, 0x4b, 0xbc, 0xb0, 0x10, 0x5c, 0x96, 0x42, 0x3a, 0x00, 0x98, 0xcd, 0x15, 0xe8, 0xb7, 0x53 }; -static const u8 enc_assoc007[] __initconst = { }; -static const u8 enc_nonce007[] __initconst = { +static const u8 enc_assoc007[] = { }; +static const u8 enc_nonce007[] = { 0xde, 0x7b, 0xef, 0xc3, 0x65, 0x1b, 0x68, 0xb0 }; -static const u8 enc_key007[] __initconst = { +static const u8 enc_key007[] = { 0x8d, 0xb8, 0x91, 0x48, 0xf0, 0xe7, 0x0a, 0xbd, 0xf9, 0x3f, 0xcd, 0xd9, 0xa0, 0x1e, 0x42, 0x4c, 0xe7, 0xde, 0x25, 0x3d, 0xa3, 0xd7, 0x05, 0x80, 0x8d, 0xf2, 0x82, 0xac, 0x44, 0x16, 0x51, 0x01 }; -static const u8 enc_input008[] __initconst = { +static const u8 enc_input008[] = { 0xc3, 0x09, 0x94, 0x62, 0xe6, 0x46, 0x2e, 0x10, 0xbe, 0x00, 0xe4, 0xfc, 0xf3, 0x40, 0xa3, 0xe2, 0x0f, 0xc2, 0x8b, 0x28, 0xdc, 0xba, 0xb4, 0x3c, @@ -389,7 +388,7 @@ static const u8 enc_input008[] __initconst = { 0x2e, 0x79, 0xb5, 0xe2, 0xb8, 0xe8, 0xb9, 0x7b, 0xd5, 0x10, 0xcb, 0xff, 0x5d, 0x14, 0x73, 0xf3 }; -static const u8 enc_output008[] __initconst = { +static const u8 enc_output008[] = { 0x14, 0xf6, 0x41, 0x37, 0xa6, 0xd4, 0x27, 0xcd, 0xdb, 0x06, 0x3e, 0x9a, 0x4e, 0xab, 0xd5, 0xb1, 0x1e, 0x6b, 0xd2, 0xbc, 0x11, 0xf4, 0x28, 0x93, @@ -457,18 +456,18 @@ static const u8 enc_output008[] __initconst = { 0x54, 0xfd, 0x0d, 0x98, 0x1c, 0x5a, 0x6f, 0x1f, 0x9a, 0x40, 0xcd, 0xa2, 0xff, 0x6a, 0xf1, 0x54 }; -static const u8 enc_assoc008[] __initconst = { }; -static const u8 enc_nonce008[] __initconst = { +static const u8 enc_assoc008[] = { }; +static const u8 enc_nonce008[] = { 0x0e, 0x0d, 0x57, 0xbb, 0x7b, 0x40, 0x54, 0x02 }; -static const u8 enc_key008[] __initconst = { +static const u8 enc_key008[] = { 0xf2, 0xaa, 0x4f, 0x99, 0xfd, 0x3e, 0xa8, 0x53, 0xc1, 0x44, 0xe9, 0x81, 0x18, 0xdc, 0xf5, 0xf0, 0x3e, 0x44, 0x15, 0x59, 0xe0, 0xc5, 0x44, 0x86, 0xc3, 0x91, 0xa8, 0x75, 0xc0, 0x12, 0x46, 0xba }; -static const u8 enc_input009[] __initconst = { +static const u8 enc_input009[] = { 0xe6, 0xc3, 0xdb, 0x63, 0x55, 0x15, 0xe3, 0x5b, 0xb7, 0x4b, 0x27, 0x8b, 0x5a, 0xdd, 0xc2, 0xe8, 0x3a, 0x6b, 0xd7, 0x81, 0x96, 0x35, 0x97, 0xca, @@ -535,7 +534,7 @@ static const u8 enc_input009[] __initconst = { 0x74, 0xb9, 0xe2, 0xd6, 0x1c, 0x80, 0x2c, 0x52, 0x65 }; -static const u8 enc_output009[] __initconst = { +static const u8 enc_output009[] = { 0xfd, 0x81, 0x8d, 0xd0, 0x3d, 0xb4, 0xd5, 0xdf, 0xd3, 0x42, 0x47, 0x5a, 0x6d, 0x19, 0x27, 0x66, 0x4b, 0x2e, 0x0c, 0x27, 0x9c, 0x96, 0x4c, 0x72, @@ -604,21 +603,21 @@ static const u8 enc_output009[] __initconst = { 0x43, 0xb5, 0xd9, 0x90, 0xe1, 0x85, 0xf5, 0xa8, 0xae }; -static const u8 enc_assoc009[] __initconst = { +static const u8 enc_assoc009[] = { 0x5a, 0x27, 0xff, 0xeb, 0xdf, 0x84, 0xb2, 0x9e, 0xef }; -static const u8 enc_nonce009[] __initconst = { +static const u8 enc_nonce009[] = { 0xef, 0x2d, 0x63, 0xee, 0x6b, 0x80, 0x8b, 0x78 }; -static const u8 enc_key009[] __initconst = { +static const u8 enc_key009[] = { 0xea, 0xbc, 0x56, 0x99, 0xe3, 0x50, 0xff, 0xc5, 0xcc, 0x1a, 0xd7, 0xc1, 0x57, 0x72, 0xea, 0x86, 0x5b, 0x89, 0x88, 0x61, 0x3d, 0x2f, 0x9b, 0xb2, 0xe7, 0x9c, 0xec, 0x74, 0x6e, 0x3e, 0xf4, 0x3b }; -static const u8 enc_input010[] __initconst = { +static const u8 enc_input010[] = { 0x42, 0x93, 0xe4, 0xeb, 0x97, 0xb0, 0x57, 0xbf, 0x1a, 0x8b, 0x1f, 0xe4, 0x5f, 0x36, 0x20, 0x3c, 0xef, 0x0a, 0xa9, 0x48, 0x5f, 0x5f, 0x37, 0x22, @@ -748,7 +747,7 @@ static const u8 enc_input010[] __initconst = { 0x5a, 0xc0, 0x33, 0x1f, 0xcb, 0x05, 0x6d, 0x5c, 0x06, 0x87, 0x52, 0xa2, 0x8f, 0x26, 0xd5, 0x4f }; -static const u8 enc_output010[] __initconst = { +static const u8 enc_output010[] = { 0xe5, 0x26, 0xa4, 0x3d, 0xbd, 0x33, 0xd0, 0x4b, 0x6f, 0x05, 0xa7, 0x6e, 0x12, 0x7a, 0xd2, 0x74, 0xa6, 0xdd, 0xbd, 0x95, 0xeb, 0xf9, 0xa4, 0xf1, @@ -880,21 +879,21 @@ static const u8 enc_output010[] __initconst = { 0x80, 0x6b, 0xf6, 0x1f, 0xc3, 0xcc, 0x97, 0xc9, 0x24, 0x9f, 0xf3, 0xaf, 0x43, 0x14, 0xd5, 0xa0 }; -static const u8 enc_assoc010[] __initconst = { +static const u8 enc_assoc010[] = { 0xd2, 0xa1, 0x70, 0xdb, 0x7a, 0xf8, 0xfa, 0x27, 0xba, 0x73, 0x0f, 0xbf, 0x3d, 0x1e, 0x82, 0xb2 }; -static const u8 enc_nonce010[] __initconst = { +static const u8 enc_nonce010[] = { 0xdb, 0x92, 0x0f, 0x7f, 0x17, 0x54, 0x0c, 0x30 }; -static const u8 enc_key010[] __initconst = { +static const u8 enc_key010[] = { 0x47, 0x11, 0xeb, 0x86, 0x2b, 0x2c, 0xab, 0x44, 0x34, 0xda, 0x7f, 0x57, 0x03, 0x39, 0x0c, 0xaf, 0x2c, 0x14, 0xfd, 0x65, 0x23, 0xe9, 0x8e, 0x74, 0xd5, 0x08, 0x68, 0x08, 0xe7, 0xb4, 0x72, 0xd7 }; -static const u8 enc_input011[] __initconst = { +static const u8 enc_input011[] = { 0x7a, 0x57, 0xf2, 0xc7, 0x06, 0x3f, 0x50, 0x7b, 0x36, 0x1a, 0x66, 0x5c, 0xb9, 0x0e, 0x5e, 0x3b, 0x45, 0x60, 0xbe, 0x9a, 0x31, 0x9f, 0xff, 0x5d, @@ -1138,7 +1137,7 @@ static const u8 enc_input011[] __initconst = { 0x5c, 0x42, 0x16, 0xb8, 0x28, 0xeb, 0x1b, 0x61, 0x10, 0x1e, 0xbf, 0xec, 0xa8 }; -static const u8 enc_output011[] __initconst = { +static const u8 enc_output011[] = { 0x6a, 0xfc, 0x4b, 0x25, 0xdf, 0xc0, 0xe4, 0xe8, 0x17, 0x4d, 0x4c, 0xc9, 0x7e, 0xde, 0x3a, 0xcc, 0x3c, 0xba, 0x6a, 0x77, 0x47, 0xdb, 0xe3, 0x74, @@ -1384,20 +1383,20 @@ static const u8 enc_output011[] __initconst = { 0x86, 0xc4, 0xfc, 0x97, 0xae, 0x3f, 0x8f, 0x1e, 0x2b, 0xdf, 0xcd, 0xf9, 0x3c }; -static const u8 enc_assoc011[] __initconst = { +static const u8 enc_assoc011[] = { 0xd6, 0x31, 0xda, 0x5d, 0x42, 0x5e, 0xd7 }; -static const u8 enc_nonce011[] __initconst = { +static const u8 enc_nonce011[] = { 0xfd, 0x87, 0xd4, 0xd8, 0x62, 0xfd, 0xec, 0xaa }; -static const u8 enc_key011[] __initconst = { +static const u8 enc_key011[] = { 0x35, 0x4e, 0xb5, 0x70, 0x50, 0x42, 0x8a, 0x85, 0xf2, 0xfb, 0xed, 0x7b, 0xd0, 0x9e, 0x97, 0xca, 0xfa, 0x98, 0x66, 0x63, 0xee, 0x37, 0xcc, 0x52, 0xfe, 0xd1, 0xdf, 0x95, 0x15, 0x34, 0x29, 0x38 }; -static const u8 enc_input012[] __initconst = { +static const u8 enc_input012[] = { 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, 0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57, @@ -1651,7 +1650,7 @@ static const u8 enc_input012[] __initconst = { 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, 0x78, 0xec, 0x00 }; -static const u8 enc_output012[] __initconst = { +static const u8 enc_output012[] = { 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, 0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1, @@ -1907,7 +1906,7 @@ static const u8 enc_output012[] __initconst = { 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, 0x70, 0xcf, 0xd6 }; -static const u8 enc_assoc012[] __initconst = { +static const u8 enc_assoc012[] = { 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, 0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce, 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, @@ -1917,10 +1916,10 @@ static const u8 enc_assoc012[] __initconst = { 0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b, 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 }; -static const u8 enc_nonce012[] __initconst = { +static const u8 enc_nonce012[] = { 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 }; -static const u8 enc_key012[] __initconst = { +static const u8 enc_key012[] = { 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, 0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d, 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e, @@ -1928,7 +1927,7 @@ static const u8 enc_key012[] __initconst = { }; /* wycheproof - rfc7539 */ -static const u8 enc_input013[] __initconst = { +static const u8 enc_input013[] = { 0x4c, 0x61, 0x64, 0x69, 0x65, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x47, 0x65, 0x6e, 0x74, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x20, 0x6f, 0x66, 0x20, @@ -1945,7 +1944,7 @@ static const u8 enc_input013[] __initconst = { 0x75, 0x6c, 0x64, 0x20, 0x62, 0x65, 0x20, 0x69, 0x74, 0x2e }; -static const u8 enc_output013[] __initconst = { +static const u8 enc_output013[] = { 0xd3, 0x1a, 0x8d, 0x34, 0x64, 0x8e, 0x60, 0xdb, 0x7b, 0x86, 0xaf, 0xbc, 0x53, 0xef, 0x7e, 0xc2, 0xa4, 0xad, 0xed, 0x51, 0x29, 0x6e, 0x08, 0xfe, @@ -1964,15 +1963,15 @@ static const u8 enc_output013[] __initconst = { 0xe2, 0x6a, 0x7e, 0x90, 0x2e, 0xcb, 0xd0, 0x60, 0x06, 0x91 }; -static const u8 enc_assoc013[] __initconst = { +static const u8 enc_assoc013[] = { 0x50, 0x51, 0x52, 0x53, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7 }; -static const u8 enc_nonce013[] __initconst = { +static const u8 enc_nonce013[] = { 0x07, 0x00, 0x00, 0x00, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47 }; -static const u8 enc_key013[] __initconst = { +static const u8 enc_key013[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -1980,17 +1979,17 @@ static const u8 enc_key013[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input014[] __initconst = { }; -static const u8 enc_output014[] __initconst = { +static const u8 enc_input014[] = { }; +static const u8 enc_output014[] = { 0x76, 0xac, 0xb3, 0x42, 0xcf, 0x31, 0x66, 0xa5, 0xb6, 0x3c, 0x0c, 0x0e, 0xa1, 0x38, 0x3c, 0x8d }; -static const u8 enc_assoc014[] __initconst = { }; -static const u8 enc_nonce014[] __initconst = { +static const u8 enc_assoc014[] = { }; +static const u8 enc_nonce014[] = { 0x4d, 0xa5, 0xbf, 0x8d, 0xfd, 0x58, 0x52, 0xc1, 0xea, 0x12, 0x37, 0x9d }; -static const u8 enc_key014[] __initconst = { +static const u8 enc_key014[] = { 0x80, 0xba, 0x31, 0x92, 0xc8, 0x03, 0xce, 0x96, 0x5e, 0xa3, 0x71, 0xd5, 0xff, 0x07, 0x3c, 0xf0, 0xf4, 0x3b, 0x6a, 0x2a, 0xb5, 0x76, 0xb2, 0x08, @@ -1998,19 +1997,19 @@ static const u8 enc_key014[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input015[] __initconst = { }; -static const u8 enc_output015[] __initconst = { +static const u8 enc_input015[] = { }; +static const u8 enc_output015[] = { 0x90, 0x6f, 0xa6, 0x28, 0x4b, 0x52, 0xf8, 0x7b, 0x73, 0x59, 0xcb, 0xaa, 0x75, 0x63, 0xc7, 0x09 }; -static const u8 enc_assoc015[] __initconst = { +static const u8 enc_assoc015[] = { 0xbd, 0x50, 0x67, 0x64, 0xf2, 0xd2, 0xc4, 0x10 }; -static const u8 enc_nonce015[] __initconst = { +static const u8 enc_nonce015[] = { 0xa9, 0x2e, 0xf0, 0xac, 0x99, 0x1d, 0xd5, 0x16, 0xa3, 0xc6, 0xf6, 0x89 }; -static const u8 enc_key015[] __initconst = { +static const u8 enc_key015[] = { 0x7a, 0x4c, 0xd7, 0x59, 0x17, 0x2e, 0x02, 0xeb, 0x20, 0x4d, 0xb2, 0xc3, 0xf5, 0xc7, 0x46, 0x22, 0x7d, 0xf5, 0x84, 0xfc, 0x13, 0x45, 0x19, 0x63, @@ -2018,20 +2017,20 @@ static const u8 enc_key015[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input016[] __initconst = { +static const u8 enc_input016[] = { 0x2a }; -static const u8 enc_output016[] __initconst = { +static const u8 enc_output016[] = { 0x3a, 0xca, 0xc2, 0x7d, 0xec, 0x09, 0x68, 0x80, 0x1e, 0x9f, 0x6e, 0xde, 0xd6, 0x9d, 0x80, 0x75, 0x22 }; -static const u8 enc_assoc016[] __initconst = { }; -static const u8 enc_nonce016[] __initconst = { +static const u8 enc_assoc016[] = { }; +static const u8 enc_nonce016[] = { 0x99, 0xe2, 0x3e, 0xc4, 0x89, 0x85, 0xbc, 0xcd, 0xee, 0xab, 0x60, 0xf1 }; -static const u8 enc_key016[] __initconst = { +static const u8 enc_key016[] = { 0xcc, 0x56, 0xb6, 0x80, 0x55, 0x2e, 0xb7, 0x50, 0x08, 0xf5, 0x48, 0x4b, 0x4c, 0xb8, 0x03, 0xfa, 0x50, 0x63, 0xeb, 0xd6, 0xea, 0xb9, 0x1f, 0x6a, @@ -2039,22 +2038,22 @@ static const u8 enc_key016[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input017[] __initconst = { +static const u8 enc_input017[] = { 0x51 }; -static const u8 enc_output017[] __initconst = { +static const u8 enc_output017[] = { 0xc4, 0x16, 0x83, 0x10, 0xca, 0x45, 0xb1, 0xf7, 0xc6, 0x6c, 0xad, 0x4e, 0x99, 0xe4, 0x3f, 0x72, 0xb9 }; -static const u8 enc_assoc017[] __initconst = { +static const u8 enc_assoc017[] = { 0x91, 0xca, 0x6c, 0x59, 0x2c, 0xbc, 0xca, 0x53 }; -static const u8 enc_nonce017[] __initconst = { +static const u8 enc_nonce017[] = { 0xab, 0x0d, 0xca, 0x71, 0x6e, 0xe0, 0x51, 0xd2, 0x78, 0x2f, 0x44, 0x03 }; -static const u8 enc_key017[] __initconst = { +static const u8 enc_key017[] = { 0x46, 0xf0, 0x25, 0x49, 0x65, 0xf7, 0x69, 0xd5, 0x2b, 0xdb, 0x4a, 0x70, 0xb4, 0x43, 0x19, 0x9f, 0x8e, 0xf2, 0x07, 0x52, 0x0d, 0x12, 0x20, 0xc5, @@ -2062,20 +2061,20 @@ static const u8 enc_key017[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input018[] __initconst = { +static const u8 enc_input018[] = { 0x5c, 0x60 }; -static const u8 enc_output018[] __initconst = { +static const u8 enc_output018[] = { 0x4d, 0x13, 0x91, 0xe8, 0xb6, 0x1e, 0xfb, 0x39, 0xc1, 0x22, 0x19, 0x54, 0x53, 0x07, 0x7b, 0x22, 0xe5, 0xe2 }; -static const u8 enc_assoc018[] __initconst = { }; -static const u8 enc_nonce018[] __initconst = { +static const u8 enc_assoc018[] = { }; +static const u8 enc_nonce018[] = { 0x46, 0x1a, 0xf1, 0x22, 0xe9, 0xf2, 0xe0, 0x34, 0x7e, 0x03, 0xf2, 0xdb }; -static const u8 enc_key018[] __initconst = { +static const u8 enc_key018[] = { 0x2f, 0x7f, 0x7e, 0x4f, 0x59, 0x2b, 0xb3, 0x89, 0x19, 0x49, 0x89, 0x74, 0x35, 0x07, 0xbf, 0x3e, 0xe9, 0xcb, 0xde, 0x17, 0x86, 0xb6, 0x69, 0x5f, @@ -2083,22 +2082,22 @@ static const u8 enc_key018[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input019[] __initconst = { +static const u8 enc_input019[] = { 0xdd, 0xf2 }; -static const u8 enc_output019[] __initconst = { +static const u8 enc_output019[] = { 0xb6, 0x0d, 0xea, 0xd0, 0xfd, 0x46, 0x97, 0xec, 0x2e, 0x55, 0x58, 0x23, 0x77, 0x19, 0xd0, 0x24, 0x37, 0xa2 }; -static const u8 enc_assoc019[] __initconst = { +static const u8 enc_assoc019[] = { 0x88, 0x36, 0x4f, 0xc8, 0x06, 0x05, 0x18, 0xbf }; -static const u8 enc_nonce019[] __initconst = { +static const u8 enc_nonce019[] = { 0x61, 0x54, 0x6b, 0xa5, 0xf1, 0x72, 0x05, 0x90, 0xb6, 0x04, 0x0a, 0xc6 }; -static const u8 enc_key019[] __initconst = { +static const u8 enc_key019[] = { 0xc8, 0x83, 0x3d, 0xce, 0x5e, 0xa9, 0xf2, 0x48, 0xaa, 0x20, 0x30, 0xea, 0xcf, 0xe7, 0x2b, 0xff, 0xe6, 0x9a, 0x62, 0x0c, 0xaf, 0x79, 0x33, 0x44, @@ -2106,20 +2105,20 @@ static const u8 enc_key019[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input020[] __initconst = { +static const u8 enc_input020[] = { 0xab, 0x85, 0xe9, 0xc1, 0x57, 0x17, 0x31 }; -static const u8 enc_output020[] __initconst = { +static const u8 enc_output020[] = { 0x5d, 0xfe, 0x34, 0x40, 0xdb, 0xb3, 0xc3, 0xed, 0x7a, 0x43, 0x4e, 0x26, 0x02, 0xd3, 0x94, 0x28, 0x1e, 0x0a, 0xfa, 0x9f, 0xb7, 0xaa, 0x42 }; -static const u8 enc_assoc020[] __initconst = { }; -static const u8 enc_nonce020[] __initconst = { +static const u8 enc_assoc020[] = { }; +static const u8 enc_nonce020[] = { 0x3c, 0x4e, 0x65, 0x4d, 0x66, 0x3f, 0xa4, 0x59, 0x6d, 0xc5, 0x5b, 0xb7 }; -static const u8 enc_key020[] __initconst = { +static const u8 enc_key020[] = { 0x55, 0x56, 0x81, 0x58, 0xd3, 0xa6, 0x48, 0x3f, 0x1f, 0x70, 0x21, 0xea, 0xb6, 0x9b, 0x70, 0x3f, 0x61, 0x42, 0x51, 0xca, 0xdc, 0x1a, 0xf5, 0xd3, @@ -2127,22 +2126,22 @@ static const u8 enc_key020[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input021[] __initconst = { +static const u8 enc_input021[] = { 0x4e, 0xe5, 0xcd, 0xa2, 0x0d, 0x42, 0x90 }; -static const u8 enc_output021[] __initconst = { +static const u8 enc_output021[] = { 0x4b, 0xd4, 0x72, 0x12, 0x94, 0x1c, 0xe3, 0x18, 0x5f, 0x14, 0x08, 0xee, 0x7f, 0xbf, 0x18, 0xf5, 0xab, 0xad, 0x6e, 0x22, 0x53, 0xa1, 0xba }; -static const u8 enc_assoc021[] __initconst = { +static const u8 enc_assoc021[] = { 0x84, 0xe4, 0x6b, 0xe8, 0xc0, 0x91, 0x90, 0x53 }; -static const u8 enc_nonce021[] __initconst = { +static const u8 enc_nonce021[] = { 0x58, 0x38, 0x93, 0x75, 0xc6, 0x9e, 0xe3, 0x98, 0xde, 0x94, 0x83, 0x96 }; -static const u8 enc_key021[] __initconst = { +static const u8 enc_key021[] = { 0xe3, 0xc0, 0x9e, 0x7f, 0xab, 0x1a, 0xef, 0xb5, 0x16, 0xda, 0x6a, 0x33, 0x02, 0x2a, 0x1d, 0xd4, 0xeb, 0x27, 0x2c, 0x80, 0xd5, 0x40, 0xc5, 0xda, @@ -2150,20 +2149,20 @@ static const u8 enc_key021[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input022[] __initconst = { +static const u8 enc_input022[] = { 0xbe, 0x33, 0x08, 0xf7, 0x2a, 0x2c, 0x6a, 0xed }; -static const u8 enc_output022[] __initconst = { +static const u8 enc_output022[] = { 0x8e, 0x94, 0x39, 0xa5, 0x6e, 0xee, 0xc8, 0x17, 0xfb, 0xe8, 0xa6, 0xed, 0x8f, 0xab, 0xb1, 0x93, 0x75, 0x39, 0xdd, 0x6c, 0x00, 0xe9, 0x00, 0x21 }; -static const u8 enc_assoc022[] __initconst = { }; -static const u8 enc_nonce022[] __initconst = { +static const u8 enc_assoc022[] = { }; +static const u8 enc_nonce022[] = { 0x4f, 0x07, 0xaf, 0xed, 0xfd, 0xc3, 0xb6, 0xc2, 0x36, 0x18, 0x23, 0xd3 }; -static const u8 enc_key022[] __initconst = { +static const u8 enc_key022[] = { 0x51, 0xe4, 0xbf, 0x2b, 0xad, 0x92, 0xb7, 0xaf, 0xf1, 0xa4, 0xbc, 0x05, 0x55, 0x0b, 0xa8, 0x1d, 0xf4, 0xb9, 0x6f, 0xab, 0xf4, 0x1c, 0x12, 0xc7, @@ -2171,22 +2170,22 @@ static const u8 enc_key022[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input023[] __initconst = { +static const u8 enc_input023[] = { 0xa4, 0xc9, 0xc2, 0x80, 0x1b, 0x71, 0xf7, 0xdf }; -static const u8 enc_output023[] __initconst = { +static const u8 enc_output023[] = { 0xb9, 0xb9, 0x10, 0x43, 0x3a, 0xf0, 0x52, 0xb0, 0x45, 0x30, 0xf5, 0x1a, 0xee, 0xe0, 0x24, 0xe0, 0xa4, 0x45, 0xa6, 0x32, 0x8f, 0xa6, 0x7a, 0x18 }; -static const u8 enc_assoc023[] __initconst = { +static const u8 enc_assoc023[] = { 0x66, 0xc0, 0xae, 0x70, 0x07, 0x6c, 0xb1, 0x4d }; -static const u8 enc_nonce023[] __initconst = { +static const u8 enc_nonce023[] = { 0xb4, 0xea, 0x66, 0x6e, 0xe1, 0x19, 0x56, 0x33, 0x66, 0x48, 0x4a, 0x78 }; -static const u8 enc_key023[] __initconst = { +static const u8 enc_key023[] = { 0x11, 0x31, 0xc1, 0x41, 0x85, 0x77, 0xa0, 0x54, 0xde, 0x7a, 0x4a, 0xc5, 0x51, 0x95, 0x0f, 0x1a, 0x05, 0x3f, 0x9a, 0xe4, 0x6e, 0x5b, 0x75, 0xfe, @@ -2194,22 +2193,22 @@ static const u8 enc_key023[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input024[] __initconst = { +static const u8 enc_input024[] = { 0x42, 0xba, 0xae, 0x59, 0x78, 0xfe, 0xaf, 0x5c, 0x36, 0x8d, 0x14, 0xe0 }; -static const u8 enc_output024[] __initconst = { +static const u8 enc_output024[] = { 0xff, 0x7d, 0xc2, 0x03, 0xb2, 0x6c, 0x46, 0x7a, 0x6b, 0x50, 0xdb, 0x33, 0x57, 0x8c, 0x0f, 0x27, 0x58, 0xc2, 0xe1, 0x4e, 0x36, 0xd4, 0xfc, 0x10, 0x6d, 0xcb, 0x29, 0xb4 }; -static const u8 enc_assoc024[] __initconst = { }; -static const u8 enc_nonce024[] __initconst = { +static const u8 enc_assoc024[] = { }; +static const u8 enc_nonce024[] = { 0x9a, 0x59, 0xfc, 0xe2, 0x6d, 0xf0, 0x00, 0x5e, 0x07, 0x53, 0x86, 0x56 }; -static const u8 enc_key024[] __initconst = { +static const u8 enc_key024[] = { 0x99, 0xb6, 0x2b, 0xd5, 0xaf, 0xbe, 0x3f, 0xb0, 0x15, 0xbd, 0xe9, 0x3f, 0x0a, 0xbf, 0x48, 0x39, 0x57, 0xa1, 0xc3, 0xeb, 0x3c, 0xa5, 0x9c, 0xb5, @@ -2217,24 +2216,24 @@ static const u8 enc_key024[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input025[] __initconst = { +static const u8 enc_input025[] = { 0xfd, 0xc8, 0x5b, 0x94, 0xa4, 0xb2, 0xa6, 0xb7, 0x59, 0xb1, 0xa0, 0xda }; -static const u8 enc_output025[] __initconst = { +static const u8 enc_output025[] = { 0x9f, 0x88, 0x16, 0xde, 0x09, 0x94, 0xe9, 0x38, 0xd9, 0xe5, 0x3f, 0x95, 0xd0, 0x86, 0xfc, 0x6c, 0x9d, 0x8f, 0xa9, 0x15, 0xfd, 0x84, 0x23, 0xa7, 0xcf, 0x05, 0x07, 0x2f }; -static const u8 enc_assoc025[] __initconst = { +static const u8 enc_assoc025[] = { 0xa5, 0x06, 0xe1, 0xa5, 0xc6, 0x90, 0x93, 0xf9 }; -static const u8 enc_nonce025[] __initconst = { +static const u8 enc_nonce025[] = { 0x58, 0xdb, 0xd4, 0xad, 0x2c, 0x4a, 0xd3, 0x5d, 0xd9, 0x06, 0xe9, 0xce }; -static const u8 enc_key025[] __initconst = { +static const u8 enc_key025[] = { 0x85, 0xf3, 0x5b, 0x62, 0x82, 0xcf, 0xf4, 0x40, 0xbc, 0x10, 0x20, 0xc8, 0x13, 0x6f, 0xf2, 0x70, 0x31, 0x11, 0x0f, 0xa6, 0x3e, 0xc1, 0x6f, 0x1e, @@ -2242,22 +2241,22 @@ static const u8 enc_key025[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input026[] __initconst = { +static const u8 enc_input026[] = { 0x51, 0xf8, 0xc1, 0xf7, 0x31, 0xea, 0x14, 0xac, 0xdb, 0x21, 0x0a, 0x6d, 0x97, 0x3e, 0x07 }; -static const u8 enc_output026[] __initconst = { +static const u8 enc_output026[] = { 0x0b, 0x29, 0x63, 0x8e, 0x1f, 0xbd, 0xd6, 0xdf, 0x53, 0x97, 0x0b, 0xe2, 0x21, 0x00, 0x42, 0x2a, 0x91, 0x34, 0x08, 0x7d, 0x67, 0xa4, 0x6e, 0x79, 0x17, 0x8d, 0x0a, 0x93, 0xf5, 0xe1, 0xd2 }; -static const u8 enc_assoc026[] __initconst = { }; -static const u8 enc_nonce026[] __initconst = { +static const u8 enc_assoc026[] = { }; +static const u8 enc_nonce026[] = { 0x68, 0xab, 0x7f, 0xdb, 0xf6, 0x19, 0x01, 0xda, 0xd4, 0x61, 0xd2, 0x3c }; -static const u8 enc_key026[] __initconst = { +static const u8 enc_key026[] = { 0x67, 0x11, 0x96, 0x27, 0xbd, 0x98, 0x8e, 0xda, 0x90, 0x62, 0x19, 0xe0, 0x8c, 0x0d, 0x0d, 0x77, 0x9a, 0x07, 0xd2, 0x08, 0xce, 0x8a, 0x4f, 0xe0, @@ -2265,24 +2264,24 @@ static const u8 enc_key026[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input027[] __initconst = { +static const u8 enc_input027[] = { 0x97, 0x46, 0x9d, 0xa6, 0x67, 0xd6, 0x11, 0x0f, 0x9c, 0xbd, 0xa1, 0xd1, 0xa2, 0x06, 0x73 }; -static const u8 enc_output027[] __initconst = { +static const u8 enc_output027[] = { 0x32, 0xdb, 0x66, 0xc4, 0xa3, 0x81, 0x9d, 0x81, 0x55, 0x74, 0x55, 0xe5, 0x98, 0x0f, 0xed, 0xfe, 0xae, 0x30, 0xde, 0xc9, 0x4e, 0x6a, 0xd3, 0xa9, 0xee, 0xa0, 0x6a, 0x0d, 0x70, 0x39, 0x17 }; -static const u8 enc_assoc027[] __initconst = { +static const u8 enc_assoc027[] = { 0x64, 0x53, 0xa5, 0x33, 0x84, 0x63, 0x22, 0x12 }; -static const u8 enc_nonce027[] __initconst = { +static const u8 enc_nonce027[] = { 0xd9, 0x5b, 0x32, 0x43, 0xaf, 0xae, 0xf7, 0x14, 0xc5, 0x03, 0x5b, 0x6a }; -static const u8 enc_key027[] __initconst = { +static const u8 enc_key027[] = { 0xe6, 0xf1, 0x11, 0x8d, 0x41, 0xe4, 0xb4, 0x3f, 0xb5, 0x82, 0x21, 0xb7, 0xed, 0x79, 0x67, 0x38, 0x34, 0xe0, 0xd8, 0xac, 0x5c, 0x4f, 0xa6, 0x0b, @@ -2290,22 +2289,22 @@ static const u8 enc_key027[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input028[] __initconst = { +static const u8 enc_input028[] = { 0x54, 0x9b, 0x36, 0x5a, 0xf9, 0x13, 0xf3, 0xb0, 0x81, 0x13, 0x1c, 0xcb, 0x6b, 0x82, 0x55, 0x88 }; -static const u8 enc_output028[] __initconst = { +static const u8 enc_output028[] = { 0xe9, 0x11, 0x0e, 0x9f, 0x56, 0xab, 0x3c, 0xa4, 0x83, 0x50, 0x0c, 0xea, 0xba, 0xb6, 0x7a, 0x13, 0x83, 0x6c, 0xca, 0xbf, 0x15, 0xa6, 0xa2, 0x2a, 0x51, 0xc1, 0x07, 0x1c, 0xfa, 0x68, 0xfa, 0x0c }; -static const u8 enc_assoc028[] __initconst = { }; -static const u8 enc_nonce028[] __initconst = { +static const u8 enc_assoc028[] = { }; +static const u8 enc_nonce028[] = { 0x2f, 0xcb, 0x1b, 0x38, 0xa9, 0x9e, 0x71, 0xb8, 0x47, 0x40, 0xad, 0x9b }; -static const u8 enc_key028[] __initconst = { +static const u8 enc_key028[] = { 0x59, 0xd4, 0xea, 0xfb, 0x4d, 0xe0, 0xcf, 0xc7, 0xd3, 0xdb, 0x99, 0xa8, 0xf5, 0x4b, 0x15, 0xd7, 0xb3, 0x9f, 0x0a, 0xcc, 0x8d, 0xa6, 0x97, 0x63, @@ -2313,24 +2312,24 @@ static const u8 enc_key028[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input029[] __initconst = { +static const u8 enc_input029[] = { 0x55, 0xa4, 0x65, 0x64, 0x4f, 0x5b, 0x65, 0x09, 0x28, 0xcb, 0xee, 0x7c, 0x06, 0x32, 0x14, 0xd6 }; -static const u8 enc_output029[] __initconst = { +static const u8 enc_output029[] = { 0xe4, 0xb1, 0x13, 0xcb, 0x77, 0x59, 0x45, 0xf3, 0xd3, 0xa8, 0xae, 0x9e, 0xc1, 0x41, 0xc0, 0x0c, 0x7c, 0x43, 0xf1, 0x6c, 0xe0, 0x96, 0xd0, 0xdc, 0x27, 0xc9, 0x58, 0x49, 0xdc, 0x38, 0x3b, 0x7d }; -static const u8 enc_assoc029[] __initconst = { +static const u8 enc_assoc029[] = { 0x03, 0x45, 0x85, 0x62, 0x1a, 0xf8, 0xd7, 0xff }; -static const u8 enc_nonce029[] __initconst = { +static const u8 enc_nonce029[] = { 0x11, 0x8a, 0x69, 0x64, 0xc2, 0xd3, 0xe3, 0x80, 0x07, 0x1f, 0x52, 0x66 }; -static const u8 enc_key029[] __initconst = { +static const u8 enc_key029[] = { 0xb9, 0x07, 0xa4, 0x50, 0x75, 0x51, 0x3f, 0xe8, 0xa8, 0x01, 0x9e, 0xde, 0xe3, 0xf2, 0x59, 0x14, 0x87, 0xb2, 0xa0, 0x30, 0xb0, 0x3c, 0x6e, 0x1d, @@ -2338,24 +2337,24 @@ static const u8 enc_key029[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input030[] __initconst = { +static const u8 enc_input030[] = { 0x3f, 0xf1, 0x51, 0x4b, 0x1c, 0x50, 0x39, 0x15, 0x91, 0x8f, 0x0c, 0x0c, 0x31, 0x09, 0x4a, 0x6e, 0x1f }; -static const u8 enc_output030[] __initconst = { +static const u8 enc_output030[] = { 0x02, 0xcc, 0x3a, 0xcb, 0x5e, 0xe1, 0xfc, 0xdd, 0x12, 0xa0, 0x3b, 0xb8, 0x57, 0x97, 0x64, 0x74, 0xd3, 0xd8, 0x3b, 0x74, 0x63, 0xa2, 0xc3, 0x80, 0x0f, 0xe9, 0x58, 0xc2, 0x8e, 0xaa, 0x29, 0x08, 0x13 }; -static const u8 enc_assoc030[] __initconst = { }; -static const u8 enc_nonce030[] __initconst = { +static const u8 enc_assoc030[] = { }; +static const u8 enc_nonce030[] = { 0x45, 0xaa, 0xa3, 0xe5, 0xd1, 0x6d, 0x2d, 0x42, 0xdc, 0x03, 0x44, 0x5d }; -static const u8 enc_key030[] __initconst = { +static const u8 enc_key030[] = { 0x3b, 0x24, 0x58, 0xd8, 0x17, 0x6e, 0x16, 0x21, 0xc0, 0xcc, 0x24, 0xc0, 0xc0, 0xe2, 0x4c, 0x1e, 0x80, 0xd7, 0x2f, 0x7e, 0xe9, 0x14, 0x9a, 0x4b, @@ -2363,26 +2362,26 @@ static const u8 enc_key030[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input031[] __initconst = { +static const u8 enc_input031[] = { 0x63, 0x85, 0x8c, 0xa3, 0xe2, 0xce, 0x69, 0x88, 0x7b, 0x57, 0x8a, 0x3c, 0x16, 0x7b, 0x42, 0x1c, 0x9c }; -static const u8 enc_output031[] __initconst = { +static const u8 enc_output031[] = { 0x35, 0x76, 0x64, 0x88, 0xd2, 0xbc, 0x7c, 0x2b, 0x8d, 0x17, 0xcb, 0xbb, 0x9a, 0xbf, 0xad, 0x9e, 0x6d, 0x1f, 0x39, 0x1e, 0x65, 0x7b, 0x27, 0x38, 0xdd, 0xa0, 0x84, 0x48, 0xcb, 0xa2, 0x81, 0x1c, 0xeb }; -static const u8 enc_assoc031[] __initconst = { +static const u8 enc_assoc031[] = { 0x9a, 0xaf, 0x29, 0x9e, 0xee, 0xa7, 0x8f, 0x79 }; -static const u8 enc_nonce031[] __initconst = { +static const u8 enc_nonce031[] = { 0xf0, 0x38, 0x4f, 0xb8, 0x76, 0x12, 0x14, 0x10, 0x63, 0x3d, 0x99, 0x3d }; -static const u8 enc_key031[] __initconst = { +static const u8 enc_key031[] = { 0xf6, 0x0c, 0x6a, 0x1b, 0x62, 0x57, 0x25, 0xf7, 0x6c, 0x70, 0x37, 0xb4, 0x8f, 0xe3, 0x57, 0x7f, 0xa7, 0xf7, 0xb8, 0x7b, 0x1b, 0xd5, 0xa9, 0x82, @@ -2390,24 +2389,24 @@ static const u8 enc_key031[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input032[] __initconst = { +static const u8 enc_input032[] = { 0x10, 0xf1, 0xec, 0xf9, 0xc6, 0x05, 0x84, 0x66, 0x5d, 0x9a, 0xe5, 0xef, 0xe2, 0x79, 0xe7, 0xf7, 0x37, 0x7e, 0xea, 0x69, 0x16, 0xd2, 0xb1, 0x11 }; -static const u8 enc_output032[] __initconst = { +static const u8 enc_output032[] = { 0x42, 0xf2, 0x6c, 0x56, 0xcb, 0x4b, 0xe2, 0x1d, 0x9d, 0x8d, 0x0c, 0x80, 0xfc, 0x99, 0xdd, 0xe0, 0x0d, 0x75, 0xf3, 0x80, 0x74, 0xbf, 0xe7, 0x64, 0x54, 0xaa, 0x7e, 0x13, 0xd4, 0x8f, 0xff, 0x7d, 0x75, 0x57, 0x03, 0x94, 0x57, 0x04, 0x0a, 0x3a }; -static const u8 enc_assoc032[] __initconst = { }; -static const u8 enc_nonce032[] __initconst = { +static const u8 enc_assoc032[] = { }; +static const u8 enc_nonce032[] = { 0xe6, 0xb1, 0xad, 0xf2, 0xfd, 0x58, 0xa8, 0x76, 0x2c, 0x65, 0xf3, 0x1b }; -static const u8 enc_key032[] __initconst = { +static const u8 enc_key032[] = { 0x02, 0x12, 0xa8, 0xde, 0x50, 0x07, 0xed, 0x87, 0xb3, 0x3f, 0x1a, 0x70, 0x90, 0xb6, 0x11, 0x4f, 0x9e, 0x08, 0xce, 0xfd, 0x96, 0x07, 0xf2, 0xc2, @@ -2415,26 +2414,26 @@ static const u8 enc_key032[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input033[] __initconst = { +static const u8 enc_input033[] = { 0x92, 0x22, 0xf9, 0x01, 0x8e, 0x54, 0xfd, 0x6d, 0xe1, 0x20, 0x08, 0x06, 0xa9, 0xee, 0x8e, 0x4c, 0xc9, 0x04, 0xd2, 0x9f, 0x25, 0xcb, 0xa1, 0x93 }; -static const u8 enc_output033[] __initconst = { +static const u8 enc_output033[] = { 0x12, 0x30, 0x32, 0x43, 0x7b, 0x4b, 0xfd, 0x69, 0x20, 0xe8, 0xf7, 0xe7, 0xe0, 0x08, 0x7a, 0xe4, 0x88, 0x9e, 0xbe, 0x7a, 0x0a, 0xd0, 0xe9, 0x00, 0x3c, 0xf6, 0x8f, 0x17, 0x95, 0x50, 0xda, 0x63, 0xd3, 0xb9, 0x6c, 0x2d, 0x55, 0x41, 0x18, 0x65 }; -static const u8 enc_assoc033[] __initconst = { +static const u8 enc_assoc033[] = { 0x3e, 0x8b, 0xc5, 0xad, 0xe1, 0x82, 0xff, 0x08 }; -static const u8 enc_nonce033[] __initconst = { +static const u8 enc_nonce033[] = { 0x6b, 0x28, 0x2e, 0xbe, 0xcc, 0x54, 0x1b, 0xcd, 0x78, 0x34, 0xed, 0x55 }; -static const u8 enc_key033[] __initconst = { +static const u8 enc_key033[] = { 0xc5, 0xbc, 0x09, 0x56, 0x56, 0x46, 0xe7, 0xed, 0xda, 0x95, 0x4f, 0x1f, 0x73, 0x92, 0x23, 0xda, 0xda, 0x20, 0xb9, 0x5c, 0x44, 0xab, 0x03, 0x3d, @@ -2442,13 +2441,13 @@ static const u8 enc_key033[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input034[] __initconst = { +static const u8 enc_input034[] = { 0xb0, 0x53, 0x99, 0x92, 0x86, 0xa2, 0x82, 0x4f, 0x42, 0xcc, 0x8c, 0x20, 0x3a, 0xb2, 0x4e, 0x2c, 0x97, 0xa6, 0x85, 0xad, 0xcc, 0x2a, 0xd3, 0x26, 0x62, 0x55, 0x8e, 0x55, 0xa5, 0xc7, 0x29 }; -static const u8 enc_output034[] __initconst = { +static const u8 enc_output034[] = { 0x45, 0xc7, 0xd6, 0xb5, 0x3a, 0xca, 0xd4, 0xab, 0xb6, 0x88, 0x76, 0xa6, 0xe9, 0x6a, 0x48, 0xfb, 0x59, 0x52, 0x4d, 0x2c, 0x92, 0xc9, 0xd8, 0xa1, @@ -2456,12 +2455,12 @@ static const u8 enc_output034[] __initconst = { 0x6d, 0x3c, 0xa1, 0x0e, 0x31, 0x1b, 0x69, 0x5f, 0x3e, 0xae, 0x15, 0x51, 0x65, 0x24, 0x93 }; -static const u8 enc_assoc034[] __initconst = { }; -static const u8 enc_nonce034[] __initconst = { +static const u8 enc_assoc034[] = { }; +static const u8 enc_nonce034[] = { 0x04, 0xa9, 0xbe, 0x03, 0x50, 0x8a, 0x5f, 0x31, 0x37, 0x1a, 0x6f, 0xd2 }; -static const u8 enc_key034[] __initconst = { +static const u8 enc_key034[] = { 0x2e, 0xb5, 0x1c, 0x46, 0x9a, 0xa8, 0xeb, 0x9e, 0x6c, 0x54, 0xa8, 0x34, 0x9b, 0xae, 0x50, 0xa2, 0x0f, 0x0e, 0x38, 0x27, 0x11, 0xbb, 0xa1, 0x15, @@ -2469,13 +2468,13 @@ static const u8 enc_key034[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input035[] __initconst = { +static const u8 enc_input035[] = { 0xf4, 0x52, 0x06, 0xab, 0xc2, 0x55, 0x52, 0xb2, 0xab, 0xc9, 0xab, 0x7f, 0xa2, 0x43, 0x03, 0x5f, 0xed, 0xaa, 0xdd, 0xc3, 0xb2, 0x29, 0x39, 0x56, 0xf1, 0xea, 0x6e, 0x71, 0x56, 0xe7, 0xeb }; -static const u8 enc_output035[] __initconst = { +static const u8 enc_output035[] = { 0x46, 0xa8, 0x0c, 0x41, 0x87, 0x02, 0x47, 0x20, 0x08, 0x46, 0x27, 0x58, 0x00, 0x80, 0xdd, 0xe5, 0xa3, 0xf4, 0xa1, 0x10, 0x93, 0xa7, 0x07, 0x6e, @@ -2483,14 +2482,14 @@ static const u8 enc_output035[] __initconst = { 0x4d, 0x4a, 0xa2, 0x83, 0x5a, 0x52, 0xe7, 0x2d, 0x14, 0xdf, 0x0e, 0x4f, 0x47, 0xf2, 0x5f }; -static const u8 enc_assoc035[] __initconst = { +static const u8 enc_assoc035[] = { 0x37, 0x46, 0x18, 0xa0, 0x6e, 0xa9, 0x8a, 0x48 }; -static const u8 enc_nonce035[] __initconst = { +static const u8 enc_nonce035[] = { 0x47, 0x0a, 0x33, 0x9e, 0xcb, 0x32, 0x19, 0xb8, 0xb8, 0x1a, 0x1f, 0x8b }; -static const u8 enc_key035[] __initconst = { +static const u8 enc_key035[] = { 0x7f, 0x5b, 0x74, 0xc0, 0x7e, 0xd1, 0xb4, 0x0f, 0xd1, 0x43, 0x58, 0xfe, 0x2f, 0xf2, 0xa7, 0x40, 0xc1, 0x16, 0xc7, 0x70, 0x65, 0x10, 0xe6, 0xa4, @@ -2498,13 +2497,13 @@ static const u8 enc_key035[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input036[] __initconst = { +static const u8 enc_input036[] = { 0xb9, 0xc5, 0x54, 0xcb, 0xc3, 0x6a, 0xc1, 0x8a, 0xe8, 0x97, 0xdf, 0x7b, 0xee, 0xca, 0xc1, 0xdb, 0xeb, 0x4e, 0xaf, 0xa1, 0x56, 0xbb, 0x60, 0xce, 0x2e, 0x5d, 0x48, 0xf0, 0x57, 0x15, 0xe6, 0x78 }; -static const u8 enc_output036[] __initconst = { +static const u8 enc_output036[] = { 0xea, 0x29, 0xaf, 0xa4, 0x9d, 0x36, 0xe8, 0x76, 0x0f, 0x5f, 0xe1, 0x97, 0x23, 0xb9, 0x81, 0x1e, 0xd5, 0xd5, 0x19, 0x93, 0x4a, 0x44, 0x0f, 0x50, @@ -2512,12 +2511,12 @@ static const u8 enc_output036[] __initconst = { 0x22, 0x25, 0x41, 0xaf, 0x46, 0xb8, 0x65, 0x33, 0xc6, 0xb6, 0x8d, 0x2f, 0xf1, 0x08, 0xa7, 0xea }; -static const u8 enc_assoc036[] __initconst = { }; -static const u8 enc_nonce036[] __initconst = { +static const u8 enc_assoc036[] = { }; +static const u8 enc_nonce036[] = { 0x72, 0xcf, 0xd9, 0x0e, 0xf3, 0x02, 0x6c, 0xa2, 0x2b, 0x7e, 0x6e, 0x6a }; -static const u8 enc_key036[] __initconst = { +static const u8 enc_key036[] = { 0xe1, 0x73, 0x1d, 0x58, 0x54, 0xe1, 0xb7, 0x0c, 0xb3, 0xff, 0xe8, 0xb7, 0x86, 0xa2, 0xb3, 0xeb, 0xf0, 0x99, 0x43, 0x70, 0x95, 0x47, 0x57, 0xb9, @@ -2525,13 +2524,13 @@ static const u8 enc_key036[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input037[] __initconst = { +static const u8 enc_input037[] = { 0x6b, 0x26, 0x04, 0x99, 0x6c, 0xd3, 0x0c, 0x14, 0xa1, 0x3a, 0x52, 0x57, 0xed, 0x6c, 0xff, 0xd3, 0xbc, 0x5e, 0x29, 0xd6, 0xb9, 0x7e, 0xb1, 0x79, 0x9e, 0xb3, 0x35, 0xe2, 0x81, 0xea, 0x45, 0x1e }; -static const u8 enc_output037[] __initconst = { +static const u8 enc_output037[] = { 0x6d, 0xad, 0x63, 0x78, 0x97, 0x54, 0x4d, 0x8b, 0xf6, 0xbe, 0x95, 0x07, 0xed, 0x4d, 0x1b, 0xb2, 0xe9, 0x54, 0xbc, 0x42, 0x7e, 0x5d, 0xe7, 0x29, @@ -2539,14 +2538,14 @@ static const u8 enc_output037[] __initconst = { 0x7b, 0x99, 0x7d, 0x93, 0xc9, 0x82, 0x18, 0x9d, 0x70, 0x95, 0xdc, 0x79, 0x4c, 0x74, 0x62, 0x32 }; -static const u8 enc_assoc037[] __initconst = { +static const u8 enc_assoc037[] = { 0x23, 0x33, 0xe5, 0xce, 0x0f, 0x93, 0xb0, 0x59 }; -static const u8 enc_nonce037[] __initconst = { +static const u8 enc_nonce037[] = { 0x26, 0x28, 0x80, 0xd4, 0x75, 0xf3, 0xda, 0xc5, 0x34, 0x0d, 0xd1, 0xb8 }; -static const u8 enc_key037[] __initconst = { +static const u8 enc_key037[] = { 0x27, 0xd8, 0x60, 0x63, 0x1b, 0x04, 0x85, 0xa4, 0x10, 0x70, 0x2f, 0xea, 0x61, 0xbc, 0x87, 0x3f, 0x34, 0x42, 0x26, 0x0c, 0xad, 0xed, 0x4a, 0xbd, @@ -2554,7 +2553,7 @@ static const u8 enc_key037[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input038[] __initconst = { +static const u8 enc_input038[] = { 0x97, 0x3d, 0x0c, 0x75, 0x38, 0x26, 0xba, 0xe4, 0x66, 0xcf, 0x9a, 0xbb, 0x34, 0x93, 0x15, 0x2e, 0x9d, 0xe7, 0x81, 0x9e, 0x2b, 0xd0, 0xc7, 0x11, @@ -2562,7 +2561,7 @@ static const u8 enc_input038[] __initconst = { 0x1a, 0xa3, 0xce, 0xdc, 0x0d, 0xfd, 0x7b, 0x46, 0x7e, 0x26, 0x22, 0x8b, 0xc8, 0x6c, 0x9a }; -static const u8 enc_output038[] __initconst = { +static const u8 enc_output038[] = { 0xfb, 0xa7, 0x8a, 0xe4, 0xf9, 0xd8, 0x08, 0xa6, 0x2e, 0x3d, 0xa4, 0x0b, 0xe2, 0xcb, 0x77, 0x00, 0xc3, 0x61, 0x3d, 0x9e, 0xb2, 0xc5, 0x29, 0xc6, @@ -2572,12 +2571,12 @@ static const u8 enc_output038[] __initconst = { 0x04, 0x69, 0x56, 0xdb, 0x3a, 0x51, 0x29, 0x08, 0xbd, 0x7a, 0xfc, 0x8f, 0x2a, 0xb0, 0xa9 }; -static const u8 enc_assoc038[] __initconst = { }; -static const u8 enc_nonce038[] __initconst = { +static const u8 enc_assoc038[] = { }; +static const u8 enc_nonce038[] = { 0xe7, 0x4a, 0x51, 0x5e, 0x7e, 0x21, 0x02, 0xb9, 0x0b, 0xef, 0x55, 0xd2 }; -static const u8 enc_key038[] __initconst = { +static const u8 enc_key038[] = { 0xcf, 0x0d, 0x40, 0xa4, 0x64, 0x4e, 0x5f, 0x51, 0x81, 0x51, 0x65, 0xd5, 0x30, 0x1b, 0x22, 0x63, 0x1f, 0x45, 0x44, 0xc4, 0x9a, 0x18, 0x78, 0xe3, @@ -2585,7 +2584,7 @@ static const u8 enc_key038[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input039[] __initconst = { +static const u8 enc_input039[] = { 0xa9, 0x89, 0x95, 0x50, 0x4d, 0xf1, 0x6f, 0x74, 0x8b, 0xfb, 0x77, 0x85, 0xff, 0x91, 0xee, 0xb3, 0xb6, 0x60, 0xea, 0x9e, 0xd3, 0x45, 0x0c, 0x3d, @@ -2593,7 +2592,7 @@ static const u8 enc_input039[] __initconst = { 0xa9, 0x97, 0x8d, 0x75, 0x54, 0x2e, 0xf9, 0x1c, 0x45, 0x67, 0x62, 0x21, 0x56, 0x40, 0xb9 }; -static const u8 enc_output039[] __initconst = { +static const u8 enc_output039[] = { 0xa1, 0xff, 0xed, 0x80, 0x76, 0x18, 0x29, 0xec, 0xce, 0x24, 0x2e, 0x0e, 0x88, 0xb1, 0x38, 0x04, 0x90, 0x16, 0xbc, 0xa0, 0x18, 0xda, 0x2b, 0x6e, @@ -2603,14 +2602,14 @@ static const u8 enc_output039[] __initconst = { 0xcb, 0xf0, 0xbe, 0xfd, 0xa0, 0xb7, 0x02, 0x42, 0xc6, 0x40, 0xd7, 0xcd, 0x02, 0xd7, 0xa3 }; -static const u8 enc_assoc039[] __initconst = { +static const u8 enc_assoc039[] = { 0xb3, 0xe4, 0x06, 0x46, 0x83, 0xb0, 0x2d, 0x84 }; -static const u8 enc_nonce039[] __initconst = { +static const u8 enc_nonce039[] = { 0xd4, 0xd8, 0x07, 0x34, 0x16, 0x83, 0x82, 0x5b, 0x31, 0xcd, 0x4d, 0x95 }; -static const u8 enc_key039[] __initconst = { +static const u8 enc_key039[] = { 0x6c, 0xbf, 0xd7, 0x1c, 0x64, 0x5d, 0x18, 0x4c, 0xf5, 0xd2, 0x3c, 0x40, 0x2b, 0xdb, 0x0d, 0x25, 0xec, 0x54, 0x89, 0x8c, 0x8a, 0x02, 0x73, 0xd4, @@ -2618,7 +2617,7 @@ static const u8 enc_key039[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input040[] __initconst = { +static const u8 enc_input040[] = { 0xd0, 0x96, 0x80, 0x31, 0x81, 0xbe, 0xef, 0x9e, 0x00, 0x8f, 0xf8, 0x5d, 0x5d, 0xdc, 0x38, 0xdd, 0xac, 0xf0, 0xf0, 0x9e, 0xe5, 0xf7, 0xe0, 0x7f, @@ -2628,7 +2627,7 @@ static const u8 enc_input040[] __initconst = { 0x18, 0xf1, 0x18, 0x55, 0x86, 0xbf, 0xea, 0x9d, 0x4c, 0x68, 0x5d, 0x50, 0xe4, 0xbb, 0x9a, 0x82 }; -static const u8 enc_output040[] __initconst = { +static const u8 enc_output040[] = { 0x9a, 0x4e, 0xf2, 0x2b, 0x18, 0x16, 0x77, 0xb5, 0x75, 0x5c, 0x08, 0xf7, 0x47, 0xc0, 0xf8, 0xd8, 0xe8, 0xd4, 0xc1, 0x8a, 0x9c, 0xc2, 0x40, 0x5c, @@ -2640,12 +2639,12 @@ static const u8 enc_output040[] __initconst = { 0x9f, 0xf3, 0x42, 0x7a, 0x0f, 0x32, 0xfa, 0x56, 0x6d, 0x9c, 0xa0, 0xa7, 0x8a, 0xef, 0xc0, 0x13 }; -static const u8 enc_assoc040[] __initconst = { }; -static const u8 enc_nonce040[] __initconst = { +static const u8 enc_assoc040[] = { }; +static const u8 enc_nonce040[] = { 0xd6, 0x10, 0x40, 0xa3, 0x13, 0xed, 0x49, 0x28, 0x23, 0xcc, 0x06, 0x5b }; -static const u8 enc_key040[] __initconst = { +static const u8 enc_key040[] = { 0x5b, 0x1d, 0x10, 0x35, 0xc0, 0xb1, 0x7e, 0xe0, 0xb0, 0x44, 0x47, 0x67, 0xf8, 0x0a, 0x25, 0xb8, 0xc1, 0xb7, 0x41, 0xf4, 0xb5, 0x0a, 0x4d, 0x30, @@ -2653,7 +2652,7 @@ static const u8 enc_key040[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input041[] __initconst = { +static const u8 enc_input041[] = { 0x94, 0xee, 0x16, 0x6d, 0x6d, 0x6e, 0xcf, 0x88, 0x32, 0x43, 0x71, 0x36, 0xb4, 0xae, 0x80, 0x5d, 0x42, 0x88, 0x64, 0x35, 0x95, 0x86, 0xd9, 0x19, @@ -2663,7 +2662,7 @@ static const u8 enc_input041[] __initconst = { 0x4a, 0x10, 0x8c, 0x7d, 0x7c, 0xe3, 0x4e, 0x6c, 0x6f, 0x8e, 0xa1, 0xbe, 0xc0, 0x56, 0x73, 0x17 }; -static const u8 enc_output041[] __initconst = { +static const u8 enc_output041[] = { 0x5f, 0xbb, 0xde, 0xcc, 0x34, 0xbe, 0x20, 0x16, 0x14, 0xf6, 0x36, 0x03, 0x1e, 0xeb, 0x42, 0xf1, 0xca, 0xce, 0x3c, 0x79, 0xa1, 0x2c, 0xff, 0xd8, @@ -2675,14 +2674,14 @@ static const u8 enc_output041[] __initconst = { 0x09, 0x26, 0x3d, 0xa7, 0xb4, 0xcb, 0x92, 0x14, 0x52, 0xf9, 0x7d, 0xca, 0x40, 0xf5, 0x80, 0xec }; -static const u8 enc_assoc041[] __initconst = { +static const u8 enc_assoc041[] = { 0x71, 0x93, 0xf6, 0x23, 0x66, 0x33, 0x21, 0xa2 }; -static const u8 enc_nonce041[] __initconst = { +static const u8 enc_nonce041[] = { 0xd3, 0x1c, 0x21, 0xab, 0xa1, 0x75, 0xb7, 0x0d, 0xe4, 0xeb, 0xb1, 0x9c }; -static const u8 enc_key041[] __initconst = { +static const u8 enc_key041[] = { 0x97, 0xd6, 0x35, 0xc4, 0xf4, 0x75, 0x74, 0xd9, 0x99, 0x8a, 0x90, 0x87, 0x5d, 0xa1, 0xd3, 0xa2, 0x84, 0xb7, 0x55, 0xb2, 0xd3, 0x92, 0x97, 0xa5, @@ -2690,7 +2689,7 @@ static const u8 enc_key041[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input042[] __initconst = { +static const u8 enc_input042[] = { 0xb4, 0x29, 0xeb, 0x80, 0xfb, 0x8f, 0xe8, 0xba, 0xed, 0xa0, 0xc8, 0x5b, 0x9c, 0x33, 0x34, 0x58, 0xe7, 0xc2, 0x99, 0x2e, 0x55, 0x84, 0x75, 0x06, @@ -2705,7 +2704,7 @@ static const u8 enc_input042[] __initconst = { 0xc3, 0xea, 0x46, 0xc8, 0x76, 0x1d, 0x50, 0xf5, 0x92 }; -static const u8 enc_output042[] __initconst = { +static const u8 enc_output042[] = { 0xd0, 0x10, 0x2f, 0x6c, 0x25, 0x8b, 0xf4, 0x97, 0x42, 0xce, 0xc3, 0x4c, 0xf2, 0xd0, 0xfe, 0xdf, 0x23, 0xd1, 0x05, 0xfb, 0x4c, 0x84, 0xcf, 0x98, @@ -2722,12 +2721,12 @@ static const u8 enc_output042[] __initconst = { 0x19, 0x61, 0x21, 0x27, 0xce, 0x49, 0x99, 0x3b, 0xb0 }; -static const u8 enc_assoc042[] __initconst = { }; -static const u8 enc_nonce042[] __initconst = { +static const u8 enc_assoc042[] = { }; +static const u8 enc_nonce042[] = { 0x17, 0xc8, 0x6a, 0x8a, 0xbb, 0xb7, 0xe0, 0x03, 0xac, 0xde, 0x27, 0x99 }; -static const u8 enc_key042[] __initconst = { +static const u8 enc_key042[] = { 0xfe, 0x6e, 0x55, 0xbd, 0xae, 0xd1, 0xf7, 0x28, 0x4c, 0xa5, 0xfc, 0x0f, 0x8c, 0x5f, 0x2b, 0x8d, 0xf5, 0x6d, 0xc0, 0xf4, 0x9e, 0x8c, 0xa6, 0x6a, @@ -2735,7 +2734,7 @@ static const u8 enc_key042[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input043[] __initconst = { +static const u8 enc_input043[] = { 0xce, 0xb5, 0x34, 0xce, 0x50, 0xdc, 0x23, 0xff, 0x63, 0x8a, 0xce, 0x3e, 0xf6, 0x3a, 0xb2, 0xcc, 0x29, 0x73, 0xee, 0xad, 0xa8, 0x07, 0x85, 0xfc, @@ -2750,7 +2749,7 @@ static const u8 enc_input043[] __initconst = { 0xdc, 0x47, 0x97, 0x0d, 0x84, 0xa6, 0xad, 0x31, 0x76 }; -static const u8 enc_output043[] __initconst = { +static const u8 enc_output043[] = { 0x75, 0x45, 0x39, 0x1b, 0x51, 0xde, 0x01, 0xd5, 0xc5, 0x3d, 0xfa, 0xca, 0x77, 0x79, 0x09, 0x06, 0x3e, 0x58, 0xed, 0xee, 0x4b, 0xb1, 0x22, 0x7e, @@ -2767,14 +2766,14 @@ static const u8 enc_output043[] __initconst = { 0xc7, 0xbb, 0xf1, 0x0d, 0xca, 0xdd, 0x7f, 0xac, 0xf6 }; -static const u8 enc_assoc043[] __initconst = { +static const u8 enc_assoc043[] = { 0xa1, 0x1c, 0x40, 0xb6, 0x03, 0x76, 0x73, 0x30 }; -static const u8 enc_nonce043[] __initconst = { +static const u8 enc_nonce043[] = { 0x46, 0x36, 0x2f, 0x45, 0xd6, 0x37, 0x9e, 0x63, 0xe5, 0x22, 0x94, 0x60 }; -static const u8 enc_key043[] __initconst = { +static const u8 enc_key043[] = { 0xaa, 0xbc, 0x06, 0x34, 0x74, 0xe6, 0x5c, 0x4c, 0x3e, 0x9b, 0xdc, 0x48, 0x0d, 0xea, 0x97, 0xb4, 0x51, 0x10, 0xc8, 0x61, 0x88, 0x46, 0xff, 0x6b, @@ -2782,24 +2781,24 @@ static const u8 enc_key043[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input044[] __initconst = { +static const u8 enc_input044[] = { 0xe5, 0xcc, 0xaa, 0x44, 0x1b, 0xc8, 0x14, 0x68, 0x8f, 0x8f, 0x6e, 0x8f, 0x28, 0xb5, 0x00, 0xb2 }; -static const u8 enc_output044[] __initconst = { +static const u8 enc_output044[] = { 0x7e, 0x72, 0xf5, 0xa1, 0x85, 0xaf, 0x16, 0xa6, 0x11, 0x92, 0x1b, 0x43, 0x8f, 0x74, 0x9f, 0x0b, 0x12, 0x42, 0xc6, 0x70, 0x73, 0x23, 0x34, 0x02, 0x9a, 0xdf, 0xe1, 0xc5, 0x00, 0x16, 0x51, 0xe4 }; -static const u8 enc_assoc044[] __initconst = { +static const u8 enc_assoc044[] = { 0x02 }; -static const u8 enc_nonce044[] __initconst = { +static const u8 enc_nonce044[] = { 0x87, 0x34, 0x5f, 0x10, 0x55, 0xfd, 0x9e, 0x21, 0x02, 0xd5, 0x06, 0x56 }; -static const u8 enc_key044[] __initconst = { +static const u8 enc_key044[] = { 0x7d, 0x00, 0xb4, 0x80, 0x95, 0xad, 0xfa, 0x32, 0x72, 0x05, 0x06, 0x07, 0xb2, 0x64, 0x18, 0x50, 0x02, 0xba, 0x99, 0x95, 0x7c, 0x49, 0x8b, 0xe0, @@ -2807,24 +2806,24 @@ static const u8 enc_key044[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input045[] __initconst = { +static const u8 enc_input045[] = { 0x02, 0xcd, 0xe1, 0x68, 0xfb, 0xa3, 0xf5, 0x44, 0xbb, 0xd0, 0x33, 0x2f, 0x7a, 0xde, 0xad, 0xa8 }; -static const u8 enc_output045[] __initconst = { +static const u8 enc_output045[] = { 0x85, 0xf2, 0x9a, 0x71, 0x95, 0x57, 0xcd, 0xd1, 0x4d, 0x1f, 0x8f, 0xff, 0xab, 0x6d, 0x9e, 0x60, 0x73, 0x2c, 0xa3, 0x2b, 0xec, 0xd5, 0x15, 0xa1, 0xed, 0x35, 0x3f, 0x54, 0x2e, 0x99, 0x98, 0x58 }; -static const u8 enc_assoc045[] __initconst = { +static const u8 enc_assoc045[] = { 0xb6, 0x48 }; -static const u8 enc_nonce045[] __initconst = { +static const u8 enc_nonce045[] = { 0x87, 0xa3, 0x16, 0x3e, 0xc0, 0x59, 0x8a, 0xd9, 0x5b, 0x3a, 0xa7, 0x13 }; -static const u8 enc_key045[] __initconst = { +static const u8 enc_key045[] = { 0x64, 0x32, 0x71, 0x7f, 0x1d, 0xb8, 0x5e, 0x41, 0xac, 0x78, 0x36, 0xbc, 0xe2, 0x51, 0x85, 0xa0, 0x80, 0xd5, 0x76, 0x2b, 0x9e, 0x2b, 0x18, 0x44, @@ -2832,25 +2831,25 @@ static const u8 enc_key045[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input046[] __initconst = { +static const u8 enc_input046[] = { 0x16, 0xdd, 0xd2, 0x3f, 0xf5, 0x3f, 0x3d, 0x23, 0xc0, 0x63, 0x34, 0x48, 0x70, 0x40, 0xeb, 0x47 }; -static const u8 enc_output046[] __initconst = { +static const u8 enc_output046[] = { 0xc1, 0xb2, 0x95, 0x93, 0x6d, 0x56, 0xfa, 0xda, 0xc0, 0x3e, 0x5f, 0x74, 0x2b, 0xff, 0x73, 0xa1, 0x39, 0xc4, 0x57, 0xdb, 0xab, 0x66, 0x38, 0x2b, 0xab, 0xb3, 0xb5, 0x58, 0x00, 0xcd, 0xa5, 0xb8 }; -static const u8 enc_assoc046[] __initconst = { +static const u8 enc_assoc046[] = { 0xbd, 0x4c, 0xd0, 0x2f, 0xc7, 0x50, 0x2b, 0xbd, 0xbd, 0xf6, 0xc9, 0xa3, 0xcb, 0xe8, 0xf0 }; -static const u8 enc_nonce046[] __initconst = { +static const u8 enc_nonce046[] = { 0x6f, 0x57, 0x3a, 0xa8, 0x6b, 0xaa, 0x49, 0x2b, 0xa4, 0x65, 0x96, 0xdf }; -static const u8 enc_key046[] __initconst = { +static const u8 enc_key046[] = { 0x8e, 0x34, 0xcf, 0x73, 0xd2, 0x45, 0xa1, 0x08, 0x2a, 0x92, 0x0b, 0x86, 0x36, 0x4e, 0xb8, 0x96, 0xc4, 0x94, 0x64, 0x67, 0xbc, 0xb3, 0xd5, 0x89, @@ -2858,25 +2857,25 @@ static const u8 enc_key046[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input047[] __initconst = { +static const u8 enc_input047[] = { 0x62, 0x3b, 0x78, 0x50, 0xc3, 0x21, 0xe2, 0xcf, 0x0c, 0x6f, 0xbc, 0xc8, 0xdf, 0xd1, 0xaf, 0xf2 }; -static const u8 enc_output047[] __initconst = { +static const u8 enc_output047[] = { 0xc8, 0x4c, 0x9b, 0xb7, 0xc6, 0x1c, 0x1b, 0xcb, 0x17, 0x77, 0x2a, 0x1c, 0x50, 0x0c, 0x50, 0x95, 0xdb, 0xad, 0xf7, 0xa5, 0x13, 0x8c, 0xa0, 0x34, 0x59, 0xa2, 0xcd, 0x65, 0x83, 0x1e, 0x09, 0x2f }; -static const u8 enc_assoc047[] __initconst = { +static const u8 enc_assoc047[] = { 0x89, 0xcc, 0xe9, 0xfb, 0x47, 0x44, 0x1d, 0x07, 0xe0, 0x24, 0x5a, 0x66, 0xfe, 0x8b, 0x77, 0x8b }; -static const u8 enc_nonce047[] __initconst = { +static const u8 enc_nonce047[] = { 0x1a, 0x65, 0x18, 0xf0, 0x2e, 0xde, 0x1d, 0xa6, 0x80, 0x92, 0x66, 0xd9 }; -static const u8 enc_key047[] __initconst = { +static const u8 enc_key047[] = { 0xcb, 0x55, 0x75, 0xf5, 0xc7, 0xc4, 0x5c, 0x91, 0xcf, 0x32, 0x0b, 0x13, 0x9f, 0xb5, 0x94, 0x23, 0x75, 0x60, 0xd0, 0xa3, 0xe6, 0xf8, 0x65, 0xa6, @@ -2884,26 +2883,26 @@ static const u8 enc_key047[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input048[] __initconst = { +static const u8 enc_input048[] = { 0x87, 0xb3, 0xa4, 0xd7, 0xb2, 0x6d, 0x8d, 0x32, 0x03, 0xa0, 0xde, 0x1d, 0x64, 0xef, 0x82, 0xe3 }; -static const u8 enc_output048[] __initconst = { +static const u8 enc_output048[] = { 0x94, 0xbc, 0x80, 0x62, 0x1e, 0xd1, 0xe7, 0x1b, 0x1f, 0xd2, 0xb5, 0xc3, 0xa1, 0x5e, 0x35, 0x68, 0x33, 0x35, 0x11, 0x86, 0x17, 0x96, 0x97, 0x84, 0x01, 0x59, 0x8b, 0x96, 0x37, 0x22, 0xf5, 0xb3 }; -static const u8 enc_assoc048[] __initconst = { +static const u8 enc_assoc048[] = { 0xd1, 0x9f, 0x2d, 0x98, 0x90, 0x95, 0xf7, 0xab, 0x03, 0xa5, 0xfd, 0xe8, 0x44, 0x16, 0xe0, 0x0c, 0x0e }; -static const u8 enc_nonce048[] __initconst = { +static const u8 enc_nonce048[] = { 0x56, 0x4d, 0xee, 0x49, 0xab, 0x00, 0xd2, 0x40, 0xfc, 0x10, 0x68, 0xc3 }; -static const u8 enc_key048[] __initconst = { +static const u8 enc_key048[] = { 0xa5, 0x56, 0x9e, 0x72, 0x9a, 0x69, 0xb2, 0x4b, 0xa6, 0xe0, 0xff, 0x15, 0xc4, 0x62, 0x78, 0x97, 0x43, 0x68, 0x24, 0xc9, 0x41, 0xe9, 0xd0, 0x0b, @@ -2911,27 +2910,27 @@ static const u8 enc_key048[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input049[] __initconst = { +static const u8 enc_input049[] = { 0xe6, 0x01, 0xb3, 0x85, 0x57, 0x79, 0x7d, 0xa2, 0xf8, 0xa4, 0x10, 0x6a, 0x08, 0x9d, 0x1d, 0xa6 }; -static const u8 enc_output049[] __initconst = { +static const u8 enc_output049[] = { 0x29, 0x9b, 0x5d, 0x3f, 0x3d, 0x03, 0xc0, 0x87, 0x20, 0x9a, 0x16, 0xe2, 0x85, 0x14, 0x31, 0x11, 0x4b, 0x45, 0x4e, 0xd1, 0x98, 0xde, 0x11, 0x7e, 0x83, 0xec, 0x49, 0xfa, 0x8d, 0x85, 0x08, 0xd6 }; -static const u8 enc_assoc049[] __initconst = { +static const u8 enc_assoc049[] = { 0x5e, 0x64, 0x70, 0xfa, 0xcd, 0x99, 0xc1, 0xd8, 0x1e, 0x37, 0xcd, 0x44, 0x01, 0x5f, 0xe1, 0x94, 0x80, 0xa2, 0xa4, 0xd3, 0x35, 0x2a, 0x4f, 0xf5, 0x60, 0xc0, 0x64, 0x0f, 0xdb, 0xda }; -static const u8 enc_nonce049[] __initconst = { +static const u8 enc_nonce049[] = { 0xdf, 0x87, 0x13, 0xe8, 0x7e, 0xc3, 0xdb, 0xcf, 0xad, 0x14, 0xd5, 0x3e }; -static const u8 enc_key049[] __initconst = { +static const u8 enc_key049[] = { 0x56, 0x20, 0x74, 0x65, 0xb4, 0xe4, 0x8e, 0x6d, 0x04, 0x63, 0x0f, 0x4a, 0x42, 0xf3, 0x5c, 0xfc, 0x16, 0x3a, 0xb2, 0x89, 0xc2, 0x2a, 0x2b, 0x47, @@ -2939,27 +2938,27 @@ static const u8 enc_key049[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input050[] __initconst = { +static const u8 enc_input050[] = { 0xdc, 0x9e, 0x9e, 0xaf, 0x11, 0xe3, 0x14, 0x18, 0x2d, 0xf6, 0xa4, 0xeb, 0xa1, 0x7a, 0xec, 0x9c }; -static const u8 enc_output050[] __initconst = { +static const u8 enc_output050[] = { 0x60, 0x5b, 0xbf, 0x90, 0xae, 0xb9, 0x74, 0xf6, 0x60, 0x2b, 0xc7, 0x78, 0x05, 0x6f, 0x0d, 0xca, 0x38, 0xea, 0x23, 0xd9, 0x90, 0x54, 0xb4, 0x6b, 0x42, 0xff, 0xe0, 0x04, 0x12, 0x9d, 0x22, 0x04 }; -static const u8 enc_assoc050[] __initconst = { +static const u8 enc_assoc050[] = { 0xba, 0x44, 0x6f, 0x6f, 0x9a, 0x0c, 0xed, 0x22, 0x45, 0x0f, 0xeb, 0x10, 0x73, 0x7d, 0x90, 0x07, 0xfd, 0x69, 0xab, 0xc1, 0x9b, 0x1d, 0x4d, 0x90, 0x49, 0xa5, 0x55, 0x1e, 0x86, 0xec, 0x2b, 0x37 }; -static const u8 enc_nonce050[] __initconst = { +static const u8 enc_nonce050[] = { 0x8d, 0xf4, 0xb1, 0x5a, 0x88, 0x8c, 0x33, 0x28, 0x6a, 0x7b, 0x76, 0x51 }; -static const u8 enc_key050[] __initconst = { +static const u8 enc_key050[] = { 0x39, 0x37, 0x98, 0x6a, 0xf8, 0x6d, 0xaf, 0xc1, 0xba, 0x0c, 0x46, 0x72, 0xd8, 0xab, 0xc4, 0x6c, 0x20, 0x70, 0x62, 0x68, 0x2d, 0x9c, 0x26, 0x4a, @@ -2967,28 +2966,28 @@ static const u8 enc_key050[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input051[] __initconst = { +static const u8 enc_input051[] = { 0x81, 0xce, 0x84, 0xed, 0xe9, 0xb3, 0x58, 0x59, 0xcc, 0x8c, 0x49, 0xa8, 0xf6, 0xbe, 0x7d, 0xc6 }; -static const u8 enc_output051[] __initconst = { +static const u8 enc_output051[] = { 0x7b, 0x7c, 0xe0, 0xd8, 0x24, 0x80, 0x9a, 0x70, 0xde, 0x32, 0x56, 0x2c, 0xcf, 0x2c, 0x2b, 0xbd, 0x15, 0xd4, 0x4a, 0x00, 0xce, 0x0d, 0x19, 0xb4, 0x23, 0x1f, 0x92, 0x1e, 0x22, 0xbc, 0x0a, 0x43 }; -static const u8 enc_assoc051[] __initconst = { +static const u8 enc_assoc051[] = { 0xd4, 0x1a, 0x82, 0x8d, 0x5e, 0x71, 0x82, 0x92, 0x47, 0x02, 0x19, 0x05, 0x40, 0x2e, 0xa2, 0x57, 0xdc, 0xcb, 0xc3, 0xb8, 0x0f, 0xcd, 0x56, 0x75, 0x05, 0x6b, 0x68, 0xbb, 0x59, 0xe6, 0x2e, 0x88, 0x73 }; -static const u8 enc_nonce051[] __initconst = { +static const u8 enc_nonce051[] = { 0xbe, 0x40, 0xe5, 0xf1, 0xa1, 0x18, 0x17, 0xa0, 0xa8, 0xfa, 0x89, 0x49 }; -static const u8 enc_key051[] __initconst = { +static const u8 enc_key051[] = { 0x36, 0x37, 0x2a, 0xbc, 0xdb, 0x78, 0xe0, 0x27, 0x96, 0x46, 0xac, 0x3d, 0x17, 0x6b, 0x96, 0x74, 0xe9, 0x15, 0x4e, 0xec, 0xf0, 0xd5, 0x46, 0x9c, @@ -2996,17 +2995,17 @@ static const u8 enc_key051[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input052[] __initconst = { +static const u8 enc_input052[] = { 0xa6, 0x67, 0x47, 0xc8, 0x9e, 0x85, 0x7a, 0xf3, 0xa1, 0x8e, 0x2c, 0x79, 0x50, 0x00, 0x87, 0xed }; -static const u8 enc_output052[] __initconst = { +static const u8 enc_output052[] = { 0xca, 0x82, 0xbf, 0xf3, 0xe2, 0xf3, 0x10, 0xcc, 0xc9, 0x76, 0x67, 0x2c, 0x44, 0x15, 0xe6, 0x9b, 0x57, 0x63, 0x8c, 0x62, 0xa5, 0xd8, 0x5d, 0xed, 0x77, 0x4f, 0x91, 0x3c, 0x81, 0x3e, 0xa0, 0x32 }; -static const u8 enc_assoc052[] __initconst = { +static const u8 enc_assoc052[] = { 0x3f, 0x2d, 0xd4, 0x9b, 0xbf, 0x09, 0xd6, 0x9a, 0x78, 0xa3, 0xd8, 0x0e, 0xa2, 0x56, 0x66, 0x14, 0xfc, 0x37, 0x94, 0x74, 0x19, 0x6c, 0x1a, 0xae, @@ -3014,11 +3013,11 @@ static const u8 enc_assoc052[] __initconst = { 0x6f, 0x42, 0xca, 0x42, 0x05, 0x6a, 0x97, 0x92, 0xcc, 0x1b, 0x9f, 0xb3, 0xc7, 0xd2, 0x61 }; -static const u8 enc_nonce052[] __initconst = { +static const u8 enc_nonce052[] = { 0x84, 0xc8, 0x7d, 0xae, 0x4e, 0xee, 0x27, 0x73, 0x0e, 0xc3, 0x5d, 0x12 }; -static const u8 enc_key052[] __initconst = { +static const u8 enc_key052[] = { 0x9f, 0x14, 0x79, 0xed, 0x09, 0x7d, 0x7f, 0xe5, 0x29, 0xc1, 0x1f, 0x2f, 0x5a, 0xdd, 0x9a, 0xaf, 0xf4, 0xa1, 0xca, 0x0b, 0x68, 0x99, 0x7a, 0x2c, @@ -3026,13 +3025,13 @@ static const u8 enc_key052[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input053[] __initconst = { +static const u8 enc_input053[] = { 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, 0x80, 0xe0, 0x30, 0xbe, 0xeb, 0xd3, 0x29, 0xbe }; -static const u8 enc_output053[] __initconst = { +static const u8 enc_output053[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -3040,14 +3039,14 @@ static const u8 enc_output053[] __initconst = { 0xe6, 0xd3, 0xd7, 0x32, 0x4a, 0x1c, 0xbb, 0xa7, 0x77, 0xbb, 0xb0, 0xec, 0xdd, 0xa3, 0x78, 0x07 }; -static const u8 enc_assoc053[] __initconst = { +static const u8 enc_assoc053[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce053[] __initconst = { +static const u8 enc_nonce053[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key053[] __initconst = { +static const u8 enc_key053[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3055,7 +3054,7 @@ static const u8 enc_key053[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input054[] __initconst = { +static const u8 enc_input054[] = { 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, @@ -3065,7 +3064,7 @@ static const u8 enc_input054[] __initconst = { 0x08, 0x69, 0xff, 0xd2, 0xec, 0x5e, 0x26, 0xe5, 0x53, 0xb7, 0xb2, 0x27, 0xfe, 0x87, 0xfd, 0xbd }; -static const u8 enc_output054[] __initconst = { +static const u8 enc_output054[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -3077,14 +3076,14 @@ static const u8 enc_output054[] __initconst = { 0x06, 0x2d, 0xe6, 0x79, 0x5f, 0x27, 0x4f, 0xd2, 0xa3, 0x05, 0xd7, 0x69, 0x80, 0xbc, 0x9c, 0xce }; -static const u8 enc_assoc054[] __initconst = { +static const u8 enc_assoc054[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce054[] __initconst = { +static const u8 enc_nonce054[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key054[] __initconst = { +static const u8 enc_key054[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3092,7 +3091,7 @@ static const u8 enc_key054[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input055[] __initconst = { +static const u8 enc_input055[] = { 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, @@ -3110,7 +3109,7 @@ static const u8 enc_input055[] __initconst = { 0x77, 0xd3, 0x0b, 0xc5, 0x76, 0x92, 0xed, 0x38, 0xfb, 0xac, 0x01, 0x88, 0x38, 0x04, 0x88, 0xc7 }; -static const u8 enc_output055[] __initconst = { +static const u8 enc_output055[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -3130,14 +3129,14 @@ static const u8 enc_output055[] __initconst = { 0xd8, 0xb4, 0x79, 0x02, 0xba, 0xae, 0xaf, 0xb3, 0x42, 0x03, 0x05, 0x15, 0x29, 0xaf, 0x28, 0x2e }; -static const u8 enc_assoc055[] __initconst = { +static const u8 enc_assoc055[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce055[] __initconst = { +static const u8 enc_nonce055[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key055[] __initconst = { +static const u8 enc_key055[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3145,13 +3144,13 @@ static const u8 enc_key055[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input056[] __initconst = { +static const u8 enc_input056[] = { 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, 0x7f, 0x1f, 0xcf, 0x41, 0x14, 0x2c, 0xd6, 0x41 }; -static const u8 enc_output056[] __initconst = { +static const u8 enc_output056[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3159,14 +3158,14 @@ static const u8 enc_output056[] __initconst = { 0xb3, 0x89, 0x1c, 0x84, 0x9c, 0xb5, 0x2c, 0x27, 0x74, 0x7e, 0xdf, 0xcf, 0x31, 0x21, 0x3b, 0xb6 }; -static const u8 enc_assoc056[] __initconst = { +static const u8 enc_assoc056[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce056[] __initconst = { +static const u8 enc_nonce056[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key056[] __initconst = { +static const u8 enc_key056[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3174,7 +3173,7 @@ static const u8 enc_key056[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input057[] __initconst = { +static const u8 enc_input057[] = { 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, @@ -3184,7 +3183,7 @@ static const u8 enc_input057[] __initconst = { 0xf7, 0x96, 0x00, 0x2d, 0x13, 0xa1, 0xd9, 0x1a, 0xac, 0x48, 0x4d, 0xd8, 0x01, 0x78, 0x02, 0x42 }; -static const u8 enc_output057[] __initconst = { +static const u8 enc_output057[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3196,14 +3195,14 @@ static const u8 enc_output057[] __initconst = { 0xf0, 0xc1, 0x2d, 0x26, 0xef, 0x03, 0x02, 0x9b, 0x62, 0xc0, 0x08, 0xda, 0x27, 0xc5, 0xdc, 0x68 }; -static const u8 enc_assoc057[] __initconst = { +static const u8 enc_assoc057[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce057[] __initconst = { +static const u8 enc_nonce057[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key057[] __initconst = { +static const u8 enc_key057[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3211,7 +3210,7 @@ static const u8 enc_key057[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input058[] __initconst = { +static const u8 enc_input058[] = { 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, @@ -3229,7 +3228,7 @@ static const u8 enc_input058[] __initconst = { 0x88, 0x2c, 0xf4, 0x3a, 0x89, 0x6d, 0x12, 0xc7, 0x04, 0x53, 0xfe, 0x77, 0xc7, 0xfb, 0x77, 0x38 }; -static const u8 enc_output058[] __initconst = { +static const u8 enc_output058[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3249,14 +3248,14 @@ static const u8 enc_output058[] __initconst = { 0xee, 0x65, 0x78, 0x30, 0x01, 0xc2, 0x56, 0x91, 0xfa, 0x28, 0xd0, 0xf5, 0xf1, 0xc1, 0xd7, 0x62 }; -static const u8 enc_assoc058[] __initconst = { +static const u8 enc_assoc058[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce058[] __initconst = { +static const u8 enc_nonce058[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key058[] __initconst = { +static const u8 enc_key058[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3264,13 +3263,13 @@ static const u8 enc_key058[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input059[] __initconst = { +static const u8 enc_input059[] = { 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, 0x80, 0xe0, 0x30, 0x3e, 0xeb, 0xd3, 0x29, 0x3e }; -static const u8 enc_output059[] __initconst = { +static const u8 enc_output059[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, @@ -3278,14 +3277,14 @@ static const u8 enc_output059[] __initconst = { 0x79, 0xba, 0x7a, 0x29, 0xf5, 0xa7, 0xbb, 0x75, 0x79, 0x7a, 0xf8, 0x7a, 0x61, 0x01, 0x29, 0xa4 }; -static const u8 enc_assoc059[] __initconst = { +static const u8 enc_assoc059[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 }; -static const u8 enc_nonce059[] __initconst = { +static const u8 enc_nonce059[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key059[] __initconst = { +static const u8 enc_key059[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3293,7 +3292,7 @@ static const u8 enc_key059[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input060[] __initconst = { +static const u8 enc_input060[] = { 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, @@ -3303,7 +3302,7 @@ static const u8 enc_input060[] __initconst = { 0x08, 0x69, 0xff, 0x52, 0xec, 0x5e, 0x26, 0x65, 0x53, 0xb7, 0xb2, 0xa7, 0xfe, 0x87, 0xfd, 0x3d }; -static const u8 enc_output060[] __initconst = { +static const u8 enc_output060[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, @@ -3315,14 +3314,14 @@ static const u8 enc_output060[] __initconst = { 0x36, 0xb1, 0x74, 0x38, 0x19, 0xe1, 0xb9, 0xba, 0x15, 0x51, 0xe8, 0xed, 0x92, 0x2a, 0x95, 0x9a }; -static const u8 enc_assoc060[] __initconst = { +static const u8 enc_assoc060[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 }; -static const u8 enc_nonce060[] __initconst = { +static const u8 enc_nonce060[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key060[] __initconst = { +static const u8 enc_key060[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3330,7 +3329,7 @@ static const u8 enc_key060[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input061[] __initconst = { +static const u8 enc_input061[] = { 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, @@ -3348,7 +3347,7 @@ static const u8 enc_input061[] __initconst = { 0x77, 0xd3, 0x0b, 0x45, 0x76, 0x92, 0xed, 0xb8, 0xfb, 0xac, 0x01, 0x08, 0x38, 0x04, 0x88, 0x47 }; -static const u8 enc_output061[] __initconst = { +static const u8 enc_output061[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, @@ -3368,14 +3367,14 @@ static const u8 enc_output061[] __initconst = { 0xfe, 0xac, 0x49, 0x55, 0x55, 0x4e, 0x80, 0x6f, 0x3a, 0x19, 0x02, 0xe2, 0x44, 0x32, 0xc0, 0x8a }; -static const u8 enc_assoc061[] __initconst = { +static const u8 enc_assoc061[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 }; -static const u8 enc_nonce061[] __initconst = { +static const u8 enc_nonce061[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key061[] __initconst = { +static const u8 enc_key061[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3383,13 +3382,13 @@ static const u8 enc_key061[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input062[] __initconst = { +static const u8 enc_input062[] = { 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, 0x7f, 0x1f, 0xcf, 0xc1, 0x14, 0x2c, 0xd6, 0xc1 }; -static const u8 enc_output062[] __initconst = { +static const u8 enc_output062[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, @@ -3397,14 +3396,14 @@ static const u8 enc_output062[] __initconst = { 0x20, 0xa3, 0x79, 0x8d, 0xf1, 0x29, 0x2c, 0x59, 0x72, 0xbf, 0x97, 0x41, 0xae, 0xc3, 0x8a, 0x19 }; -static const u8 enc_assoc062[] __initconst = { +static const u8 enc_assoc062[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f }; -static const u8 enc_nonce062[] __initconst = { +static const u8 enc_nonce062[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key062[] __initconst = { +static const u8 enc_key062[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3412,7 +3411,7 @@ static const u8 enc_key062[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input063[] __initconst = { +static const u8 enc_input063[] = { 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, @@ -3422,7 +3421,7 @@ static const u8 enc_input063[] __initconst = { 0xf7, 0x96, 0x00, 0xad, 0x13, 0xa1, 0xd9, 0x9a, 0xac, 0x48, 0x4d, 0x58, 0x01, 0x78, 0x02, 0xc2 }; -static const u8 enc_output063[] __initconst = { +static const u8 enc_output063[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, @@ -3434,14 +3433,14 @@ static const u8 enc_output063[] __initconst = { 0xc0, 0x3d, 0x9f, 0x67, 0x35, 0x4a, 0x97, 0xb2, 0xf0, 0x74, 0xf7, 0x55, 0x15, 0x57, 0xe4, 0x9c }; -static const u8 enc_assoc063[] __initconst = { +static const u8 enc_assoc063[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f }; -static const u8 enc_nonce063[] __initconst = { +static const u8 enc_nonce063[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key063[] __initconst = { +static const u8 enc_key063[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3449,7 +3448,7 @@ static const u8 enc_key063[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input064[] __initconst = { +static const u8 enc_input064[] = { 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, @@ -3467,7 +3466,7 @@ static const u8 enc_input064[] __initconst = { 0x88, 0x2c, 0xf4, 0xba, 0x89, 0x6d, 0x12, 0x47, 0x04, 0x53, 0xfe, 0xf7, 0xc7, 0xfb, 0x77, 0xb8 }; -static const u8 enc_output064[] __initconst = { +static const u8 enc_output064[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, @@ -3487,14 +3486,14 @@ static const u8 enc_output064[] __initconst = { 0xc8, 0x6d, 0xa8, 0xdd, 0x65, 0x22, 0x86, 0xd5, 0x02, 0x13, 0xd3, 0x28, 0xd6, 0x3e, 0x40, 0x06 }; -static const u8 enc_assoc064[] __initconst = { +static const u8 enc_assoc064[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f }; -static const u8 enc_nonce064[] __initconst = { +static const u8 enc_nonce064[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key064[] __initconst = { +static const u8 enc_key064[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3502,13 +3501,13 @@ static const u8 enc_key064[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input065[] __initconst = { +static const u8 enc_input065[] = { 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, 0xff, 0x1f, 0xcf, 0x41, 0x94, 0x2c, 0xd6, 0x41 }; -static const u8 enc_output065[] __initconst = { +static const u8 enc_output065[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, @@ -3516,14 +3515,14 @@ static const u8 enc_output065[] __initconst = { 0xbe, 0xde, 0x90, 0x83, 0xce, 0xb3, 0x6d, 0xdf, 0xe5, 0xfa, 0x81, 0x1f, 0x95, 0x47, 0x1c, 0x67 }; -static const u8 enc_assoc065[] __initconst = { +static const u8 enc_assoc065[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff }; -static const u8 enc_nonce065[] __initconst = { +static const u8 enc_nonce065[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key065[] __initconst = { +static const u8 enc_key065[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3531,7 +3530,7 @@ static const u8 enc_key065[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input066[] __initconst = { +static const u8 enc_input066[] = { 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, @@ -3541,7 +3540,7 @@ static const u8 enc_input066[] __initconst = { 0x77, 0x96, 0x00, 0x2d, 0x93, 0xa1, 0xd9, 0x1a, 0x2c, 0x48, 0x4d, 0xd8, 0x81, 0x78, 0x02, 0x42 }; -static const u8 enc_output066[] __initconst = { +static const u8 enc_output066[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, @@ -3553,14 +3552,14 @@ static const u8 enc_output066[] __initconst = { 0x30, 0x08, 0x74, 0xbb, 0x06, 0x92, 0xb6, 0x89, 0xde, 0xad, 0x9a, 0xe1, 0x5b, 0x06, 0x73, 0x90 }; -static const u8 enc_assoc066[] __initconst = { +static const u8 enc_assoc066[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff }; -static const u8 enc_nonce066[] __initconst = { +static const u8 enc_nonce066[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key066[] __initconst = { +static const u8 enc_key066[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3568,7 +3567,7 @@ static const u8 enc_key066[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input067[] __initconst = { +static const u8 enc_input067[] = { 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, @@ -3586,7 +3585,7 @@ static const u8 enc_input067[] __initconst = { 0x08, 0x2c, 0xf4, 0x3a, 0x09, 0x6d, 0x12, 0xc7, 0x84, 0x53, 0xfe, 0x77, 0x47, 0xfb, 0x77, 0x38 }; -static const u8 enc_output067[] __initconst = { +static const u8 enc_output067[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, @@ -3606,14 +3605,14 @@ static const u8 enc_output067[] __initconst = { 0x99, 0xca, 0xd8, 0x5f, 0x45, 0xca, 0x40, 0x94, 0x2d, 0x0d, 0x4d, 0x5e, 0x95, 0x0a, 0xde, 0x22 }; -static const u8 enc_assoc067[] __initconst = { +static const u8 enc_assoc067[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff }; -static const u8 enc_nonce067[] __initconst = { +static const u8 enc_nonce067[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key067[] __initconst = { +static const u8 enc_key067[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3621,13 +3620,13 @@ static const u8 enc_key067[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input068[] __initconst = { +static const u8 enc_input068[] = { 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, 0x80, 0xe0, 0x30, 0xbe, 0x14, 0x2c, 0xd6, 0x41 }; -static const u8 enc_output068[] __initconst = { +static const u8 enc_output068[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, @@ -3635,14 +3634,14 @@ static const u8 enc_output068[] __initconst = { 0x8b, 0xbe, 0x14, 0x52, 0x72, 0xe7, 0xc2, 0xd9, 0xa1, 0x89, 0x1a, 0x3a, 0xb0, 0x98, 0x3d, 0x9d }; -static const u8 enc_assoc068[] __initconst = { +static const u8 enc_assoc068[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce068[] __initconst = { +static const u8 enc_nonce068[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key068[] __initconst = { +static const u8 enc_key068[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3650,7 +3649,7 @@ static const u8 enc_key068[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input069[] __initconst = { +static const u8 enc_input069[] = { 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, @@ -3660,7 +3659,7 @@ static const u8 enc_input069[] __initconst = { 0x08, 0x69, 0xff, 0xd2, 0x13, 0xa1, 0xd9, 0x1a, 0x53, 0xb7, 0xb2, 0x27, 0x01, 0x78, 0x02, 0x42 }; -static const u8 enc_output069[] __initconst = { +static const u8 enc_output069[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, @@ -3672,14 +3671,14 @@ static const u8 enc_output069[] __initconst = { 0x3b, 0x41, 0x86, 0x19, 0x13, 0xa8, 0xf6, 0xde, 0x7f, 0x61, 0xe2, 0x25, 0x63, 0x1b, 0xc3, 0x82 }; -static const u8 enc_assoc069[] __initconst = { +static const u8 enc_assoc069[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce069[] __initconst = { +static const u8 enc_nonce069[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key069[] __initconst = { +static const u8 enc_key069[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3687,7 +3686,7 @@ static const u8 enc_key069[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input070[] __initconst = { +static const u8 enc_input070[] = { 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, @@ -3705,7 +3704,7 @@ static const u8 enc_input070[] __initconst = { 0x77, 0xd3, 0x0b, 0xc5, 0x89, 0x6d, 0x12, 0xc7, 0xfb, 0xac, 0x01, 0x88, 0xc7, 0xfb, 0x77, 0x38 }; -static const u8 enc_output070[] __initconst = { +static const u8 enc_output070[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, @@ -3725,14 +3724,14 @@ static const u8 enc_output070[] __initconst = { 0x84, 0x28, 0xbc, 0xf0, 0x23, 0xec, 0x6b, 0xf3, 0x1f, 0xd9, 0xef, 0xb2, 0x03, 0xff, 0x08, 0x71 }; -static const u8 enc_assoc070[] __initconst = { +static const u8 enc_assoc070[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce070[] __initconst = { +static const u8 enc_nonce070[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key070[] __initconst = { +static const u8 enc_key070[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3740,13 +3739,13 @@ static const u8 enc_key070[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input071[] __initconst = { +static const u8 enc_input071[] = { 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, 0x7f, 0x1f, 0xcf, 0x41, 0xeb, 0xd3, 0x29, 0xbe }; -static const u8 enc_output071[] __initconst = { +static const u8 enc_output071[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, @@ -3754,14 +3753,14 @@ static const u8 enc_output071[] __initconst = { 0x13, 0x9f, 0xdf, 0x64, 0x74, 0xea, 0x24, 0xf5, 0x49, 0xb0, 0x75, 0x82, 0x5f, 0x2c, 0x76, 0x20 }; -static const u8 enc_assoc071[] __initconst = { +static const u8 enc_assoc071[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce071[] __initconst = { +static const u8 enc_nonce071[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key071[] __initconst = { +static const u8 enc_key071[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3769,7 +3768,7 @@ static const u8 enc_key071[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input072[] __initconst = { +static const u8 enc_input072[] = { 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, @@ -3779,7 +3778,7 @@ static const u8 enc_input072[] __initconst = { 0xf7, 0x96, 0x00, 0x2d, 0xec, 0x5e, 0x26, 0xe5, 0xac, 0x48, 0x4d, 0xd8, 0xfe, 0x87, 0xfd, 0xbd }; -static const u8 enc_output072[] __initconst = { +static const u8 enc_output072[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, @@ -3791,14 +3790,14 @@ static const u8 enc_output072[] __initconst = { 0xbb, 0xad, 0x8d, 0x86, 0x3b, 0x83, 0x5a, 0x8e, 0x86, 0x64, 0xfd, 0x1d, 0x45, 0x66, 0xb6, 0xb4 }; -static const u8 enc_assoc072[] __initconst = { +static const u8 enc_assoc072[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce072[] __initconst = { +static const u8 enc_nonce072[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key072[] __initconst = { +static const u8 enc_key072[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3806,7 +3805,7 @@ static const u8 enc_key072[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input073[] __initconst = { +static const u8 enc_input073[] = { 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, @@ -3824,7 +3823,7 @@ static const u8 enc_input073[] __initconst = { 0x88, 0x2c, 0xf4, 0x3a, 0x76, 0x92, 0xed, 0x38, 0x04, 0x53, 0xfe, 0x77, 0x38, 0x04, 0x88, 0xc7 }; -static const u8 enc_output073[] __initconst = { +static const u8 enc_output073[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, @@ -3844,14 +3843,14 @@ static const u8 enc_output073[] __initconst = { 0x42, 0xf2, 0x35, 0x42, 0x97, 0x84, 0x9a, 0x51, 0x1d, 0x53, 0xe5, 0x57, 0x17, 0x72, 0xf7, 0x1f }; -static const u8 enc_assoc073[] __initconst = { +static const u8 enc_assoc073[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce073[] __initconst = { +static const u8 enc_nonce073[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key073[] __initconst = { +static const u8 enc_key073[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3859,7 +3858,7 @@ static const u8 enc_key073[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input074[] __initconst = { +static const u8 enc_input074[] = { 0xd4, 0x50, 0x0b, 0xf0, 0x09, 0x49, 0x35, 0x51, 0xc3, 0x80, 0xad, 0xf5, 0x2c, 0x57, 0x3a, 0x69, 0xdf, 0x7e, 0x8b, 0x76, 0x24, 0x63, 0x33, 0x0f, @@ -3877,7 +3876,7 @@ static const u8 enc_input074[] __initconst = { 0x93, 0x76, 0x71, 0xa0, 0x62, 0x9b, 0xd9, 0x5c, 0x99, 0x15, 0xc7, 0x85, 0x55, 0x77, 0x1e, 0x7a }; -static const u8 enc_output074[] __initconst = { +static const u8 enc_output074[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3897,7 +3896,7 @@ static const u8 enc_output074[] __initconst = { 0x0b, 0x30, 0x0d, 0x8d, 0xa5, 0x6c, 0x21, 0x85, 0x75, 0x52, 0x79, 0x55, 0x3c, 0x4c, 0x82, 0xca }; -static const u8 enc_assoc074[] __initconst = { +static const u8 enc_assoc074[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3907,11 +3906,11 @@ static const u8 enc_assoc074[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce074[] __initconst = { +static const u8 enc_nonce074[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x00, 0x02, 0x50, 0x6e }; -static const u8 enc_key074[] __initconst = { +static const u8 enc_key074[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -3919,7 +3918,7 @@ static const u8 enc_key074[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input075[] __initconst = { +static const u8 enc_input075[] = { 0x7d, 0xe8, 0x7f, 0x67, 0x29, 0x94, 0x52, 0x75, 0xd0, 0x65, 0x5d, 0xa4, 0xc7, 0xfd, 0xe4, 0x56, 0x9e, 0x16, 0xf1, 0x11, 0xb5, 0xeb, 0x26, 0xc2, @@ -3937,7 +3936,7 @@ static const u8 enc_input075[] __initconst = { 0x3a, 0xc1, 0xee, 0x54, 0xc2, 0x9e, 0xe4, 0xc1, 0x70, 0xde, 0x40, 0x8f, 0x66, 0x69, 0x21, 0x94 }; -static const u8 enc_output075[] __initconst = { +static const u8 enc_output075[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3957,7 +3956,7 @@ static const u8 enc_output075[] __initconst = { 0xc5, 0x78, 0xe2, 0xaa, 0x44, 0xd3, 0x09, 0xb7, 0xb6, 0xa5, 0x19, 0x3b, 0xdc, 0x61, 0x18, 0xf5 }; -static const u8 enc_assoc075[] __initconst = { +static const u8 enc_assoc075[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3967,11 +3966,11 @@ static const u8 enc_assoc075[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce075[] __initconst = { +static const u8 enc_nonce075[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x00, 0x03, 0x18, 0xa5 }; -static const u8 enc_key075[] __initconst = { +static const u8 enc_key075[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -3979,7 +3978,7 @@ static const u8 enc_key075[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input076[] __initconst = { +static const u8 enc_input076[] = { 0x1b, 0x99, 0x6f, 0x9a, 0x3c, 0xcc, 0x67, 0x85, 0xde, 0x22, 0xff, 0x5b, 0x8a, 0xdd, 0x95, 0x02, 0xce, 0x03, 0xa0, 0xfa, 0xf5, 0x99, 0x2a, 0x09, @@ -3997,7 +3996,7 @@ static const u8 enc_input076[] __initconst = { 0xc3, 0x53, 0xd0, 0xcf, 0x93, 0x8d, 0xcc, 0xb9, 0xef, 0xad, 0x8f, 0xed, 0xbe, 0x46, 0xda, 0xa5 }; -static const u8 enc_output076[] __initconst = { +static const u8 enc_output076[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4017,7 +4016,7 @@ static const u8 enc_output076[] __initconst = { 0x4b, 0x0b, 0xda, 0x8a, 0xd0, 0x43, 0x83, 0x0d, 0x83, 0x19, 0xab, 0x82, 0xc5, 0x0c, 0x76, 0x63 }; -static const u8 enc_assoc076[] __initconst = { +static const u8 enc_assoc076[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4027,10 +4026,10 @@ static const u8 enc_assoc076[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce076[] __initconst = { +static const u8 enc_nonce076[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xb4, 0xf0 }; -static const u8 enc_key076[] __initconst = { +static const u8 enc_key076[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4038,7 +4037,7 @@ static const u8 enc_key076[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input077[] __initconst = { +static const u8 enc_input077[] = { 0x86, 0xcb, 0xac, 0xae, 0x4d, 0x3f, 0x74, 0xae, 0x01, 0x21, 0x3e, 0x05, 0x51, 0xcc, 0x15, 0x16, 0x0e, 0xa1, 0xbe, 0x84, 0x08, 0xe3, 0xd5, 0xd7, @@ -4056,7 +4055,7 @@ static const u8 enc_input077[] __initconst = { 0x0e, 0x76, 0x2b, 0x43, 0x0c, 0x4d, 0x51, 0x7c, 0x97, 0x10, 0x70, 0x68, 0xf4, 0x98, 0xef, 0x7f }; -static const u8 enc_output077[] __initconst = { +static const u8 enc_output077[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4076,7 +4075,7 @@ static const u8 enc_output077[] __initconst = { 0x4b, 0xc9, 0x8f, 0x72, 0xc4, 0x94, 0xc2, 0xa4, 0x3c, 0x2b, 0x15, 0xa1, 0x04, 0x3f, 0x1c, 0xfa }; -static const u8 enc_assoc077[] __initconst = { +static const u8 enc_assoc077[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4086,10 +4085,10 @@ static const u8 enc_assoc077[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce077[] __initconst = { +static const u8 enc_nonce077[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0xfb, 0x66 }; -static const u8 enc_key077[] __initconst = { +static const u8 enc_key077[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4097,7 +4096,7 @@ static const u8 enc_key077[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input078[] __initconst = { +static const u8 enc_input078[] = { 0xfa, 0xb1, 0xcd, 0xdf, 0x4f, 0xe1, 0x98, 0xef, 0x63, 0xad, 0xd8, 0x81, 0xd6, 0xea, 0xd6, 0xc5, 0x76, 0x37, 0xbb, 0xe9, 0x20, 0x18, 0xca, 0x7c, @@ -4115,7 +4114,7 @@ static const u8 enc_input078[] __initconst = { 0xc3, 0x24, 0x60, 0x41, 0x43, 0x21, 0x43, 0xe9, 0xab, 0x3a, 0x6d, 0x2c, 0xcc, 0x2f, 0x4d, 0x62 }; -static const u8 enc_output078[] __initconst = { +static const u8 enc_output078[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4135,7 +4134,7 @@ static const u8 enc_output078[] __initconst = { 0xf7, 0xe9, 0xe1, 0x51, 0xb0, 0x25, 0x33, 0xc7, 0x46, 0x58, 0xbf, 0xc7, 0x73, 0x7c, 0x68, 0x0d }; -static const u8 enc_assoc078[] __initconst = { +static const u8 enc_assoc078[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4145,10 +4144,10 @@ static const u8 enc_assoc078[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce078[] __initconst = { +static const u8 enc_nonce078[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0xbb, 0x90 }; -static const u8 enc_key078[] __initconst = { +static const u8 enc_key078[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4156,7 +4155,7 @@ static const u8 enc_key078[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input079[] __initconst = { +static const u8 enc_input079[] = { 0x22, 0x72, 0x02, 0xbe, 0x7f, 0x35, 0x15, 0xe9, 0xd1, 0xc0, 0x2e, 0xea, 0x2f, 0x19, 0x50, 0xb6, 0x48, 0x1b, 0x04, 0x8a, 0x4c, 0x91, 0x50, 0x6c, @@ -4174,7 +4173,7 @@ static const u8 enc_input079[] __initconst = { 0xfb, 0xe1, 0xee, 0x45, 0xb1, 0xb2, 0x1f, 0x71, 0x62, 0xe2, 0xfc, 0xaa, 0x74, 0x2a, 0xbe, 0xfd }; -static const u8 enc_output079[] __initconst = { +static const u8 enc_output079[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4194,7 +4193,7 @@ static const u8 enc_output079[] __initconst = { 0x79, 0x5b, 0xcf, 0xf6, 0x47, 0xc5, 0x53, 0xc2, 0xe4, 0xeb, 0x6e, 0x0e, 0xaf, 0xd9, 0xe0, 0x4e }; -static const u8 enc_assoc079[] __initconst = { +static const u8 enc_assoc079[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4204,10 +4203,10 @@ static const u8 enc_assoc079[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce079[] __initconst = { +static const u8 enc_nonce079[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x48, 0x4a }; -static const u8 enc_key079[] __initconst = { +static const u8 enc_key079[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4215,7 +4214,7 @@ static const u8 enc_key079[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input080[] __initconst = { +static const u8 enc_input080[] = { 0xfa, 0xe5, 0x83, 0x45, 0xc1, 0x6c, 0xb0, 0xf5, 0xcc, 0x53, 0x7f, 0x2b, 0x1b, 0x34, 0x69, 0xc9, 0x69, 0x46, 0x3b, 0x3e, 0xa7, 0x1b, 0xcf, 0x6b, @@ -4233,7 +4232,7 @@ static const u8 enc_input080[] __initconst = { 0x18, 0x01, 0xce, 0x33, 0xc4, 0xe4, 0xa7, 0x7d, 0x83, 0x1d, 0x3c, 0xe3, 0x4e, 0x84, 0x10, 0xe1 }; -static const u8 enc_output080[] __initconst = { +static const u8 enc_output080[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4253,7 +4252,7 @@ static const u8 enc_output080[] __initconst = { 0x19, 0x46, 0xd6, 0x53, 0x96, 0x0f, 0x94, 0x7a, 0x74, 0xd3, 0xe8, 0x09, 0x3c, 0xf4, 0x85, 0x02 }; -static const u8 enc_assoc080[] __initconst = { +static const u8 enc_assoc080[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4263,10 +4262,10 @@ static const u8 enc_assoc080[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce080[] __initconst = { +static const u8 enc_nonce080[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0x2f, 0x40 }; -static const u8 enc_key080[] __initconst = { +static const u8 enc_key080[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4274,7 +4273,7 @@ static const u8 enc_key080[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input081[] __initconst = { +static const u8 enc_input081[] = { 0xeb, 0xb2, 0x16, 0xdd, 0xd7, 0xca, 0x70, 0x92, 0x15, 0xf5, 0x03, 0xdf, 0x9c, 0xe6, 0x3c, 0x5c, 0xd2, 0x19, 0x4e, 0x7d, 0x90, 0x99, 0xe8, 0xa9, @@ -4292,7 +4291,7 @@ static const u8 enc_input081[] __initconst = { 0x83, 0xfd, 0xca, 0x39, 0xd2, 0xe1, 0x4f, 0x23, 0xd0, 0x0a, 0x58, 0x26, 0x64, 0xf4, 0xec, 0xb1 }; -static const u8 enc_output081[] __initconst = { +static const u8 enc_output081[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4312,7 +4311,7 @@ static const u8 enc_output081[] __initconst = { 0x36, 0xc3, 0x00, 0x29, 0x85, 0xdd, 0x21, 0xba, 0xf8, 0x95, 0xd6, 0x33, 0x57, 0x3f, 0x12, 0xc0 }; -static const u8 enc_assoc081[] __initconst = { +static const u8 enc_assoc081[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4322,10 +4321,10 @@ static const u8 enc_assoc081[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce081[] __initconst = { +static const u8 enc_nonce081[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0xe2, 0x93, 0x35 }; -static const u8 enc_key081[] __initconst = { +static const u8 enc_key081[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4333,7 +4332,7 @@ static const u8 enc_key081[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input082[] __initconst = { +static const u8 enc_input082[] = { 0x40, 0x8a, 0xe6, 0xef, 0x1c, 0x7e, 0xf0, 0xfb, 0x2c, 0x2d, 0x61, 0x08, 0x16, 0xfc, 0x78, 0x49, 0xef, 0xa5, 0x8f, 0x78, 0x27, 0x3f, 0x5f, 0x16, @@ -4351,7 +4350,7 @@ static const u8 enc_input082[] __initconst = { 0x2b, 0x68, 0x9f, 0x93, 0x55, 0xd9, 0xc1, 0x83, 0x80, 0x1f, 0x6a, 0xcc, 0x31, 0x3f, 0x89, 0x07 }; -static const u8 enc_output082[] __initconst = { +static const u8 enc_output082[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4371,7 +4370,7 @@ static const u8 enc_output082[] __initconst = { 0x65, 0x14, 0x51, 0x8e, 0x0a, 0x26, 0x41, 0x42, 0xe0, 0xb7, 0x35, 0x1f, 0x96, 0x7f, 0xc2, 0xae }; -static const u8 enc_assoc082[] __initconst = { +static const u8 enc_assoc082[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4381,10 +4380,10 @@ static const u8 enc_assoc082[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce082[] __initconst = { +static const u8 enc_nonce082[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0xf7, 0xd5 }; -static const u8 enc_key082[] __initconst = { +static const u8 enc_key082[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4392,7 +4391,7 @@ static const u8 enc_key082[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input083[] __initconst = { +static const u8 enc_input083[] = { 0x0a, 0x0a, 0x24, 0x49, 0x9b, 0xca, 0xde, 0x58, 0xcf, 0x15, 0x76, 0xc3, 0x12, 0xac, 0xa9, 0x84, 0x71, 0x8c, 0xb4, 0xcc, 0x7e, 0x01, 0x53, 0xf5, @@ -4410,7 +4409,7 @@ static const u8 enc_input083[] __initconst = { 0xb3, 0x06, 0xa2, 0x1b, 0x42, 0xd4, 0xc3, 0xba, 0x6e, 0x6f, 0x0c, 0xbc, 0xc8, 0x1e, 0x87, 0x7a }; -static const u8 enc_output083[] __initconst = { +static const u8 enc_output083[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4430,7 +4429,7 @@ static const u8 enc_output083[] __initconst = { 0x4c, 0x19, 0x4d, 0xa6, 0xa9, 0x9f, 0xd6, 0x5b, 0x40, 0xe9, 0xca, 0xd7, 0x98, 0xf4, 0x4b, 0x19 }; -static const u8 enc_assoc083[] __initconst = { +static const u8 enc_assoc083[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4440,10 +4439,10 @@ static const u8 enc_assoc083[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce083[] __initconst = { +static const u8 enc_nonce083[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0xfc, 0xe4 }; -static const u8 enc_key083[] __initconst = { +static const u8 enc_key083[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4451,7 +4450,7 @@ static const u8 enc_key083[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input084[] __initconst = { +static const u8 enc_input084[] = { 0x4a, 0x0a, 0xaf, 0xf8, 0x49, 0x47, 0x29, 0x18, 0x86, 0x91, 0x70, 0x13, 0x40, 0xf3, 0xce, 0x2b, 0x8a, 0x78, 0xee, 0xd3, 0xa0, 0xf0, 0x65, 0x99, @@ -4469,7 +4468,7 @@ static const u8 enc_input084[] __initconst = { 0x5d, 0x7e, 0x51, 0x3b, 0xe5, 0xb8, 0xea, 0x97, 0x13, 0x10, 0xd5, 0xbf, 0x16, 0xba, 0x7a, 0xee }; -static const u8 enc_output084[] __initconst = { +static const u8 enc_output084[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4489,7 +4488,7 @@ static const u8 enc_output084[] __initconst = { 0xc8, 0xae, 0x77, 0x88, 0xcd, 0x28, 0x74, 0xab, 0xc1, 0x38, 0x54, 0x1e, 0x11, 0xfd, 0x05, 0x87 }; -static const u8 enc_assoc084[] __initconst = { +static const u8 enc_assoc084[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4499,10 +4498,10 @@ static const u8 enc_assoc084[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce084[] __initconst = { +static const u8 enc_nonce084[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0x84, 0x86, 0xa8 }; -static const u8 enc_key084[] __initconst = { +static const u8 enc_key084[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4510,7 +4509,7 @@ static const u8 enc_key084[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input085[] __initconst = { +static const u8 enc_input085[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x78, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -4528,7 +4527,7 @@ static const u8 enc_input085[] __initconst = { 0xf7, 0x4d, 0x5b, 0xf8, 0x67, 0x1c, 0x5a, 0x8a, 0x50, 0x92, 0xf6, 0x1d, 0x54, 0xc9, 0xaa, 0x5b }; -static const u8 enc_output085[] __initconst = { +static const u8 enc_output085[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4548,7 +4547,7 @@ static const u8 enc_output085[] __initconst = { 0x93, 0x3a, 0x51, 0x63, 0xc7, 0xf6, 0x23, 0x68, 0x32, 0x7b, 0x3f, 0xbc, 0x10, 0x36, 0xc9, 0x43 }; -static const u8 enc_assoc085[] __initconst = { +static const u8 enc_assoc085[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4558,10 +4557,10 @@ static const u8 enc_assoc085[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce085[] __initconst = { +static const u8 enc_nonce085[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key085[] __initconst = { +static const u8 enc_key085[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4569,7 +4568,7 @@ static const u8 enc_key085[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input086[] __initconst = { +static const u8 enc_input086[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4579,7 +4578,7 @@ static const u8 enc_input086[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output086[] __initconst = { +static const u8 enc_output086[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4591,17 +4590,17 @@ static const u8 enc_output086[] __initconst = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }; -static const u8 enc_assoc086[] __initconst = { +static const u8 enc_assoc086[] = { 0x85, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa6, 0x90, 0x2f, 0xcb, 0xc8, 0x83, 0xbb, 0xc1, 0x80, 0xb2, 0x56, 0xae, 0x34, 0xad, 0x7f, 0x00 }; -static const u8 enc_nonce086[] __initconst = { +static const u8 enc_nonce086[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key086[] __initconst = { +static const u8 enc_key086[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4609,7 +4608,7 @@ static const u8 enc_key086[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input087[] __initconst = { +static const u8 enc_input087[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4619,7 +4618,7 @@ static const u8 enc_input087[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output087[] __initconst = { +static const u8 enc_output087[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4631,17 +4630,17 @@ static const u8 enc_output087[] __initconst = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_assoc087[] __initconst = { +static const u8 enc_assoc087[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x24, 0x7e, 0x50, 0x64, 0x2a, 0x1c, 0x0a, 0x2f, 0x8f, 0x77, 0x21, 0x96, 0x09, 0xdb, 0xa9, 0x58 }; -static const u8 enc_nonce087[] __initconst = { +static const u8 enc_nonce087[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key087[] __initconst = { +static const u8 enc_key087[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4649,7 +4648,7 @@ static const u8 enc_key087[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input088[] __initconst = { +static const u8 enc_input088[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4659,7 +4658,7 @@ static const u8 enc_input088[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output088[] __initconst = { +static const u8 enc_output088[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4671,17 +4670,17 @@ static const u8 enc_output088[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_assoc088[] __initconst = { +static const u8 enc_assoc088[] = { 0x7c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xd9, 0xe7, 0x2c, 0x06, 0x4a, 0xc8, 0x96, 0x1f, 0x3f, 0xa5, 0x85, 0xe0, 0xe2, 0xab, 0xd6, 0x00 }; -static const u8 enc_nonce088[] __initconst = { +static const u8 enc_nonce088[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key088[] __initconst = { +static const u8 enc_key088[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4689,7 +4688,7 @@ static const u8 enc_key088[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input089[] __initconst = { +static const u8 enc_input089[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4699,7 +4698,7 @@ static const u8 enc_input089[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output089[] __initconst = { +static const u8 enc_output089[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4711,17 +4710,17 @@ static const u8 enc_output089[] __initconst = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 }; -static const u8 enc_assoc089[] __initconst = { +static const u8 enc_assoc089[] = { 0x65, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x95, 0xaf, 0x0f, 0x4d, 0x0b, 0x68, 0x6e, 0xae, 0xcc, 0xca, 0x43, 0x07, 0xd5, 0x96, 0xf5, 0x02 }; -static const u8 enc_nonce089[] __initconst = { +static const u8 enc_nonce089[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key089[] __initconst = { +static const u8 enc_key089[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4729,7 +4728,7 @@ static const u8 enc_key089[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input090[] __initconst = { +static const u8 enc_input090[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4739,7 +4738,7 @@ static const u8 enc_input090[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output090[] __initconst = { +static const u8 enc_output090[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4751,17 +4750,17 @@ static const u8 enc_output090[] __initconst = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f }; -static const u8 enc_assoc090[] __initconst = { +static const u8 enc_assoc090[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x85, 0x40, 0xb4, 0x64, 0x35, 0x77, 0x07, 0xbe, 0x3a, 0x39, 0xd5, 0x5c, 0x34, 0xf8, 0xbc, 0xb3 }; -static const u8 enc_nonce090[] __initconst = { +static const u8 enc_nonce090[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key090[] __initconst = { +static const u8 enc_key090[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4769,7 +4768,7 @@ static const u8 enc_key090[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input091[] __initconst = { +static const u8 enc_input091[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4779,7 +4778,7 @@ static const u8 enc_input091[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output091[] __initconst = { +static const u8 enc_output091[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4791,17 +4790,17 @@ static const u8 enc_output091[] __initconst = { 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00 }; -static const u8 enc_assoc091[] __initconst = { +static const u8 enc_assoc091[] = { 0x4f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x66, 0x23, 0xd9, 0x90, 0xb8, 0x98, 0xd8, 0x30, 0xd2, 0x12, 0xaf, 0x23, 0x83, 0x33, 0x07, 0x01 }; -static const u8 enc_nonce091[] __initconst = { +static const u8 enc_nonce091[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key091[] __initconst = { +static const u8 enc_key091[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4809,7 +4808,7 @@ static const u8 enc_key091[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input092[] __initconst = { +static const u8 enc_input092[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4819,7 +4818,7 @@ static const u8 enc_input092[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output092[] __initconst = { +static const u8 enc_output092[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4831,17 +4830,17 @@ static const u8 enc_output092[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_assoc092[] __initconst = { +static const u8 enc_assoc092[] = { 0x83, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x5f, 0x16, 0xd0, 0x9f, 0x17, 0x78, 0x72, 0x11, 0xb7, 0xd4, 0x84, 0xe0, 0x24, 0xf8, 0x97, 0x01 }; -static const u8 enc_nonce092[] __initconst = { +static const u8 enc_nonce092[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key092[] __initconst = { +static const u8 enc_key092[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4849,7 +4848,7 @@ static const u8 enc_key092[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input093[] __initconst = { +static const u8 enc_input093[] = { 0x00, 0x52, 0x35, 0xd2, 0xa9, 0x19, 0xf2, 0x8d, 0x3d, 0xb7, 0x66, 0x4a, 0x34, 0xae, 0x6b, 0x44, 0x4d, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -4861,7 +4860,7 @@ static const u8 enc_input093[] __initconst = { 0x83, 0xdc, 0xe9, 0xf3, 0x07, 0x3e, 0xfa, 0xdb, 0x7d, 0x23, 0xb8, 0x7a, 0xce, 0x35, 0x16, 0x8c }; -static const u8 enc_output093[] __initconst = { +static const u8 enc_output093[] = { 0x00, 0x39, 0xe2, 0xfd, 0x2f, 0xd3, 0x12, 0x14, 0x9e, 0x98, 0x98, 0x80, 0x88, 0x48, 0x13, 0xe7, 0xca, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4875,13 +4874,13 @@ static const u8 enc_output093[] __initconst = { 0xa5, 0x19, 0xac, 0x1a, 0x35, 0xb4, 0xa5, 0x77, 0x87, 0x51, 0x0a, 0xf7, 0x8d, 0x8d, 0x20, 0x0a }; -static const u8 enc_assoc093[] __initconst = { +static const u8 enc_assoc093[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce093[] __initconst = { +static const u8 enc_nonce093[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key093[] __initconst = { +static const u8 enc_key093[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4889,7 +4888,7 @@ static const u8 enc_key093[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input094[] __initconst = { +static const u8 enc_input094[] = { 0xd3, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xe5, 0xda, 0x78, 0x76, 0x6f, 0xa1, 0x92, 0x90, @@ -4903,7 +4902,7 @@ static const u8 enc_input094[] __initconst = { 0x01, 0x49, 0xef, 0x50, 0x4b, 0x71, 0xb1, 0x20, 0xca, 0x4f, 0xf3, 0x95, 0x19, 0xc2, 0xc2, 0x10 }; -static const u8 enc_output094[] __initconst = { +static const u8 enc_output094[] = { 0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x62, 0x18, 0xb2, 0x7f, 0x83, 0xb8, 0xb4, 0x66, @@ -4919,13 +4918,13 @@ static const u8 enc_output094[] __initconst = { 0x30, 0x2f, 0xe8, 0x2a, 0xb0, 0xa0, 0x9a, 0xf6, 0x44, 0x00, 0xd0, 0x15, 0xae, 0x83, 0xd9, 0xcc }; -static const u8 enc_assoc094[] __initconst = { +static const u8 enc_assoc094[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce094[] __initconst = { +static const u8 enc_nonce094[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key094[] __initconst = { +static const u8 enc_key094[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4933,7 +4932,7 @@ static const u8 enc_key094[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input095[] __initconst = { +static const u8 enc_input095[] = { 0xe9, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x6d, 0xf1, 0x39, 0x4e, 0xdc, 0x53, 0x9b, 0x5b, @@ -4947,7 +4946,7 @@ static const u8 enc_input095[] __initconst = { 0x99, 0x52, 0xae, 0x08, 0x18, 0xc3, 0x89, 0x79, 0xc0, 0x74, 0x13, 0x71, 0x1a, 0x9a, 0xf7, 0x13 }; -static const u8 enc_output095[] __initconst = { +static const u8 enc_output095[] = { 0xe9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xea, 0x33, 0xf3, 0x47, 0x30, 0x4a, 0xbd, 0xad, @@ -4963,13 +4962,13 @@ static const u8 enc_output095[] __initconst = { 0x98, 0xa7, 0xe8, 0x36, 0xe0, 0xee, 0x4d, 0x02, 0x35, 0x00, 0xd0, 0x55, 0x7e, 0xc2, 0xcb, 0xe0 }; -static const u8 enc_assoc095[] __initconst = { +static const u8 enc_assoc095[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce095[] __initconst = { +static const u8 enc_nonce095[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key095[] __initconst = { +static const u8 enc_key095[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4977,7 +4976,7 @@ static const u8 enc_key095[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input096[] __initconst = { +static const u8 enc_input096[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x64, 0xf9, 0x0f, 0x5b, 0x26, 0x92, 0xb8, 0x60, @@ -4987,7 +4986,7 @@ static const u8 enc_input096[] __initconst = { 0x03, 0x35, 0x61, 0xe7, 0xca, 0xca, 0x6d, 0x94, 0x1d, 0xc3, 0xcd, 0x69, 0x14, 0xad, 0x69, 0x04 }; -static const u8 enc_output096[] __initconst = { +static const u8 enc_output096[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe3, 0x3b, 0xc5, 0x52, 0xca, 0x8b, 0x9e, 0x96, @@ -4999,13 +4998,13 @@ static const u8 enc_output096[] __initconst = { 0x6a, 0xb8, 0xdc, 0xe2, 0xc5, 0x9d, 0xa4, 0x73, 0x71, 0x30, 0xb0, 0x25, 0x2f, 0x68, 0xa8, 0xd8 }; -static const u8 enc_assoc096[] __initconst = { +static const u8 enc_assoc096[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce096[] __initconst = { +static const u8 enc_nonce096[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key096[] __initconst = { +static const u8 enc_key096[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5013,7 +5012,7 @@ static const u8 enc_key096[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input097[] __initconst = { +static const u8 enc_input097[] = { 0x68, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xb0, 0x8f, 0x25, 0x67, 0x5b, 0x9b, 0xcb, 0xf6, @@ -5027,7 +5026,7 @@ static const u8 enc_input097[] __initconst = { 0x65, 0x0e, 0xc6, 0x2d, 0x75, 0x70, 0x72, 0xce, 0xe6, 0xff, 0x23, 0x31, 0x86, 0xdd, 0x1c, 0x8f }; -static const u8 enc_output097[] __initconst = { +static const u8 enc_output097[] = { 0x68, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x37, 0x4d, 0xef, 0x6e, 0xb7, 0x82, 0xed, 0x00, @@ -5043,13 +5042,13 @@ static const u8 enc_output097[] __initconst = { 0x04, 0x4d, 0xea, 0x60, 0x88, 0x80, 0x41, 0x2b, 0xfd, 0xff, 0xcf, 0x35, 0x57, 0x9e, 0x9b, 0x26 }; -static const u8 enc_assoc097[] __initconst = { +static const u8 enc_assoc097[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce097[] __initconst = { +static const u8 enc_nonce097[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key097[] __initconst = { +static const u8 enc_key097[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5057,7 +5056,7 @@ static const u8 enc_key097[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input098[] __initconst = { +static const u8 enc_input098[] = { 0x6d, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xa1, 0x61, 0xb5, 0xab, 0x04, 0x09, 0x00, 0x62, @@ -5071,7 +5070,7 @@ static const u8 enc_input098[] __initconst = { 0x8e, 0xdc, 0x36, 0x6c, 0xd6, 0x97, 0x65, 0x6f, 0xca, 0x81, 0xfb, 0x13, 0x3c, 0xed, 0x79, 0xa1 }; -static const u8 enc_output098[] __initconst = { +static const u8 enc_output098[] = { 0x6d, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x26, 0xa3, 0x7f, 0xa2, 0xe8, 0x10, 0x26, 0x94, @@ -5087,13 +5086,13 @@ static const u8 enc_output098[] __initconst = { 0x1e, 0x6b, 0xea, 0x63, 0x14, 0x54, 0x2e, 0x2e, 0xf9, 0xff, 0xcf, 0x45, 0x0b, 0x2e, 0x98, 0x2b }; -static const u8 enc_assoc098[] __initconst = { +static const u8 enc_assoc098[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce098[] __initconst = { +static const u8 enc_nonce098[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key098[] __initconst = { +static const u8 enc_key098[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5101,7 +5100,7 @@ static const u8 enc_key098[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input099[] __initconst = { +static const u8 enc_input099[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xfc, 0x01, 0xb8, 0x91, 0xe5, 0xf0, 0xf9, 0x12, @@ -5111,7 +5110,7 @@ static const u8 enc_input099[] __initconst = { 0x60, 0xcd, 0x9e, 0xa1, 0x0c, 0x29, 0xa3, 0x66, 0x54, 0xe7, 0xa2, 0x8e, 0x76, 0x1b, 0xec, 0xd8 }; -static const u8 enc_output099[] __initconst = { +static const u8 enc_output099[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7b, 0xc3, 0x72, 0x98, 0x09, 0xe9, 0xdf, 0xe4, @@ -5123,13 +5122,13 @@ static const u8 enc_output099[] __initconst = { 0xed, 0x20, 0x17, 0xc8, 0xdb, 0xa4, 0x77, 0x56, 0x29, 0x04, 0x9d, 0x78, 0x6e, 0x3b, 0xce, 0xb1 }; -static const u8 enc_assoc099[] __initconst = { +static const u8 enc_assoc099[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce099[] __initconst = { +static const u8 enc_nonce099[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key099[] __initconst = { +static const u8 enc_key099[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5137,7 +5136,7 @@ static const u8 enc_key099[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input100[] __initconst = { +static const u8 enc_input100[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x6b, 0x6d, 0xc9, 0xd2, 0x1a, 0x81, 0x9e, 0x70, @@ -5147,7 +5146,7 @@ static const u8 enc_input100[] __initconst = { 0x10, 0xb9, 0x2f, 0x5f, 0xfe, 0xf9, 0x8b, 0x84, 0x7c, 0xf1, 0x7a, 0x9c, 0x98, 0xd8, 0x83, 0xe5 }; -static const u8 enc_output100[] __initconst = { +static const u8 enc_output100[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xec, 0xaf, 0x03, 0xdb, 0xf6, 0x98, 0xb8, 0x86, @@ -5159,13 +5158,13 @@ static const u8 enc_output100[] __initconst = { 0x07, 0x3f, 0x17, 0xcb, 0x67, 0x78, 0x64, 0x59, 0x25, 0x04, 0x9d, 0x88, 0x22, 0xcb, 0xca, 0xb6 }; -static const u8 enc_assoc100[] __initconst = { +static const u8 enc_assoc100[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce100[] __initconst = { +static const u8 enc_nonce100[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key100[] __initconst = { +static const u8 enc_key100[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5173,7 +5172,7 @@ static const u8 enc_key100[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input101[] __initconst = { +static const u8 enc_input101[] = { 0xff, 0xcb, 0x2b, 0x11, 0x06, 0xf8, 0x23, 0x4c, 0x5e, 0x99, 0xd4, 0xdb, 0x4c, 0x70, 0x48, 0xde, 0x32, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5185,7 +5184,7 @@ static const u8 enc_input101[] __initconst = { 0xce, 0xbe, 0xf5, 0xe9, 0x88, 0x5a, 0x80, 0xea, 0x76, 0xd9, 0x75, 0xc1, 0x44, 0xa4, 0x18, 0x88 }; -static const u8 enc_output101[] __initconst = { +static const u8 enc_output101[] = { 0xff, 0xa0, 0xfc, 0x3e, 0x80, 0x32, 0xc3, 0xd5, 0xfd, 0xb6, 0x2a, 0x11, 0xf0, 0x96, 0x30, 0x7d, 0xb5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5199,13 +5198,13 @@ static const u8 enc_output101[] __initconst = { 0x8b, 0x9b, 0xb4, 0xb4, 0x86, 0x12, 0x89, 0x65, 0x8c, 0x69, 0x6a, 0x83, 0x40, 0x15, 0x04, 0x05 }; -static const u8 enc_assoc101[] __initconst = { +static const u8 enc_assoc101[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce101[] __initconst = { +static const u8 enc_nonce101[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key101[] __initconst = { +static const u8 enc_key101[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5213,7 +5212,7 @@ static const u8 enc_key101[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input102[] __initconst = { +static const u8 enc_input102[] = { 0x6f, 0x9e, 0x70, 0xed, 0x3b, 0x8b, 0xac, 0xa0, 0x26, 0xe4, 0x6a, 0x5a, 0x09, 0x43, 0x15, 0x8d, 0x21, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5225,7 +5224,7 @@ static const u8 enc_input102[] __initconst = { 0xd4, 0x36, 0x51, 0xfd, 0x14, 0x9c, 0x26, 0x0b, 0xcb, 0xdd, 0x7b, 0x12, 0x68, 0x01, 0x31, 0x8c }; -static const u8 enc_output102[] __initconst = { +static const u8 enc_output102[] = { 0x6f, 0xf5, 0xa7, 0xc2, 0xbd, 0x41, 0x4c, 0x39, 0x85, 0xcb, 0x94, 0x90, 0xb5, 0xa5, 0x6d, 0x2e, 0xa6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5239,13 +5238,13 @@ static const u8 enc_output102[] __initconst = { 0x8b, 0x3b, 0xbd, 0x51, 0x64, 0x44, 0x59, 0x56, 0x8d, 0x81, 0xca, 0x1f, 0xa7, 0x2c, 0xe4, 0x04 }; -static const u8 enc_assoc102[] __initconst = { +static const u8 enc_assoc102[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce102[] __initconst = { +static const u8 enc_nonce102[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key102[] __initconst = { +static const u8 enc_key102[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5253,7 +5252,7 @@ static const u8 enc_key102[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input103[] __initconst = { +static const u8 enc_input103[] = { 0x41, 0x2b, 0x08, 0x0a, 0x3e, 0x19, 0xc1, 0x0d, 0x44, 0xa1, 0xaf, 0x1e, 0xab, 0xde, 0xb4, 0xce, 0x35, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5265,7 +5264,7 @@ static const u8 enc_input103[] __initconst = { 0xb3, 0xd4, 0xe9, 0x90, 0x90, 0x34, 0xc6, 0x14, 0xb1, 0x0a, 0xff, 0x55, 0x25, 0xd0, 0x9d, 0x8d }; -static const u8 enc_output103[] __initconst = { +static const u8 enc_output103[] = { 0x41, 0x40, 0xdf, 0x25, 0xb8, 0xd3, 0x21, 0x94, 0xe7, 0x8e, 0x51, 0xd4, 0x17, 0x38, 0xcc, 0x6d, 0xb2, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5279,13 +5278,13 @@ static const u8 enc_output103[] __initconst = { 0x86, 0xfb, 0xab, 0x2b, 0x4a, 0x94, 0xf4, 0x7a, 0xa5, 0x6f, 0x0a, 0xea, 0x65, 0xd1, 0x10, 0x08 }; -static const u8 enc_assoc103[] __initconst = { +static const u8 enc_assoc103[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce103[] __initconst = { +static const u8 enc_nonce103[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key103[] __initconst = { +static const u8 enc_key103[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5293,7 +5292,7 @@ static const u8 enc_key103[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input104[] __initconst = { +static const u8 enc_input104[] = { 0xb2, 0x47, 0xa7, 0x47, 0x23, 0x49, 0x1a, 0xac, 0xac, 0xaa, 0xd7, 0x09, 0xc9, 0x1e, 0x93, 0x2b, 0x31, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5305,7 +5304,7 @@ static const u8 enc_input104[] __initconst = { 0x42, 0x89, 0x79, 0x44, 0xc2, 0xa2, 0x8f, 0xa1, 0x76, 0x11, 0xd7, 0xfa, 0x5c, 0x22, 0xad, 0x8f }; -static const u8 enc_output104[] __initconst = { +static const u8 enc_output104[] = { 0xb2, 0x2c, 0x70, 0x68, 0xa5, 0x83, 0xfa, 0x35, 0x0f, 0x85, 0x29, 0xc3, 0x75, 0xf8, 0xeb, 0x88, 0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5319,13 +5318,13 @@ static const u8 enc_output104[] __initconst = { 0xa0, 0x19, 0xac, 0x2e, 0xd6, 0x67, 0xe1, 0x7d, 0xa1, 0x6f, 0x0a, 0xfa, 0x19, 0x61, 0x0d, 0x0d }; -static const u8 enc_assoc104[] __initconst = { +static const u8 enc_assoc104[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce104[] __initconst = { +static const u8 enc_nonce104[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key104[] __initconst = { +static const u8 enc_key104[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5333,7 +5332,7 @@ static const u8 enc_key104[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input105[] __initconst = { +static const u8 enc_input105[] = { 0x74, 0x0f, 0x9e, 0x49, 0xf6, 0x10, 0xef, 0xa5, 0x85, 0xb6, 0x59, 0xca, 0x6e, 0xd8, 0xb4, 0x99, 0x2d, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5345,7 +5344,7 @@ static const u8 enc_input105[] __initconst = { 0x99, 0x7a, 0xeb, 0x0c, 0x27, 0x95, 0x62, 0x46, 0x69, 0xc3, 0x87, 0xf9, 0x11, 0x6a, 0xc1, 0x8d }; -static const u8 enc_output105[] __initconst = { +static const u8 enc_output105[] = { 0x74, 0x64, 0x49, 0x66, 0x70, 0xda, 0x0f, 0x3c, 0x26, 0x99, 0xa7, 0x00, 0xd2, 0x3e, 0xcc, 0x3a, 0xaa, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5359,13 +5358,13 @@ static const u8 enc_output105[] __initconst = { 0x73, 0x6e, 0x18, 0x18, 0x16, 0x96, 0xa5, 0x88, 0x9c, 0x31, 0x59, 0xfa, 0xab, 0xab, 0x20, 0xfd }; -static const u8 enc_assoc105[] __initconst = { +static const u8 enc_assoc105[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce105[] __initconst = { +static const u8 enc_nonce105[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key105[] __initconst = { +static const u8 enc_key105[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5373,7 +5372,7 @@ static const u8 enc_key105[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input106[] __initconst = { +static const u8 enc_input106[] = { 0xad, 0xba, 0x5d, 0x10, 0x5b, 0xc8, 0xaa, 0x06, 0x2c, 0x23, 0x36, 0xcb, 0x88, 0x9d, 0xdb, 0xd5, 0x37, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5385,7 +5384,7 @@ static const u8 enc_input106[] __initconst = { 0xcf, 0x2b, 0x22, 0x5d, 0xb1, 0x60, 0x7a, 0x10, 0xe6, 0xd5, 0x40, 0x1e, 0x53, 0xb4, 0x2a, 0x8d }; -static const u8 enc_output106[] __initconst = { +static const u8 enc_output106[] = { 0xad, 0xd1, 0x8a, 0x3f, 0xdd, 0x02, 0x4a, 0x9f, 0x8f, 0x0c, 0xc8, 0x01, 0x34, 0x7b, 0xa3, 0x76, 0xb0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5399,13 +5398,13 @@ static const u8 enc_output106[] __initconst = { 0xba, 0xd5, 0x8f, 0x10, 0xa9, 0x1e, 0x6a, 0x88, 0x9a, 0xba, 0x32, 0xfd, 0x17, 0xd8, 0x33, 0x1a }; -static const u8 enc_assoc106[] __initconst = { +static const u8 enc_assoc106[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce106[] __initconst = { +static const u8 enc_nonce106[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key106[] __initconst = { +static const u8 enc_key106[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5413,7 +5412,7 @@ static const u8 enc_key106[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input107[] __initconst = { +static const u8 enc_input107[] = { 0xfe, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xc0, 0x01, 0xed, 0xc5, 0xda, 0x44, 0x2e, 0x71, @@ -5427,7 +5426,7 @@ static const u8 enc_input107[] __initconst = { 0x00, 0x26, 0x6e, 0xa1, 0xe4, 0x36, 0x44, 0xa3, 0x4d, 0x8d, 0xd1, 0xdc, 0x93, 0xf2, 0xfa, 0x13 }; -static const u8 enc_output107[] __initconst = { +static const u8 enc_output107[] = { 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x47, 0xc3, 0x27, 0xcc, 0x36, 0x5d, 0x08, 0x87, @@ -5443,13 +5442,13 @@ static const u8 enc_output107[] __initconst = { 0xd6, 0x8c, 0xe1, 0x74, 0x07, 0x9a, 0xdd, 0x02, 0x8d, 0xd0, 0x5c, 0xf8, 0x14, 0x63, 0x04, 0x88 }; -static const u8 enc_assoc107[] __initconst = { +static const u8 enc_assoc107[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce107[] __initconst = { +static const u8 enc_nonce107[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key107[] __initconst = { +static const u8 enc_key107[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5457,7 +5456,7 @@ static const u8 enc_key107[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input108[] __initconst = { +static const u8 enc_input108[] = { 0xb5, 0x13, 0xb0, 0x6a, 0xb9, 0xac, 0x14, 0x43, 0x5a, 0xcb, 0x8a, 0xa3, 0xa3, 0x7a, 0xfd, 0xb6, 0x54, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5469,7 +5468,7 @@ static const u8 enc_input108[] __initconst = { 0xb9, 0xc2, 0x7c, 0x30, 0x28, 0xaa, 0x8d, 0x69, 0xef, 0x06, 0xaf, 0xc0, 0xb5, 0x9e, 0xda, 0x8e }; -static const u8 enc_output108[] __initconst = { +static const u8 enc_output108[] = { 0xb5, 0x78, 0x67, 0x45, 0x3f, 0x66, 0xf4, 0xda, 0xf9, 0xe4, 0x74, 0x69, 0x1f, 0x9c, 0x85, 0x15, 0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5483,13 +5482,13 @@ static const u8 enc_output108[] __initconst = { 0xaa, 0x48, 0xa3, 0x88, 0x7d, 0x4b, 0x05, 0x96, 0x99, 0xc2, 0xfd, 0xf9, 0xc6, 0x78, 0x7e, 0x0a }; -static const u8 enc_assoc108[] __initconst = { +static const u8 enc_assoc108[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce108[] __initconst = { +static const u8 enc_nonce108[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key108[] __initconst = { +static const u8 enc_key108[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5497,7 +5496,7 @@ static const u8 enc_key108[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input109[] __initconst = { +static const u8 enc_input109[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xd4, 0xf1, 0x09, 0xe8, 0x14, 0xce, 0xa8, 0x5a, @@ -5511,7 +5510,7 @@ static const u8 enc_input109[] __initconst = { 0x1b, 0x64, 0x89, 0xba, 0x84, 0xd8, 0xf5, 0x59, 0x82, 0x9e, 0xd9, 0xbd, 0xa2, 0x29, 0x0f, 0x16 }; -static const u8 enc_output109[] __initconst = { +static const u8 enc_output109[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x53, 0x33, 0xc3, 0xe1, 0xf8, 0xd7, 0x8e, 0xac, @@ -5527,13 +5526,13 @@ static const u8 enc_output109[] __initconst = { 0xb9, 0x36, 0xa8, 0x17, 0xf2, 0x21, 0x1a, 0xf1, 0x29, 0xe2, 0xcf, 0x16, 0x0f, 0xd4, 0x2b, 0xcb }; -static const u8 enc_assoc109[] __initconst = { +static const u8 enc_assoc109[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce109[] __initconst = { +static const u8 enc_nonce109[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key109[] __initconst = { +static const u8 enc_key109[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5541,7 +5540,7 @@ static const u8 enc_key109[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input110[] __initconst = { +static const u8 enc_input110[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xdf, 0x4c, 0x62, 0x03, 0x2d, 0x41, 0x19, 0xb5, @@ -5555,7 +5554,7 @@ static const u8 enc_input110[] __initconst = { 0xb2, 0xa0, 0xc1, 0x84, 0x4b, 0x4e, 0x35, 0xd4, 0x1e, 0x5d, 0xa2, 0x10, 0xf6, 0x2f, 0x84, 0x12 }; -static const u8 enc_output110[] __initconst = { +static const u8 enc_output110[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x58, 0x8e, 0xa8, 0x0a, 0xc1, 0x58, 0x3f, 0x43, @@ -5571,13 +5570,13 @@ static const u8 enc_output110[] __initconst = { 0x9f, 0x7a, 0xc4, 0x35, 0x1f, 0x6b, 0x91, 0xe6, 0x30, 0x97, 0xa7, 0x13, 0x11, 0x5d, 0x05, 0xbe }; -static const u8 enc_assoc110[] __initconst = { +static const u8 enc_assoc110[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce110[] __initconst = { +static const u8 enc_nonce110[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key110[] __initconst = { +static const u8 enc_key110[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5585,7 +5584,7 @@ static const u8 enc_key110[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input111[] __initconst = { +static const u8 enc_input111[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x13, 0xf8, 0x0a, 0x00, 0x6d, 0xc1, 0xbb, 0xda, @@ -5599,7 +5598,7 @@ static const u8 enc_input111[] __initconst = { 0x2b, 0x6c, 0x89, 0x1d, 0x37, 0xc7, 0xe1, 0x1a, 0x56, 0x41, 0x91, 0x9c, 0x49, 0x4d, 0x95, 0x16 }; -static const u8 enc_output111[] __initconst = { +static const u8 enc_output111[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x94, 0x3a, 0xc0, 0x09, 0x81, 0xd8, 0x9d, 0x2c, @@ -5615,13 +5614,13 @@ static const u8 enc_output111[] __initconst = { 0x9a, 0x18, 0xa8, 0x28, 0x07, 0x02, 0x69, 0xf4, 0x47, 0x00, 0xd0, 0x09, 0xe7, 0x17, 0x1c, 0xc9 }; -static const u8 enc_assoc111[] __initconst = { +static const u8 enc_assoc111[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce111[] __initconst = { +static const u8 enc_nonce111[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key111[] __initconst = { +static const u8 enc_key111[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5629,7 +5628,7 @@ static const u8 enc_key111[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input112[] __initconst = { +static const u8 enc_input112[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x82, 0xe5, 0x9b, 0x45, 0x82, 0x91, 0x50, 0x38, @@ -5643,7 +5642,7 @@ static const u8 enc_input112[] __initconst = { 0xfe, 0x55, 0xf9, 0x2a, 0xdc, 0x08, 0xb5, 0xaa, 0x95, 0x48, 0xa9, 0x2d, 0x63, 0xaf, 0xe1, 0x13 }; -static const u8 enc_output112[] __initconst = { +static const u8 enc_output112[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x05, 0x27, 0x51, 0x4c, 0x6e, 0x88, 0x76, 0xce, @@ -5659,13 +5658,13 @@ static const u8 enc_output112[] __initconst = { 0xb4, 0x36, 0xa8, 0x2b, 0x93, 0xd5, 0x55, 0xf7, 0x43, 0x00, 0xd0, 0x19, 0x9b, 0xa7, 0x18, 0xce }; -static const u8 enc_assoc112[] __initconst = { +static const u8 enc_assoc112[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce112[] __initconst = { +static const u8 enc_nonce112[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key112[] __initconst = { +static const u8 enc_key112[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5673,7 +5672,7 @@ static const u8 enc_key112[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input113[] __initconst = { +static const u8 enc_input113[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xf1, 0xd1, 0x28, 0x87, 0xb7, 0x21, 0x69, 0x86, @@ -5687,7 +5686,7 @@ static const u8 enc_input113[] __initconst = { 0xef, 0xe3, 0x69, 0x79, 0xed, 0x9e, 0x7d, 0x3e, 0xc9, 0x52, 0x41, 0x4e, 0x49, 0xb1, 0x30, 0x16 }; -static const u8 enc_output113[] __initconst = { +static const u8 enc_output113[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x76, 0x13, 0xe2, 0x8e, 0x5b, 0x38, 0x4f, 0x70, @@ -5703,13 +5702,13 @@ static const u8 enc_output113[] __initconst = { 0xce, 0x54, 0xa8, 0x2e, 0x1f, 0xa9, 0x42, 0xfa, 0x3f, 0x00, 0xd0, 0x29, 0x4f, 0x37, 0x15, 0xd3 }; -static const u8 enc_assoc113[] __initconst = { +static const u8 enc_assoc113[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce113[] __initconst = { +static const u8 enc_nonce113[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key113[] __initconst = { +static const u8 enc_key113[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5717,7 +5716,7 @@ static const u8 enc_key113[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input114[] __initconst = { +static const u8 enc_input114[] = { 0xcb, 0xf1, 0xda, 0x9e, 0x0b, 0xa9, 0x37, 0x73, 0x74, 0xe6, 0x9e, 0x1c, 0x0e, 0x60, 0x0c, 0xfc, 0x34, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5729,7 +5728,7 @@ static const u8 enc_input114[] __initconst = { 0x66, 0x68, 0xdb, 0xc8, 0xf5, 0xf2, 0x0e, 0xf2, 0xb3, 0xf3, 0x8f, 0x00, 0xe2, 0x03, 0x17, 0x88 }; -static const u8 enc_output114[] __initconst = { +static const u8 enc_output114[] = { 0xcb, 0x9a, 0x0d, 0xb1, 0x8d, 0x63, 0xd7, 0xea, 0xd7, 0xc9, 0x60, 0xd6, 0xb2, 0x86, 0x74, 0x5f, 0xb3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5743,13 +5742,13 @@ static const u8 enc_output114[] __initconst = { 0x23, 0x83, 0xab, 0x0b, 0x79, 0x92, 0x05, 0x69, 0x9b, 0x51, 0x0a, 0xa7, 0x09, 0xbf, 0x31, 0xf1 }; -static const u8 enc_assoc114[] __initconst = { +static const u8 enc_assoc114[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce114[] __initconst = { +static const u8 enc_nonce114[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key114[] __initconst = { +static const u8 enc_key114[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5757,7 +5756,7 @@ static const u8 enc_key114[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input115[] __initconst = { +static const u8 enc_input115[] = { 0x8f, 0x27, 0x86, 0x94, 0xc4, 0xe9, 0xda, 0xeb, 0xd5, 0x8d, 0x3e, 0x5b, 0x96, 0x6e, 0x8b, 0x68, 0x42, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5769,7 +5768,7 @@ static const u8 enc_input115[] __initconst = { 0xde, 0x04, 0x9a, 0x00, 0xa8, 0x64, 0x06, 0x4b, 0xbc, 0xd4, 0x6f, 0xe4, 0xe4, 0x5b, 0x42, 0x8f }; -static const u8 enc_output115[] __initconst = { +static const u8 enc_output115[] = { 0x8f, 0x4c, 0x51, 0xbb, 0x42, 0x23, 0x3a, 0x72, 0x76, 0xa2, 0xc0, 0x91, 0x2a, 0x88, 0xf3, 0xcb, 0xc5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5783,13 +5782,13 @@ static const u8 enc_output115[] __initconst = { 0x8b, 0xfb, 0xab, 0x17, 0xa9, 0xe0, 0xb8, 0x74, 0x8b, 0x51, 0x0a, 0xe7, 0xd9, 0xfd, 0x23, 0x05 }; -static const u8 enc_assoc115[] __initconst = { +static const u8 enc_assoc115[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce115[] __initconst = { +static const u8 enc_nonce115[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key115[] __initconst = { +static const u8 enc_key115[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5797,7 +5796,7 @@ static const u8 enc_key115[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input116[] __initconst = { +static const u8 enc_input116[] = { 0xd5, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x9a, 0x22, 0xd7, 0x0a, 0x48, 0xe2, 0x4f, 0xdd, @@ -5811,7 +5810,7 @@ static const u8 enc_input116[] __initconst = { 0x3f, 0x91, 0xf8, 0xe7, 0xc7, 0xb1, 0x96, 0x25, 0x64, 0x61, 0x9c, 0x5e, 0x7e, 0x9b, 0xf6, 0x13 }; -static const u8 enc_output116[] __initconst = { +static const u8 enc_output116[] = { 0xd5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x1d, 0xe0, 0x1d, 0x03, 0xa4, 0xfb, 0x69, 0x2b, @@ -5827,13 +5826,13 @@ static const u8 enc_output116[] __initconst = { 0x49, 0xbc, 0x6e, 0x9f, 0xc5, 0x1c, 0x4d, 0x50, 0x30, 0x36, 0x64, 0x4d, 0x84, 0x27, 0x73, 0xd2 }; -static const u8 enc_assoc116[] __initconst = { +static const u8 enc_assoc116[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce116[] __initconst = { +static const u8 enc_nonce116[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key116[] __initconst = { +static const u8 enc_key116[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5841,7 +5840,7 @@ static const u8 enc_key116[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input117[] __initconst = { +static const u8 enc_input117[] = { 0xdb, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x75, 0xd5, 0x64, 0x3a, 0xa5, 0xaf, 0x93, 0x4d, @@ -5855,7 +5854,7 @@ static const u8 enc_input117[] __initconst = { 0x28, 0x3f, 0x6b, 0x32, 0x18, 0x07, 0x5f, 0xc9, 0x5f, 0x6b, 0xb4, 0xff, 0x45, 0x6d, 0xc1, 0x11 }; -static const u8 enc_output117[] __initconst = { +static const u8 enc_output117[] = { 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf2, 0x17, 0xae, 0x33, 0x49, 0xb6, 0xb5, 0xbb, @@ -5871,13 +5870,13 @@ static const u8 enc_output117[] __initconst = { 0x63, 0xda, 0x6e, 0xa2, 0x51, 0xf0, 0x39, 0x53, 0x2c, 0x36, 0x64, 0x5d, 0x38, 0xb7, 0x6f, 0xd7 }; -static const u8 enc_assoc117[] __initconst = { +static const u8 enc_assoc117[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce117[] __initconst = { +static const u8 enc_nonce117[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key117[] __initconst = { +static const u8 enc_key117[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5885,7 +5884,7 @@ static const u8 enc_key117[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input118[] __initconst = { +static const u8 enc_input118[] = { 0x93, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x62, 0x48, 0x39, 0x60, 0x42, 0x16, 0xe4, 0x03, @@ -5899,7 +5898,7 @@ static const u8 enc_input118[] __initconst = { 0x90, 0xec, 0xf2, 0x1a, 0x04, 0xe6, 0x30, 0x85, 0x8b, 0xb6, 0x56, 0x52, 0xb5, 0xb1, 0x80, 0x16 }; -static const u8 enc_output118[] __initconst = { +static const u8 enc_output118[] = { 0x93, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe5, 0x8a, 0xf3, 0x69, 0xae, 0x0f, 0xc2, 0xf5, @@ -5915,13 +5914,13 @@ static const u8 enc_output118[] __initconst = { 0x73, 0xeb, 0x27, 0x24, 0xb5, 0xc4, 0x05, 0xf0, 0x4d, 0x00, 0xd0, 0xf1, 0x58, 0x40, 0xa1, 0xc1 }; -static const u8 enc_assoc118[] __initconst = { +static const u8 enc_assoc118[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce118[] __initconst = { +static const u8 enc_nonce118[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key118[] __initconst = { +static const u8 enc_key118[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5929,7 +5928,7 @@ static const u8 enc_key118[] __initconst = { }; static const struct chacha20poly1305_testvec -chacha20poly1305_enc_vectors[] __initconst = { +chacha20poly1305_enc_vectors[] = { { enc_input001, enc_output001, enc_assoc001, enc_nonce001, enc_key001, sizeof(enc_input001), sizeof(enc_assoc001), sizeof(enc_nonce001) }, { enc_input002, enc_output002, enc_assoc002, enc_nonce002, enc_key002, @@ -6168,7 +6167,7 @@ chacha20poly1305_enc_vectors[] __initconst = { sizeof(enc_input118), sizeof(enc_assoc118), sizeof(enc_nonce118) } }; -static const u8 dec_input001[] __initconst = { +static const u8 dec_input001[] = { 0x64, 0xa0, 0x86, 0x15, 0x75, 0x86, 0x1a, 0xf4, 0x60, 0xf0, 0x62, 0xc7, 0x9b, 0xe6, 0x43, 0xbd, 0x5e, 0x80, 0x5c, 0xfd, 0x34, 0x5c, 0xf3, 0x89, @@ -6206,7 +6205,7 @@ static const u8 dec_input001[] __initconst = { 0x22, 0x39, 0x23, 0x36, 0xfe, 0xa1, 0x85, 0x1f, 0x38 }; -static const u8 dec_output001[] __initconst = { +static const u8 dec_output001[] = { 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, @@ -6242,95 +6241,95 @@ static const u8 dec_output001[] __initconst = { 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, 0x9d }; -static const u8 dec_assoc001[] __initconst = { +static const u8 dec_assoc001[] = { 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x91 }; -static const u8 dec_nonce001[] __initconst = { +static const u8 dec_nonce001[] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 }; -static const u8 dec_key001[] __initconst = { +static const u8 dec_key001[] = { 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 }; -static const u8 dec_input002[] __initconst = { +static const u8 dec_input002[] = { 0xea, 0xe0, 0x1e, 0x9e, 0x2c, 0x91, 0xaa, 0xe1, 0xdb, 0x5d, 0x99, 0x3f, 0x8a, 0xf7, 0x69, 0x92 }; -static const u8 dec_output002[] __initconst = { }; -static const u8 dec_assoc002[] __initconst = { }; -static const u8 dec_nonce002[] __initconst = { +static const u8 dec_output002[] = { }; +static const u8 dec_assoc002[] = { }; +static const u8 dec_nonce002[] = { 0xca, 0xbf, 0x33, 0x71, 0x32, 0x45, 0x77, 0x8e }; -static const u8 dec_key002[] __initconst = { +static const u8 dec_key002[] = { 0x4c, 0xf5, 0x96, 0x83, 0x38, 0xe6, 0xae, 0x7f, 0x2d, 0x29, 0x25, 0x76, 0xd5, 0x75, 0x27, 0x86, 0x91, 0x9a, 0x27, 0x7a, 0xfb, 0x46, 0xc5, 0xef, 0x94, 0x81, 0x79, 0x57, 0x14, 0x59, 0x40, 0x68 }; -static const u8 dec_input003[] __initconst = { +static const u8 dec_input003[] = { 0xdd, 0x6b, 0x3b, 0x82, 0xce, 0x5a, 0xbd, 0xd6, 0xa9, 0x35, 0x83, 0xd8, 0x8c, 0x3d, 0x85, 0x77 }; -static const u8 dec_output003[] __initconst = { }; -static const u8 dec_assoc003[] __initconst = { +static const u8 dec_output003[] = { }; +static const u8 dec_assoc003[] = { 0x33, 0x10, 0x41, 0x12, 0x1f, 0xf3, 0xd2, 0x6b }; -static const u8 dec_nonce003[] __initconst = { +static const u8 dec_nonce003[] = { 0x3d, 0x86, 0xb5, 0x6b, 0xc8, 0xa3, 0x1f, 0x1d }; -static const u8 dec_key003[] __initconst = { +static const u8 dec_key003[] = { 0x2d, 0xb0, 0x5d, 0x40, 0xc8, 0xed, 0x44, 0x88, 0x34, 0xd1, 0x13, 0xaf, 0x57, 0xa1, 0xeb, 0x3a, 0x2a, 0x80, 0x51, 0x36, 0xec, 0x5b, 0xbc, 0x08, 0x93, 0x84, 0x21, 0xb5, 0x13, 0x88, 0x3c, 0x0d }; -static const u8 dec_input004[] __initconst = { +static const u8 dec_input004[] = { 0xb7, 0x1b, 0xb0, 0x73, 0x59, 0xb0, 0x84, 0xb2, 0x6d, 0x8e, 0xab, 0x94, 0x31, 0xa1, 0xae, 0xac, 0x89 }; -static const u8 dec_output004[] __initconst = { +static const u8 dec_output004[] = { 0xa4 }; -static const u8 dec_assoc004[] __initconst = { +static const u8 dec_assoc004[] = { 0x6a, 0xe2, 0xad, 0x3f, 0x88, 0x39, 0x5a, 0x40 }; -static const u8 dec_nonce004[] __initconst = { +static const u8 dec_nonce004[] = { 0xd2, 0x32, 0x1f, 0x29, 0x28, 0xc6, 0xc4, 0xc4 }; -static const u8 dec_key004[] __initconst = { +static const u8 dec_key004[] = { 0x4b, 0x28, 0x4b, 0xa3, 0x7b, 0xbe, 0xe9, 0xf8, 0x31, 0x80, 0x82, 0xd7, 0xd8, 0xe8, 0xb5, 0xa1, 0xe2, 0x18, 0x18, 0x8a, 0x9c, 0xfa, 0xa3, 0x3d, 0x25, 0x71, 0x3e, 0x40, 0xbc, 0x54, 0x7a, 0x3e }; -static const u8 dec_input005[] __initconst = { +static const u8 dec_input005[] = { 0xbf, 0xe1, 0x5b, 0x0b, 0xdb, 0x6b, 0xf5, 0x5e, 0x6c, 0x5d, 0x84, 0x44, 0x39, 0x81, 0xc1, 0x9c, 0xac }; -static const u8 dec_output005[] __initconst = { +static const u8 dec_output005[] = { 0x2d }; -static const u8 dec_assoc005[] __initconst = { }; -static const u8 dec_nonce005[] __initconst = { +static const u8 dec_assoc005[] = { }; +static const u8 dec_nonce005[] = { 0x20, 0x1c, 0xaa, 0x5f, 0x9c, 0xbf, 0x92, 0x30 }; -static const u8 dec_key005[] __initconst = { +static const u8 dec_key005[] = { 0x66, 0xca, 0x9c, 0x23, 0x2a, 0x4b, 0x4b, 0x31, 0x0e, 0x92, 0x89, 0x8b, 0xf4, 0x93, 0xc7, 0x87, 0x98, 0xa3, 0xd8, 0x39, 0xf8, 0xf4, 0xa7, 0x01, 0xc0, 0x2e, 0x0a, 0xa6, 0x7e, 0x5a, 0x78, 0x87 }; -static const u8 dec_input006[] __initconst = { +static const u8 dec_input006[] = { 0x8b, 0x06, 0xd3, 0x31, 0xb0, 0x93, 0x45, 0xb1, 0x75, 0x6e, 0x26, 0xf9, 0x67, 0xbc, 0x90, 0x15, 0x81, 0x2c, 0xb5, 0xf0, 0xc6, 0x2b, 0xc7, 0x8c, @@ -6351,7 +6350,7 @@ static const u8 dec_input006[] __initconst = { 0x8d, 0x31, 0xf7, 0x7a, 0x39, 0x4d, 0x8f, 0x9a, 0xeb }; -static const u8 dec_output006[] __initconst = { +static const u8 dec_output006[] = { 0x33, 0x2f, 0x94, 0xc1, 0xa4, 0xef, 0xcc, 0x2a, 0x5b, 0xa6, 0xe5, 0x8f, 0x1d, 0x40, 0xf0, 0x92, 0x3c, 0xd9, 0x24, 0x11, 0xa9, 0x71, 0xf9, 0x37, @@ -6370,20 +6369,20 @@ static const u8 dec_output006[] __initconst = { 0x69, 0x21, 0x70, 0xd8, 0xa4, 0x4b, 0xc8, 0xde, 0x8f }; -static const u8 dec_assoc006[] __initconst = { +static const u8 dec_assoc006[] = { 0x70, 0xd3, 0x33, 0xf3, 0x8b, 0x18, 0x0b }; -static const u8 dec_nonce006[] __initconst = { +static const u8 dec_nonce006[] = { 0xdf, 0x51, 0x84, 0x82, 0x42, 0x0c, 0x75, 0x9c }; -static const u8 dec_key006[] __initconst = { +static const u8 dec_key006[] = { 0x68, 0x7b, 0x8d, 0x8e, 0xe3, 0xc4, 0xdd, 0xae, 0xdf, 0x72, 0x7f, 0x53, 0x72, 0x25, 0x1e, 0x78, 0x91, 0xcb, 0x69, 0x76, 0x1f, 0x49, 0x93, 0xf9, 0x6f, 0x21, 0xcc, 0x39, 0x9c, 0xad, 0xb1, 0x01 }; -static const u8 dec_input007[] __initconst = { +static const u8 dec_input007[] = { 0x85, 0x04, 0xc2, 0xed, 0x8d, 0xfd, 0x97, 0x5c, 0xd2, 0xb7, 0xe2, 0xc1, 0x6b, 0xa3, 0xba, 0xf8, 0xc9, 0x50, 0xc3, 0xc6, 0xa5, 0xe3, 0xa4, 0x7c, @@ -6419,7 +6418,7 @@ static const u8 dec_input007[] __initconst = { 0x7a, 0x4b, 0xbc, 0xb0, 0x10, 0x5c, 0x96, 0x42, 0x3a, 0x00, 0x98, 0xcd, 0x15, 0xe8, 0xb7, 0x53 }; -static const u8 dec_output007[] __initconst = { +static const u8 dec_output007[] = { 0x9b, 0x18, 0xdb, 0xdd, 0x9a, 0x0f, 0x3e, 0xa5, 0x15, 0x17, 0xde, 0xdf, 0x08, 0x9d, 0x65, 0x0a, 0x67, 0x30, 0x12, 0xe2, 0x34, 0x77, 0x4b, 0xc1, @@ -6453,18 +6452,18 @@ static const u8 dec_output007[] __initconst = { 0x40, 0x5f, 0x99, 0xb7, 0x73, 0xec, 0x9b, 0x2b, 0xf0, 0x65, 0x11, 0xc8, 0xd0, 0x0a, 0x9f, 0xd3 }; -static const u8 dec_assoc007[] __initconst = { }; -static const u8 dec_nonce007[] __initconst = { +static const u8 dec_assoc007[] = { }; +static const u8 dec_nonce007[] = { 0xde, 0x7b, 0xef, 0xc3, 0x65, 0x1b, 0x68, 0xb0 }; -static const u8 dec_key007[] __initconst = { +static const u8 dec_key007[] = { 0x8d, 0xb8, 0x91, 0x48, 0xf0, 0xe7, 0x0a, 0xbd, 0xf9, 0x3f, 0xcd, 0xd9, 0xa0, 0x1e, 0x42, 0x4c, 0xe7, 0xde, 0x25, 0x3d, 0xa3, 0xd7, 0x05, 0x80, 0x8d, 0xf2, 0x82, 0xac, 0x44, 0x16, 0x51, 0x01 }; -static const u8 dec_input008[] __initconst = { +static const u8 dec_input008[] = { 0x14, 0xf6, 0x41, 0x37, 0xa6, 0xd4, 0x27, 0xcd, 0xdb, 0x06, 0x3e, 0x9a, 0x4e, 0xab, 0xd5, 0xb1, 0x1e, 0x6b, 0xd2, 0xbc, 0x11, 0xf4, 0x28, 0x93, @@ -6532,7 +6531,7 @@ static const u8 dec_input008[] __initconst = { 0x54, 0xfd, 0x0d, 0x98, 0x1c, 0x5a, 0x6f, 0x1f, 0x9a, 0x40, 0xcd, 0xa2, 0xff, 0x6a, 0xf1, 0x54 }; -static const u8 dec_output008[] __initconst = { +static const u8 dec_output008[] = { 0xc3, 0x09, 0x94, 0x62, 0xe6, 0x46, 0x2e, 0x10, 0xbe, 0x00, 0xe4, 0xfc, 0xf3, 0x40, 0xa3, 0xe2, 0x0f, 0xc2, 0x8b, 0x28, 0xdc, 0xba, 0xb4, 0x3c, @@ -6598,18 +6597,18 @@ static const u8 dec_output008[] __initconst = { 0x2e, 0x79, 0xb5, 0xe2, 0xb8, 0xe8, 0xb9, 0x7b, 0xd5, 0x10, 0xcb, 0xff, 0x5d, 0x14, 0x73, 0xf3 }; -static const u8 dec_assoc008[] __initconst = { }; -static const u8 dec_nonce008[] __initconst = { +static const u8 dec_assoc008[] = { }; +static const u8 dec_nonce008[] = { 0x0e, 0x0d, 0x57, 0xbb, 0x7b, 0x40, 0x54, 0x02 }; -static const u8 dec_key008[] __initconst = { +static const u8 dec_key008[] = { 0xf2, 0xaa, 0x4f, 0x99, 0xfd, 0x3e, 0xa8, 0x53, 0xc1, 0x44, 0xe9, 0x81, 0x18, 0xdc, 0xf5, 0xf0, 0x3e, 0x44, 0x15, 0x59, 0xe0, 0xc5, 0x44, 0x86, 0xc3, 0x91, 0xa8, 0x75, 0xc0, 0x12, 0x46, 0xba }; -static const u8 dec_input009[] __initconst = { +static const u8 dec_input009[] = { 0xfd, 0x81, 0x8d, 0xd0, 0x3d, 0xb4, 0xd5, 0xdf, 0xd3, 0x42, 0x47, 0x5a, 0x6d, 0x19, 0x27, 0x66, 0x4b, 0x2e, 0x0c, 0x27, 0x9c, 0x96, 0x4c, 0x72, @@ -6678,7 +6677,7 @@ static const u8 dec_input009[] __initconst = { 0x43, 0xb5, 0xd9, 0x90, 0xe1, 0x85, 0xf5, 0xa8, 0xae }; -static const u8 dec_output009[] __initconst = { +static const u8 dec_output009[] = { 0xe6, 0xc3, 0xdb, 0x63, 0x55, 0x15, 0xe3, 0x5b, 0xb7, 0x4b, 0x27, 0x8b, 0x5a, 0xdd, 0xc2, 0xe8, 0x3a, 0x6b, 0xd7, 0x81, 0x96, 0x35, 0x97, 0xca, @@ -6745,21 +6744,21 @@ static const u8 dec_output009[] __initconst = { 0x74, 0xb9, 0xe2, 0xd6, 0x1c, 0x80, 0x2c, 0x52, 0x65 }; -static const u8 dec_assoc009[] __initconst = { +static const u8 dec_assoc009[] = { 0x5a, 0x27, 0xff, 0xeb, 0xdf, 0x84, 0xb2, 0x9e, 0xef }; -static const u8 dec_nonce009[] __initconst = { +static const u8 dec_nonce009[] = { 0xef, 0x2d, 0x63, 0xee, 0x6b, 0x80, 0x8b, 0x78 }; -static const u8 dec_key009[] __initconst = { +static const u8 dec_key009[] = { 0xea, 0xbc, 0x56, 0x99, 0xe3, 0x50, 0xff, 0xc5, 0xcc, 0x1a, 0xd7, 0xc1, 0x57, 0x72, 0xea, 0x86, 0x5b, 0x89, 0x88, 0x61, 0x3d, 0x2f, 0x9b, 0xb2, 0xe7, 0x9c, 0xec, 0x74, 0x6e, 0x3e, 0xf4, 0x3b }; -static const u8 dec_input010[] __initconst = { +static const u8 dec_input010[] = { 0xe5, 0x26, 0xa4, 0x3d, 0xbd, 0x33, 0xd0, 0x4b, 0x6f, 0x05, 0xa7, 0x6e, 0x12, 0x7a, 0xd2, 0x74, 0xa6, 0xdd, 0xbd, 0x95, 0xeb, 0xf9, 0xa4, 0xf1, @@ -6891,7 +6890,7 @@ static const u8 dec_input010[] __initconst = { 0x80, 0x6b, 0xf6, 0x1f, 0xc3, 0xcc, 0x97, 0xc9, 0x24, 0x9f, 0xf3, 0xaf, 0x43, 0x14, 0xd5, 0xa0 }; -static const u8 dec_output010[] __initconst = { +static const u8 dec_output010[] = { 0x42, 0x93, 0xe4, 0xeb, 0x97, 0xb0, 0x57, 0xbf, 0x1a, 0x8b, 0x1f, 0xe4, 0x5f, 0x36, 0x20, 0x3c, 0xef, 0x0a, 0xa9, 0x48, 0x5f, 0x5f, 0x37, 0x22, @@ -7021,21 +7020,21 @@ static const u8 dec_output010[] __initconst = { 0x5a, 0xc0, 0x33, 0x1f, 0xcb, 0x05, 0x6d, 0x5c, 0x06, 0x87, 0x52, 0xa2, 0x8f, 0x26, 0xd5, 0x4f }; -static const u8 dec_assoc010[] __initconst = { +static const u8 dec_assoc010[] = { 0xd2, 0xa1, 0x70, 0xdb, 0x7a, 0xf8, 0xfa, 0x27, 0xba, 0x73, 0x0f, 0xbf, 0x3d, 0x1e, 0x82, 0xb2 }; -static const u8 dec_nonce010[] __initconst = { +static const u8 dec_nonce010[] = { 0xdb, 0x92, 0x0f, 0x7f, 0x17, 0x54, 0x0c, 0x30 }; -static const u8 dec_key010[] __initconst = { +static const u8 dec_key010[] = { 0x47, 0x11, 0xeb, 0x86, 0x2b, 0x2c, 0xab, 0x44, 0x34, 0xda, 0x7f, 0x57, 0x03, 0x39, 0x0c, 0xaf, 0x2c, 0x14, 0xfd, 0x65, 0x23, 0xe9, 0x8e, 0x74, 0xd5, 0x08, 0x68, 0x08, 0xe7, 0xb4, 0x72, 0xd7 }; -static const u8 dec_input011[] __initconst = { +static const u8 dec_input011[] = { 0x6a, 0xfc, 0x4b, 0x25, 0xdf, 0xc0, 0xe4, 0xe8, 0x17, 0x4d, 0x4c, 0xc9, 0x7e, 0xde, 0x3a, 0xcc, 0x3c, 0xba, 0x6a, 0x77, 0x47, 0xdb, 0xe3, 0x74, @@ -7281,7 +7280,7 @@ static const u8 dec_input011[] __initconst = { 0x86, 0xc4, 0xfc, 0x97, 0xae, 0x3f, 0x8f, 0x1e, 0x2b, 0xdf, 0xcd, 0xf9, 0x3c }; -static const u8 dec_output011[] __initconst = { +static const u8 dec_output011[] = { 0x7a, 0x57, 0xf2, 0xc7, 0x06, 0x3f, 0x50, 0x7b, 0x36, 0x1a, 0x66, 0x5c, 0xb9, 0x0e, 0x5e, 0x3b, 0x45, 0x60, 0xbe, 0x9a, 0x31, 0x9f, 0xff, 0x5d, @@ -7525,20 +7524,20 @@ static const u8 dec_output011[] __initconst = { 0x5c, 0x42, 0x16, 0xb8, 0x28, 0xeb, 0x1b, 0x61, 0x10, 0x1e, 0xbf, 0xec, 0xa8 }; -static const u8 dec_assoc011[] __initconst = { +static const u8 dec_assoc011[] = { 0xd6, 0x31, 0xda, 0x5d, 0x42, 0x5e, 0xd7 }; -static const u8 dec_nonce011[] __initconst = { +static const u8 dec_nonce011[] = { 0xfd, 0x87, 0xd4, 0xd8, 0x62, 0xfd, 0xec, 0xaa }; -static const u8 dec_key011[] __initconst = { +static const u8 dec_key011[] = { 0x35, 0x4e, 0xb5, 0x70, 0x50, 0x42, 0x8a, 0x85, 0xf2, 0xfb, 0xed, 0x7b, 0xd0, 0x9e, 0x97, 0xca, 0xfa, 0x98, 0x66, 0x63, 0xee, 0x37, 0xcc, 0x52, 0xfe, 0xd1, 0xdf, 0x95, 0x15, 0x34, 0x29, 0x38 }; -static const u8 dec_input012[] __initconst = { +static const u8 dec_input012[] = { 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, 0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1, @@ -7794,7 +7793,7 @@ static const u8 dec_input012[] __initconst = { 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, 0x70, 0xcf, 0xd6 }; -static const u8 dec_output012[] __initconst = { +static const u8 dec_output012[] = { 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, 0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57, @@ -8048,7 +8047,7 @@ static const u8 dec_output012[] __initconst = { 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, 0x78, 0xec, 0x00 }; -static const u8 dec_assoc012[] __initconst = { +static const u8 dec_assoc012[] = { 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, 0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce, 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, @@ -8058,17 +8057,17 @@ static const u8 dec_assoc012[] __initconst = { 0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b, 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 }; -static const u8 dec_nonce012[] __initconst = { +static const u8 dec_nonce012[] = { 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 }; -static const u8 dec_key012[] __initconst = { +static const u8 dec_key012[] = { 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, 0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d, 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e, 0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64 }; -static const u8 dec_input013[] __initconst = { +static const u8 dec_input013[] = { 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, 0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1, @@ -8324,7 +8323,7 @@ static const u8 dec_input013[] __initconst = { 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, 0x70, 0xcf, 0xd7 }; -static const u8 dec_output013[] __initconst = { +static const u8 dec_output013[] = { 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, 0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57, @@ -8578,7 +8577,7 @@ static const u8 dec_output013[] __initconst = { 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, 0x78, 0xec, 0x00 }; -static const u8 dec_assoc013[] __initconst = { +static const u8 dec_assoc013[] = { 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, 0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce, 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, @@ -8588,10 +8587,10 @@ static const u8 dec_assoc013[] __initconst = { 0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b, 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 }; -static const u8 dec_nonce013[] __initconst = { +static const u8 dec_nonce013[] = { 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 }; -static const u8 dec_key013[] __initconst = { +static const u8 dec_key013[] = { 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, 0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d, 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e, @@ -8599,7 +8598,7 @@ static const u8 dec_key013[] __initconst = { }; static const struct chacha20poly1305_testvec -chacha20poly1305_dec_vectors[] __initconst = { +chacha20poly1305_dec_vectors[] = { { dec_input001, dec_output001, dec_assoc001, dec_nonce001, dec_key001, sizeof(dec_input001), sizeof(dec_assoc001), sizeof(dec_nonce001) }, { dec_input002, dec_output002, dec_assoc002, dec_nonce002, dec_key002, @@ -8629,7 +8628,7 @@ chacha20poly1305_dec_vectors[] __initconst = { true } }; -static const u8 xenc_input001[] __initconst = { +static const u8 xenc_input001[] = { 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, @@ -8665,7 +8664,7 @@ static const u8 xenc_input001[] __initconst = { 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, 0x9d }; -static const u8 xenc_output001[] __initconst = { +static const u8 xenc_output001[] = { 0x1a, 0x6e, 0x3a, 0xd9, 0xfd, 0x41, 0x3f, 0x77, 0x54, 0x72, 0x0a, 0x70, 0x9a, 0xa0, 0x29, 0x92, 0x2e, 0xed, 0x93, 0xcf, 0x0f, 0x71, 0x88, 0x18, @@ -8703,16 +8702,16 @@ static const u8 xenc_output001[] __initconst = { 0x93, 0xa6, 0x85, 0x5b, 0x40, 0x39, 0x5c, 0xc5, 0x9c }; -static const u8 xenc_assoc001[] __initconst = { +static const u8 xenc_assoc001[] = { 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x91 }; -static const u8 xenc_nonce001[] __initconst = { +static const u8 xenc_nonce001[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 }; -static const u8 xenc_key001[] __initconst = { +static const u8 xenc_key001[] = { 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, @@ -8720,12 +8719,12 @@ static const u8 xenc_key001[] __initconst = { }; static const struct chacha20poly1305_testvec -xchacha20poly1305_enc_vectors[] __initconst = { +xchacha20poly1305_enc_vectors[] = { { xenc_input001, xenc_output001, xenc_assoc001, xenc_nonce001, xenc_key001, sizeof(xenc_input001), sizeof(xenc_assoc001), sizeof(xenc_nonce001) } }; -static const u8 xdec_input001[] __initconst = { +static const u8 xdec_input001[] = { 0x1a, 0x6e, 0x3a, 0xd9, 0xfd, 0x41, 0x3f, 0x77, 0x54, 0x72, 0x0a, 0x70, 0x9a, 0xa0, 0x29, 0x92, 0x2e, 0xed, 0x93, 0xcf, 0x0f, 0x71, 0x88, 0x18, @@ -8763,7 +8762,7 @@ static const u8 xdec_input001[] __initconst = { 0x93, 0xa6, 0x85, 0x5b, 0x40, 0x39, 0x5c, 0xc5, 0x9c }; -static const u8 xdec_output001[] __initconst = { +static const u8 xdec_output001[] = { 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, @@ -8799,16 +8798,16 @@ static const u8 xdec_output001[] __initconst = { 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, 0x9d }; -static const u8 xdec_assoc001[] __initconst = { +static const u8 xdec_assoc001[] = { 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x91 }; -static const u8 xdec_nonce001[] __initconst = { +static const u8 xdec_nonce001[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 }; -static const u8 xdec_key001[] __initconst = { +static const u8 xdec_key001[] = { 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, @@ -8816,15 +8815,15 @@ static const u8 xdec_key001[] __initconst = { }; static const struct chacha20poly1305_testvec -xchacha20poly1305_dec_vectors[] __initconst = { +xchacha20poly1305_dec_vectors[] = { { xdec_input001, xdec_output001, xdec_assoc001, xdec_nonce001, xdec_key001, sizeof(xdec_input001), sizeof(xdec_assoc001), sizeof(xdec_nonce001) } }; -/* This is for the selftests-only, since it is only useful for the purpose of +/* This is for the tests only, since it is only useful for the purpose of * testing the underlying primitives and interactions. */ -static void __init +static void chacha20poly1305_encrypt_bignonce(u8 *dst, const u8 *src, const size_t src_len, const u8 *ad, const size_t ad_len, const u8 nonce[12], @@ -8858,11 +8857,12 @@ chacha20poly1305_encrypt_bignonce(u8 *dst, const u8 *src, const size_t src_len, poly1305_final(&poly1305_state, dst + src_len); } -static void __init -chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len, - const u8 *ad, const size_t ad_len, - const u8 *nonce, const size_t nonce_len, - const u8 key[CHACHA20POLY1305_KEY_SIZE]) +static void +chacha20poly1305_test_encrypt(struct kunit *test, u8 *dst, + const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u8 *nonce, const size_t nonce_len, + const u8 key[CHACHA20POLY1305_KEY_SIZE]) { if (nonce_len == 8) chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, @@ -8871,10 +8871,10 @@ chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len, chacha20poly1305_encrypt_bignonce(dst, src, src_len, ad, ad_len, nonce, key); else - BUG(); + KUNIT_FAIL(test, "bad nonce_len: %zu", nonce_len); } -static bool __init +static bool decryption_success(bool func_ret, bool expect_failure, int memcmp_result) { if (expect_failure) @@ -8882,25 +8882,23 @@ decryption_success(bool func_ret, bool expect_failure, int memcmp_result) return func_ret && !memcmp_result; } -bool __init chacha20poly1305_selftest(void) +static void test_chacha20poly1305(struct kunit *test) { enum { MAXIMUM_TEST_BUFFER_LEN = 1UL << 12 }; size_t i, j, k, total_len; u8 *computed_output = NULL, *input = NULL; - bool success = true, ret; + bool ret; struct scatterlist sg_src[3]; - computed_output = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); - input = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); - if (!computed_output || !input) { - pr_err("chacha20poly1305 self-test malloc: FAIL\n"); - success = false; - goto out; - } + computed_output = kunit_kmalloc(test, MAXIMUM_TEST_BUFFER_LEN, + GFP_KERNEL); + input = kunit_kmalloc(test, MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, computed_output); + KUNIT_ASSERT_NOT_NULL(test, input); for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) { memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); - chacha20poly1305_selftest_encrypt(computed_output, + chacha20poly1305_test_encrypt(test, computed_output, chacha20poly1305_enc_vectors[i].input, chacha20poly1305_enc_vectors[i].ilen, chacha20poly1305_enc_vectors[i].assoc, @@ -8908,14 +8906,13 @@ bool __init chacha20poly1305_selftest(void) chacha20poly1305_enc_vectors[i].nonce, chacha20poly1305_enc_vectors[i].nlen, chacha20poly1305_enc_vectors[i].key); - if (memcmp(computed_output, - chacha20poly1305_enc_vectors[i].output, - chacha20poly1305_enc_vectors[i].ilen + - POLY1305_DIGEST_SIZE)) { - pr_err("chacha20poly1305 encryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + KUNIT_EXPECT_TRUE_MSG( + test, + memcmp(computed_output, + chacha20poly1305_enc_vectors[i].output, + chacha20poly1305_enc_vectors[i].ilen + + POLY1305_DIGEST_SIZE) == 0, + "chacha20poly1305 encryption test %zu: FAIL", i + 1); } for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) { @@ -8931,14 +8928,13 @@ bool __init chacha20poly1305_selftest(void) chacha20poly1305_enc_vectors[i].alen, get_unaligned_le64(chacha20poly1305_enc_vectors[i].nonce), chacha20poly1305_enc_vectors[i].key); - if (!ret || memcmp(computed_output, - chacha20poly1305_enc_vectors[i].output, - chacha20poly1305_enc_vectors[i].ilen + - POLY1305_DIGEST_SIZE)) { - pr_err("chacha20poly1305 sg encryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + KUNIT_EXPECT_TRUE_MSG( + test, + ret && memcmp(computed_output, + chacha20poly1305_enc_vectors[i].output, + chacha20poly1305_enc_vectors[i].ilen + + POLY1305_DIGEST_SIZE) == 0, + "chacha20poly1305 sg encryption test %zu: FAIL", i + 1); } for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) { @@ -8950,16 +8946,15 @@ bool __init chacha20poly1305_selftest(void) chacha20poly1305_dec_vectors[i].alen, get_unaligned_le64(chacha20poly1305_dec_vectors[i].nonce), chacha20poly1305_dec_vectors[i].key); - if (!decryption_success(ret, - chacha20poly1305_dec_vectors[i].failure, + KUNIT_EXPECT_TRUE_MSG( + test, + decryption_success( + ret, chacha20poly1305_dec_vectors[i].failure, memcmp(computed_output, chacha20poly1305_dec_vectors[i].output, chacha20poly1305_dec_vectors[i].ilen - - POLY1305_DIGEST_SIZE))) { - pr_err("chacha20poly1305 decryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + POLY1305_DIGEST_SIZE)), + "chacha20poly1305 decryption test %zu: FAIL", i + 1); } for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) { @@ -8973,15 +8968,15 @@ bool __init chacha20poly1305_selftest(void) chacha20poly1305_dec_vectors[i].alen, get_unaligned_le64(chacha20poly1305_dec_vectors[i].nonce), chacha20poly1305_dec_vectors[i].key); - if (!decryption_success(ret, - chacha20poly1305_dec_vectors[i].failure, - memcmp(computed_output, chacha20poly1305_dec_vectors[i].output, - chacha20poly1305_dec_vectors[i].ilen - - POLY1305_DIGEST_SIZE))) { - pr_err("chacha20poly1305 sg decryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + KUNIT_EXPECT_TRUE_MSG( + test, + decryption_success( + ret, chacha20poly1305_dec_vectors[i].failure, + memcmp(computed_output, + chacha20poly1305_dec_vectors[i].output, + chacha20poly1305_dec_vectors[i].ilen - + POLY1305_DIGEST_SIZE)), + "chacha20poly1305 sg decryption test %zu: FAIL", i + 1); } for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_enc_vectors); ++i) { @@ -8993,14 +8988,13 @@ bool __init chacha20poly1305_selftest(void) xchacha20poly1305_enc_vectors[i].alen, xchacha20poly1305_enc_vectors[i].nonce, xchacha20poly1305_enc_vectors[i].key); - if (memcmp(computed_output, - xchacha20poly1305_enc_vectors[i].output, - xchacha20poly1305_enc_vectors[i].ilen + - POLY1305_DIGEST_SIZE)) { - pr_err("xchacha20poly1305 encryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + KUNIT_EXPECT_TRUE_MSG( + test, + memcmp(computed_output, + xchacha20poly1305_enc_vectors[i].output, + xchacha20poly1305_enc_vectors[i].ilen + + POLY1305_DIGEST_SIZE) == 0, + "xchacha20poly1305 encryption test %zu: FAIL", i + 1); } for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_dec_vectors); ++i) { @@ -9012,16 +9006,15 @@ bool __init chacha20poly1305_selftest(void) xchacha20poly1305_dec_vectors[i].alen, xchacha20poly1305_dec_vectors[i].nonce, xchacha20poly1305_dec_vectors[i].key); - if (!decryption_success(ret, - xchacha20poly1305_dec_vectors[i].failure, + KUNIT_EXPECT_TRUE_MSG( + test, + decryption_success( + ret, xchacha20poly1305_dec_vectors[i].failure, memcmp(computed_output, xchacha20poly1305_dec_vectors[i].output, xchacha20poly1305_dec_vectors[i].ilen - - POLY1305_DIGEST_SIZE))) { - pr_err("xchacha20poly1305 decryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + POLY1305_DIGEST_SIZE)), + "xchacha20poly1305 decryption test %zu: FAIL", i + 1); } for (total_len = POLY1305_DIGEST_SIZE; IS_ENABLED(DEBUG_CHACHA20POLY1305_SLOW_CHUNK_TEST) @@ -9067,16 +9060,26 @@ bool __init chacha20poly1305_selftest(void) continue; chunkfail: - pr_err("chacha20poly1305 chunked self-test %zu/%zu/%zu: FAIL\n", - total_len, i, j); - success = false; + KUNIT_FAIL( + test, + "chacha20poly1305 chunked test %zu/%zu/%zu: FAIL\n", + total_len, i, j); } } } - -out: - kfree(computed_output); - kfree(input); - return success; } + +static struct kunit_case chacha20poly1305_test_cases[] = { + KUNIT_CASE(test_chacha20poly1305), + {}, +}; + +static struct kunit_suite chacha20poly1305_test_suite = { + .name = "chacha20poly1305", + .test_cases = chacha20poly1305_test_cases, +}; +kunit_test_suite(chacha20poly1305_test_suite); + +MODULE_DESCRIPTION("KUnit tests for ChaCha20Poly1305"); +MODULE_LICENSE("GPL"); diff --git a/lib/crypto/tests/ghash-testvecs.h b/lib/crypto/tests/ghash-testvecs.h new file mode 100644 index 000000000000..759eb4072336 --- /dev/null +++ b/lib/crypto/tests/ghash-testvecs.h @@ -0,0 +1,186 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* This file was generated by: ./scripts/crypto/gen-hash-testvecs.py ghash */ + +static const struct { + size_t data_len; + u8 digest[GHASH_DIGEST_SIZE]; +} hash_testvecs[] = { + { + .data_len = 0, + .digest = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, + }, + { + .data_len = 1, + .digest = { + 0x13, 0x91, 0xa1, 0x11, 0x08, 0xc3, 0x7e, 0xeb, + 0x21, 0x42, 0x4a, 0xd6, 0x45, 0x0f, 0x41, 0xa7, + }, + }, + { + .data_len = 2, + .digest = { + 0xde, 0x00, 0x63, 0x3f, 0x71, 0x0f, 0xc6, 0x29, + 0x53, 0x2e, 0x49, 0xd9, 0xc2, 0xb7, 0x73, 0xce, + }, + }, + { + .data_len = 3, + .digest = { + 0xcf, 0xc7, 0xa8, 0x20, 0x24, 0xe9, 0x7a, 0x6c, + 0x2c, 0x2a, 0x34, 0x70, 0x26, 0xba, 0xd5, 0x9a, + }, + }, + { + .data_len = 16, + .digest = { + 0xaa, 0xe0, 0xdc, 0x7f, 0xcf, 0x8b, 0xe6, 0x0c, + 0x2e, 0x93, 0x89, 0x7d, 0x68, 0x4e, 0xc2, 0x63, + }, + }, + { + .data_len = 32, + .digest = { + 0x4b, 0x8b, 0x93, 0x5c, 0x79, 0xad, 0x85, 0x08, + 0xd3, 0x8a, 0xcd, 0xdd, 0x4c, 0x6e, 0x0e, 0x6f, + }, + }, + { + .data_len = 48, + .digest = { + 0xfa, 0xa0, 0x25, 0xdd, 0x61, 0x9a, 0x52, 0x9a, + 0xea, 0xee, 0xc6, 0x62, 0xb2, 0xba, 0x11, 0x49, + }, + }, + { + .data_len = 49, + .digest = { + 0x23, 0xf1, 0x05, 0xeb, 0x30, 0x40, 0xb9, 0x1d, + 0xe6, 0x35, 0x51, 0x4e, 0x0f, 0xc0, 0x1b, 0x9e, + }, + }, + { + .data_len = 63, + .digest = { + 0x8d, 0xcf, 0xa0, 0xc8, 0x83, 0x21, 0x06, 0x81, + 0xc6, 0x36, 0xd5, 0x62, 0xbf, 0xa0, 0xcd, 0x9c, + }, + }, + { + .data_len = 64, + .digest = { + 0xe7, 0xca, 0xbe, 0xe7, 0x66, 0xc8, 0x85, 0xad, + 0xbc, 0xaf, 0x58, 0x21, 0xd7, 0x67, 0x82, 0x15, + }, + }, + { + .data_len = 65, + .digest = { + 0x9f, 0x48, 0x10, 0xd9, 0xa2, 0x6b, 0x9d, 0xe0, + 0xb1, 0x87, 0xe1, 0x39, 0xc3, 0xd7, 0xee, 0x09, + }, + }, + { + .data_len = 127, + .digest = { + 0xa4, 0x36, 0xb7, 0x82, 0xd2, 0x67, 0x7e, 0xaf, + 0x5d, 0xfd, 0x67, 0x9c, 0x1d, 0x9f, 0xe4, 0xf7, + }, + }, + { + .data_len = 128, + .digest = { + 0x57, 0xe7, 0x1d, 0x78, 0xf0, 0x8e, 0xc7, 0x0c, + 0x15, 0xee, 0x18, 0xc4, 0xd1, 0x75, 0x90, 0xaa, + }, + }, + { + .data_len = 129, + .digest = { + 0x9b, 0xad, 0x81, 0xa9, 0x22, 0xb2, 0x19, 0x53, + 0x60, 0x30, 0xe7, 0xa0, 0x4f, 0xd6, 0x72, 0x42, + }, + }, + { + .data_len = 256, + .digest = { + 0xf7, 0x33, 0x42, 0xbf, 0x58, 0xde, 0x88, 0x0f, + 0x8d, 0x3d, 0xa6, 0x11, 0x14, 0xc3, 0xf1, 0xdc, + }, + }, + { + .data_len = 511, + .digest = { + 0x59, 0xdc, 0xa9, 0xc0, 0x4e, 0xd6, 0x97, 0xb3, + 0x60, 0xaf, 0xa8, 0xa0, 0xea, 0x54, 0x8e, 0xc3, + }, + }, + { + .data_len = 513, + .digest = { + 0xa2, 0x23, 0x37, 0xcc, 0x97, 0xec, 0xea, 0xbe, + 0xd6, 0xc7, 0x13, 0xf7, 0x93, 0x73, 0xc0, 0x64, + }, + }, + { + .data_len = 1000, + .digest = { + 0x46, 0x8b, 0x43, 0x77, 0x9b, 0xc2, 0xfc, 0xa4, + 0x68, 0x6a, 0x6c, 0x07, 0xa4, 0x6f, 0x47, 0x65, + }, + }, + { + .data_len = 3333, + .digest = { + 0x69, 0x7f, 0x19, 0xc3, 0xb9, 0xa4, 0xff, 0x40, + 0xe3, 0x03, 0x71, 0xa3, 0x88, 0x8a, 0xf1, 0xbd, + }, + }, + { + .data_len = 4096, + .digest = { + 0x4d, 0x65, 0xe6, 0x9c, 0xeb, 0x6a, 0x46, 0x8d, + 0xe9, 0x32, 0x96, 0x72, 0xb3, 0x0d, 0x08, 0xa9, + }, + }, + { + .data_len = 4128, + .digest = { + 0xfc, 0xa1, 0x74, 0x46, 0x21, 0x64, 0xa7, 0x64, + 0xbe, 0x47, 0x03, 0x1e, 0x05, 0xf7, 0xd8, 0x37, + }, + }, + { + .data_len = 4160, + .digest = { + 0x70, 0x5b, 0xe9, 0x17, 0xab, 0xd5, 0xa2, 0xee, + 0xcb, 0x39, 0xa4, 0x81, 0x2f, 0x41, 0x70, 0xae, + }, + }, + { + .data_len = 4224, + .digest = { + 0x07, 0xbd, 0xb6, 0x52, 0xe2, 0x75, 0x2c, 0x33, + 0x6d, 0x1b, 0x63, 0x56, 0x58, 0xda, 0x98, 0x55, + }, + }, + { + .data_len = 16384, + .digest = { + 0x9c, 0xb5, 0xf4, 0x14, 0xe8, 0xa8, 0x4a, 0xde, + 0xee, 0x7b, 0xbb, 0xd6, 0x21, 0x6d, 0x6a, 0x69, + }, + }, +}; + +static const u8 hash_testvec_consolidated[GHASH_DIGEST_SIZE] = { + 0x08, 0xef, 0xf5, 0x27, 0xb1, 0xca, 0xd4, 0x1d, + 0xad, 0x38, 0x69, 0x88, 0x6b, 0x16, 0xdf, 0xa8, +}; + +static const u8 ghash_allones_hashofhashes[GHASH_DIGEST_SIZE] = { + 0xef, 0x85, 0x58, 0xf8, 0x54, 0x9c, 0x5e, 0x54, + 0xd9, 0xbe, 0x04, 0x1f, 0xff, 0xff, 0xff, 0xff, +}; diff --git a/lib/crypto/tests/ghash_kunit.c b/lib/crypto/tests/ghash_kunit.c new file mode 100644 index 000000000000..68b3837a3607 --- /dev/null +++ b/lib/crypto/tests/ghash_kunit.c @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2026 Google LLC + */ +#include <crypto/gf128hash.h> +#include "ghash-testvecs.h" + +/* + * A fixed key used when presenting GHASH as an unkeyed hash function in order + * to reuse hash-test-template.h. At the beginning of the test suite, this is + * initialized to a key prepared from bytes generated from a fixed seed. + */ +static struct ghash_key test_key; + +static void ghash_init_withtestkey(struct ghash_ctx *ctx) +{ + ghash_init(ctx, &test_key); +} + +static void ghash_withtestkey(const u8 *data, size_t len, + u8 out[GHASH_BLOCK_SIZE]) +{ + ghash(&test_key, data, len, out); +} + +/* Generate the HASH_KUNIT_CASES using hash-test-template.h. */ +#define HASH ghash_withtestkey +#define HASH_CTX ghash_ctx +#define HASH_SIZE GHASH_BLOCK_SIZE +#define HASH_INIT ghash_init_withtestkey +#define HASH_UPDATE ghash_update +#define HASH_FINAL ghash_final +#include "hash-test-template.h" + +/* + * Test a key and messages containing all one bits. This is useful to detect + * overflow bugs in implementations that emulate carryless multiplication using + * a series of standard multiplications with the bits spread out. + */ +static void test_ghash_allones_key_and_message(struct kunit *test) +{ + struct ghash_key key; + struct ghash_ctx hashofhashes_ctx; + u8 hash[GHASH_BLOCK_SIZE]; + + static_assert(TEST_BUF_LEN >= 4096); + memset(test_buf, 0xff, 4096); + + ghash_preparekey(&key, test_buf); + ghash_init(&hashofhashes_ctx, &key); + for (size_t len = 0; len <= 4096; len += 16) { + ghash(&key, test_buf, len, hash); + ghash_update(&hashofhashes_ctx, hash, sizeof(hash)); + } + ghash_final(&hashofhashes_ctx, hash); + KUNIT_ASSERT_MEMEQ(test, hash, ghash_allones_hashofhashes, + sizeof(hash)); +} + +#define MAX_LEN_FOR_KEY_CHECK 1024 + +/* + * Given two prepared keys which should be identical (but may differ in + * alignment and/or whether they are followed by a guard page or not), verify + * that they produce consistent results on various data lengths. + */ +static void check_key_consistency(struct kunit *test, + const struct ghash_key *key1, + const struct ghash_key *key2) +{ + u8 *data = test_buf; + u8 hash1[GHASH_BLOCK_SIZE]; + u8 hash2[GHASH_BLOCK_SIZE]; + + rand_bytes(data, MAX_LEN_FOR_KEY_CHECK); + KUNIT_ASSERT_MEMEQ(test, key1, key2, sizeof(*key1)); + + for (int i = 0; i < 100; i++) { + size_t len = rand_length(MAX_LEN_FOR_KEY_CHECK); + + ghash(key1, data, len, hash1); + ghash(key2, data, len, hash2); + KUNIT_ASSERT_MEMEQ(test, hash1, hash2, sizeof(hash1)); + } +} + +/* Test that no buffer overreads occur on either raw_key or ghash_key. */ +static void test_ghash_with_guarded_key(struct kunit *test) +{ + u8 raw_key[GHASH_BLOCK_SIZE]; + u8 *guarded_raw_key = &test_buf[TEST_BUF_LEN - sizeof(raw_key)]; + struct ghash_key key1, key2; + struct ghash_key *guarded_key = + (struct ghash_key *)&test_buf[TEST_BUF_LEN - sizeof(key1)]; + + /* Prepare with regular buffers. */ + rand_bytes(raw_key, sizeof(raw_key)); + ghash_preparekey(&key1, raw_key); + + /* Prepare with guarded raw_key, then check that it works. */ + memcpy(guarded_raw_key, raw_key, sizeof(raw_key)); + ghash_preparekey(&key2, guarded_raw_key); + check_key_consistency(test, &key1, &key2); + + /* Prepare guarded ghash_key, then check that it works. */ + ghash_preparekey(guarded_key, raw_key); + check_key_consistency(test, &key1, guarded_key); +} + +/* + * Test that ghash_key only needs to be aligned to + * __alignof__(struct ghash_key), i.e. 8 bytes. The assembly code may prefer + * 16-byte or higher alignment, but it mustn't require it. + */ +static void test_ghash_with_minimally_aligned_key(struct kunit *test) +{ + u8 raw_key[GHASH_BLOCK_SIZE]; + struct ghash_key key; + struct ghash_key *minaligned_key = + (struct ghash_key *)&test_buf[MAX_LEN_FOR_KEY_CHECK + + __alignof__(struct ghash_key)]; + + KUNIT_ASSERT_TRUE(test, IS_ALIGNED((uintptr_t)minaligned_key, + __alignof__(struct ghash_key))); + KUNIT_ASSERT_TRUE(test, !IS_ALIGNED((uintptr_t)minaligned_key, + 2 * __alignof__(struct ghash_key))); + + rand_bytes(raw_key, sizeof(raw_key)); + ghash_preparekey(&key, raw_key); + ghash_preparekey(minaligned_key, raw_key); + check_key_consistency(test, &key, minaligned_key); +} + +struct ghash_irq_test_state { + struct ghash_key expected_key; + u8 raw_key[GHASH_BLOCK_SIZE]; +}; + +static bool ghash_irq_test_func(void *state_) +{ + struct ghash_irq_test_state *state = state_; + struct ghash_key key; + + ghash_preparekey(&key, state->raw_key); + return memcmp(&key, &state->expected_key, sizeof(key)) == 0; +} + +/* + * Test that ghash_preparekey() produces the same output regardless of whether + * FPU or vector registers are usable when it is called. + */ +static void test_ghash_preparekey_in_irqs(struct kunit *test) +{ + struct ghash_irq_test_state state; + + rand_bytes(state.raw_key, sizeof(state.raw_key)); + ghash_preparekey(&state.expected_key, state.raw_key); + kunit_run_irq_test(test, ghash_irq_test_func, 200000, &state); +} + +static int ghash_suite_init(struct kunit_suite *suite) +{ + u8 raw_key[GHASH_BLOCK_SIZE]; + + rand_bytes_seeded_from_len(raw_key, sizeof(raw_key)); + ghash_preparekey(&test_key, raw_key); + return hash_suite_init(suite); +} + +static void ghash_suite_exit(struct kunit_suite *suite) +{ + hash_suite_exit(suite); +} + +static struct kunit_case ghash_test_cases[] = { + HASH_KUNIT_CASES, + KUNIT_CASE(test_ghash_allones_key_and_message), + KUNIT_CASE(test_ghash_with_guarded_key), + KUNIT_CASE(test_ghash_with_minimally_aligned_key), + KUNIT_CASE(test_ghash_preparekey_in_irqs), + KUNIT_CASE(benchmark_hash), + {}, +}; + +static struct kunit_suite ghash_test_suite = { + .name = "ghash", + .test_cases = ghash_test_cases, + .suite_init = ghash_suite_init, + .suite_exit = ghash_suite_exit, +}; +kunit_test_suite(ghash_test_suite); + +MODULE_DESCRIPTION("KUnit tests and benchmark for GHASH"); +MODULE_LICENSE("GPL"); diff --git a/lib/crypto/tests/polyval_kunit.c b/lib/crypto/tests/polyval_kunit.c index f47f41a39a41..d1f53a690ab8 100644 --- a/lib/crypto/tests/polyval_kunit.c +++ b/lib/crypto/tests/polyval_kunit.c @@ -2,7 +2,7 @@ /* * Copyright 2025 Google LLC */ -#include <crypto/polyval.h> +#include <crypto/gf128hash.h> #include "polyval-testvecs.h" /* diff --git a/lib/crypto/tests/sm3-testvecs.h b/lib/crypto/tests/sm3-testvecs.h new file mode 100644 index 000000000000..5e788c29f487 --- /dev/null +++ b/lib/crypto/tests/sm3-testvecs.h @@ -0,0 +1,231 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* This file was generated by: ./scripts/crypto/gen-hash-testvecs.py sm3 */ + +static const struct { + size_t data_len; + u8 digest[SM3_DIGEST_SIZE]; +} hash_testvecs[] = { + { + .data_len = 0, + .digest = { + 0x1a, 0xb2, 0x1d, 0x83, 0x55, 0xcf, 0xa1, 0x7f, + 0x8e, 0x61, 0x19, 0x48, 0x31, 0xe8, 0x1a, 0x8f, + 0x22, 0xbe, 0xc8, 0xc7, 0x28, 0xfe, 0xfb, 0x74, + 0x7e, 0xd0, 0x35, 0xeb, 0x50, 0x82, 0xaa, 0x2b, + }, + }, + { + .data_len = 1, + .digest = { + 0xb6, 0x22, 0x2c, 0x39, 0xdc, 0x14, 0x9a, 0xee, + 0x01, 0x9a, 0xcb, 0x0d, 0xe6, 0xc6, 0x75, 0x6e, + 0x8f, 0x18, 0x7b, 0x0e, 0xe8, 0x98, 0x61, 0x71, + 0x2b, 0xd8, 0x38, 0xa9, 0xee, 0x2c, 0x1e, 0x93, + }, + }, + { + .data_len = 2, + .digest = { + 0x62, 0x0c, 0x66, 0x77, 0x67, 0x28, 0x74, 0x8a, + 0xe3, 0x64, 0xea, 0x44, 0x6a, 0x3f, 0x34, 0x61, + 0x55, 0xc5, 0xaa, 0xb2, 0x6c, 0x67, 0x97, 0x68, + 0x68, 0xae, 0x4d, 0x64, 0xa8, 0xb6, 0x72, 0x3e, + }, + }, + { + .data_len = 3, + .digest = { + 0x71, 0xd4, 0x63, 0xb1, 0xfa, 0x27, 0xc7, 0xae, + 0x65, 0xed, 0x5c, 0x93, 0x70, 0xe0, 0x09, 0x34, + 0x2f, 0x42, 0xe6, 0x71, 0x16, 0x8e, 0x90, 0x90, + 0x9a, 0xdd, 0xa6, 0x44, 0x66, 0x71, 0x18, 0xf9, + }, + }, + { + .data_len = 16, + .digest = { + 0x79, 0x0b, 0x68, 0xb5, 0x41, 0xc1, 0x97, 0xa0, + 0x50, 0xe6, 0x93, 0x70, 0xf6, 0x98, 0x49, 0xea, + 0x92, 0xc9, 0xd0, 0xb1, 0x46, 0xbd, 0x4a, 0x0c, + 0x8e, 0xe8, 0xf3, 0xe4, 0x8f, 0x90, 0x33, 0x3c, + }, + }, + { + .data_len = 32, + .digest = { + 0x32, 0x9f, 0xa3, 0x18, 0x18, 0x45, 0xe0, 0x28, + 0xd3, 0xa4, 0x41, 0x3a, 0x25, 0x62, 0x9c, 0x95, + 0xab, 0xfe, 0x02, 0xe0, 0x37, 0x7d, 0x3c, 0xc4, + 0xce, 0x69, 0x57, 0x5a, 0x07, 0x0e, 0xb9, 0xf5, + }, + }, + { + .data_len = 48, + .digest = { + 0x0c, 0xcf, 0x7c, 0x48, 0x44, 0xa0, 0xb0, 0x8d, + 0xdf, 0xbe, 0x22, 0x14, 0x7e, 0xd4, 0xc3, 0x8d, + 0x6a, 0x23, 0xfc, 0x44, 0x0e, 0x0f, 0xde, 0xa5, + 0x7c, 0x8b, 0xc4, 0x8b, 0xab, 0x8c, 0x87, 0x41, + }, + }, + { + .data_len = 49, + .digest = { + 0xb3, 0x76, 0x8b, 0x19, 0xf9, 0x10, 0xa9, 0x56, + 0x4f, 0xce, 0x27, 0xaa, 0x65, 0x96, 0xe5, 0xdb, + 0x90, 0x9b, 0x92, 0xcd, 0x32, 0x0d, 0x16, 0xac, + 0xf8, 0xd0, 0x66, 0x62, 0x10, 0xf0, 0x44, 0xdf, + }, + }, + { + .data_len = 63, + .digest = { + 0x07, 0xc9, 0x45, 0x65, 0x9f, 0x68, 0x75, 0xc3, + 0x74, 0xb2, 0x3b, 0x0c, 0x97, 0x05, 0xd3, 0x13, + 0xc0, 0xb6, 0x21, 0xed, 0xf6, 0x10, 0x7a, 0xed, + 0xec, 0xd8, 0x10, 0x29, 0xbf, 0x7a, 0x78, 0x37, + }, + }, + { + .data_len = 64, + .digest = { + 0x3e, 0x69, 0x18, 0x45, 0xd8, 0x25, 0x6f, 0x44, + 0xc0, 0x02, 0xe5, 0xcf, 0xcd, 0x94, 0x42, 0xa9, + 0xd5, 0x12, 0x62, 0x10, 0x15, 0xa0, 0xf9, 0x16, + 0x19, 0x2d, 0x8d, 0x63, 0x31, 0xf2, 0x2f, 0x36, + }, + }, + { + .data_len = 65, + .digest = { + 0x6b, 0x3e, 0xc0, 0x20, 0xb7, 0x74, 0x30, 0xa0, + 0xc6, 0x5c, 0xee, 0xbe, 0xdc, 0xe6, 0xe5, 0x4f, + 0x3c, 0x61, 0x8d, 0x91, 0xac, 0x31, 0x4b, 0x2a, + 0xdf, 0x1c, 0xef, 0x24, 0xdc, 0x0a, 0x10, 0xe8, + }, + }, + { + .data_len = 127, + .digest = { + 0xab, 0xd6, 0xa1, 0xbf, 0x39, 0x43, 0x75, 0xda, + 0xbf, 0xc7, 0x22, 0xcc, 0x4e, 0xfc, 0xe4, 0x42, + 0x6d, 0x1b, 0x87, 0x25, 0x64, 0x7f, 0x88, 0xf7, + 0xc3, 0x0a, 0x0a, 0x4c, 0xd6, 0xa7, 0x68, 0xae, + }, + }, + { + .data_len = 128, + .digest = { + 0x1b, 0x70, 0xd4, 0x5f, 0x6c, 0xe4, 0x2d, 0x58, + 0x2d, 0x0f, 0x9a, 0x12, 0x34, 0xbb, 0x5e, 0x38, + 0xd8, 0x1f, 0x6a, 0x46, 0x8a, 0xef, 0xdb, 0x68, + 0x18, 0x62, 0xbb, 0x85, 0xfc, 0xc4, 0x6e, 0x2e, + }, + }, + { + .data_len = 129, + .digest = { + 0x33, 0x62, 0xba, 0xa7, 0x4a, 0xbc, 0xd7, 0x7b, + 0xd4, 0x67, 0x6d, 0x3e, 0xea, 0xe8, 0xb0, 0x64, + 0x0d, 0xf3, 0xae, 0x1d, 0x52, 0x24, 0x11, 0x9f, + 0xda, 0xa9, 0x7f, 0xd5, 0x22, 0x1a, 0xde, 0x8a, + }, + }, + { + .data_len = 256, + .digest = { + 0x70, 0xa8, 0xa6, 0x2b, 0xfb, 0x1f, 0x3b, 0x5a, + 0xcc, 0x71, 0x76, 0x9e, 0x25, 0x4c, 0xfa, 0x8f, + 0x39, 0x4a, 0x21, 0x8a, 0x9d, 0x74, 0x8d, 0x2c, + 0x31, 0xa5, 0xb5, 0xff, 0x30, 0xc1, 0x14, 0xc4, + }, + }, + { + .data_len = 511, + .digest = { + 0x39, 0xd0, 0x8c, 0x5f, 0xfc, 0x36, 0xc2, 0x7c, + 0xdb, 0x8b, 0x2e, 0xdc, 0x9d, 0x1b, 0xd1, 0xba, + 0x9b, 0x52, 0x6b, 0x35, 0x46, 0x46, 0x75, 0x73, + 0xe5, 0x62, 0x96, 0x6e, 0xf3, 0xba, 0xd9, 0x19, + }, + }, + { + .data_len = 513, + .digest = { + 0x76, 0xa0, 0x3d, 0xa2, 0x5f, 0xd4, 0xa6, 0xbe, + 0x6b, 0xdb, 0xed, 0x14, 0x9e, 0xa8, 0x15, 0x77, + 0xa9, 0x38, 0x30, 0x6b, 0x68, 0xfa, 0xb6, 0xe2, + 0x93, 0x19, 0x24, 0x72, 0x67, 0x20, 0x72, 0xc3, + }, + }, + { + .data_len = 1000, + .digest = { + 0x16, 0xbc, 0x33, 0x77, 0x0b, 0xcf, 0x93, 0x5e, + 0xec, 0x7d, 0x8d, 0x3c, 0xae, 0xd9, 0x75, 0xdf, + 0x46, 0x24, 0x17, 0x7e, 0x03, 0x88, 0xf2, 0x75, + 0xa9, 0x18, 0xa6, 0x1c, 0x7a, 0x74, 0x0d, 0xf3, + }, + }, + { + .data_len = 3333, + .digest = { + 0xdb, 0x54, 0x89, 0xe7, 0x1c, 0x50, 0xf2, 0xbf, + 0xde, 0x3a, 0xbf, 0x5b, 0xee, 0x5a, 0x46, 0x62, + 0x20, 0xb1, 0x80, 0x48, 0xac, 0x56, 0x33, 0xb3, + 0xbb, 0x3f, 0xfa, 0x02, 0xc6, 0x43, 0xb5, 0x8c, + }, + }, + { + .data_len = 4096, + .digest = { + 0xdf, 0x0d, 0xed, 0x3b, 0x8f, 0xea, 0x81, 0xfd, + 0xd6, 0x34, 0xae, 0x74, 0x24, 0x3a, 0x15, 0x38, + 0xe7, 0xcf, 0x45, 0x7e, 0x8f, 0xf5, 0x50, 0x6c, + 0xaa, 0x27, 0x23, 0x92, 0x6d, 0xab, 0x3b, 0xde, + }, + }, + { + .data_len = 4128, + .digest = { + 0x6a, 0xbd, 0x56, 0x5a, 0xf1, 0xc6, 0x40, 0x4d, + 0xf3, 0x50, 0x77, 0x87, 0x86, 0x63, 0x1b, 0x4d, + 0x21, 0x99, 0x96, 0xad, 0x24, 0x62, 0xce, 0xc0, + 0x3e, 0xb7, 0x35, 0x52, 0x56, 0x0e, 0x55, 0x85, + }, + }, + { + .data_len = 4160, + .digest = { + 0x5b, 0xc1, 0x1f, 0x25, 0x43, 0xa3, 0x1c, 0xa0, + 0x8c, 0xfc, 0x41, 0xf1, 0xcc, 0xb3, 0x95, 0x95, + 0xe0, 0xb9, 0xd3, 0x29, 0xf4, 0x08, 0x31, 0x47, + 0x6d, 0x09, 0xa8, 0x2e, 0xa5, 0xf4, 0xf1, 0x8d, + }, + }, + { + .data_len = 4224, + .digest = { + 0xec, 0x56, 0x1e, 0xa6, 0x1f, 0xb2, 0x87, 0xb2, + 0x7e, 0x15, 0xd6, 0x30, 0x08, 0x74, 0xb0, 0x48, + 0x90, 0x2a, 0xbe, 0x2f, 0x80, 0x3a, 0x88, 0xcc, + 0xd7, 0xc5, 0x87, 0x8c, 0x04, 0xef, 0x78, 0x71, + }, + }, + { + .data_len = 16384, + .digest = { + 0xe7, 0xb8, 0x84, 0x20, 0xff, 0xd5, 0x53, 0xe6, + 0x14, 0x31, 0x12, 0x75, 0xb7, 0x9a, 0x4f, 0x63, + 0x63, 0x00, 0xfe, 0x2c, 0x54, 0xee, 0x06, 0xfc, + 0x12, 0x16, 0xe5, 0xdc, 0xa4, 0x40, 0x62, 0x12, + }, + }, +}; + +static const u8 hash_testvec_consolidated[SM3_DIGEST_SIZE] = { + 0xe6, 0x58, 0xd4, 0x8e, 0x74, 0x92, 0xdf, 0xfe, + 0x58, 0x05, 0xe5, 0x29, 0x83, 0xfb, 0xb7, 0x51, + 0x7e, 0x66, 0x0c, 0x49, 0x3e, 0x11, 0x7e, 0x9b, + 0xb1, 0x83, 0x3a, 0xa6, 0xb0, 0x3c, 0xf5, 0xe0, +}; diff --git a/lib/crypto/tests/sm3_kunit.c b/lib/crypto/tests/sm3_kunit.c new file mode 100644 index 000000000000..dc8136acdff6 --- /dev/null +++ b/lib/crypto/tests/sm3_kunit.c @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2026 Google LLC + */ +#include <crypto/sm3.h> +#include "sm3-testvecs.h" + +#define HASH sm3 +#define HASH_CTX sm3_ctx +#define HASH_SIZE SM3_DIGEST_SIZE +#define HASH_INIT sm3_init +#define HASH_UPDATE sm3_update +#define HASH_FINAL sm3_final +#include "hash-test-template.h" + +static struct kunit_case sm3_test_cases[] = { + HASH_KUNIT_CASES, + KUNIT_CASE(benchmark_hash), + {}, +}; + +static struct kunit_suite sm3_test_suite = { + .name = "sm3", + .test_cases = sm3_test_cases, + .suite_init = hash_suite_init, + .suite_exit = hash_suite_exit, +}; +kunit_test_suite(sm3_test_suite); + +MODULE_DESCRIPTION("KUnit tests and benchmark for SM3"); +MODULE_LICENSE("GPL"); diff --git a/lib/crypto/x86/polyval.h b/lib/crypto/x86/gf128hash.h index ef8797521420..6b79b06caab0 100644 --- a/lib/crypto/x86/polyval.h +++ b/lib/crypto/x86/gf128hash.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * POLYVAL library functions, x86_64 optimized + * GHASH and POLYVAL, x86_64 optimized * * Copyright 2025 Google LLC */ @@ -9,14 +9,22 @@ #define NUM_H_POWERS 8 +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul_avx); +asmlinkage void polyval_mul_pclmul(struct polyval_elem *a, + const struct polyval_elem *b); asmlinkage void polyval_mul_pclmul_avx(struct polyval_elem *a, const struct polyval_elem *b); + +asmlinkage void ghash_blocks_pclmul(struct polyval_elem *acc, + const struct polyval_elem *key, + const u8 *data, size_t nblocks); asmlinkage void polyval_blocks_pclmul_avx(struct polyval_elem *acc, const struct polyval_key *key, const u8 *data, size_t nblocks); +#define polyval_preparekey_arch polyval_preparekey_arch static void polyval_preparekey_arch(struct polyval_key *key, const u8 raw_key[POLYVAL_BLOCK_SIZE]) { @@ -40,18 +48,58 @@ static void polyval_preparekey_arch(struct polyval_key *key, } } -static void polyval_mul_arch(struct polyval_elem *acc, - const struct polyval_key *key) +static void polyval_mul_x86(struct polyval_elem *a, + const struct polyval_elem *b) { - if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) { + if (static_branch_likely(&have_pclmul) && irq_fpu_usable()) { kernel_fpu_begin(); - polyval_mul_pclmul_avx(acc, &key->h_powers[NUM_H_POWERS - 1]); + if (static_branch_likely(&have_pclmul_avx)) + polyval_mul_pclmul_avx(a, b); + else + polyval_mul_pclmul(a, b); kernel_fpu_end(); } else { - polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]); + polyval_mul_generic(a, b); + } +} + +#define ghash_mul_arch ghash_mul_arch +static void ghash_mul_arch(struct polyval_elem *acc, + const struct ghash_key *key) +{ + polyval_mul_x86(acc, &key->h); +} + +#define polyval_mul_arch polyval_mul_arch +static void polyval_mul_arch(struct polyval_elem *acc, + const struct polyval_key *key) +{ + polyval_mul_x86(acc, &key->h_powers[NUM_H_POWERS - 1]); +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_pclmul) && irq_fpu_usable()) { + do { + /* Allow rescheduling every 4 KiB. */ + size_t n = min_t(size_t, nblocks, + 4096 / GHASH_BLOCK_SIZE); + + kernel_fpu_begin(); + ghash_blocks_pclmul(acc, &key->h, data, n); + kernel_fpu_end(); + data += n * GHASH_BLOCK_SIZE; + nblocks -= n; + } while (nblocks); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); } } +#define polyval_blocks_arch polyval_blocks_arch static void polyval_blocks_arch(struct polyval_elem *acc, const struct polyval_key *key, const u8 *data, size_t nblocks) @@ -74,10 +122,12 @@ static void polyval_blocks_arch(struct polyval_elem *acc, } } -#define polyval_mod_init_arch polyval_mod_init_arch -static void polyval_mod_init_arch(void) +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) { - if (boot_cpu_has(X86_FEATURE_PCLMULQDQ) && - boot_cpu_has(X86_FEATURE_AVX)) - static_branch_enable(&have_pclmul_avx); + if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) { + static_branch_enable(&have_pclmul); + if (boot_cpu_has(X86_FEATURE_AVX)) + static_branch_enable(&have_pclmul_avx); + } } diff --git a/lib/crypto/x86/ghash-pclmul.S b/lib/crypto/x86/ghash-pclmul.S new file mode 100644 index 000000000000..6ffb5aea6063 --- /dev/null +++ b/lib/crypto/x86/ghash-pclmul.S @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Accelerated GHASH implementation with Intel PCLMULQDQ-NI + * instructions. This file contains accelerated part of ghash + * implementation. More information about PCLMULQDQ can be found at: + * + * https://www.intel.com/content/dam/develop/external/us/en/documents/clmul-wp-rev-2-02-2014-04-20.pdf + * + * Copyright (c) 2009 Intel Corp. + * Author: Huang Ying <ying.huang@intel.com> + * Vinodh Gopal + * Erdinc Ozturk + * Deniz Karakoyunlu + */ + +#include <linux/linkage.h> +#include <asm/frame.h> + +.section .rodata.cst16.bswap_mask, "aM", @progbits, 16 +.align 16 +.Lbswap_mask: + .octa 0x000102030405060708090a0b0c0d0e0f + +#define ACC %xmm0 +#define KEY %xmm1 +#define T1 %xmm2 +#define T2 %xmm3 +#define T3 %xmm4 +#define BSWAP %xmm5 +#define IN1 %xmm6 + +.text + +/* + * __clmul_gf128mul_ble: internal ABI + * input: + * ACC: operand1 + * KEY: operand2, hash_key << 1 mod poly + * output: + * ACC: operand1 * operand2 mod poly + * changed: + * T1 + * T2 + * T3 + */ +SYM_FUNC_START_LOCAL(__clmul_gf128mul_ble) + movaps ACC, T1 + pshufd $0b01001110, ACC, T2 + pshufd $0b01001110, KEY, T3 + pxor ACC, T2 + pxor KEY, T3 + + pclmulqdq $0x00, KEY, ACC # ACC = a0 * b0 + pclmulqdq $0x11, KEY, T1 # T1 = a1 * b1 + pclmulqdq $0x00, T3, T2 # T2 = (a1 + a0) * (b1 + b0) + pxor ACC, T2 + pxor T1, T2 # T2 = a0 * b1 + a1 * b0 + + movaps T2, T3 + pslldq $8, T3 + psrldq $8, T2 + pxor T3, ACC + pxor T2, T1 # <T1:ACC> is result of + # carry-less multiplication + + # first phase of the reduction + movaps ACC, T3 + psllq $1, T3 + pxor ACC, T3 + psllq $5, T3 + pxor ACC, T3 + psllq $57, T3 + movaps T3, T2 + pslldq $8, T2 + psrldq $8, T3 + pxor T2, ACC + pxor T3, T1 + + # second phase of the reduction + movaps ACC, T2 + psrlq $5, T2 + pxor ACC, T2 + psrlq $1, T2 + pxor ACC, T2 + psrlq $1, T2 + pxor T2, T1 + pxor T1, ACC + RET +SYM_FUNC_END(__clmul_gf128mul_ble) + +/* + * void polyval_mul_pclmul(struct polyval_elem *a, + * const struct polyval_elem *b) + */ +SYM_FUNC_START(polyval_mul_pclmul) + FRAME_BEGIN + movups (%rdi), ACC + movups (%rsi), KEY + call __clmul_gf128mul_ble + movups ACC, (%rdi) + FRAME_END + RET +SYM_FUNC_END(polyval_mul_pclmul) + +/* + * void ghash_blocks_pclmul(struct polyval_elem *acc, + * const struct polyval_elem *key, + * const u8 *data, size_t nblocks) + */ +SYM_FUNC_START(ghash_blocks_pclmul) + FRAME_BEGIN + movaps .Lbswap_mask(%rip), BSWAP + movups (%rdi), ACC + movups (%rsi), KEY +.align 4 +.Lnext_block: + movups (%rdx), IN1 + pshufb BSWAP, IN1 + pxor IN1, ACC + call __clmul_gf128mul_ble + add $16, %rdx + dec %rcx + jnz .Lnext_block + movups ACC, (%rdi) + FRAME_END + RET +SYM_FUNC_END(ghash_blocks_pclmul) diff --git a/lib/crypto/x86/sha256.h b/lib/crypto/x86/sha256.h index 38e33b22a092..0ee69d8e39fe 100644 --- a/lib/crypto/x86/sha256.h +++ b/lib/crypto/x86/sha256.h @@ -31,6 +31,27 @@ DEFINE_X86_SHA256_FN(sha256_blocks_avx, sha256_transform_avx); DEFINE_X86_SHA256_FN(sha256_blocks_avx2, sha256_transform_rorx); DEFINE_X86_SHA256_FN(sha256_blocks_ni, sha256_ni_transform); +#define PHE_ALIGNMENT 16 +static void sha256_blocks_phe(struct sha256_block_state *state, + const u8 *data, size_t nblocks) +{ + /* + * On Zhaoxin processors, XSHA256 requires the %rdi register + * in 64-bit mode (or %edi in 32-bit mode) to point to + * a 32-byte, 16-byte-aligned buffer. + */ + u8 buf[32 + PHE_ALIGNMENT - 1]; + u8 *dst = PTR_ALIGN(&buf[0], PHE_ALIGNMENT); + size_t padding = -1; + + memcpy(dst, state, SHA256_DIGEST_SIZE); + asm volatile(".byte 0xf3,0x0f,0xa6,0xd0" /* REP XSHA256 */ + : "+a"(padding), "+c"(nblocks), "+S"(data) + : "D"(dst) + : "memory"); + memcpy(state, dst, SHA256_DIGEST_SIZE); +} + static void sha256_blocks(struct sha256_block_state *state, const u8 *data, size_t nblocks) { @@ -79,6 +100,10 @@ static void sha256_mod_init_arch(void) if (boot_cpu_has(X86_FEATURE_SHA_NI)) { static_call_update(sha256_blocks_x86, sha256_blocks_ni); static_branch_enable(&have_sha_ni); + } else if (IS_ENABLED(CONFIG_CPU_SUP_ZHAOXIN) && + boot_cpu_has(X86_FEATURE_PHE_EN) && + boot_cpu_data.x86 >= 0x07) { + static_call_update(sha256_blocks_x86, sha256_blocks_phe); } else if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL) && boot_cpu_has(X86_FEATURE_AVX)) { diff --git a/lib/crypto/x86/sm3-avx-asm_64.S b/lib/crypto/x86/sm3-avx-asm_64.S new file mode 100644 index 000000000000..a1925b136010 --- /dev/null +++ b/lib/crypto/x86/sm3-avx-asm_64.S @@ -0,0 +1,516 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SM3 AVX accelerated transform. + * specified in: https://datatracker.ietf.org/doc/html/draft-sca-cfrg-sm3-02 + * + * Copyright (C) 2021 Jussi Kivilinna <jussi.kivilinna@iki.fi> + * Copyright (C) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> + */ + +/* Based on SM3 AES/BMI2 accelerated work by libgcrypt at: + * https://gnupg.org/software/libgcrypt/index.html + */ + +#include <linux/linkage.h> +#include <asm/frame.h> + +/* State structure */ + +#define state_h0 0 +#define state_h1 4 +#define state_h2 8 +#define state_h3 12 +#define state_h4 16 +#define state_h5 20 +#define state_h6 24 +#define state_h7 28 + +/* Constants */ + +/* Round constant macros */ + +#define K0 2043430169 /* 0x79cc4519 */ +#define K1 -208106958 /* 0xf3988a32 */ +#define K2 -416213915 /* 0xe7311465 */ +#define K3 -832427829 /* 0xce6228cb */ +#define K4 -1664855657 /* 0x9cc45197 */ +#define K5 965255983 /* 0x3988a32f */ +#define K6 1930511966 /* 0x7311465e */ +#define K7 -433943364 /* 0xe6228cbc */ +#define K8 -867886727 /* 0xcc451979 */ +#define K9 -1735773453 /* 0x988a32f3 */ +#define K10 823420391 /* 0x311465e7 */ +#define K11 1646840782 /* 0x6228cbce */ +#define K12 -1001285732 /* 0xc451979c */ +#define K13 -2002571463 /* 0x88a32f39 */ +#define K14 289824371 /* 0x11465e73 */ +#define K15 579648742 /* 0x228cbce6 */ +#define K16 -1651869049 /* 0x9d8a7a87 */ +#define K17 991229199 /* 0x3b14f50f */ +#define K18 1982458398 /* 0x7629ea1e */ +#define K19 -330050500 /* 0xec53d43c */ +#define K20 -660100999 /* 0xd8a7a879 */ +#define K21 -1320201997 /* 0xb14f50f3 */ +#define K22 1654563303 /* 0x629ea1e7 */ +#define K23 -985840690 /* 0xc53d43ce */ +#define K24 -1971681379 /* 0x8a7a879d */ +#define K25 351604539 /* 0x14f50f3b */ +#define K26 703209078 /* 0x29ea1e76 */ +#define K27 1406418156 /* 0x53d43cec */ +#define K28 -1482130984 /* 0xa7a879d8 */ +#define K29 1330705329 /* 0x4f50f3b1 */ +#define K30 -1633556638 /* 0x9ea1e762 */ +#define K31 1027854021 /* 0x3d43cec5 */ +#define K32 2055708042 /* 0x7a879d8a */ +#define K33 -183551212 /* 0xf50f3b14 */ +#define K34 -367102423 /* 0xea1e7629 */ +#define K35 -734204845 /* 0xd43cec53 */ +#define K36 -1468409689 /* 0xa879d8a7 */ +#define K37 1358147919 /* 0x50f3b14f */ +#define K38 -1578671458 /* 0xa1e7629e */ +#define K39 1137624381 /* 0x43cec53d */ +#define K40 -2019718534 /* 0x879d8a7a */ +#define K41 255530229 /* 0x0f3b14f5 */ +#define K42 511060458 /* 0x1e7629ea */ +#define K43 1022120916 /* 0x3cec53d4 */ +#define K44 2044241832 /* 0x79d8a7a8 */ +#define K45 -206483632 /* 0xf3b14f50 */ +#define K46 -412967263 /* 0xe7629ea1 */ +#define K47 -825934525 /* 0xcec53d43 */ +#define K48 -1651869049 /* 0x9d8a7a87 */ +#define K49 991229199 /* 0x3b14f50f */ +#define K50 1982458398 /* 0x7629ea1e */ +#define K51 -330050500 /* 0xec53d43c */ +#define K52 -660100999 /* 0xd8a7a879 */ +#define K53 -1320201997 /* 0xb14f50f3 */ +#define K54 1654563303 /* 0x629ea1e7 */ +#define K55 -985840690 /* 0xc53d43ce */ +#define K56 -1971681379 /* 0x8a7a879d */ +#define K57 351604539 /* 0x14f50f3b */ +#define K58 703209078 /* 0x29ea1e76 */ +#define K59 1406418156 /* 0x53d43cec */ +#define K60 -1482130984 /* 0xa7a879d8 */ +#define K61 1330705329 /* 0x4f50f3b1 */ +#define K62 -1633556638 /* 0x9ea1e762 */ +#define K63 1027854021 /* 0x3d43cec5 */ + +/* Register macros */ + +#define RSTATE %rdi +#define RDATA %rsi +#define RNBLKS %rdx + +#define t0 %eax +#define t1 %ebx +#define t2 %ecx + +#define a %r8d +#define b %r9d +#define c %r10d +#define d %r11d +#define e %r12d +#define f %r13d +#define g %r14d +#define h %r15d + +#define W0 %xmm0 +#define W1 %xmm1 +#define W2 %xmm2 +#define W3 %xmm3 +#define W4 %xmm4 +#define W5 %xmm5 + +#define XTMP0 %xmm6 +#define XTMP1 %xmm7 +#define XTMP2 %xmm8 +#define XTMP3 %xmm9 +#define XTMP4 %xmm10 +#define XTMP5 %xmm11 +#define XTMP6 %xmm12 + +#define BSWAP_REG %xmm15 + +/* Stack structure */ + +#define STACK_W_SIZE (32 * 2 * 3) +#define STACK_REG_SAVE_SIZE (64) + +#define STACK_W (0) +#define STACK_REG_SAVE (STACK_W + STACK_W_SIZE) +#define STACK_SIZE (STACK_REG_SAVE + STACK_REG_SAVE_SIZE) + +/* Instruction helpers. */ + +#define roll2(v, reg) \ + roll $(v), reg; + +#define roll3mov(v, src, dst) \ + movl src, dst; \ + roll $(v), dst; + +#define roll3(v, src, dst) \ + rorxl $(32-(v)), src, dst; + +#define addl2(a, out) \ + leal (a, out), out; + +/* Round function macros. */ + +#define GG1(x, y, z, o, t) \ + movl x, o; \ + xorl y, o; \ + xorl z, o; + +#define FF1(x, y, z, o, t) GG1(x, y, z, o, t) + +#define GG2(x, y, z, o, t) \ + andnl z, x, o; \ + movl y, t; \ + andl x, t; \ + addl2(t, o); + +#define FF2(x, y, z, o, t) \ + movl y, o; \ + xorl x, o; \ + movl y, t; \ + andl x, t; \ + andl z, o; \ + xorl t, o; + +#define R(i, a, b, c, d, e, f, g, h, round, widx, wtype) \ + /* rol(a, 12) => t0 */ \ + roll3mov(12, a, t0); /* rorxl here would reduce perf by 6% on zen3 */ \ + /* rol (t0 + e + t), 7) => t1 */ \ + leal K##round(t0, e, 1), t1; \ + roll2(7, t1); \ + /* h + w1 => h */ \ + addl wtype##_W1_ADDR(round, widx), h; \ + /* h + t1 => h */ \ + addl2(t1, h); \ + /* t1 ^ t0 => t0 */ \ + xorl t1, t0; \ + /* w1w2 + d => d */ \ + addl wtype##_W1W2_ADDR(round, widx), d; \ + /* FF##i(a,b,c) => t1 */ \ + FF##i(a, b, c, t1, t2); \ + /* d + t1 => d */ \ + addl2(t1, d); \ + /* GG#i(e,f,g) => t2 */ \ + GG##i(e, f, g, t2, t1); \ + /* h + t2 => h */ \ + addl2(t2, h); \ + /* rol (f, 19) => f */ \ + roll2(19, f); \ + /* d + t0 => d */ \ + addl2(t0, d); \ + /* rol (b, 9) => b */ \ + roll2(9, b); \ + /* P0(h) => h */ \ + roll3(9, h, t2); \ + roll3(17, h, t1); \ + xorl t2, h; \ + xorl t1, h; + +#define R1(a, b, c, d, e, f, g, h, round, widx, wtype) \ + R(1, a, b, c, d, e, f, g, h, round, widx, wtype) + +#define R2(a, b, c, d, e, f, g, h, round, widx, wtype) \ + R(2, a, b, c, d, e, f, g, h, round, widx, wtype) + +/* Input expansion macros. */ + +/* Byte-swapped input address. */ +#define IW_W_ADDR(round, widx, offs) \ + (STACK_W + ((round) / 4) * 64 + (offs) + ((widx) * 4))(%rsp) + +/* Expanded input address. */ +#define XW_W_ADDR(round, widx, offs) \ + (STACK_W + ((((round) / 3) - 4) % 2) * 64 + (offs) + ((widx) * 4))(%rsp) + +/* Rounds 1-12, byte-swapped input block addresses. */ +#define IW_W1_ADDR(round, widx) IW_W_ADDR(round, widx, 0) +#define IW_W1W2_ADDR(round, widx) IW_W_ADDR(round, widx, 32) + +/* Rounds 1-12, expanded input block addresses. */ +#define XW_W1_ADDR(round, widx) XW_W_ADDR(round, widx, 0) +#define XW_W1W2_ADDR(round, widx) XW_W_ADDR(round, widx, 32) + +/* Input block loading. */ +#define LOAD_W_XMM_1() \ + vmovdqu 0*16(RDATA), XTMP0; /* XTMP0: w3, w2, w1, w0 */ \ + vmovdqu 1*16(RDATA), XTMP1; /* XTMP1: w7, w6, w5, w4 */ \ + vmovdqu 2*16(RDATA), XTMP2; /* XTMP2: w11, w10, w9, w8 */ \ + vmovdqu 3*16(RDATA), XTMP3; /* XTMP3: w15, w14, w13, w12 */ \ + vpshufb BSWAP_REG, XTMP0, XTMP0; \ + vpshufb BSWAP_REG, XTMP1, XTMP1; \ + vpshufb BSWAP_REG, XTMP2, XTMP2; \ + vpshufb BSWAP_REG, XTMP3, XTMP3; \ + vpxor XTMP0, XTMP1, XTMP4; \ + vpxor XTMP1, XTMP2, XTMP5; \ + vpxor XTMP2, XTMP3, XTMP6; \ + leaq 64(RDATA), RDATA; \ + vmovdqa XTMP0, IW_W1_ADDR(0, 0); \ + vmovdqa XTMP4, IW_W1W2_ADDR(0, 0); \ + vmovdqa XTMP1, IW_W1_ADDR(4, 0); \ + vmovdqa XTMP5, IW_W1W2_ADDR(4, 0); + +#define LOAD_W_XMM_2() \ + vmovdqa XTMP2, IW_W1_ADDR(8, 0); \ + vmovdqa XTMP6, IW_W1W2_ADDR(8, 0); + +#define LOAD_W_XMM_3() \ + vpshufd $0b00000000, XTMP0, W0; /* W0: xx, w0, xx, xx */ \ + vpshufd $0b11111001, XTMP0, W1; /* W1: xx, w3, w2, w1 */ \ + vmovdqa XTMP1, W2; /* W2: xx, w6, w5, w4 */ \ + vpalignr $12, XTMP1, XTMP2, W3; /* W3: xx, w9, w8, w7 */ \ + vpalignr $8, XTMP2, XTMP3, W4; /* W4: xx, w12, w11, w10 */ \ + vpshufd $0b11111001, XTMP3, W5; /* W5: xx, w15, w14, w13 */ + +/* Message scheduling. Note: 3 words per XMM register. */ +#define SCHED_W_0(round, w0, w1, w2, w3, w4, w5) \ + /* Load (w[i - 16]) => XTMP0 */ \ + vpshufd $0b10111111, w0, XTMP0; \ + vpalignr $12, XTMP0, w1, XTMP0; /* XTMP0: xx, w2, w1, w0 */ \ + /* Load (w[i - 13]) => XTMP1 */ \ + vpshufd $0b10111111, w1, XTMP1; \ + vpalignr $12, XTMP1, w2, XTMP1; \ + /* w[i - 9] == w3 */ \ + /* XMM3 ^ XTMP0 => XTMP0 */ \ + vpxor w3, XTMP0, XTMP0; + +#define SCHED_W_1(round, w0, w1, w2, w3, w4, w5) \ + /* w[i - 3] == w5 */ \ + /* rol(XMM5, 15) ^ XTMP0 => XTMP0 */ \ + vpslld $15, w5, XTMP2; \ + vpsrld $(32-15), w5, XTMP3; \ + vpxor XTMP2, XTMP3, XTMP3; \ + vpxor XTMP3, XTMP0, XTMP0; \ + /* rol(XTMP1, 7) => XTMP1 */ \ + vpslld $7, XTMP1, XTMP5; \ + vpsrld $(32-7), XTMP1, XTMP1; \ + vpxor XTMP5, XTMP1, XTMP1; \ + /* XMM4 ^ XTMP1 => XTMP1 */ \ + vpxor w4, XTMP1, XTMP1; \ + /* w[i - 6] == XMM4 */ \ + /* P1(XTMP0) ^ XTMP1 => XMM0 */ \ + vpslld $15, XTMP0, XTMP5; \ + vpsrld $(32-15), XTMP0, XTMP6; \ + vpslld $23, XTMP0, XTMP2; \ + vpsrld $(32-23), XTMP0, XTMP3; \ + vpxor XTMP0, XTMP1, XTMP1; \ + vpxor XTMP6, XTMP5, XTMP5; \ + vpxor XTMP3, XTMP2, XTMP2; \ + vpxor XTMP2, XTMP5, XTMP5; \ + vpxor XTMP5, XTMP1, w0; + +#define SCHED_W_2(round, w0, w1, w2, w3, w4, w5) \ + /* W1 in XMM12 */ \ + vpshufd $0b10111111, w4, XTMP4; \ + vpalignr $12, XTMP4, w5, XTMP4; \ + vmovdqa XTMP4, XW_W1_ADDR((round), 0); \ + /* W1 ^ W2 => XTMP1 */ \ + vpxor w0, XTMP4, XTMP1; \ + vmovdqa XTMP1, XW_W1W2_ADDR((round), 0); + + +.section .rodata.cst16, "aM", @progbits, 16 +.align 16 + +.Lbe32mask: + .long 0x00010203, 0x04050607, 0x08090a0b, 0x0c0d0e0f + +.text + +/* + * Transform nblocks*64 bytes (nblocks*16 32-bit words) at DATA. + * + * void sm3_transform_avx(struct sm3_block_state *state, + * const u8 *data, size_t nblocks); + */ +SYM_FUNC_START(sm3_transform_avx) + /* input: + * %rdi: state + * %rsi: data + * %rdx: nblocks + */ + vzeroupper; + + pushq %rbp; + movq %rsp, %rbp; + + movq %rdx, RNBLKS; + + subq $STACK_SIZE, %rsp; + andq $(~63), %rsp; + + movq %rbx, (STACK_REG_SAVE + 0 * 8)(%rsp); + movq %r15, (STACK_REG_SAVE + 1 * 8)(%rsp); + movq %r14, (STACK_REG_SAVE + 2 * 8)(%rsp); + movq %r13, (STACK_REG_SAVE + 3 * 8)(%rsp); + movq %r12, (STACK_REG_SAVE + 4 * 8)(%rsp); + + vmovdqa .Lbe32mask (%rip), BSWAP_REG; + + /* Get the values of the chaining variables. */ + movl state_h0(RSTATE), a; + movl state_h1(RSTATE), b; + movl state_h2(RSTATE), c; + movl state_h3(RSTATE), d; + movl state_h4(RSTATE), e; + movl state_h5(RSTATE), f; + movl state_h6(RSTATE), g; + movl state_h7(RSTATE), h; + +.align 16 +.Loop: + /* Load data part1. */ + LOAD_W_XMM_1(); + + leaq -1(RNBLKS), RNBLKS; + + /* Transform 0-3 + Load data part2. */ + R1(a, b, c, d, e, f, g, h, 0, 0, IW); LOAD_W_XMM_2(); + R1(d, a, b, c, h, e, f, g, 1, 1, IW); + R1(c, d, a, b, g, h, e, f, 2, 2, IW); + R1(b, c, d, a, f, g, h, e, 3, 3, IW); LOAD_W_XMM_3(); + + /* Transform 4-7 + Precalc 12-14. */ + R1(a, b, c, d, e, f, g, h, 4, 0, IW); + R1(d, a, b, c, h, e, f, g, 5, 1, IW); + R1(c, d, a, b, g, h, e, f, 6, 2, IW); SCHED_W_0(12, W0, W1, W2, W3, W4, W5); + R1(b, c, d, a, f, g, h, e, 7, 3, IW); SCHED_W_1(12, W0, W1, W2, W3, W4, W5); + + /* Transform 8-11 + Precalc 12-17. */ + R1(a, b, c, d, e, f, g, h, 8, 0, IW); SCHED_W_2(12, W0, W1, W2, W3, W4, W5); + R1(d, a, b, c, h, e, f, g, 9, 1, IW); SCHED_W_0(15, W1, W2, W3, W4, W5, W0); + R1(c, d, a, b, g, h, e, f, 10, 2, IW); SCHED_W_1(15, W1, W2, W3, W4, W5, W0); + R1(b, c, d, a, f, g, h, e, 11, 3, IW); SCHED_W_2(15, W1, W2, W3, W4, W5, W0); + + /* Transform 12-14 + Precalc 18-20 */ + R1(a, b, c, d, e, f, g, h, 12, 0, XW); SCHED_W_0(18, W2, W3, W4, W5, W0, W1); + R1(d, a, b, c, h, e, f, g, 13, 1, XW); SCHED_W_1(18, W2, W3, W4, W5, W0, W1); + R1(c, d, a, b, g, h, e, f, 14, 2, XW); SCHED_W_2(18, W2, W3, W4, W5, W0, W1); + + /* Transform 15-17 + Precalc 21-23 */ + R1(b, c, d, a, f, g, h, e, 15, 0, XW); SCHED_W_0(21, W3, W4, W5, W0, W1, W2); + R2(a, b, c, d, e, f, g, h, 16, 1, XW); SCHED_W_1(21, W3, W4, W5, W0, W1, W2); + R2(d, a, b, c, h, e, f, g, 17, 2, XW); SCHED_W_2(21, W3, W4, W5, W0, W1, W2); + + /* Transform 18-20 + Precalc 24-26 */ + R2(c, d, a, b, g, h, e, f, 18, 0, XW); SCHED_W_0(24, W4, W5, W0, W1, W2, W3); + R2(b, c, d, a, f, g, h, e, 19, 1, XW); SCHED_W_1(24, W4, W5, W0, W1, W2, W3); + R2(a, b, c, d, e, f, g, h, 20, 2, XW); SCHED_W_2(24, W4, W5, W0, W1, W2, W3); + + /* Transform 21-23 + Precalc 27-29 */ + R2(d, a, b, c, h, e, f, g, 21, 0, XW); SCHED_W_0(27, W5, W0, W1, W2, W3, W4); + R2(c, d, a, b, g, h, e, f, 22, 1, XW); SCHED_W_1(27, W5, W0, W1, W2, W3, W4); + R2(b, c, d, a, f, g, h, e, 23, 2, XW); SCHED_W_2(27, W5, W0, W1, W2, W3, W4); + + /* Transform 24-26 + Precalc 30-32 */ + R2(a, b, c, d, e, f, g, h, 24, 0, XW); SCHED_W_0(30, W0, W1, W2, W3, W4, W5); + R2(d, a, b, c, h, e, f, g, 25, 1, XW); SCHED_W_1(30, W0, W1, W2, W3, W4, W5); + R2(c, d, a, b, g, h, e, f, 26, 2, XW); SCHED_W_2(30, W0, W1, W2, W3, W4, W5); + + /* Transform 27-29 + Precalc 33-35 */ + R2(b, c, d, a, f, g, h, e, 27, 0, XW); SCHED_W_0(33, W1, W2, W3, W4, W5, W0); + R2(a, b, c, d, e, f, g, h, 28, 1, XW); SCHED_W_1(33, W1, W2, W3, W4, W5, W0); + R2(d, a, b, c, h, e, f, g, 29, 2, XW); SCHED_W_2(33, W1, W2, W3, W4, W5, W0); + + /* Transform 30-32 + Precalc 36-38 */ + R2(c, d, a, b, g, h, e, f, 30, 0, XW); SCHED_W_0(36, W2, W3, W4, W5, W0, W1); + R2(b, c, d, a, f, g, h, e, 31, 1, XW); SCHED_W_1(36, W2, W3, W4, W5, W0, W1); + R2(a, b, c, d, e, f, g, h, 32, 2, XW); SCHED_W_2(36, W2, W3, W4, W5, W0, W1); + + /* Transform 33-35 + Precalc 39-41 */ + R2(d, a, b, c, h, e, f, g, 33, 0, XW); SCHED_W_0(39, W3, W4, W5, W0, W1, W2); + R2(c, d, a, b, g, h, e, f, 34, 1, XW); SCHED_W_1(39, W3, W4, W5, W0, W1, W2); + R2(b, c, d, a, f, g, h, e, 35, 2, XW); SCHED_W_2(39, W3, W4, W5, W0, W1, W2); + + /* Transform 36-38 + Precalc 42-44 */ + R2(a, b, c, d, e, f, g, h, 36, 0, XW); SCHED_W_0(42, W4, W5, W0, W1, W2, W3); + R2(d, a, b, c, h, e, f, g, 37, 1, XW); SCHED_W_1(42, W4, W5, W0, W1, W2, W3); + R2(c, d, a, b, g, h, e, f, 38, 2, XW); SCHED_W_2(42, W4, W5, W0, W1, W2, W3); + + /* Transform 39-41 + Precalc 45-47 */ + R2(b, c, d, a, f, g, h, e, 39, 0, XW); SCHED_W_0(45, W5, W0, W1, W2, W3, W4); + R2(a, b, c, d, e, f, g, h, 40, 1, XW); SCHED_W_1(45, W5, W0, W1, W2, W3, W4); + R2(d, a, b, c, h, e, f, g, 41, 2, XW); SCHED_W_2(45, W5, W0, W1, W2, W3, W4); + + /* Transform 42-44 + Precalc 48-50 */ + R2(c, d, a, b, g, h, e, f, 42, 0, XW); SCHED_W_0(48, W0, W1, W2, W3, W4, W5); + R2(b, c, d, a, f, g, h, e, 43, 1, XW); SCHED_W_1(48, W0, W1, W2, W3, W4, W5); + R2(a, b, c, d, e, f, g, h, 44, 2, XW); SCHED_W_2(48, W0, W1, W2, W3, W4, W5); + + /* Transform 45-47 + Precalc 51-53 */ + R2(d, a, b, c, h, e, f, g, 45, 0, XW); SCHED_W_0(51, W1, W2, W3, W4, W5, W0); + R2(c, d, a, b, g, h, e, f, 46, 1, XW); SCHED_W_1(51, W1, W2, W3, W4, W5, W0); + R2(b, c, d, a, f, g, h, e, 47, 2, XW); SCHED_W_2(51, W1, W2, W3, W4, W5, W0); + + /* Transform 48-50 + Precalc 54-56 */ + R2(a, b, c, d, e, f, g, h, 48, 0, XW); SCHED_W_0(54, W2, W3, W4, W5, W0, W1); + R2(d, a, b, c, h, e, f, g, 49, 1, XW); SCHED_W_1(54, W2, W3, W4, W5, W0, W1); + R2(c, d, a, b, g, h, e, f, 50, 2, XW); SCHED_W_2(54, W2, W3, W4, W5, W0, W1); + + /* Transform 51-53 + Precalc 57-59 */ + R2(b, c, d, a, f, g, h, e, 51, 0, XW); SCHED_W_0(57, W3, W4, W5, W0, W1, W2); + R2(a, b, c, d, e, f, g, h, 52, 1, XW); SCHED_W_1(57, W3, W4, W5, W0, W1, W2); + R2(d, a, b, c, h, e, f, g, 53, 2, XW); SCHED_W_2(57, W3, W4, W5, W0, W1, W2); + + /* Transform 54-56 + Precalc 60-62 */ + R2(c, d, a, b, g, h, e, f, 54, 0, XW); SCHED_W_0(60, W4, W5, W0, W1, W2, W3); + R2(b, c, d, a, f, g, h, e, 55, 1, XW); SCHED_W_1(60, W4, W5, W0, W1, W2, W3); + R2(a, b, c, d, e, f, g, h, 56, 2, XW); SCHED_W_2(60, W4, W5, W0, W1, W2, W3); + + /* Transform 57-59 + Precalc 63 */ + R2(d, a, b, c, h, e, f, g, 57, 0, XW); SCHED_W_0(63, W5, W0, W1, W2, W3, W4); + R2(c, d, a, b, g, h, e, f, 58, 1, XW); + R2(b, c, d, a, f, g, h, e, 59, 2, XW); SCHED_W_1(63, W5, W0, W1, W2, W3, W4); + + /* Transform 60-62 + Precalc 63 */ + R2(a, b, c, d, e, f, g, h, 60, 0, XW); + R2(d, a, b, c, h, e, f, g, 61, 1, XW); SCHED_W_2(63, W5, W0, W1, W2, W3, W4); + R2(c, d, a, b, g, h, e, f, 62, 2, XW); + + /* Transform 63 */ + R2(b, c, d, a, f, g, h, e, 63, 0, XW); + + /* Update the chaining variables. */ + xorl state_h0(RSTATE), a; + xorl state_h1(RSTATE), b; + xorl state_h2(RSTATE), c; + xorl state_h3(RSTATE), d; + movl a, state_h0(RSTATE); + movl b, state_h1(RSTATE); + movl c, state_h2(RSTATE); + movl d, state_h3(RSTATE); + xorl state_h4(RSTATE), e; + xorl state_h5(RSTATE), f; + xorl state_h6(RSTATE), g; + xorl state_h7(RSTATE), h; + movl e, state_h4(RSTATE); + movl f, state_h5(RSTATE); + movl g, state_h6(RSTATE); + movl h, state_h7(RSTATE); + + cmpq $0, RNBLKS; + jne .Loop; + + vzeroall; + + movq (STACK_REG_SAVE + 0 * 8)(%rsp), %rbx; + movq (STACK_REG_SAVE + 1 * 8)(%rsp), %r15; + movq (STACK_REG_SAVE + 2 * 8)(%rsp), %r14; + movq (STACK_REG_SAVE + 3 * 8)(%rsp), %r13; + movq (STACK_REG_SAVE + 4 * 8)(%rsp), %r12; + + vmovdqa %xmm0, IW_W1_ADDR(0, 0); + vmovdqa %xmm0, IW_W1W2_ADDR(0, 0); + vmovdqa %xmm0, IW_W1_ADDR(4, 0); + vmovdqa %xmm0, IW_W1W2_ADDR(4, 0); + vmovdqa %xmm0, IW_W1_ADDR(8, 0); + vmovdqa %xmm0, IW_W1W2_ADDR(8, 0); + + movq %rbp, %rsp; + popq %rbp; + RET; +SYM_FUNC_END(sm3_transform_avx) diff --git a/lib/crypto/x86/sm3.h b/lib/crypto/x86/sm3.h new file mode 100644 index 000000000000..3834780f2f6a --- /dev/null +++ b/lib/crypto/x86/sm3.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SM3 optimized for x86_64 + * + * Copyright 2026 Google LLC + */ +#include <asm/fpu/api.h> +#include <linux/static_call.h> + +asmlinkage void sm3_transform_avx(struct sm3_block_state *state, + const u8 *data, size_t nblocks); + +static void sm3_blocks_avx(struct sm3_block_state *state, + const u8 *data, size_t nblocks) +{ + if (likely(irq_fpu_usable())) { + kernel_fpu_begin(); + sm3_transform_avx(state, data, nblocks); + kernel_fpu_end(); + } else { + sm3_blocks_generic(state, data, nblocks); + } +} + +DEFINE_STATIC_CALL(sm3_blocks_x86, sm3_blocks_generic); + +static void sm3_blocks(struct sm3_block_state *state, + const u8 *data, size_t nblocks) +{ + static_call(sm3_blocks_x86)(state, data, nblocks); +} + +#define sm3_mod_init_arch sm3_mod_init_arch +static void sm3_mod_init_arch(void) +{ + if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_BMI2) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) + static_call_update(sm3_blocks_x86, sm3_blocks_avx); +} |