From c7c18c94a6becc42c71a17fdc6a551aa8efb2999 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 22 Apr 2025 08:27:14 -0700 Subject: crypto: x86 - move library functions to arch/x86/lib/crypto/ Continue disentangling the crypto library functions from the generic crypto infrastructure by moving the x86 BLAKE2s, ChaCha, and Poly1305 library functions into a new directory arch/x86/lib/crypto/ that does not depend on CRYPTO. This mirrors the distinction between crypto/ and lib/crypto/. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/lib/crypto/blake2s-glue.c | 72 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 arch/x86/lib/crypto/blake2s-glue.c (limited to 'arch/x86/lib/crypto/blake2s-glue.c') diff --git a/arch/x86/lib/crypto/blake2s-glue.c b/arch/x86/lib/crypto/blake2s-glue.c new file mode 100644 index 000000000000..00f84f29cc8c --- /dev/null +++ b/arch/x86/lib/crypto/blake2s-glue.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +asmlinkage void blake2s_compress_ssse3(struct blake2s_state *state, + const u8 *block, const size_t nblocks, + const u32 inc); +asmlinkage void blake2s_compress_avx512(struct blake2s_state *state, + const u8 *block, const size_t nblocks, + const u32 inc); + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_ssse3); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_avx512); + +void blake2s_compress(struct blake2s_state *state, const u8 *block, + size_t nblocks, const u32 inc) +{ + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(SZ_4K / BLAKE2S_BLOCK_SIZE < 8); + + if (!static_branch_likely(&blake2s_use_ssse3) || !may_use_simd()) { + blake2s_compress_generic(state, block, nblocks, inc); + return; + } + + do { + const size_t blocks = min_t(size_t, nblocks, + SZ_4K / BLAKE2S_BLOCK_SIZE); + + kernel_fpu_begin(); + if (static_branch_likely(&blake2s_use_avx512)) + blake2s_compress_avx512(state, block, blocks, inc); + else + blake2s_compress_ssse3(state, block, blocks, inc); + kernel_fpu_end(); + + nblocks -= blocks; + block += blocks * BLAKE2S_BLOCK_SIZE; + } while (nblocks); +} +EXPORT_SYMBOL(blake2s_compress); + +static int __init blake2s_mod_init(void) +{ + if (boot_cpu_has(X86_FEATURE_SSSE3)) + static_branch_enable(&blake2s_use_ssse3); + + if (boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_AVX512F) && + boot_cpu_has(X86_FEATURE_AVX512VL) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | + XFEATURE_MASK_AVX512, NULL)) + static_branch_enable(&blake2s_use_avx512); + + return 0; +} + +subsys_initcall(blake2s_mod_init); -- cgit v1.2.3 From ce026b35b7b62094b640d3781d09bbd5b132b326 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 30 Apr 2025 16:25:55 +0800 Subject: crypto: x86/blake2s - Include linux/init.h Explicitly include linux/init.h rather than pulling it through potluck. Signed-off-by: Herbert Xu --- arch/x86/lib/crypto/blake2s-glue.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) (limited to 'arch/x86/lib/crypto/blake2s-glue.c') diff --git a/arch/x86/lib/crypto/blake2s-glue.c b/arch/x86/lib/crypto/blake2s-glue.c index 00f84f29cc8c..adc296cd17c9 100644 --- a/arch/x86/lib/crypto/blake2s-glue.c +++ b/arch/x86/lib/crypto/blake2s-glue.c @@ -3,17 +3,15 @@ * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. */ -#include - -#include -#include -#include -#include - #include #include #include #include +#include +#include +#include +#include +#include asmlinkage void blake2s_compress_ssse3(struct blake2s_state *state, const u8 *block, const size_t nblocks, -- cgit v1.2.3