From af413d71f09d4dde28277319926c1c3a6ec8b8d4 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:13 -0700 Subject: lib/crypto: riscv/ghash: Migrate optimized code into library Remove the "ghash-riscv64-zvkg" crypto_shash algorithm. Move the corresponding assembly code into lib/crypto/, modify it to take the length in blocks instead of bytes, and wire it up to the GHASH library. This makes the GHASH library be optimized with the RISC-V Vector Cryptography Extension. It also greatly reduces the amount of riscv-specific glue code that is needed, and it fixes the issue where this optimized GHASH code was disabled by default. Note that this RISC-V code has multiple opportunities for improvement, such as adding more parallelism, providing an optimized multiplication function, and directly supporting POLYVAL. But for now, this commit simply tweaks ghash_zvkg() slightly to make it compatible with the library, then wires it up to ghash_blocks_arch(). ghash_preparekey_arch() is also implemented to store the copy of the raw key needed by the vghsh.vv instruction. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-13-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/riscv/crypto/Kconfig | 11 --- arch/riscv/crypto/Makefile | 3 - arch/riscv/crypto/ghash-riscv64-glue.c | 146 --------------------------------- arch/riscv/crypto/ghash-riscv64-zvkg.S | 72 ---------------- include/crypto/gf128hash.h | 3 + lib/crypto/Kconfig | 2 + lib/crypto/Makefile | 1 + lib/crypto/riscv/gf128hash.h | 57 +++++++++++++ lib/crypto/riscv/ghash-riscv64-zvkg.S | 73 +++++++++++++++++ 9 files changed, 136 insertions(+), 232 deletions(-) delete mode 100644 arch/riscv/crypto/ghash-riscv64-glue.c delete mode 100644 arch/riscv/crypto/ghash-riscv64-zvkg.S create mode 100644 lib/crypto/riscv/gf128hash.h create mode 100644 lib/crypto/riscv/ghash-riscv64-zvkg.S diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig index 22d4eaab15f3..c208f54afbcd 100644 --- a/arch/riscv/crypto/Kconfig +++ b/arch/riscv/crypto/Kconfig @@ -17,17 +17,6 @@ config CRYPTO_AES_RISCV64 - Zvkb vector crypto extension (CTR) - Zvkg vector crypto extension (XTS) -config CRYPTO_GHASH_RISCV64 - tristate "Hash functions: GHASH" - depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ - RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS - select CRYPTO_GCM - help - GCM GHASH function (NIST SP 800-38D) - - Architecture: riscv64 using: - - Zvkg vector crypto extension - config CRYPTO_SM3_RISCV64 tristate "Hash functions: SM3 (ShangMi 3)" depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile index 183495a95cc0..5c9ee1b876fa 100644 --- a/arch/riscv/crypto/Makefile +++ b/arch/riscv/crypto/Makefile @@ -4,9 +4,6 @@ obj-$(CONFIG_CRYPTO_AES_RISCV64) += aes-riscv64.o aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \ aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o -obj-$(CONFIG_CRYPTO_GHASH_RISCV64) += ghash-riscv64.o -ghash-riscv64-y := ghash-riscv64-glue.o ghash-riscv64-zvkg.o - obj-$(CONFIG_CRYPTO_SM3_RISCV64) += sm3-riscv64.o sm3-riscv64-y := sm3-riscv64-glue.o sm3-riscv64-zvksh-zvkb.o diff --git a/arch/riscv/crypto/ghash-riscv64-glue.c b/arch/riscv/crypto/ghash-riscv64-glue.c deleted file mode 100644 index d86073d25387..000000000000 --- a/arch/riscv/crypto/ghash-riscv64-glue.c +++ /dev/null @@ -1,146 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * GHASH using the RISC-V vector crypto extensions - * - * Copyright (C) 2023 VRULL GmbH - * Author: Heiko Stuebner - * - * Copyright (C) 2023 SiFive, Inc. - * Author: Jerry Shih - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -asmlinkage void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data, - size_t len); - -struct riscv64_ghash_tfm_ctx { - be128 key; -}; - -struct riscv64_ghash_desc_ctx { - be128 accumulator; -}; - -static int riscv64_ghash_setkey(struct crypto_shash *tfm, const u8 *key, - unsigned int keylen) -{ - struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(tfm); - - if (keylen != GHASH_BLOCK_SIZE) - return -EINVAL; - - memcpy(&tctx->key, key, GHASH_BLOCK_SIZE); - - return 0; -} - -static int riscv64_ghash_init(struct shash_desc *desc) -{ - struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - *dctx = (struct riscv64_ghash_desc_ctx){}; - - return 0; -} - -static inline void -riscv64_ghash_blocks(const struct riscv64_ghash_tfm_ctx *tctx, - struct riscv64_ghash_desc_ctx *dctx, - const u8 *src, size_t srclen) -{ - /* The srclen is nonzero and a multiple of 16. */ - if (crypto_simd_usable()) { - kernel_vector_begin(); - ghash_zvkg(&dctx->accumulator, &tctx->key, src, srclen); - kernel_vector_end(); - } else { - do { - crypto_xor((u8 *)&dctx->accumulator, src, - GHASH_BLOCK_SIZE); - gf128mul_lle(&dctx->accumulator, &tctx->key); - src += GHASH_BLOCK_SIZE; - srclen -= GHASH_BLOCK_SIZE; - } while (srclen); - } -} - -static int riscv64_ghash_update(struct shash_desc *desc, const u8 *src, - unsigned int srclen) -{ - const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - riscv64_ghash_blocks(tctx, dctx, src, - round_down(srclen, GHASH_BLOCK_SIZE)); - return srclen - round_down(srclen, GHASH_BLOCK_SIZE); -} - -static int riscv64_ghash_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *out) -{ - const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - if (len) { - u8 buf[GHASH_BLOCK_SIZE] = {}; - - memcpy(buf, src, len); - riscv64_ghash_blocks(tctx, dctx, buf, GHASH_BLOCK_SIZE); - memzero_explicit(buf, sizeof(buf)); - } - - memcpy(out, &dctx->accumulator, GHASH_DIGEST_SIZE); - return 0; -} - -static struct shash_alg riscv64_ghash_alg = { - .init = riscv64_ghash_init, - .update = riscv64_ghash_update, - .finup = riscv64_ghash_finup, - .setkey = riscv64_ghash_setkey, - .descsize = sizeof(struct riscv64_ghash_desc_ctx), - .digestsize = GHASH_DIGEST_SIZE, - .base = { - .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct riscv64_ghash_tfm_ctx), - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_name = "ghash", - .cra_driver_name = "ghash-riscv64-zvkg", - .cra_module = THIS_MODULE, - }, -}; - -static int __init riscv64_ghash_mod_init(void) -{ - if (riscv_isa_extension_available(NULL, ZVKG) && - riscv_vector_vlen() >= 128) - return crypto_register_shash(&riscv64_ghash_alg); - - return -ENODEV; -} - -static void __exit riscv64_ghash_mod_exit(void) -{ - crypto_unregister_shash(&riscv64_ghash_alg); -} - -module_init(riscv64_ghash_mod_init); -module_exit(riscv64_ghash_mod_exit); - -MODULE_DESCRIPTION("GHASH (RISC-V accelerated)"); -MODULE_AUTHOR("Heiko Stuebner "); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("ghash"); diff --git a/arch/riscv/crypto/ghash-riscv64-zvkg.S b/arch/riscv/crypto/ghash-riscv64-zvkg.S deleted file mode 100644 index f2b43fb4d434..000000000000 --- a/arch/riscv/crypto/ghash-riscv64-zvkg.S +++ /dev/null @@ -1,72 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */ -// -// This file is dual-licensed, meaning that you can use it under your -// choice of either of the following two licenses: -// -// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. -// -// Licensed under the Apache License 2.0 (the "License"). You can obtain -// a copy in the file LICENSE in the source distribution or at -// https://www.openssl.org/source/license.html -// -// or -// -// Copyright (c) 2023, Christoph Müllner -// Copyright (c) 2023, Jerry Shih -// Copyright 2024 Google LLC -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// The generated code of this file depends on the following RISC-V extensions: -// - RV64I -// - RISC-V Vector ('V') with VLEN >= 128 -// - RISC-V Vector GCM/GMAC extension ('Zvkg') - -#include - -.text -.option arch, +zvkg - -#define ACCUMULATOR a0 -#define KEY a1 -#define DATA a2 -#define LEN a3 - -// void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data, -// size_t len); -// -// |len| must be nonzero and a multiple of 16 (GHASH_BLOCK_SIZE). -SYM_FUNC_START(ghash_zvkg) - vsetivli zero, 4, e32, m1, ta, ma - vle32.v v1, (ACCUMULATOR) - vle32.v v2, (KEY) -.Lnext_block: - vle32.v v3, (DATA) - vghsh.vv v1, v2, v3 - addi DATA, DATA, 16 - addi LEN, LEN, -16 - bnez LEN, .Lnext_block - - vse32.v v1, (ACCUMULATOR) - ret -SYM_FUNC_END(ghash_zvkg) diff --git a/include/crypto/gf128hash.h b/include/crypto/gf128hash.h index 650652dd6003..b798438cce23 100644 --- a/include/crypto/gf128hash.h +++ b/include/crypto/gf128hash.h @@ -44,6 +44,9 @@ struct ghash_key { #if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_PPC64) /** @htable: GHASH key format used by the POWER8 assembly code */ u64 htable[4][2]; +#elif defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_RISCV) + /** @h_raw: The hash key H, in GHASH format */ + u8 h_raw[GHASH_BLOCK_SIZE]; #endif /** @h: The hash key H, in POLYVAL format */ struct polyval_elem h; diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index f54add7d9070..027802e0de33 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -122,6 +122,8 @@ config CRYPTO_LIB_GF128HASH_ARCH default y if ARM && KERNEL_MODE_NEON default y if ARM64 default y if PPC64 && VSX + default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ + RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS default y if X86_64 config CRYPTO_LIB_MD5 diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 8a9084188778..8950509833af 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -173,6 +173,7 @@ targets += powerpc/ghashp8-ppc.S OBJECT_FILES_NON_STANDARD_powerpc/ghashp8-ppc.o := y endif +libgf128hash-$(CONFIG_RISCV) += riscv/ghash-riscv64-zvkg.o libgf128hash-$(CONFIG_X86) += x86/polyval-pclmul-avx.o endif # CONFIG_CRYPTO_LIB_GF128HASH_ARCH diff --git a/lib/crypto/riscv/gf128hash.h b/lib/crypto/riscv/gf128hash.h new file mode 100644 index 000000000000..4301a0384f60 --- /dev/null +++ b/lib/crypto/riscv/gf128hash.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * GHASH, RISC-V optimized + * + * Copyright (C) 2023 VRULL GmbH + * Copyright (C) 2023 SiFive, Inc. + * Copyright 2026 Google LLC + */ + +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkg); + +asmlinkage void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE], + const u8 key[GHASH_BLOCK_SIZE], + const u8 *data, size_t nblocks); + +#define ghash_preparekey_arch ghash_preparekey_arch +static void ghash_preparekey_arch(struct ghash_key *key, + const u8 raw_key[GHASH_BLOCK_SIZE]) +{ + /* Save key in POLYVAL format for fallback */ + ghash_key_to_polyval(raw_key, &key->h); + + /* Save key in GHASH format for zvkg */ + memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE); +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_zvkg) && likely(may_use_simd())) { + u8 ghash_acc[GHASH_BLOCK_SIZE]; + + polyval_acc_to_ghash(acc, ghash_acc); + + kernel_vector_begin(); + ghash_zvkg(ghash_acc, key->h_raw, data, nblocks); + kernel_vector_end(); + + ghash_acc_to_polyval(ghash_acc, acc); + memzero_explicit(ghash_acc, sizeof(ghash_acc)); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (riscv_isa_extension_available(NULL, ZVKG) && + riscv_vector_vlen() >= 128) + static_branch_enable(&have_zvkg); +} diff --git a/lib/crypto/riscv/ghash-riscv64-zvkg.S b/lib/crypto/riscv/ghash-riscv64-zvkg.S new file mode 100644 index 000000000000..6a2a2f2bc7c8 --- /dev/null +++ b/lib/crypto/riscv/ghash-riscv64-zvkg.S @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */ +// +// This file is dual-licensed, meaning that you can use it under your +// choice of either of the following two licenses: +// +// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. +// +// Licensed under the Apache License 2.0 (the "License"). You can obtain +// a copy in the file LICENSE in the source distribution or at +// https://www.openssl.org/source/license.html +// +// or +// +// Copyright (c) 2023, Christoph Müllner +// Copyright (c) 2023, Jerry Shih +// Copyright 2024 Google LLC +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// The generated code of this file depends on the following RISC-V extensions: +// - RV64I +// - RISC-V Vector ('V') with VLEN >= 128 +// - RISC-V Vector GCM/GMAC extension ('Zvkg') + +#include + +.text +.option arch, +zvkg + +#define ACCUMULATOR a0 +#define KEY a1 +#define DATA a2 +#define NBLOCKS a3 + +// void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE], +// const u8 key[GHASH_BLOCK_SIZE], +// const u8 *data, size_t nblocks); +// +// |nblocks| must be nonzero. +SYM_FUNC_START(ghash_zvkg) + vsetivli zero, 4, e32, m1, ta, ma + vle32.v v1, (ACCUMULATOR) + vle32.v v2, (KEY) +.Lnext_block: + vle32.v v3, (DATA) + vghsh.vv v1, v2, v3 + addi DATA, DATA, 16 + addi NBLOCKS, NBLOCKS, -1 + bnez NBLOCKS, .Lnext_block + + vse32.v v1, (ACCUMULATOR) + ret +SYM_FUNC_END(ghash_zvkg) -- cgit v1.2.3