lib/crypto: x86/ghash: Migrate optimized code into library

Remove the "ghash-pclmulqdqni" crypto_shash algorithm.  Move the
corresponding assembly code into lib/crypto/, and wire it up to the
GHASH library.

This makes the GHASH library be optimized with x86's carryless
multiplication instructions.  It also greatly reduces the amount of
x86-specific glue code that is needed, and it fixes the issue where this
GHASH optimization was disabled by default.

Rename and adjust the prototypes of the assembly functions to make them
fit better with the library.  Remove the byte-swaps (pshufb
instructions) that are no longer necessary because the library keeps the
accumulator in POLYVAL format rather than GHASH format.

Rename clmul_ghash_mul() to polyval_mul_pclmul() to reflect that it
really does a POLYVAL style multiplication.  Wire it up to both
ghash_mul_arch() and polyval_mul_arch().

Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260319061723.1140720-15-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>

3 files changed, 185 insertions, 10 deletions

diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
index 8950509833af..19c67f70fb38 100644
--- a/lib/crypto/Makefile
+++ b/lib/crypto/Makefile
@@ -174,7 +174,8 @@ OBJECT_FILES_NON_STANDARD_powerpc/ghashp8-ppc.o := y
 endif
 
 libgf128hash-$(CONFIG_RISCV) += riscv/ghash-riscv64-zvkg.o
-libgf128hash-$(CONFIG_X86) += x86/polyval-pclmul-avx.o
+libgf128hash-$(CONFIG_X86) += x86/ghash-pclmul.o \
+			      x86/polyval-pclmul-avx.o
 endif # CONFIG_CRYPTO_LIB_GF128HASH_ARCH
 
 # clean-files must be defined unconditionally
diff --git a/lib/crypto/x86/gf128hash.h b/lib/crypto/x86/gf128hash.h
index adf6147ea677..6b79b06caab0 100644
--- a/lib/crypto/x86/gf128hash.h
+++ b/lib/crypto/x86/gf128hash.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-or-later */
 /*
- * POLYVAL library functions, x86_64 optimized
+ * GHASH and POLYVAL, x86_64 optimized
  *
  * Copyright 2025 Google LLC
  */
@@ -9,10 +9,17 @@
 
 #define NUM_H_POWERS 8
 
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul);
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul_avx);
 
+asmlinkage void polyval_mul_pclmul(struct polyval_elem *a,
+				   const struct polyval_elem *b);
 asmlinkage void polyval_mul_pclmul_avx(struct polyval_elem *a,
 				       const struct polyval_elem *b);
+
+asmlinkage void ghash_blocks_pclmul(struct polyval_elem *acc,
+				    const struct polyval_elem *key,
+				    const u8 *data, size_t nblocks);
 asmlinkage void polyval_blocks_pclmul_avx(struct polyval_elem *acc,
 					  const struct polyval_key *key,
 					  const u8 *data, size_t nblocks);
@@ -41,16 +48,54 @@ static void polyval_preparekey_arch(struct polyval_key *key,
 	}
 }
 
+static void polyval_mul_x86(struct polyval_elem *a,
+			    const struct polyval_elem *b)
+{
+	if (static_branch_likely(&have_pclmul) && irq_fpu_usable()) {
+		kernel_fpu_begin();
+		if (static_branch_likely(&have_pclmul_avx))
+			polyval_mul_pclmul_avx(a, b);
+		else
+			polyval_mul_pclmul(a, b);
+		kernel_fpu_end();
+	} else {
+		polyval_mul_generic(a, b);
+	}
+}
+
+#define ghash_mul_arch ghash_mul_arch
+static void ghash_mul_arch(struct polyval_elem *acc,
+			   const struct ghash_key *key)
+{
+	polyval_mul_x86(acc, &key->h);
+}
+
 #define polyval_mul_arch polyval_mul_arch
 static void polyval_mul_arch(struct polyval_elem *acc,
 			     const struct polyval_key *key)
 {
-	if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) {
-		kernel_fpu_begin();
-		polyval_mul_pclmul_avx(acc, &key->h_powers[NUM_H_POWERS - 1]);
-		kernel_fpu_end();
+	polyval_mul_x86(acc, &key->h_powers[NUM_H_POWERS - 1]);
+}
+
+#define ghash_blocks_arch ghash_blocks_arch
+static void ghash_blocks_arch(struct polyval_elem *acc,
+			      const struct ghash_key *key,
+			      const u8 *data, size_t nblocks)
+{
+	if (static_branch_likely(&have_pclmul) && irq_fpu_usable()) {
+		do {
+			/* Allow rescheduling every 4 KiB. */
+			size_t n = min_t(size_t, nblocks,
+					 4096 / GHASH_BLOCK_SIZE);
+
+			kernel_fpu_begin();
+			ghash_blocks_pclmul(acc, &key->h, data, n);
+			kernel_fpu_end();
+			data += n * GHASH_BLOCK_SIZE;
+			nblocks -= n;
+		} while (nblocks);
 	} else {
-		polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]);
+		ghash_blocks_generic(acc, &key->h, data, nblocks);
 	}
 }
 
@@ -80,7 +125,9 @@ static void polyval_blocks_arch(struct polyval_elem *acc,
 #define gf128hash_mod_init_arch gf128hash_mod_init_arch
 static void gf128hash_mod_init_arch(void)
 {
-	if (boot_cpu_has(X86_FEATURE_PCLMULQDQ) &&
-	    boot_cpu_has(X86_FEATURE_AVX))
-		static_branch_enable(&have_pclmul_avx);
+	if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) {
+		static_branch_enable(&have_pclmul);
+		if (boot_cpu_has(X86_FEATURE_AVX))
+			static_branch_enable(&have_pclmul_avx);
+	}
 }
diff --git a/lib/crypto/x86/ghash-pclmul.S b/lib/crypto/x86/ghash-pclmul.S
new file mode 100644
index 000000000000..6ffb5aea6063
--- /dev/null
+++ b/lib/crypto/x86/ghash-pclmul.S
@@ -0,0 +1,127 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Accelerated GHASH implementation with Intel PCLMULQDQ-NI
+ * instructions. This file contains accelerated part of ghash
+ * implementation. More information about PCLMULQDQ can be found at:
+ *
+ * https://www.intel.com/content/dam/develop/external/us/en/documents/clmul-wp-rev-2-02-2014-04-20.pdf
+ *
+ * Copyright (c) 2009 Intel Corp.
+ *   Author: Huang Ying <ying.huang@intel.com>
+ *	     Vinodh Gopal
+ *	     Erdinc Ozturk
+ *	     Deniz Karakoyunlu
+ */
+
+#include <linux/linkage.h>
+#include <asm/frame.h>
+
+.section	.rodata.cst16.bswap_mask, "aM", @progbits, 16
+.align 16
+.Lbswap_mask:
+	.octa 0x000102030405060708090a0b0c0d0e0f
+
+#define ACC	%xmm0
+#define KEY	%xmm1
+#define T1	%xmm2
+#define T2	%xmm3
+#define T3	%xmm4
+#define BSWAP	%xmm5
+#define IN1	%xmm6
+
+.text
+
+/*
+ * __clmul_gf128mul_ble:	internal ABI
+ * input:
+ *	ACC:			operand1
+ *	KEY:			operand2, hash_key << 1 mod poly
+ * output:
+ *	ACC:			operand1 * operand2 mod poly
+ * changed:
+ *	T1
+ *	T2
+ *	T3
+ */
+SYM_FUNC_START_LOCAL(__clmul_gf128mul_ble)
+	movaps ACC, T1
+	pshufd $0b01001110, ACC, T2
+	pshufd $0b01001110, KEY, T3
+	pxor ACC, T2
+	pxor KEY, T3
+
+	pclmulqdq $0x00, KEY, ACC	# ACC = a0 * b0
+	pclmulqdq $0x11, KEY, T1	# T1 = a1 * b1
+	pclmulqdq $0x00, T3, T2		# T2 = (a1 + a0) * (b1 + b0)
+	pxor ACC, T2
+	pxor T1, T2			# T2 = a0 * b1 + a1 * b0
+
+	movaps T2, T3
+	pslldq $8, T3
+	psrldq $8, T2
+	pxor T3, ACC
+	pxor T2, T1			# <T1:ACC> is result of
+					# carry-less multiplication
+
+	# first phase of the reduction
+	movaps ACC, T3
+	psllq $1, T3
+	pxor ACC, T3
+	psllq $5, T3
+	pxor ACC, T3
+	psllq $57, T3
+	movaps T3, T2
+	pslldq $8, T2
+	psrldq $8, T3
+	pxor T2, ACC
+	pxor T3, T1
+
+	# second phase of the reduction
+	movaps ACC, T2
+	psrlq $5, T2
+	pxor ACC, T2
+	psrlq $1, T2
+	pxor ACC, T2
+	psrlq $1, T2
+	pxor T2, T1
+	pxor T1, ACC
+	RET
+SYM_FUNC_END(__clmul_gf128mul_ble)
+
+/*
+ * void polyval_mul_pclmul(struct polyval_elem *a,
+ *			   const struct polyval_elem *b)
+ */
+SYM_FUNC_START(polyval_mul_pclmul)
+	FRAME_BEGIN
+	movups (%rdi), ACC
+	movups (%rsi), KEY
+	call __clmul_gf128mul_ble
+	movups ACC, (%rdi)
+	FRAME_END
+	RET
+SYM_FUNC_END(polyval_mul_pclmul)
+
+/*
+ * void ghash_blocks_pclmul(struct polyval_elem *acc,
+ *			    const struct polyval_elem *key,
+ *			    const u8 *data, size_t nblocks)
+ */
+SYM_FUNC_START(ghash_blocks_pclmul)
+	FRAME_BEGIN
+	movaps .Lbswap_mask(%rip), BSWAP
+	movups (%rdi), ACC
+	movups (%rsi), KEY
+.align 4
+.Lnext_block:
+	movups (%rdx), IN1
+	pshufb BSWAP, IN1
+	pxor IN1, ACC
+	call __clmul_gf128mul_ble
+	add $16, %rdx
+	dec %rcx
+	jnz .Lnext_block
+	movups ACC, (%rdi)
+	FRAME_END
+	RET
+SYM_FUNC_END(ghash_blocks_pclmul)