diff options
Diffstat (limited to 'lib')
330 files changed, 24991 insertions, 7535 deletions
diff --git a/lib/.gitignore b/lib/.gitignore index 101a4aa92fb5..42aee3ddb374 100644 --- a/lib/.gitignore +++ b/lib/.gitignore @@ -5,3 +5,4 @@ /gen_crc32table /gen_crc64table /oid_registry_data.c +/raid6 diff --git a/lib/Kconfig b/lib/Kconfig index 0f2fb9610647..4e6b34c3346d 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -8,17 +8,6 @@ config BINARY_PRINTF menu "Library routines" -config RAID6_PQ - tristate - -config RAID6_PQ_BENCHMARK - bool "Automatically choose fastest RAID6 PQ functions" - depends on RAID6_PQ - default y - help - Benchmark all available RAID6 PQ functions on init and choose the - fastest one. - config LINEAR_RANGES tristate @@ -54,14 +43,27 @@ config PACKING_KUNIT_TEST config BITREVERSE tristate + select GENERIC_BITREVERSE if !HAVE_ARCH_BITREVERSE config HAVE_ARCH_BITREVERSE bool default n + depends on BITREVERSE help This option enables the use of hardware bit-reversal instructions on architectures which support such operations. +config GENERIC_BITREVERSE + tristate + depends on BITREVERSE + help + Generic bit reversal implementation. Drivers should never enable + it explicitly. Instead, enable BITREVERSE. + + Architectures may want to select it as a fall-back option for + HAVE_ARCH_BITREVERSE, when the hardware-accelerated bit reverse + instruction set is optional, like RISC-V ZBKB extension. + config ARCH_HAS_STRNCPY_FROM_USER bool @@ -138,6 +140,7 @@ config TRACE_MMIO_ACCESS source "lib/crc/Kconfig" source "lib/crypto/Kconfig" +source "lib/raid/Kconfig" config XXHASH tristate @@ -156,12 +159,6 @@ config AUDIT_COMPAT_GENERIC depends on AUDIT_GENERIC && AUDIT_ARCH_COMPAT_GENERIC && COMPAT default y -config RANDOM32_SELFTEST - bool "PRNG perform self test on init" - help - This option enables the 32 bit PRNG library functions to perform a - self test on initialization. - # # compression support is select'ed if needed # @@ -589,7 +586,7 @@ config OBJAGG config LWQ_TEST bool "Boot-time test for lwq queuing" help - Run boot-time test of light-weight queuing. + Run boot-time test of light-weight queuing. endmenu @@ -625,9 +622,6 @@ config PLDMFW config ASN1_ENCODER tristate -config POLYNOMIAL - tristate - config FIRMWARE_TABLE bool diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 93f356d2b3d9..134b15a44625 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -613,24 +613,9 @@ config BUILTIN_MODULE_RANGES It also records an anchor symbol to determine the load address of the section. -config DEBUG_FORCE_WEAK_PER_CPU - bool "Force weak per-cpu definitions" - depends on DEBUG_KERNEL - help - s390 and alpha require percpu variables in modules to be - defined weak to work around addressing range issue which - puts the following two restrictions on percpu variable - definitions. - - 1. percpu symbols must be unique whether static or not - 2. percpu variables can't be defined inside a function - - To ensure that generic code follows the above rules, this - option forces all percpu variables to be defined as weak. - config WARN_CONTEXT_ANALYSIS bool "Compiler context-analysis warnings" - depends on CC_IS_CLANG && CLANG_VERSION >= 220100 + depends on CC_IS_CLANG && CLANG_VERSION >= 230000 # Branch profiling re-defines "if", which messes with the compiler's # ability to analyze __cond_acquires(..), resulting in false positives. depends on !TRACE_BRANCH_PROFILING @@ -641,7 +626,7 @@ config WARN_CONTEXT_ANALYSIS and releasing user-definable "context locks". Clang's name of the feature is "Thread Safety Analysis". Requires - Clang 22.1.0 or later. + Clang 23 or later. Produces warnings by default. Select CONFIG_WERROR if you wish to turn these warnings into errors. @@ -1048,34 +1033,6 @@ config CODE_TAGGING bool select KALLSYMS -config MEM_ALLOC_PROFILING - bool "Enable memory allocation profiling" - default n - depends on MMU - depends on PROC_FS - depends on !DEBUG_FORCE_WEAK_PER_CPU - select CODE_TAGGING - select PAGE_EXTENSION - select SLAB_OBJ_EXT - help - Track allocation source code and record total allocation size - initiated at that code location. The mechanism can be used to track - memory leaks with a low performance and memory impact. - -config MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT - bool "Enable memory allocation profiling by default" - default y - depends on MEM_ALLOC_PROFILING - -config MEM_ALLOC_PROFILING_DEBUG - bool "Memory allocation profiler debugging" - default n - depends on MEM_ALLOC_PROFILING - select MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT - help - Adds warnings with helpful error messages for memory allocation - profiling. - source "lib/Kconfig.kasan" source "lib/Kconfig.kfence" source "lib/Kconfig.kmsan" @@ -1320,10 +1277,10 @@ config DETECT_HUNG_TASK_BLOCKER depends on !PREEMPT_RT default y help - Say Y here to show the blocker task's stacktrace who acquires - the mutex lock which "hung tasks" are waiting. - This will add overhead a bit but shows suspicious tasks and - call trace if it comes from waiting a mutex. + Say Y here to show a possible blocker task's stacktrace when + a hung task is waiting on a mutex, semaphore, or rwsem. + This will add a bit of overhead but shows suspicious tasks and + call traces for waits on one of these locks. config WQ_WATCHDOG bool "Detect Workqueue Stalls" @@ -1618,14 +1575,22 @@ config LOCKDEP_STACK_TRACE_BITS int "Size for MAX_STACK_TRACE_ENTRIES (as Nth power of 2)" depends on LOCKDEP && !LOCKDEP_SMALL range 10 26 + default 21 if KASAN default 19 help Try increasing this value if you hit "BUG: MAX_STACK_TRACE_ENTRIES too low!" message. + KASAN significantly increases stack trace consumption because its + slab tracking interacts with lockdep's dependency validation under + PREEMPT_FULL, creating a feedback loop. The higher default when + KASAN is enabled costs ~12MB extra, which is negligible compared to + KASAN's own shadow memory overhead. + config LOCKDEP_STACK_TRACE_HASH_BITS int "Size for STACK_TRACE_HASH_SIZE (as Nth power of 2)" depends on LOCKDEP && !LOCKDEP_SMALL range 10 26 + default 16 if KASAN default 14 help Try increasing this value if you need large STACK_TRACE_HASH_SIZE. @@ -2239,10 +2204,11 @@ config KCOV_INSTRUMENT_ALL config KCOV_IRQ_AREA_SIZE hex "Size of interrupt coverage collection area in words" depends on KCOV + range 0x80 0x1000000 default 0x40000 help - KCOV uses preallocated per-cpu areas to collect coverage from - soft interrupts. This specifies the size of those areas in the + KCOV uses preallocated areas to collect coverage from soft + interrupts. This specifies the size of those areas in the number of unsigned long words. config KCOV_SELFTEST @@ -2518,6 +2484,17 @@ config STRING_HELPERS_KUNIT_TEST depends on KUNIT default KUNIT_ALL_TESTS +config STRING_KUNIT_BENCH + bool "Benchmark string functions at runtime" + depends on STRING_KUNIT_TEST + help + Enable performance measurement for string functions. + + This measures the execution efficiency of string functions + during the KUnit test run. + + If unsure, say N. + config FFS_KUNIT_TEST tristate "KUnit test ffs-family functions at runtime" if !KUNIT_ALL_TESTS depends on KUNIT @@ -2628,6 +2605,21 @@ config TEST_VMALLOC If unsure, say N. +config TEST_WORKQUEUE + tristate "Test module for stress/performance analysis of workqueue" + depends on m + default n + help + This builds the "test_workqueue" module for benchmarking + workqueue throughput under contention. Useful for evaluating + affinity scope changes (e.g., cache_shard vs cache). + + The test drives sysfs to switch affinity scopes, so it must be + loaded after userspace has mounted sysfs; building it in (=y) + would run module_init before /sys is available. + + If unsure, say N. + config TEST_BPF tristate "Test BPF filter functionality" depends on m && NET @@ -2649,6 +2641,19 @@ config FIND_BIT_BENCHMARK If unsure, say N. +config REGION_ALLOC_BENCHMARK + tristate "Benchmark bitmap, IDA and Maple Tree region allocation" + help + This builds a microbenchmark comparing variable-sized region + allocation using bitmaps, IDA and Maple Tree. The benchmark + runs at initialization time. + + Usage: + insmod region_alloc_benchmark.ko + insmod region_alloc_benchmark.ko capacities=1024,2048,4096,65536 + + If unsure, say N. + config FIND_BIT_BENCHMARK_RUST tristate "Test find_bit functions in Rust" depends on RUST @@ -2676,7 +2681,7 @@ config TEST_FIRMWARE config TEST_SYSCTL tristate "sysctl test driver" - depends on PROC_SYSCTL + depends on SYSCTL help This builds the "test_sysctl" module. This driver enables to test the proc sysctl interfaces available to drivers safely without affecting @@ -2942,6 +2947,16 @@ config BITS_TEST If unsure, say N. +config SHDI3_KUNIT_TEST + tristate "KUnit test for __ashldi3(), __ashrdi3(), and __lshrdi3()" + depends on KUNIT + depends on ARM || XTENSA || MICROBLAZE || ((RISCV || SPARC) && !64BIT) + help + This builds the unit test for __ashldi3(), __ashrdi3(), and + __lshrdi3() helper functions used to implement 64-bit arithmetic + shift left, arithmetic shift right and logical shift right, + respectively, on a 32-bit CPUs. + config SLUB_KUNIT_TEST tristate "KUnit test for SLUB cache error detection" if !KUNIT_ALL_TESTS depends on SLUB_DEBUG && KUNIT @@ -3041,7 +3056,7 @@ config FORTIFY_KUNIT_TEST config LONGEST_SYM_KUNIT_TEST tristate "Test the longest symbol possible" if !KUNIT_ALL_TESTS depends on KUNIT && KPROBES - depends on !PREFIX_SYMBOLS && !CFI && !GCOV_KERNEL + depends on !CALL_PADDING && !CFI && !GCOV_KERNEL default KUNIT_ALL_TESTS help Tests the longest symbol possible @@ -3058,6 +3073,8 @@ config HW_BREAKPOINT_KUNIT_TEST If unsure, say N. +source "lib/crypto/tests/Kconfig" + config SIPHASH_KUNIT_TEST tristate "Perform selftest on siphash functions" if !KUNIT_ALL_TESTS depends on KUNIT @@ -3458,6 +3475,23 @@ config GCD_KUNIT_TEST If unsure, say N +config POLYNOMIAL_KUNIT_TEST + tristate "Polynomial calculation (polynomial_calc) test" if !KUNIT_ALL_TESTS + depends on KUNIT + select POLYNOMIAL + default KUNIT_ALL_TESTS + help + This option enables the KUnit test suite for the polynomial_calc() + function, which evaluates integer polynomials using factor + redistribution to avoid overflow. + + The test suite verifies correctness for constant, linear, and + quadratic polynomials, negative coefficients, per-step dividers, + divider_leftover, total_divider scaling, and a real sensor + N-to-temperature conversion polynomial. + + If unsure, say N + config PRIME_NUMBERS_KUNIT_TEST tristate "Prime number generator test" if !KUNIT_ALL_TESTS depends on KUNIT @@ -3485,6 +3519,15 @@ config GLOB_KUNIT_TEST If unsure, say N +config PRANDOM_KUNIT_TEST + tristate "KUnit test for prandom" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Enable this option to test the prandom functions at runtime. + + If unsure, say N + endif # RUNTIME_TESTING_MENU config ARCH_USE_MEMTEST @@ -3577,6 +3620,23 @@ config RUST_KERNEL_DOCTESTS If unsure, say N. +config RUST_INLINE_HELPERS + bool "Inline C helpers into Rust code (EXPERIMENTAL)" + depends on RUST && RUSTC_CLANG_LLVM_COMPATIBLE + depends on EXPERT + depends on ARM64 || X86_64 + depends on !UML + help + Inlines C helpers into Rust code using Link Time Optimization. + + If this option is enabled, C helper functions declared in + rust/helpers/ are inlined into Rust code, which is helpful for + performance of Rust code. This requires a matching LLVM version for + Clang and rustc. + + If you are sure that you're using Clang and rustc with matching LLVM + versions, say Y. Otherwise say N. + endmenu # "Rust" endmenu # Kernel hacking diff --git a/lib/Makefile b/lib/Makefile index 1b9ee167517f..dfab958327c5 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -64,6 +64,7 @@ obj-y += hexdump.o obj-$(CONFIG_TEST_HEXDUMP) += test_hexdump.o obj-y += kstrtox.o obj-$(CONFIG_FIND_BIT_BENCHMARK) += find_bit_benchmark.o +obj-$(CONFIG_REGION_ALLOC_BENCHMARK) += region_alloc_benchmark.o obj-$(CONFIG_FIND_BIT_BENCHMARK_RUST) += find_bit_benchmark_rust.o obj-$(CONFIG_TEST_BPF) += test_bpf.o test_dhry-objs := dhry_1.o dhry_2.o dhry_run.o @@ -79,6 +80,7 @@ UBSAN_SANITIZE_test_ubsan.o := y obj-$(CONFIG_TEST_KSTRTOX) += test-kstrtox.o obj-$(CONFIG_TEST_LKM) += test_module.o obj-$(CONFIG_TEST_VMALLOC) += test_vmalloc.o +obj-$(CONFIG_TEST_WORKQUEUE) += test_workqueue.o obj-$(CONFIG_TEST_RHASHTABLE) += test_rhashtable.o obj-$(CONFIG_TEST_STATIC_KEYS) += test_static_keys.o obj-$(CONFIG_TEST_STATIC_KEYS) += test_static_key_base.o @@ -120,7 +122,7 @@ endif obj-$(CONFIG_DEBUG_INFO_REDUCED) += debug_info.o CFLAGS_debug_info.o += $(call cc-option, -femit-struct-debug-detailed=any) -obj-y += math/ crc/ crypto/ tests/ vdso/ +obj-y += math/ crc/ crypto/ tests/ vdso/ raid/ obj-$(CONFIG_GENERIC_IOMAP) += iomap.o obj-$(CONFIG_HAS_IOMEM) += iomap_copy.o devres.o @@ -144,7 +146,7 @@ obj-$(CONFIG_DEBUG_PREEMPT) += smp_processor_id.o obj-$(CONFIG_LIST_HARDENED) += list_debug.o obj-$(CONFIG_DEBUG_OBJECTS) += debugobjects.o -obj-$(CONFIG_BITREVERSE) += bitrev.o +obj-$(CONFIG_GENERIC_BITREVERSE) += bitrev.o obj-$(CONFIG_LINEAR_RANGES) += linear_ranges.o obj-$(CONFIG_PACKING) += packing.o obj-$(CONFIG_PACKING_KUNIT_TEST) += packing_test.o @@ -166,7 +168,6 @@ obj-$(CONFIG_LZ4_DECOMPRESS) += lz4/ obj-$(CONFIG_ZSTD_COMPRESS) += zstd/ obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd/ obj-$(CONFIG_XZ_DEC) += xz/ -obj-$(CONFIG_RAID6_PQ) += raid6/ lib-$(CONFIG_DECOMPRESS_GZIP) += decompress_inflate.o lib-$(CONFIG_DECOMPRESS_BZIP2) += decompress_bunzip2.o @@ -196,7 +197,6 @@ obj-$(CONFIG_OF_RECONFIG_NOTIFIER_ERROR_INJECT) += \ obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o obj-$(CONFIG_CODE_TAGGING) += codetag.o -obj-$(CONFIG_MEM_ALLOC_PROFILING) += alloc_tag.o lib-$(CONFIG_GENERIC_BUG) += bug.o @@ -243,8 +243,6 @@ obj-$(CONFIG_MEMREGION) += memregion.o obj-$(CONFIG_STMP_DEVICE) += stmp_device.o obj-$(CONFIG_IRQ_POLL) += irq_poll.o -obj-$(CONFIG_POLYNOMIAL) += polynomial.o - # stackdepot.c should not be instrumented or call instrumented functions. # Prevent the compiler from calling builtins like memcmp() or bcmp() from this # file. @@ -275,6 +273,22 @@ filechk_defbconf = cat $(or $(real-prereqs), /dev/null) $(obj)/default.bconf: $(CONFIG_BOOT_CONFIG_EMBED_FILE) FORCE $(call filechk,defbconf) +obj-$(CONFIG_CMDLINE_FROM_BOOTCONFIG) += embedded-cmdline.o +$(obj)/embedded-cmdline.o: $(obj)/embedded_cmdline.bin + +# Render the bootconfig "kernel" subtree to a flat cmdline string using +# the userspace tools/bootconfig parser (-C mode). The runtime prepend +# helper enforces COMMAND_LINE_SIZE at boot, so no build-time size +# check is performed here (COMMAND_LINE_SIZE is an arch header +# constant, not a Kconfig value). +quiet_cmd_render_cmdline = BCONF2C $@ + cmd_render_cmdline = \ + $(objtree)/tools/bootconfig/bootconfig -C $< > $@ + +targets += embedded_cmdline.bin +$(obj)/embedded_cmdline.bin: $(obj)/default.bconf $(objtree)/tools/bootconfig/bootconfig FORCE + $(call if_changed,render_cmdline) + obj-$(CONFIG_RBTREE_TEST) += rbtree_test.o obj-$(CONFIG_INTERVAL_TREE_TEST) += interval_tree_test.o @@ -307,6 +321,7 @@ obj-$(CONFIG_UBSAN) += ubsan.o UBSAN_SANITIZE_ubsan.o := n KASAN_SANITIZE_ubsan.o := n KCSAN_SANITIZE_ubsan.o := n +KMSAN_SANITIZE_ubsan.o := n CFLAGS_ubsan.o := -fno-stack-protector $(DISABLE_KSTACK_ERASE) obj-$(CONFIG_SBITMAP) += sbitmap.o diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c deleted file mode 100644 index 58991ab09d84..000000000000 --- a/lib/alloc_tag.c +++ /dev/null @@ -1,864 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -#include <linux/alloc_tag.h> -#include <linux/execmem.h> -#include <linux/fs.h> -#include <linux/gfp.h> -#include <linux/kallsyms.h> -#include <linux/module.h> -#include <linux/page_ext.h> -#include <linux/proc_fs.h> -#include <linux/seq_buf.h> -#include <linux/seq_file.h> -#include <linux/string_choices.h> -#include <linux/vmalloc.h> -#include <linux/kmemleak.h> - -#define ALLOCINFO_FILE_NAME "allocinfo" -#define MODULE_ALLOC_TAG_VMAP_SIZE (100000UL * sizeof(struct alloc_tag)) -#define SECTION_START(NAME) (CODETAG_SECTION_START_PREFIX NAME) -#define SECTION_STOP(NAME) (CODETAG_SECTION_STOP_PREFIX NAME) - -#ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT -static bool mem_profiling_support = true; -#else -static bool mem_profiling_support; -#endif - -static struct codetag_type *alloc_tag_cttype; - -#ifdef CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU -DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag); -EXPORT_SYMBOL(_shared_alloc_tag); -#endif - -DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT, - mem_alloc_profiling_key); -EXPORT_SYMBOL(mem_alloc_profiling_key); - -DEFINE_STATIC_KEY_FALSE(mem_profiling_compressed); - -struct alloc_tag_kernel_section kernel_tags = { NULL, 0 }; -unsigned long alloc_tag_ref_mask; -int alloc_tag_ref_offs; - -struct allocinfo_private { - struct codetag_iterator iter; - bool print_header; -}; - -static void *allocinfo_start(struct seq_file *m, loff_t *pos) -{ - struct allocinfo_private *priv; - loff_t node = *pos; - - priv = (struct allocinfo_private *)m->private; - codetag_lock_module_list(alloc_tag_cttype, true); - if (node == 0) { - priv->print_header = true; - priv->iter = codetag_get_ct_iter(alloc_tag_cttype); - codetag_next_ct(&priv->iter); - } - return priv->iter.ct ? priv : NULL; -} - -static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos) -{ - struct allocinfo_private *priv = (struct allocinfo_private *)arg; - struct codetag *ct = codetag_next_ct(&priv->iter); - - (*pos)++; - if (!ct) - return NULL; - - return priv; -} - -static void allocinfo_stop(struct seq_file *m, void *arg) -{ - codetag_lock_module_list(alloc_tag_cttype, false); -} - -static void print_allocinfo_header(struct seq_buf *buf) -{ - /* Output format version, so we can change it. */ - seq_buf_printf(buf, "allocinfo - version: 2.0\n"); - seq_buf_printf(buf, "# <size> <calls> <tag info>\n"); -} - -static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct) -{ - struct alloc_tag *tag = ct_to_alloc_tag(ct); - struct alloc_tag_counters counter = alloc_tag_read(tag); - s64 bytes = counter.bytes; - - seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls); - codetag_to_text(out, ct); - if (unlikely(alloc_tag_is_inaccurate(tag))) - seq_buf_printf(out, " accurate:no"); - seq_buf_putc(out, ' '); - seq_buf_putc(out, '\n'); -} - -static int allocinfo_show(struct seq_file *m, void *arg) -{ - struct allocinfo_private *priv = (struct allocinfo_private *)arg; - char *bufp; - size_t n = seq_get_buf(m, &bufp); - struct seq_buf buf; - - seq_buf_init(&buf, bufp, n); - if (priv->print_header) { - print_allocinfo_header(&buf); - priv->print_header = false; - } - alloc_tag_to_text(&buf, priv->iter.ct); - seq_commit(m, seq_buf_used(&buf)); - return 0; -} - -static const struct seq_operations allocinfo_seq_op = { - .start = allocinfo_start, - .next = allocinfo_next, - .stop = allocinfo_stop, - .show = allocinfo_show, -}; - -size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep) -{ - struct codetag_iterator iter; - struct codetag *ct; - struct codetag_bytes n; - unsigned int i, nr = 0; - - if (IS_ERR_OR_NULL(alloc_tag_cttype)) - return 0; - - if (can_sleep) - codetag_lock_module_list(alloc_tag_cttype, true); - else if (!codetag_trylock_module_list(alloc_tag_cttype)) - return 0; - - iter = codetag_get_ct_iter(alloc_tag_cttype); - while ((ct = codetag_next_ct(&iter))) { - struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct)); - - n.ct = ct; - n.bytes = counter.bytes; - - for (i = 0; i < nr; i++) - if (n.bytes > tags[i].bytes) - break; - - if (i < count) { - nr -= nr == count; - memmove(&tags[i + 1], - &tags[i], - sizeof(tags[0]) * (nr - i)); - nr++; - tags[i] = n; - } - } - - codetag_lock_module_list(alloc_tag_cttype, false); - - return nr; -} - -void pgalloc_tag_split(struct folio *folio, int old_order, int new_order) -{ - int i; - struct alloc_tag *tag; - unsigned int nr_pages = 1 << new_order; - - if (!mem_alloc_profiling_enabled()) - return; - - tag = __pgalloc_tag_get(&folio->page); - if (!tag) - return; - - for (i = nr_pages; i < (1 << old_order); i += nr_pages) { - union pgtag_ref_handle handle; - union codetag_ref ref; - - if (get_page_tag_ref(folio_page(folio, i), &ref, &handle)) { - /* Set new reference to point to the original tag */ - alloc_tag_ref_set(&ref, tag); - update_page_tag_ref(handle, &ref); - put_page_tag_ref(handle); - } - } -} - -void pgalloc_tag_swap(struct folio *new, struct folio *old) -{ - union pgtag_ref_handle handle_old, handle_new; - union codetag_ref ref_old, ref_new; - struct alloc_tag *tag_old, *tag_new; - - if (!mem_alloc_profiling_enabled()) - return; - - tag_old = __pgalloc_tag_get(&old->page); - if (!tag_old) - return; - tag_new = __pgalloc_tag_get(&new->page); - if (!tag_new) - return; - - if (!get_page_tag_ref(&old->page, &ref_old, &handle_old)) - return; - if (!get_page_tag_ref(&new->page, &ref_new, &handle_new)) { - put_page_tag_ref(handle_old); - return; - } - - /* - * Clear tag references to avoid debug warning when using - * __alloc_tag_ref_set() with non-empty reference. - */ - set_codetag_empty(&ref_old); - set_codetag_empty(&ref_new); - - /* swap tags */ - __alloc_tag_ref_set(&ref_old, tag_new); - update_page_tag_ref(handle_old, &ref_old); - __alloc_tag_ref_set(&ref_new, tag_old); - update_page_tag_ref(handle_new, &ref_new); - - put_page_tag_ref(handle_old); - put_page_tag_ref(handle_new); -} - -static void shutdown_mem_profiling(bool remove_file) -{ - if (mem_alloc_profiling_enabled()) - static_branch_disable(&mem_alloc_profiling_key); - - if (!mem_profiling_support) - return; - - if (remove_file) - remove_proc_entry(ALLOCINFO_FILE_NAME, NULL); - mem_profiling_support = false; -} - -void __init alloc_tag_sec_init(void) -{ - struct alloc_tag *last_codetag; - - if (!mem_profiling_support) - return; - - if (!static_key_enabled(&mem_profiling_compressed)) - return; - - kernel_tags.first_tag = (struct alloc_tag *)kallsyms_lookup_name( - SECTION_START(ALLOC_TAG_SECTION_NAME)); - last_codetag = (struct alloc_tag *)kallsyms_lookup_name( - SECTION_STOP(ALLOC_TAG_SECTION_NAME)); - kernel_tags.count = last_codetag - kernel_tags.first_tag; - - /* Check if kernel tags fit into page flags */ - if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) { - shutdown_mem_profiling(false); /* allocinfo file does not exist yet */ - pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n", - kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS); - return; - } - - alloc_tag_ref_offs = (LRU_REFS_PGOFF - NR_UNUSED_PAGEFLAG_BITS); - alloc_tag_ref_mask = ((1UL << NR_UNUSED_PAGEFLAG_BITS) - 1); - pr_debug("Memory allocation profiling compression is using %d page flag bits!\n", - NR_UNUSED_PAGEFLAG_BITS); -} - -#ifdef CONFIG_MODULES - -static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE); -static struct vm_struct *vm_module_tags; -/* A dummy object used to indicate an unloaded module */ -static struct module unloaded_mod; -/* A dummy object used to indicate a module prepended area */ -static struct module prepend_mod; - -struct alloc_tag_module_section module_tags; - -static inline unsigned long alloc_tag_align(unsigned long val) -{ - if (!static_key_enabled(&mem_profiling_compressed)) { - /* No alignment requirements when we are not indexing the tags */ - return val; - } - - if (val % sizeof(struct alloc_tag) == 0) - return val; - return ((val / sizeof(struct alloc_tag)) + 1) * sizeof(struct alloc_tag); -} - -static bool ensure_alignment(unsigned long align, unsigned int *prepend) -{ - if (!static_key_enabled(&mem_profiling_compressed)) { - /* No alignment requirements when we are not indexing the tags */ - return true; - } - - /* - * If alloc_tag size is not a multiple of required alignment, tag - * indexing does not work. - */ - if (!IS_ALIGNED(sizeof(struct alloc_tag), align)) - return false; - - /* Ensure prepend consumes multiple of alloc_tag-sized blocks */ - if (*prepend) - *prepend = alloc_tag_align(*prepend); - - return true; -} - -static inline bool tags_addressable(void) -{ - unsigned long tag_idx_count; - - if (!static_key_enabled(&mem_profiling_compressed)) - return true; /* with page_ext tags are always addressable */ - - tag_idx_count = CODETAG_ID_FIRST + kernel_tags.count + - module_tags.size / sizeof(struct alloc_tag); - - return tag_idx_count < (1UL << NR_UNUSED_PAGEFLAG_BITS); -} - -static bool needs_section_mem(struct module *mod, unsigned long size) -{ - if (!mem_profiling_support) - return false; - - return size >= sizeof(struct alloc_tag); -} - -static bool clean_unused_counters(struct alloc_tag *start_tag, - struct alloc_tag *end_tag) -{ - struct alloc_tag *tag; - bool ret = true; - - for (tag = start_tag; tag <= end_tag; tag++) { - struct alloc_tag_counters counter; - - if (!tag->counters) - continue; - - counter = alloc_tag_read(tag); - if (!counter.bytes) { - free_percpu(tag->counters); - tag->counters = NULL; - } else { - ret = false; - } - } - - return ret; -} - -/* Called with mod_area_mt locked */ -static void clean_unused_module_areas_locked(void) -{ - MA_STATE(mas, &mod_area_mt, 0, module_tags.size); - struct module *val; - - mas_for_each(&mas, val, module_tags.size) { - struct alloc_tag *start_tag; - struct alloc_tag *end_tag; - - if (val != &unloaded_mod) - continue; - - /* Release area if all tags are unused */ - start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index); - end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last); - if (clean_unused_counters(start_tag, end_tag)) - mas_erase(&mas); - } -} - -/* Called with mod_area_mt locked */ -static bool find_aligned_area(struct ma_state *mas, unsigned long section_size, - unsigned long size, unsigned int prepend, unsigned long align) -{ - bool cleanup_done = false; - -repeat: - /* Try finding exact size and hope the start is aligned */ - if (!mas_empty_area(mas, 0, section_size - 1, prepend + size)) { - if (IS_ALIGNED(mas->index + prepend, align)) - return true; - - /* Try finding larger area to align later */ - mas_reset(mas); - if (!mas_empty_area(mas, 0, section_size - 1, - size + prepend + align - 1)) - return true; - } - - /* No free area, try cleanup stale data and repeat the search once */ - if (!cleanup_done) { - clean_unused_module_areas_locked(); - cleanup_done = true; - mas_reset(mas); - goto repeat; - } - - return false; -} - -static int vm_module_tags_populate(void) -{ - unsigned long phys_end = ALIGN_DOWN(module_tags.start_addr, PAGE_SIZE) + - (vm_module_tags->nr_pages << PAGE_SHIFT); - unsigned long new_end = module_tags.start_addr + module_tags.size; - - if (phys_end < new_end) { - struct page **next_page = vm_module_tags->pages + vm_module_tags->nr_pages; - unsigned long old_shadow_end = ALIGN(phys_end, MODULE_ALIGN); - unsigned long new_shadow_end = ALIGN(new_end, MODULE_ALIGN); - unsigned long more_pages; - unsigned long nr = 0; - - more_pages = ALIGN(new_end - phys_end, PAGE_SIZE) >> PAGE_SHIFT; - while (nr < more_pages) { - unsigned long allocated; - - allocated = alloc_pages_bulk_node(GFP_KERNEL | __GFP_NOWARN, - NUMA_NO_NODE, more_pages - nr, next_page + nr); - - if (!allocated) - break; - nr += allocated; - } - - if (nr < more_pages || - vmap_pages_range(phys_end, phys_end + (nr << PAGE_SHIFT), PAGE_KERNEL, - next_page, PAGE_SHIFT) < 0) { - release_pages_arg arg = { .pages = next_page }; - - /* Clean up and error out */ - release_pages(arg, nr); - return -ENOMEM; - } - - vm_module_tags->nr_pages += nr; - - /* - * Kasan allocates 1 byte of shadow for every 8 bytes of data. - * When kasan_alloc_module_shadow allocates shadow memory, - * its unit of allocation is a page. - * Therefore, here we need to align to MODULE_ALIGN. - */ - if (old_shadow_end < new_shadow_end) - kasan_alloc_module_shadow((void *)old_shadow_end, - new_shadow_end - old_shadow_end, - GFP_KERNEL); - } - - /* - * Mark the pages as accessible, now that they are mapped. - * With hardware tag-based KASAN, marking is skipped for - * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). - */ - kasan_unpoison_vmalloc((void *)module_tags.start_addr, - new_end - module_tags.start_addr, - KASAN_VMALLOC_PROT_NORMAL); - - return 0; -} - -static void *reserve_module_tags(struct module *mod, unsigned long size, - unsigned int prepend, unsigned long align) -{ - unsigned long section_size = module_tags.end_addr - module_tags.start_addr; - MA_STATE(mas, &mod_area_mt, 0, section_size - 1); - unsigned long offset; - void *ret = NULL; - - /* If no tags return error */ - if (size < sizeof(struct alloc_tag)) - return ERR_PTR(-EINVAL); - - /* - * align is always power of 2, so we can use IS_ALIGNED and ALIGN. - * align 0 or 1 means no alignment, to simplify set to 1. - */ - if (!align) - align = 1; - - if (!ensure_alignment(align, &prepend)) { - shutdown_mem_profiling(true); - pr_err("%s: alignment %lu is incompatible with allocation tag indexing. Memory allocation profiling is disabled!\n", - mod->name, align); - return ERR_PTR(-EINVAL); - } - - mas_lock(&mas); - if (!find_aligned_area(&mas, section_size, size, prepend, align)) { - ret = ERR_PTR(-ENOMEM); - goto unlock; - } - - /* Mark found area as reserved */ - offset = mas.index; - offset += prepend; - offset = ALIGN(offset, align); - if (offset != mas.index) { - unsigned long pad_start = mas.index; - - mas.last = offset - 1; - mas_store(&mas, &prepend_mod); - if (mas_is_err(&mas)) { - ret = ERR_PTR(xa_err(mas.node)); - goto unlock; - } - mas.index = offset; - mas.last = offset + size - 1; - mas_store(&mas, mod); - if (mas_is_err(&mas)) { - mas.index = pad_start; - mas_erase(&mas); - ret = ERR_PTR(xa_err(mas.node)); - } - } else { - mas.last = offset + size - 1; - mas_store(&mas, mod); - if (mas_is_err(&mas)) - ret = ERR_PTR(xa_err(mas.node)); - } -unlock: - mas_unlock(&mas); - - if (IS_ERR(ret)) - return ret; - - if (module_tags.size < offset + size) { - int grow_res; - - module_tags.size = offset + size; - if (mem_alloc_profiling_enabled() && !tags_addressable()) { - shutdown_mem_profiling(true); - pr_warn("With module %s there are too many tags to fit in %d page flag bits. Memory allocation profiling is disabled!\n", - mod->name, NR_UNUSED_PAGEFLAG_BITS); - } - - grow_res = vm_module_tags_populate(); - if (grow_res) { - shutdown_mem_profiling(true); - pr_err("Failed to allocate memory for allocation tags in the module %s. Memory allocation profiling is disabled!\n", - mod->name); - return ERR_PTR(grow_res); - } - } - - return (struct alloc_tag *)(module_tags.start_addr + offset); -} - -static void release_module_tags(struct module *mod, bool used) -{ - MA_STATE(mas, &mod_area_mt, module_tags.size, module_tags.size); - struct alloc_tag *start_tag; - struct alloc_tag *end_tag; - struct module *val; - - mas_lock(&mas); - mas_for_each_rev(&mas, val, 0) - if (val == mod) - break; - - if (!val) /* module not found */ - goto out; - - if (!used) - goto release_area; - - start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index); - end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last); - if (!clean_unused_counters(start_tag, end_tag)) { - struct alloc_tag *tag; - - for (tag = start_tag; tag <= end_tag; tag++) { - struct alloc_tag_counters counter; - - if (!tag->counters) - continue; - - counter = alloc_tag_read(tag); - pr_info("%s:%u module %s func:%s has %llu allocated at module unload\n", - tag->ct.filename, tag->ct.lineno, tag->ct.modname, - tag->ct.function, counter.bytes); - } - } else { - used = false; - } -release_area: - mas_store(&mas, used ? &unloaded_mod : NULL); - val = mas_prev_range(&mas, 0); - if (val == &prepend_mod) - mas_store(&mas, NULL); -out: - mas_unlock(&mas); -} - -static int load_module(struct module *mod, struct codetag *start, struct codetag *stop) -{ - /* Allocate module alloc_tag percpu counters */ - struct alloc_tag *start_tag; - struct alloc_tag *stop_tag; - struct alloc_tag *tag; - - /* percpu counters for core allocations are already statically allocated */ - if (!mod) - return 0; - - start_tag = ct_to_alloc_tag(start); - stop_tag = ct_to_alloc_tag(stop); - for (tag = start_tag; tag < stop_tag; tag++) { - WARN_ON(tag->counters); - tag->counters = alloc_percpu(struct alloc_tag_counters); - if (!tag->counters) { - while (--tag >= start_tag) { - free_percpu(tag->counters); - tag->counters = NULL; - } - pr_err("Failed to allocate memory for allocation tag percpu counters in the module %s\n", - mod->name); - return -ENOMEM; - } - - /* - * Avoid a kmemleak false positive. The pointer to the counters is stored - * in the alloc_tag section of the module and cannot be directly accessed. - */ - kmemleak_ignore_percpu(tag->counters); - } - return 0; -} - -static void replace_module(struct module *mod, struct module *new_mod) -{ - MA_STATE(mas, &mod_area_mt, 0, module_tags.size); - struct module *val; - - mas_lock(&mas); - mas_for_each(&mas, val, module_tags.size) { - if (val != mod) - continue; - - mas_store_gfp(&mas, new_mod, GFP_KERNEL); - break; - } - mas_unlock(&mas); -} - -static int __init alloc_mod_tags_mem(void) -{ - /* Map space to copy allocation tags */ - vm_module_tags = execmem_vmap(MODULE_ALLOC_TAG_VMAP_SIZE); - if (!vm_module_tags) { - pr_err("Failed to map %lu bytes for module allocation tags\n", - MODULE_ALLOC_TAG_VMAP_SIZE); - module_tags.start_addr = 0; - return -ENOMEM; - } - - vm_module_tags->pages = kmalloc_objs(struct page *, - get_vm_area_size(vm_module_tags) >> PAGE_SHIFT, - GFP_KERNEL | __GFP_ZERO); - if (!vm_module_tags->pages) { - free_vm_area(vm_module_tags); - return -ENOMEM; - } - - module_tags.start_addr = (unsigned long)vm_module_tags->addr; - module_tags.end_addr = module_tags.start_addr + MODULE_ALLOC_TAG_VMAP_SIZE; - /* Ensure the base is alloc_tag aligned when required for indexing */ - module_tags.start_addr = alloc_tag_align(module_tags.start_addr); - - return 0; -} - -static void __init free_mod_tags_mem(void) -{ - release_pages_arg arg = { .pages = vm_module_tags->pages }; - - module_tags.start_addr = 0; - release_pages(arg, vm_module_tags->nr_pages); - kfree(vm_module_tags->pages); - free_vm_area(vm_module_tags); -} - -#else /* CONFIG_MODULES */ - -static inline int alloc_mod_tags_mem(void) { return 0; } -static inline void free_mod_tags_mem(void) {} - -#endif /* CONFIG_MODULES */ - -/* See: Documentation/mm/allocation-profiling.rst */ -static int __init setup_early_mem_profiling(char *str) -{ - bool compressed = false; - bool enable; - - if (!str || !str[0]) - return -EINVAL; - - if (!strncmp(str, "never", 5)) { - enable = false; - mem_profiling_support = false; - pr_info("Memory allocation profiling is disabled!\n"); - } else { - char *token = strsep(&str, ","); - - if (kstrtobool(token, &enable)) - return -EINVAL; - - if (str) { - - if (strcmp(str, "compressed")) - return -EINVAL; - - compressed = true; - } - mem_profiling_support = true; - pr_info("Memory allocation profiling is enabled %s compression and is turned %s!\n", - compressed ? "with" : "without", str_on_off(enable)); - } - - if (enable != mem_alloc_profiling_enabled()) { - if (enable) - static_branch_enable(&mem_alloc_profiling_key); - else - static_branch_disable(&mem_alloc_profiling_key); - } - if (compressed != static_key_enabled(&mem_profiling_compressed)) { - if (compressed) - static_branch_enable(&mem_profiling_compressed); - else - static_branch_disable(&mem_profiling_compressed); - } - - return 0; -} -early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling); - -static __init bool need_page_alloc_tagging(void) -{ - if (static_key_enabled(&mem_profiling_compressed)) - return false; - - return mem_profiling_support; -} - -static __init void init_page_alloc_tagging(void) -{ -} - -struct page_ext_operations page_alloc_tagging_ops = { - .size = sizeof(union codetag_ref), - .need = need_page_alloc_tagging, - .init = init_page_alloc_tagging, -}; -EXPORT_SYMBOL(page_alloc_tagging_ops); - -#ifdef CONFIG_SYSCTL -/* - * Not using proc_do_static_key() directly to prevent enabling profiling - * after it was shut down. - */ -static int proc_mem_profiling_handler(const struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) -{ - if (write) { - /* - * Call from do_sysctl_args() which is a no-op since the same - * value was already set by setup_early_mem_profiling. - * Return success to avoid warnings from do_sysctl_args(). - */ - if (!current->mm) - return 0; - -#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG - /* User can't toggle profiling while debugging */ - return -EACCES; -#endif - if (!mem_profiling_support) - return -EINVAL; - } - - return proc_do_static_key(table, write, buffer, lenp, ppos); -} - - -static const struct ctl_table memory_allocation_profiling_sysctls[] = { - { - .procname = "mem_profiling", - .data = &mem_alloc_profiling_key, - .mode = 0644, - .proc_handler = proc_mem_profiling_handler, - }, -}; - -static void __init sysctl_init(void) -{ - register_sysctl_init("vm", memory_allocation_profiling_sysctls); -} -#else /* CONFIG_SYSCTL */ -static inline void sysctl_init(void) {} -#endif /* CONFIG_SYSCTL */ - -static int __init alloc_tag_init(void) -{ - const struct codetag_type_desc desc = { - .section = ALLOC_TAG_SECTION_NAME, - .tag_size = sizeof(struct alloc_tag), -#ifdef CONFIG_MODULES - .needs_section_mem = needs_section_mem, - .alloc_section_mem = reserve_module_tags, - .free_section_mem = release_module_tags, - .module_load = load_module, - .module_replaced = replace_module, -#endif - }; - int res; - - sysctl_init(); - - if (!mem_profiling_support) { - pr_info("Memory allocation profiling is not supported!\n"); - return 0; - } - - if (!proc_create_seq_private(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_seq_op, - sizeof(struct allocinfo_private), NULL)) { - pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME); - shutdown_mem_profiling(false); - return -ENOMEM; - } - - res = alloc_mod_tags_mem(); - if (res) { - pr_err("Failed to reserve address space for module tags, errno = %d\n", res); - shutdown_mem_profiling(true); - return res; - } - - alloc_tag_cttype = codetag_register_type(&desc); - if (IS_ERR(alloc_tag_cttype)) { - pr_err("Allocation tags registration failed, errno = %pe\n", alloc_tag_cttype); - free_mod_tags_mem(); - shutdown_mem_profiling(true); - return PTR_ERR(alloc_tag_cttype); - } - - return 0; -} -module_init(alloc_tag_init); diff --git a/lib/assoc_array.c b/lib/assoc_array.c index bcc6e0a013eb..b6c9723e12ce 100644 --- a/lib/assoc_array.c +++ b/lib/assoc_array.c @@ -255,7 +255,8 @@ follow_shortcut: sc_segments = shortcut->index_key[sc_level >> ASSOC_ARRAY_KEY_CHUNK_SHIFT]; dissimilarity = segments ^ sc_segments; - if (round_up(sc_level, ASSOC_ARRAY_KEY_CHUNK_SIZE) > shortcut->skip_to_level) { + if (shortcut->skip_to_level < round_down(sc_level, + ASSOC_ARRAY_KEY_CHUNK_SIZE) + ASSOC_ARRAY_KEY_CHUNK_SIZE) { /* Trim segments that are beyond the shortcut */ int shift = shortcut->skip_to_level & ASSOC_ARRAY_KEY_CHUNK_MASK; dissimilarity &= ~(ULONG_MAX << shift); diff --git a/lib/audit.c b/lib/audit.c index 738bda22dd39..bc07fbd3a698 100644 --- a/lib/audit.c +++ b/lib/audit.c @@ -42,7 +42,7 @@ int audit_classify_syscall(int abi, unsigned syscall) if (audit_is_compat(abi)) return audit_classify_compat_syscall(abi, syscall); - switch(syscall) { + switch (syscall) { #ifdef __NR_open case __NR_open: return AUDITSC_OPEN; diff --git a/lib/base64.c b/lib/base64.c index 41961a444028..325c7332b049 100644 --- a/lib/base64.c +++ b/lib/base64.c @@ -122,7 +122,7 @@ EXPORT_SYMBOL_GPL(base64_encode); * @src: the string to decode. Doesn't need to be NUL-terminated. * @srclen: the length of @src in bytes * @dst: (output) the decoded binary data - * @padding: whether to append '=' padding characters + * @padding: whether the input is expected to include '=' padding characters * @variant: which base64 variant to use * * Decodes a string using the selected Base64 variant. @@ -168,15 +168,16 @@ int base64_decode(const char *src, int srclen, u8 *dst, bool padding, enum base6 return -1; val = (base64_rev_tables[s[0]] << 12) | (base64_rev_tables[s[1]] << 6); - *bp++ = val >> 10; if (srclen == 2) { if (val & 0x800003ff) return -1; + *bp++ = val >> 10; } else { val |= base64_rev_tables[s[2]]; if (val & 0x80000003) return -1; + *bp++ = val >> 10; *bp++ = val >> 2; } return bp - dst; diff --git a/lib/bch.c b/lib/bch.c index 9561c0828802..c991c71c4cbd 100644 --- a/lib/bch.c +++ b/lib/bch.c @@ -392,7 +392,7 @@ static void compute_syndromes(struct bch_control *bch, uint32_t *ecc, for (j = 0; j < 2*t; j += 2) syn[j] ^= a_pow(bch, (j+1)*(i+s)); - poly ^= (1 << i); + poly ^= (1u << i); } } while (s > 0); @@ -612,7 +612,7 @@ static int find_poly_deg2_roots(struct bch_control *bch, struct gf_poly *poly, while (v) { i = deg(v); r ^= bch->xi_tab[i]; - v ^= (1 << i); + v ^= (1u << i); } /* verify root */ if ((gf_sqr(bch, r)^r) == u) { @@ -1116,7 +1116,7 @@ static void build_mod8_tables(struct bch_control *bch, const uint32_t *g) for (b = 0; b < 4; b++) { /* we want to compute (p(X).X^(8*b+deg(g))) mod g(X) */ tab = bch->mod8_tab + (b*256+i)*l; - data = i << (8*b); + data = (unsigned int)i << (8*b); while (data) { d = deg(data); /* subtract X^d.g(X) from p(X).X^(8*b+deg(g)) */ diff --git a/lib/bitmap-str.c b/lib/bitmap-str.c index be745209507a..dd9aa0635fa5 100644 --- a/lib/bitmap-str.c +++ b/lib/bitmap-str.c @@ -40,32 +40,6 @@ int bitmap_parse_user(const char __user *ubuf, EXPORT_SYMBOL(bitmap_parse_user); /** - * bitmap_print_to_pagebuf - convert bitmap to list or hex format ASCII string - * @list: indicates whether the bitmap must be list - * @buf: page aligned buffer into which string is placed - * @maskp: pointer to bitmap to convert - * @nmaskbits: size of bitmap, in bits - * - * Output format is a comma-separated list of decimal numbers and - * ranges if list is specified or hex digits grouped into comma-separated - * sets of 8 digits/set. Returns the number of characters written to buf. - * - * It is assumed that @buf is a pointer into a PAGE_SIZE, page-aligned - * area and that sufficient storage remains at @buf to accommodate the - * bitmap_print_to_pagebuf() output. Returns the number of characters - * actually printed to @buf, excluding terminating '\0'. - */ -int bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, - int nmaskbits) -{ - ptrdiff_t len = PAGE_SIZE - offset_in_page(buf); - - return list ? scnprintf(buf, len, "%*pbl\n", nmaskbits, maskp) : - scnprintf(buf, len, "%*pb\n", nmaskbits, maskp); -} -EXPORT_SYMBOL(bitmap_print_to_pagebuf); - -/** * bitmap_print_to_buf - convert bitmap to list or hex format ASCII string * @list: indicates whether the bitmap must be list * true: print in decimal list format @@ -101,8 +75,7 @@ static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp, * @off: in the string from which we are copying, We copy to @buf * @count: the maximum number of bytes to print * - * The bitmap_print_to_pagebuf() is used indirectly via its cpumap wrapper - * cpumap_print_to_pagebuf() or directly by drivers to export hexadecimal + * The sprintf("%*pb[l]") format is used by drivers to export hexadecimal * bitmask and decimal list to userspace by sysfs ABI. * Drivers might be using a normal attribute for this kind of ABIs. A * normal attribute typically has show entry as below:: @@ -111,18 +84,11 @@ static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp, * struct device_attribute *attr, char *buf) * { * ... - * return bitmap_print_to_pagebuf(true, buf, &mask, nr_trig_max); + * return scnprintf(buf, PAGE_SIZE - offset_in_page(buf), nr_trig_max, &mask); * } * * show entry of attribute has no offset and count parameters and this * means the file is limited to one page only. - * bitmap_print_to_pagebuf() API works terribly well for this kind of - * normal attribute with buf parameter and without offset, count:: - * - * bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, - * int nmaskbits) - * { - * } * * The problem is once we have a large bitmap, we have a chance to get a * bitmask or list more than one page. Especially for list, it could be @@ -148,9 +114,9 @@ static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp, * parameters such as off, count from bin_attribute show entry to this API. * * The role of cpumap_print_bitmask_to_buf() and cpumap_print_list_to_buf() - * is similar with cpumap_print_to_pagebuf(), the difference is that - * bitmap_print_to_pagebuf() mainly serves sysfs attribute with the assumption - * the destination buffer is exactly one page and won't be more than one page. + * is similar to direct sysfs_emit("%*pb[l]") formatting, but the latter + * assumes the destination buffer is exactly one page and won't be more than + * one page. * cpumap_print_bitmask_to_buf() and cpumap_print_list_to_buf(), on the other * hand, mainly serves bin_attribute which doesn't work with exact one page, * and it can break the size limit of converted decimal list and hexadecimal @@ -158,7 +124,8 @@ static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp, * * WARNING! * - * This function is not a replacement for sprintf() or bitmap_print_to_pagebuf(). + * This function is not a replacement for sprintf(). + * * It is intended to workaround sysfs limitations discussed above and should be * used carefully in general case for the following reasons: * diff --git a/lib/bitmap.c b/lib/bitmap.c index 9dc526507875..ed685127a107 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -69,6 +69,7 @@ bool __bitmap_or_equal(const unsigned long *bitmap1, tmp = (bitmap1[k] | bitmap2[k]) ^ bitmap3[k]; return (tmp & BITMAP_LAST_WORD_MASK(bits)) == 0; } +EXPORT_SYMBOL(__bitmap_or_equal); void __bitmap_complement(unsigned long *dst, const unsigned long *src, unsigned int bits) { @@ -360,6 +361,14 @@ unsigned int __bitmap_weighted_or(unsigned long *dst, const unsigned long *bitma { return BITMAP_WEIGHT(({dst[idx] = bitmap1[idx] | bitmap2[idx]; dst[idx]; }), bits); } +EXPORT_SYMBOL(__bitmap_weighted_or); + +unsigned int __bitmap_weighted_xor(unsigned long *dst, const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int bits) +{ + return BITMAP_WEIGHT(({dst[idx] = bitmap1[idx] ^ bitmap2[idx]; dst[idx]; }), bits); +} +EXPORT_SYMBOL(__bitmap_weighted_xor); void __bitmap_set(unsigned long *map, unsigned int start, int len) { @@ -415,6 +424,9 @@ EXPORT_SYMBOL(__bitmap_clear); * The @align_mask should be one less than a power of 2; the effect is that * the bit offset of all zero areas this function finds plus @align_offset * is multiple of that power of 2. + * + * Return: The bit offset of the found area or a value greater than or equal + * to @size if no area is found. */ unsigned long bitmap_find_next_zero_area_off(unsigned long *map, unsigned long size, @@ -423,22 +435,23 @@ unsigned long bitmap_find_next_zero_area_off(unsigned long *map, unsigned long align_mask, unsigned long align_offset) { - unsigned long index, end, i; -again: - index = find_next_zero_bit(map, size, start); + unsigned long end, i, off; + + for_each_clear_bit_from(start, map, size) { + start = __ALIGN_MASK(start + align_offset, align_mask) - align_offset; + end = start + nr; + if (end > size) + break; - /* Align allocation */ - index = __ALIGN_MASK(index + align_offset, align_mask) - align_offset; + off = round_down(start, BITS_PER_LONG); + i = find_last_bit(map + start / BITS_PER_LONG, end - off) + off; + if (i >= end || i < start) + return start; - end = index + nr; - if (end > size) - return end; - i = find_next_bit(map, end, index); - if (i < end) { - start = i + 1; - goto again; + start = i; } - return index; + + return size; } EXPORT_SYMBOL(bitmap_find_next_zero_area_off); diff --git a/lib/bitrev.c b/lib/bitrev.c index 81b56e0a7f32..05088231f31f 100644 --- a/lib/bitrev.c +++ b/lib/bitrev.c @@ -1,5 +1,4 @@ // SPDX-License-Identifier: GPL-2.0-only -#ifndef CONFIG_HAVE_ARCH_BITREVERSE #include <linux/types.h> #include <linux/module.h> #include <linux/bitrev.h> @@ -43,5 +42,3 @@ const u8 byte_rev_table[256] = { 0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff, }; EXPORT_SYMBOL_GPL(byte_rev_table); - -#endif /* CONFIG_HAVE_ARCH_BITREVERSE */ diff --git a/lib/bootconfig.c b/lib/bootconfig.c index e88d0221a826..89c88e359179 100644 --- a/lib/bootconfig.c +++ b/lib/bootconfig.c @@ -17,9 +17,15 @@ #include <linux/bug.h> #include <linux/ctype.h> #include <linux/errno.h> -#include <linux/kernel.h> +#include <linux/cache.h> +#include <linux/compiler.h> +#include <linux/init.h> +#include <linux/moduleparam.h> +#include <linux/printk.h> +#include <linux/sprintf.h> #include <linux/memblock.h> #include <linux/string.h> +#include <asm/setup.h> /* COMMAND_LINE_SIZE */ #ifdef CONFIG_BOOT_CONFIG_EMBED /* embedded_bootconfig_data is defined in bootconfig-data.S */ @@ -32,7 +38,129 @@ const char * __init xbc_get_embedded_bootconfig(size_t *size) return (*size) ? embedded_bootconfig_data : NULL; } #endif -#endif + +#ifdef CONFIG_CMDLINE_FROM_BOOTCONFIG +/* embedded_kernel_cmdline is defined in embedded-cmdline.S */ +extern __visible const char embedded_kernel_cmdline[]; +extern __visible const char embedded_kernel_cmdline_end[]; + +/* Set once the embedded cmdline has actually been prepended. */ +static bool xbc_cmdline_applied __initdata; + +/* + * str_prepend() - Prepend @src in front of the string in @dst, in place + * @dst: NUL-terminated destination buffer, currently @dst_len bytes long + * @dst_len: length of the current @dst string (excluding its NUL) + * @src: bytes to prepend (not NUL-terminated) + * @src_len: number of bytes from @src to prepend + * + * The caller must guarantee @dst has room for src_len + dst_len + 1 bytes. + * Moving dst_len + 1 bytes carries @dst's NUL terminator too, so an empty + * @dst needs no special case. + */ +static void __init str_prepend(char *dst, size_t dst_len, + const char *src, size_t src_len) +{ + memmove(dst + src_len, dst, dst_len + 1); + memcpy(dst, src, src_len); +} + +/** + * xbc_prepend_embedded_cmdline() - Prepend embedded bootconfig cmdline + * @dst: cmdline buffer to prepend into (must already contain a NUL byte) + * @size: total capacity of @dst in bytes + * + * Prepend the build-time-rendered "kernel" subtree of the embedded + * bootconfig to @dst. The rendered string already ends with a single + * space (the xbc_snprint_cmdline() invariant), which serves as the + * separator between the embedded keys and any existing content of @dst. + * On overflow, log an error and leave @dst untouched rather than + * silently truncating: booting without the embedded values is better + * than refusing to boot, and the error message tells the user why + * their embedded keys are missing. + * + * Intended to be called from setup_arch() before parse_early_param() so + * that early_param() handlers see the embedded values. + */ +void __init xbc_prepend_embedded_cmdline(char *dst, size_t size) +{ + size_t embed_len = embedded_kernel_cmdline_end - embedded_kernel_cmdline; + size_t dst_len; + + if (!size || embed_len <= 1) /* trailing NUL only */ + return; + embed_len--; /* exclude trailing NUL byte */ + + dst_len = strnlen(dst, size); + if (embed_len + dst_len + 1 > size) { + pr_err("embedded bootconfig cmdline (%zu bytes) does not fit in COMMAND_LINE_SIZE with %zu bytes already used; ignoring embedded values\n", + embed_len, dst_len); + return; + } + + str_prepend(dst, dst_len, embedded_kernel_cmdline, embed_len); + xbc_cmdline_applied = true; +} + +/** + * xbc_embedded_cmdline_applied() - Did the embedded cmdline get prepended? + * + * Return true if xbc_prepend_embedded_cmdline() actually prepended the + * embedded "kernel" subtree. setup_boot_config() uses this to avoid + * rendering the same keys a second time. + */ +bool __init xbc_embedded_cmdline_applied(void) +{ + return xbc_cmdline_applied; +} +#endif /* CONFIG_CMDLINE_FROM_BOOTCONFIG */ + +/* parse_args() callback: flag when the "bootconfig" parameter is present. */ +static int __init bootconfig_optin(char *param, char *val, + const char *unused, void *arg) +{ + if (!strcmp(param, "bootconfig")) + *(bool *)arg = true; + return 0; +} + +/** + * bootconfig_cmdline_requested() - Was "bootconfig" passed on the cmdline? + * @boot_cmdline: kernel command line to inspect (not modified) + * @end_offset: if non-NULL, set to the offset of the init arguments that + * follow a "--" separator, or 0 when there is none + * + * Parse a private copy of @boot_cmdline (parse_args() is destructive) and + * report whether "bootconfig" is present before the "--" separator. + * setup_arch() uses this to gate prepending the build-time embedded cmdline; + * setup_boot_config() uses it for the runtime opt-in and to locate the init + * arguments via @end_offset. Sharing one parser keeps the early and late + * paths agreeing on what counts as opt-in. CONFIG_BOOT_CONFIG_FORCE is not + * folded in here; callers apply it where they need it. + */ +bool __init bootconfig_cmdline_requested(const char *boot_cmdline, int *end_offset) +{ + static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata; + bool found = false; + char *err; + + if (end_offset) + *end_offset = 0; + + strscpy(tmp_cmdline, boot_cmdline, COMMAND_LINE_SIZE); + err = parse_args("bootconfig", tmp_cmdline, NULL, 0, 0, 0, + &found, bootconfig_optin); + if (IS_ERR(err)) + return false; + + /* parse_args() stops at "--" and returns the address of the rest. */ + if (end_offset && err) + *end_offset = err - tmp_cmdline; + + return found; +} + +#endif /* __KERNEL__ */ /* * Extra Boot Config (XBC) is given as tree-structured ascii text of @@ -64,14 +192,14 @@ static inline void __init xbc_free_mem(void *addr, size_t size, bool early) if (early) memblock_free(addr, size); else if (addr) - memblock_free_late(__pa(addr), size); + memblock_free(addr, size); } #else /* !__KERNEL__ */ static inline void *xbc_alloc_mem(size_t size) { - return malloc(size); + return calloc(1, size); } static inline void xbc_free_mem(void *addr, size_t size, bool early) @@ -79,6 +207,7 @@ static inline void xbc_free_mem(void *addr, size_t size, bool early) free(addr); } #endif + /** * xbc_get_info() - Get the information of loaded boot config * @node_size: A pointer to store the number of nodes. @@ -112,7 +241,7 @@ static int __init xbc_parse_error(const char *msg, const char *p) * xbc_root_node() - Get the root node of extended boot config * * Return the address of root node of extended boot config. If the - * extended boot config is not initiized, return NULL. + * extended boot config is not initialized, return NULL. */ struct xbc_node * __init xbc_root_node(void) { @@ -128,9 +257,9 @@ struct xbc_node * __init xbc_root_node(void) * * Return the index number of @node in XBC node list. */ -int __init xbc_node_index(struct xbc_node *node) +uint16_t __init xbc_node_index(struct xbc_node *node) { - return node - &xbc_nodes[0]; + return (uint16_t)(node - &xbc_nodes[0]); } /** @@ -180,7 +309,7 @@ struct xbc_node * __init xbc_node_get_next(struct xbc_node *node) */ const char * __init xbc_node_get_data(struct xbc_node *node) { - int offset = node->data & ~XBC_VALUE; + size_t offset = node->data & ~XBC_VALUE; if (WARN_ON(offset >= xbc_data_size)) return NULL; @@ -192,7 +321,7 @@ static bool __init xbc_node_match_prefix(struct xbc_node *node, const char **prefix) { const char *p = xbc_node_get_data(node); - int len = strlen(p); + size_t len = strlen(p); if (strncmp(*prefix, p, len)) return false; @@ -364,7 +493,7 @@ struct xbc_node * __init xbc_node_find_next_leaf(struct xbc_node *root, node = xbc_node_get_parent(node); if (node == root) return NULL; - /* User passed a node which is not uder parent */ + /* User passed a node which is not under parent */ if (WARN_ON(!node)) return NULL; } @@ -405,13 +534,89 @@ const char * __init xbc_node_find_next_key_value(struct xbc_node *root, return ""; /* No value key */ } +static char xbc_namebuf[XBC_KEYLEN_MAX] __initdata; + +#define rest(dst, end) ((end) > (dst) ? (end) - (dst) : 0) + +/** + * xbc_snprint_cmdline() - Render bootconfig keys under @root as a cmdline string + * @buf: Destination buffer (may be NULL when @size is 0 to query the length) + * @size: Size of @buf in bytes + * @root: Subtree root whose key=value pairs should be rendered + * + * Walk all key/value pairs under @root and emit them as a space-separated + * cmdline string into @buf. Values containing whitespace are quoted with + * double quotes. Returns the number of bytes that would be written if @buf + * were large enough (matching snprintf semantics), or a negative errno on + * failure. + */ +int __init xbc_snprint_cmdline(char *buf, size_t size, struct xbc_node *root) +{ + struct xbc_node *knode, *vnode; + const char *val, *q; + size_t len = 0; + int ret; + + /* + * Track the running written length rather than advancing @buf, so we + * never form "buf + size" or "buf += ret" while @buf is NULL (the + * size-probe call passes buf=NULL, size=0). NULL pointer arithmetic + * is undefined behavior and trips host UBSan / FORTIFY_SOURCE when + * this renderer runs at kernel build time. snprintf(NULL, 0, ...) + * itself is well defined and returns the would-be length. + */ + xbc_node_for_each_key_value(root, knode, val) { + /* + * An empty or value-only @root (e.g. "kernel {}" or + * "kernel = x", possibly alongside "kernel.foo = bar") + * yields @root itself here. Skip it: composing a key for it + * would fail with -EINVAL, yet any real descendant keys must + * still be rendered. An entirely empty subtree then renders + * nothing and returns 0 rather than an error. + */ + if (knode == root) + continue; + + ret = xbc_node_compose_key_after(root, knode, + xbc_namebuf, XBC_KEYLEN_MAX); + if (ret < 0) + return ret; + + vnode = xbc_node_get_child(knode); + if (!vnode) { + ret = snprintf(buf ? buf + len : NULL, rest(len, size), + "%s ", xbc_namebuf); + if (ret < 0) + return ret; + len += ret; + continue; + } + xbc_array_for_each_value(vnode, val) { + /* + * For prettier and more readable /proc/cmdline, only + * quote the value when necessary, i.e. when it contains + * whitespace. + */ + q = strpbrk(val, " \t\r\n") ? "\"" : ""; + ret = snprintf(buf ? buf + len : NULL, rest(len, size), + "%s=%s%s%s ", xbc_namebuf, q, val, q); + if (ret < 0) + return ret; + len += ret; + } + } + + return len; +} +#undef rest + /* XBC parse and tree build */ -static int __init xbc_init_node(struct xbc_node *node, char *data, uint32_t flag) +static int __init xbc_init_node(struct xbc_node *node, char *data, uint16_t flag) { - unsigned long offset = data - xbc_data; + long offset = data - xbc_data; - if (WARN_ON(offset >= XBC_DATA_MAX)) + if (WARN_ON(offset < 0 || offset >= XBC_DATA_MAX)) return -EINVAL; node->data = (uint16_t)offset | flag; @@ -421,16 +626,17 @@ static int __init xbc_init_node(struct xbc_node *node, char *data, uint32_t flag return 0; } -static struct xbc_node * __init xbc_add_node(char *data, uint32_t flag) +static struct xbc_node * __init xbc_add_node(char *data, uint16_t flag) { struct xbc_node *node; if (xbc_node_num == XBC_NODE_MAX) return NULL; - node = &xbc_nodes[xbc_node_num++]; + node = &xbc_nodes[xbc_node_num]; if (xbc_init_node(node, data, flag) < 0) return NULL; + xbc_node_num++; return node; } @@ -451,7 +657,7 @@ static inline __init struct xbc_node *xbc_last_child(struct xbc_node *node) return node; } -static struct xbc_node * __init __xbc_add_sibling(char *data, uint32_t flag, bool head) +static struct xbc_node * __init __xbc_add_sibling(char *data, uint16_t flag, bool head) { struct xbc_node *sib, *node = xbc_add_node(data, flag); @@ -472,23 +678,24 @@ static struct xbc_node * __init __xbc_add_sibling(char *data, uint32_t flag, boo sib->next = xbc_node_index(node); } } - } else + } else { xbc_parse_error("Too many nodes", data); + } return node; } -static inline struct xbc_node * __init xbc_add_sibling(char *data, uint32_t flag) +static inline struct xbc_node * __init xbc_add_sibling(char *data, uint16_t flag) { return __xbc_add_sibling(data, flag, false); } -static inline struct xbc_node * __init xbc_add_head_sibling(char *data, uint32_t flag) +static inline struct xbc_node * __init xbc_add_head_sibling(char *data, uint16_t flag) { return __xbc_add_sibling(data, flag, true); } -static inline __init struct xbc_node *xbc_add_child(char *data, uint32_t flag) +static inline __init struct xbc_node *xbc_add_child(char *data, uint16_t flag) { struct xbc_node *node = xbc_add_sibling(data, flag); @@ -655,9 +862,9 @@ static int __init __xbc_add_key(char *k) if (unlikely(xbc_node_num == 0)) goto add_node; - if (!last_parent) /* the first level */ + if (!last_parent) { /* the first level */ node = find_match_node(xbc_nodes, k); - else { + } else { child = xbc_node_get_child(last_parent); /* Since the value node is the first child, skip it. */ if (child && xbc_node_is_value(child)) @@ -665,9 +872,9 @@ static int __init __xbc_add_key(char *k) node = find_match_node(child, k); } - if (node) + if (node) { last_parent = node; - else { + } else { add_node: node = xbc_add_child(k, XBC_KEY); if (!node) @@ -798,7 +1005,8 @@ static int __init xbc_close_brace(char **k, char *n) static int __init xbc_verify_tree(void) { - int i, depth, len, wlen; + int i, depth; + size_t len, wlen; struct xbc_node *n, *m; /* Brace closing */ @@ -815,10 +1023,14 @@ static int __init xbc_verify_tree(void) } for (i = 0; i < xbc_node_num; i++) { - if (xbc_nodes[i].next > xbc_node_num) { + if (xbc_nodes[i].next >= xbc_node_num) { return xbc_parse_error("No closing brace", xbc_node_get_data(xbc_nodes + i)); } + if (xbc_nodes[i].child >= xbc_node_num) { + return xbc_parse_error("Broken child node", + xbc_node_get_data(xbc_nodes + i)); + } } /* Key tree limitation check */ @@ -980,7 +1192,6 @@ int __init xbc_init(const char *data, size_t size, const char **emsg, int *epos) _xbc_exit(true); return -ENOMEM; } - memset(xbc_nodes, 0, sizeof(struct xbc_node) * XBC_NODE_MAX); ret = xbc_parse_tree(); if (!ret) @@ -992,8 +1203,9 @@ int __init xbc_init(const char *data, size_t size, const char **emsg, int *epos) if (emsg) *emsg = xbc_err_msg; _xbc_exit(true); - } else + } else { ret = xbc_node_num; + } return ret; } diff --git a/lib/bug.c b/lib/bug.c index 623c467a8b76..7c1c2c27f58e 100644 --- a/lib/bug.c +++ b/lib/bug.c @@ -1,41 +1,41 @@ // SPDX-License-Identifier: GPL-2.0 /* - Generic support for BUG() - - This respects the following config options: - - CONFIG_BUG - emit BUG traps. Nothing happens without this. - CONFIG_GENERIC_BUG - enable this code. - CONFIG_GENERIC_BUG_RELATIVE_POINTERS - use 32-bit relative pointers for bug_addr and file - CONFIG_DEBUG_BUGVERBOSE - emit full file+line information for each BUG - - CONFIG_BUG and CONFIG_DEBUG_BUGVERBOSE are potentially user-settable - (though they're generally always on). - - CONFIG_GENERIC_BUG is set by each architecture using this code. - - To use this, your architecture must: - - 1. Set up the config options: - - Enable CONFIG_GENERIC_BUG if CONFIG_BUG - - 2. Implement BUG (and optionally BUG_ON, WARN, WARN_ON) - - Define HAVE_ARCH_BUG - - Implement BUG() to generate a faulting instruction - - NOTE: struct bug_entry does not have "file" or "line" entries - when CONFIG_DEBUG_BUGVERBOSE is not enabled, so you must generate - the values accordingly. - - 3. Implement the trap - - In the illegal instruction trap handler (typically), verify - that the fault was in kernel mode, and call report_bug() - - report_bug() will return whether it was a false alarm, a warning, - or an actual bug. - - You must implement the is_valid_bugaddr(bugaddr) callback which - returns true if the eip is a real kernel address, and it points - to the expected BUG trap instruction. - - Jeremy Fitzhardinge <jeremy@goop.org> 2006 + * Generic support for BUG() + * + * This respects the following config options: + * + * CONFIG_BUG - emit BUG traps. Nothing happens without this. + * CONFIG_GENERIC_BUG - enable this code. + * CONFIG_GENERIC_BUG_RELATIVE_POINTERS - use 32-bit relative pointers for bug_addr and file + * CONFIG_DEBUG_BUGVERBOSE - emit full file+line information for each BUG + * + * CONFIG_BUG and CONFIG_DEBUG_BUGVERBOSE are potentially user-settable + * (though they're generally always on). + * + * CONFIG_GENERIC_BUG is set by each architecture using this code. + * + * To use this, your architecture must: + * + * 1. Set up the config options: + * - Enable CONFIG_GENERIC_BUG if CONFIG_BUG + * + * 2. Implement BUG (and optionally BUG_ON, WARN, WARN_ON) + * - Define HAVE_ARCH_BUG + * - Implement BUG() to generate a faulting instruction + * - NOTE: struct bug_entry does not have "file" or "line" entries + * when CONFIG_DEBUG_BUGVERBOSE is not enabled, so you must generate + * the values accordingly. + * + * 3. Implement the trap + * - In the illegal instruction trap handler (typically), verify + * that the fault was in kernel mode, and call report_bug() + * - report_bug() will return whether it was a false alarm, a warning, + * or an actual bug. + * - You must implement the is_valid_bugaddr(bugaddr) callback which + * returns true if the eip is a real kernel address, and it points + * to the expected BUG trap instruction. + * + * Jeremy Fitzhardinge <jeremy@goop.org> 2006 */ #define pr_fmt(fmt) fmt @@ -48,6 +48,7 @@ #include <linux/rculist.h> #include <linux/ftrace.h> #include <linux/context_tracking.h> +#include <kunit/test-bug.h> extern struct bug_entry __start___bug_table[], __stop___bug_table[]; @@ -71,7 +72,7 @@ static struct bug_entry *module_find_bug(unsigned long bugaddr) guard(rcu)(); list_for_each_entry_rcu(mod, &module_bug_list, bug_list) { - unsigned i; + unsigned int i; bug = mod->bug_table; for (i = 0; i < mod->num_bugs; ++i, ++bug) @@ -173,10 +174,8 @@ struct bug_entry *find_bug(unsigned long bugaddr) return module_find_bug(bugaddr); } -__diag_push(); -__diag_ignore(GCC, all, "-Wsuggest-attribute=format", - "Not a valid __printf() conversion candidate."); -static void __warn_printf(const char *fmt, struct pt_regs *regs) +static __printf(1, 0) +void __warn_printf(const char *fmt, struct pt_regs *regs) { if (!fmt) return; @@ -193,15 +192,14 @@ static void __warn_printf(const char *fmt, struct pt_regs *regs) } #endif - printk("%s", fmt); + pr_warn("%s", fmt); } -__diag_pop(); static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long bugaddr, struct pt_regs *regs) { bool warning, once, done, no_cut, has_args; const char *file, *fmt; - unsigned line; + unsigned int line; if (!bug) { if (!is_valid_bugaddr(bugaddr)) @@ -212,8 +210,6 @@ static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long buga return BUG_TRAP_TYPE_NONE; } - disable_trace_on_warning(); - bug_get_file_line(bug, &file, &line); fmt = bug_get_format(bug); @@ -223,6 +219,15 @@ static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long buga no_cut = bug->flags & BUGFLAG_NO_CUT_HERE; has_args = bug->flags & BUGFLAG_ARGS; + /* + * Before the once logic so suppressed warnings do not consume + * the single-fire budget of WARN_ON_ONCE(). + */ + if (warning && kunit_is_suppressed_warning(true)) + return BUG_TRAP_TYPE_WARN; + + disable_trace_on_warning(); + if (warning && once) { if (done) return BUG_TRAP_TYPE_WARN; @@ -240,7 +245,7 @@ static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long buga * extra debugging message it writes before triggering the handler. */ if (!no_cut) { - printk(KERN_DEFAULT CUT_HERE); + pr_info(CUT_HERE); __warn_printf(fmt, has_args ? regs : NULL); } @@ -254,7 +259,7 @@ static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long buga if (file) pr_crit("kernel BUG at %s:%u!\n", file, line); else - pr_crit("Kernel BUG at %pB [verbose debug info unavailable]\n", + pr_crit("kernel BUG at %pB [verbose debug info unavailable]\n", (void *)bugaddr); return BUG_TRAP_TYPE_BUG; @@ -263,7 +268,7 @@ static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long buga enum bug_trap_type report_bug_entry(struct bug_entry *bug, struct pt_regs *regs) { enum bug_trap_type ret; - bool rcu = false; + bool rcu; rcu = warn_rcu_enter(); ret = __report_bug(bug, bug_addr(bug), regs); @@ -275,7 +280,7 @@ enum bug_trap_type report_bug_entry(struct bug_entry *bug, struct pt_regs *regs) enum bug_trap_type report_bug(unsigned long bugaddr, struct pt_regs *regs) { enum bug_trap_type ret; - bool rcu = false; + bool rcu; rcu = warn_rcu_enter(); ret = __report_bug(NULL, bugaddr, regs); diff --git a/lib/cmdline.c b/lib/cmdline.c index 90ed997d9570..16cce6621cec 100644 --- a/lib/cmdline.c +++ b/lib/cmdline.c @@ -43,7 +43,7 @@ static int get_range(char **str, int *pint, int n) * When @pint is NULL the function can be used as a validator of * the current option in the string. * - * Return values: + * Return: * 0 - no int in string * 1 - int found, no subsequent comma * 2 - int found including a subsequent comma @@ -145,44 +145,54 @@ EXPORT_SYMBOL(get_options); * * Parses a string into a number. The number stored at @ptr is * potentially suffixed with K, M, G, T, P, E. + * + * Return: The value as recognized by simple_strtoull() multiplied + * by the value as specified by suffix, if any. */ unsigned long long memparse(const char *ptr, char **retptr) { char *endptr; /* local pointer to end of parsed string */ - unsigned long long ret = simple_strtoull(ptr, &endptr, 0); + unsigned int shl = 0; + /* Consume valid suffix even in case of overflow. */ switch (*endptr) { case 'E': case 'e': - ret <<= 10; + shl += 10; fallthrough; case 'P': case 'p': - ret <<= 10; + shl += 10; fallthrough; case 'T': case 't': - ret <<= 10; + shl += 10; fallthrough; case 'G': case 'g': - ret <<= 10; + shl += 10; fallthrough; case 'M': case 'm': - ret <<= 10; + shl += 10; fallthrough; case 'K': case 'k': - ret <<= 10; - endptr++; + shl += 10; fallthrough; default: break; } + if (shl && likely(ptr != endptr)) { + /* Have valid suffix with preceding number. */ + if (unlikely(check_shl_overflow(ret, shl, &ret))) + ret = ULLONG_MAX; + endptr++; + } + if (retptr) *retptr = endptr; @@ -198,7 +208,7 @@ EXPORT_SYMBOL(memparse); * This function parses a string containing a comma-separated list of * strings like a=b,c. * - * Return true if there's such option in the string, or return false. + * Return: True if there's such option in the string or false otherwise. */ bool parse_option_str(const char *str, const char *option) { diff --git a/lib/codetag.c b/lib/codetag.c index 304667897ad4..a9cda4c962a3 100644 --- a/lib/codetag.c +++ b/lib/codetag.c @@ -19,6 +19,8 @@ struct codetag_type { struct codetag_type_desc desc; /* generates unique sequence number for module load */ unsigned long next_mod_seq; + /* bumped on every module load and unload */ + unsigned long content_id; }; struct codetag_range { @@ -35,12 +37,9 @@ struct codetag_module { static DEFINE_MUTEX(codetag_lock); static LIST_HEAD(codetag_types); -void codetag_lock_module_list(struct codetag_type *cttype, bool lock) +void codetag_lock_module_list(struct codetag_type *cttype) { - if (lock) - down_read(&cttype->mod_lock); - else - up_read(&cttype->mod_lock); + down_read(&cttype->mod_lock); } bool codetag_trylock_module_list(struct codetag_type *cttype) @@ -48,6 +47,25 @@ bool codetag_trylock_module_list(struct codetag_type *cttype) return down_read_trylock(&cttype->mod_lock) != 0; } +void codetag_unlock_module_list(struct codetag_type *cttype) +{ + up_read(&cttype->mod_lock); +} + +unsigned long codetag_get_content_id(struct codetag_type *cttype) +{ + lockdep_assert_held(&cttype->mod_lock); + + return cttype->content_id; +} + +unsigned int codetag_get_count(struct codetag_type *cttype) +{ + lockdep_assert_held(&cttype->mod_lock); + + return cttype->count; +} + struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype) { struct codetag_iterator iter = { @@ -202,6 +220,7 @@ static int codetag_module_init(struct codetag_type *cttype, struct module *mod) down_write(&cttype->mod_lock); cmod->mod_seq = ++cttype->next_mod_seq; + ++cttype->content_id; mod_id = idr_alloc(&cttype->mod_idr, cmod, 0, 0, GFP_KERNEL); if (mod_id >= 0) { if (cttype->desc.module_load) { @@ -366,6 +385,7 @@ void codetag_unload_module(struct module *mod) cttype->count -= range_size(cttype, &cmod->range); idr_remove(&cttype->mod_idr, mod_id); kfree(cmod); + ++cttype->content_id; } up_write(&cttype->mod_lock); if (found && cttype->desc.free_section_mem) diff --git a/lib/compat_audit.c b/lib/compat_audit.c index 3d6b8996f027..fee1dfccd116 100644 --- a/lib/compat_audit.c +++ b/lib/compat_audit.c @@ -4,32 +4,32 @@ #include <linux/audit_arch.h> #include <asm/unistd32.h> -unsigned compat_dir_class[] = { +unsigned int compat_dir_class[] = { #include <asm-generic/audit_dir_write.h> ~0U }; -unsigned compat_read_class[] = { +unsigned int compat_read_class[] = { #include <asm-generic/audit_read.h> ~0U }; -unsigned compat_write_class[] = { +unsigned int compat_write_class[] = { #include <asm-generic/audit_write.h> ~0U }; -unsigned compat_chattr_class[] = { +unsigned int compat_chattr_class[] = { #include <asm-generic/audit_change_attr.h> ~0U }; -unsigned compat_signal_class[] = { +unsigned int compat_signal_class[] = { #include <asm-generic/audit_signal.h> ~0U }; -int audit_classify_compat_syscall(int abi, unsigned syscall) +int audit_classify_compat_syscall(int abi, unsigned int syscall) { switch (syscall) { #ifdef __NR_open diff --git a/lib/crc/.kunitconfig b/lib/crc/.kunitconfig new file mode 100644 index 000000000000..0a3671ba573f --- /dev/null +++ b/lib/crc/.kunitconfig @@ -0,0 +1,3 @@ +CONFIG_KUNIT=y +CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=y +CONFIG_CRC_KUNIT_TEST=y diff --git a/lib/crc/Kconfig b/lib/crc/Kconfig index 70e7a6016de3..927fc6a6b2b9 100644 --- a/lib/crc/Kconfig +++ b/lib/crc/Kconfig @@ -48,7 +48,7 @@ config CRC_T10DIF_ARCH bool depends on CRC_T10DIF && CRC_OPTIMIZATIONS default y if ARM && KERNEL_MODE_NEON - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if PPC64 && ALTIVEC default y if RISCV && RISCV_ISA_ZBC default y if X86 @@ -65,7 +65,7 @@ config CRC32_ARCH depends on CRC32 && CRC_OPTIMIZATIONS default y if ARM && KERNEL_MODE_NEON default y if ARM64 - default y if LOONGARCH + default y if LOONGARCH && 64BIT default y if MIPS && CPU_MIPSR6 default y if PPC64 && ALTIVEC default y if RISCV && RISCV_ISA_ZBC @@ -82,6 +82,8 @@ config CRC64 config CRC64_ARCH bool depends on CRC64 && CRC_OPTIMIZATIONS + default y if ARM && KERNEL_MODE_NEON && !CPU_BIG_ENDIAN + default y if ARM64 default y if RISCV && RISCV_ISA_ZBC && 64BIT default y if X86_64 @@ -99,18 +101,27 @@ config CRC_OPTIMIZATIONS config CRC_KUNIT_TEST tristate "KUnit tests for CRC functions" if !KUNIT_ALL_TESTS - depends on KUNIT + depends on KUNIT && (CRC7 || CRC16 || CRC_T10DIF || CRC32 || CRC64) default KUNIT_ALL_TESTS + help + Unit tests for the CRC library functions. + + This is intended to help people writing architecture-specific + optimized versions. If unsure, say N. + +config CRC_ENABLE_ALL_FOR_KUNIT + tristate "Enable all CRC functions for KUnit test" + depends on KUNIT select CRC7 select CRC16 select CRC_T10DIF select CRC32 select CRC64 help - Unit tests for the CRC library functions. + Enable all CRC functions that have test code in CRC_KUNIT_TEST. - This is intended to help people writing architecture-specific - optimized versions. If unsure, say N. + Enable this only if you'd like the CRC KUnit test suite to test all + the CRC variants, even ones that wouldn't otherwise need to be built. config CRC_BENCHMARK bool "Benchmark for the CRC functions" diff --git a/lib/crc/Makefile b/lib/crc/Makefile index 7543ad295ab6..386e9c175263 100644 --- a/lib/crc/Makefile +++ b/lib/crc/Makefile @@ -38,9 +38,17 @@ obj-$(CONFIG_CRC64) += crc64.o crc64-y := crc64-main.o ifeq ($(CONFIG_CRC64_ARCH),y) CFLAGS_crc64-main.o += -I$(src)/$(SRCARCH) + +crc64-cflags-$(CONFIG_ARM) += -march=armv8-a -mfpu=crypto-neon-fp-armv8 +crc64-cflags-$(CONFIG_ARM64) += -march=armv8-a+crypto +CFLAGS_REMOVE_crc64-neon.o += $(CC_FLAGS_NO_FPU) +CFLAGS_crc64-neon.o += $(CC_FLAGS_FPU) -I$(src)/$(SRCARCH) $(crc64-cflags-y) +crc64-$(CONFIG_ARM) += crc64-neon.o +crc64-$(CONFIG_ARM64) += crc64-neon.o + crc64-$(CONFIG_RISCV) += riscv/crc64_lsb.o riscv/crc64_msb.o crc64-$(CONFIG_X86) += x86/crc64-pclmul.o -endif +endif # CONFIG_CRC64_ARCH obj-y += tests/ diff --git a/lib/crc/arm/crc64-neon.h b/lib/crc/arm/crc64-neon.h new file mode 100644 index 000000000000..645f553220ff --- /dev/null +++ b/lib/crc/arm/crc64-neon.h @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0-only + +static inline uint64x2_t pmull64(uint64x2_t a, uint64x2_t b) +{ + uint64_t l = vgetq_lane_u64(a, 0); + uint64_t m = vgetq_lane_u64(b, 0); + uint64x2_t result; + + asm("vmull.p64 %q0, %P1, %P2" : "=w"(result) : "w"(l), "w"(m)); + + return result; +} + +static inline uint64x2_t pmull64_high(uint64x2_t a, uint64x2_t b) +{ + uint64_t l = vgetq_lane_u64(a, 1); + uint64_t m = vgetq_lane_u64(b, 1); + uint64x2_t result; + + asm("vmull.p64 %q0, %P1, %P2" : "=w"(result) : "w"(l), "w"(m)); + + return result; +} + +static inline uint64x2_t pmull64_hi_lo(uint64x2_t a, uint64x2_t b) +{ + uint64_t l = vgetq_lane_u64(a, 1); + uint64_t m = vgetq_lane_u64(b, 0); + uint64x2_t result; + + asm("vmull.p64 %q0, %P1, %P2" : "=w"(result) : "w"(l), "w"(m)); + + return result; +} diff --git a/lib/crc/arm/crc64.h b/lib/crc/arm/crc64.h new file mode 100644 index 000000000000..de274288af61 --- /dev/null +++ b/lib/crc/arm/crc64.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * CRC64 using ARM PMULL instructions + */ + +#include <asm/simd.h> + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); + +u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len); + +#define crc64_be_arch crc64_be_generic + +static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len) +{ + if (len >= 128 && static_branch_likely(&have_pmull) && + likely(may_use_simd())) { + do { + size_t chunk = min_t(size_t, len & ~15, SZ_4K); + + scoped_ksimd() + crc = crc64_nvme_neon(crc, p, chunk); + + p += chunk; + len -= chunk; + } while (len >= 128); + } + return crc64_nvme_generic(crc, p, len); +} + +#define crc64_mod_init_arch crc64_mod_init_arch +static void crc64_mod_init_arch(void) +{ + if (elf_hwcap2 & HWCAP2_PMULL) + static_branch_enable(&have_pmull); +} diff --git a/lib/crc/arm64/crc-t10dif-core.S b/lib/crc/arm64/crc-t10dif-core.S index 87dd6d46224d..71388466825b 100644 --- a/lib/crc/arm64/crc-t10dif-core.S +++ b/lib/crc/arm64/crc-t10dif-core.S @@ -181,13 +181,13 @@ SYM_FUNC_END(__pmull_p8_16x64) pmull16x64_\p fold_consts, \reg1, v8 -CPU_LE( rev64 v11.16b, v11.16b ) -CPU_LE( rev64 v12.16b, v12.16b ) + rev64 v11.16b, v11.16b + rev64 v12.16b, v12.16b pmull16x64_\p fold_consts, \reg2, v9 -CPU_LE( ext v11.16b, v11.16b, v11.16b, #8 ) -CPU_LE( ext v12.16b, v12.16b, v12.16b, #8 ) + ext v11.16b, v11.16b, v11.16b, #8 + ext v12.16b, v12.16b, v12.16b, #8 eor \reg1\().16b, \reg1\().16b, v8.16b eor \reg2\().16b, \reg2\().16b, v9.16b @@ -220,22 +220,22 @@ CPU_LE( ext v12.16b, v12.16b, v12.16b, #8 ) ldp q4, q5, [buf, #0x40] ldp q6, q7, [buf, #0x60] add buf, buf, #0x80 -CPU_LE( rev64 v0.16b, v0.16b ) -CPU_LE( rev64 v1.16b, v1.16b ) -CPU_LE( rev64 v2.16b, v2.16b ) -CPU_LE( rev64 v3.16b, v3.16b ) -CPU_LE( rev64 v4.16b, v4.16b ) -CPU_LE( rev64 v5.16b, v5.16b ) -CPU_LE( rev64 v6.16b, v6.16b ) -CPU_LE( rev64 v7.16b, v7.16b ) -CPU_LE( ext v0.16b, v0.16b, v0.16b, #8 ) -CPU_LE( ext v1.16b, v1.16b, v1.16b, #8 ) -CPU_LE( ext v2.16b, v2.16b, v2.16b, #8 ) -CPU_LE( ext v3.16b, v3.16b, v3.16b, #8 ) -CPU_LE( ext v4.16b, v4.16b, v4.16b, #8 ) -CPU_LE( ext v5.16b, v5.16b, v5.16b, #8 ) -CPU_LE( ext v6.16b, v6.16b, v6.16b, #8 ) -CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) + rev64 v0.16b, v0.16b + rev64 v1.16b, v1.16b + rev64 v2.16b, v2.16b + rev64 v3.16b, v3.16b + rev64 v4.16b, v4.16b + rev64 v5.16b, v5.16b + rev64 v6.16b, v6.16b + rev64 v7.16b, v7.16b + ext v0.16b, v0.16b, v0.16b, #8 + ext v1.16b, v1.16b, v1.16b, #8 + ext v2.16b, v2.16b, v2.16b, #8 + ext v3.16b, v3.16b, v3.16b, #8 + ext v4.16b, v4.16b, v4.16b, #8 + ext v5.16b, v5.16b, v5.16b, #8 + ext v6.16b, v6.16b, v6.16b, #8 + ext v7.16b, v7.16b, v7.16b, #8 // XOR the first 16 data *bits* with the initial CRC value. movi v8.16b, #0 @@ -288,8 +288,8 @@ CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) pmull16x64_\p fold_consts, v7, v8 eor v7.16b, v7.16b, v8.16b ldr q0, [buf], #16 -CPU_LE( rev64 v0.16b, v0.16b ) -CPU_LE( ext v0.16b, v0.16b, v0.16b, #8 ) + rev64 v0.16b, v0.16b + ext v0.16b, v0.16b, v0.16b, #8 eor v7.16b, v7.16b, v0.16b subs len, len, #16 b.ge .Lfold_16_bytes_loop_\@ @@ -310,8 +310,8 @@ CPU_LE( ext v0.16b, v0.16b, v0.16b, #8 ) // v0 = last 16 original data bytes add buf, buf, len ldr q0, [buf, #-16] -CPU_LE( rev64 v0.16b, v0.16b ) -CPU_LE( ext v0.16b, v0.16b, v0.16b, #8 ) + rev64 v0.16b, v0.16b + ext v0.16b, v0.16b, v0.16b, #8 // v1 = high order part of second chunk: v7 left-shifted by 'len' bytes. adr_l x4, .Lbyteshift_table + 16 @@ -344,8 +344,8 @@ CPU_LE( ext v0.16b, v0.16b, v0.16b, #8 ) // Load the first 16 data bytes. ldr q7, [buf], #0x10 -CPU_LE( rev64 v7.16b, v7.16b ) -CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) + rev64 v7.16b, v7.16b + ext v7.16b, v7.16b, v7.16b, #8 // XOR the first 16 data *bits* with the initial CRC value. movi v0.16b, #0 @@ -382,8 +382,8 @@ SYM_FUNC_START(crc_t10dif_pmull_p8) crc_t10dif_pmull p8 -CPU_LE( rev64 v7.16b, v7.16b ) -CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) + rev64 v7.16b, v7.16b + ext v7.16b, v7.16b, v7.16b, #8 str q7, [x3] frame_pop diff --git a/lib/crc/arm64/crc32-core.S b/lib/crc/arm64/crc32-core.S index 68825317460f..49d02cc485b3 100644 --- a/lib/crc/arm64/crc32-core.S +++ b/lib/crc/arm64/crc32-core.S @@ -29,24 +29,19 @@ .endm .macro hwordle, reg -CPU_BE( rev16 \reg, \reg ) .endm .macro hwordbe, reg -CPU_LE( rev \reg, \reg ) + rev \reg, \reg rbit \reg, \reg -CPU_BE( lsr \reg, \reg, #16 ) .endm .macro le, regs:vararg - .irp r, \regs -CPU_BE( rev \r, \r ) - .endr .endm .macro be, regs:vararg .irp r, \regs -CPU_LE( rev \r, \r ) + rev \r, \r .endr .irp r, \regs rbit \r, \r diff --git a/lib/crc/arm64/crc64-neon.h b/lib/crc/arm64/crc64-neon.h new file mode 100644 index 000000000000..fcd5b1e6f812 --- /dev/null +++ b/lib/crc/arm64/crc64-neon.h @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0-only + +static inline uint64x2_t pmull64(uint64x2_t a, uint64x2_t b) +{ + return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 0), + vgetq_lane_u64(b, 0))); +} + +static inline uint64x2_t pmull64_high(uint64x2_t a, uint64x2_t b) +{ + poly64x2_t l = vreinterpretq_p64_u64(a); + poly64x2_t m = vreinterpretq_p64_u64(b); + + return vreinterpretq_u64_p128(vmull_high_p64(l, m)); +} + +static inline uint64x2_t pmull64_hi_lo(uint64x2_t a, uint64x2_t b) +{ + return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 1), + vgetq_lane_u64(b, 0))); +} diff --git a/lib/crc/arm64/crc64.h b/lib/crc/arm64/crc64.h new file mode 100644 index 000000000000..c7a69e1f3d8f --- /dev/null +++ b/lib/crc/arm64/crc64.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * CRC64 using ARM64 PMULL instructions + */ + +#include <linux/cpufeature.h> +#include <asm/simd.h> +#include <linux/minmax.h> +#include <linux/sizes.h> + +u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len); + +#define crc64_be_arch crc64_be_generic + +static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len) +{ + if (len >= 128 && cpu_have_named_feature(PMULL) && + likely(may_use_simd())) { + size_t chunk = len & ~15; + + scoped_ksimd() + crc = crc64_nvme_neon(crc, p, chunk); + + p += chunk; + len &= 15; + } + return crc64_nvme_generic(crc, p, len); +} diff --git a/lib/crc/crc64-neon.c b/lib/crc/crc64-neon.c new file mode 100644 index 000000000000..4753fb94a4be --- /dev/null +++ b/lib/crc/crc64-neon.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Accelerated CRC64 (NVMe) using ARM NEON C intrinsics + */ + +#include <linux/types.h> +#include <asm/neon-intrinsics.h> + +#include "crc64-neon.h" + +u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len); + +/* x^191 mod G, x^127 mod G */ +static const u64 fold_consts_val[2] = { 0xeadc41fd2ba3d420ULL, + 0x21e9761e252621acULL }; +/* floor(x^127 / G), (G - x^64) / x */ +static const u64 bconsts_val[2] = { 0x27ecfa329aef9f77ULL, + 0x34d926535897936aULL }; + +u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len) +{ + uint64x2_t fold_consts = vld1q_u64(fold_consts_val); + uint64x2_t v0 = { crc, 0 }; + uint64x2_t zero = { }; + + for (;;) { + v0 ^= vreinterpretq_u64_u8(vld1q_u8(p)); + + p += 16; + len -= 16; + if (len < 16) + break; + + v0 = pmull64(fold_consts, v0) ^ pmull64_high(fold_consts, v0); + } + + /* Multiply the 128-bit value by x^64 and reduce it back to 128 bits. */ + v0 = vextq_u64(v0, zero, 1) ^ pmull64_hi_lo(fold_consts, v0); + + /* Final Barrett reduction */ + uint64x2_t bconsts = vld1q_u64(bconsts_val); + uint64x2_t final = pmull64(bconsts, v0); + + v0 ^= vextq_u64(zero, final, 1) ^ pmull64_hi_lo(bconsts, final); + + return vgetq_lane_u64(v0, 1); +} diff --git a/lib/crc/tests/crc_kunit.c b/lib/crc/tests/crc_kunit.c index 9a450e25ac81..9428cd913625 100644 --- a/lib/crc/tests/crc_kunit.c +++ b/lib/crc/tests/crc_kunit.c @@ -268,8 +268,7 @@ crc_benchmark(struct kunit *test, } } -/* crc7_be */ - +#if IS_REACHABLE(CONFIG_CRC7) static u64 crc7_be_wrapper(u64 crc, const u8 *p, size_t len) { /* @@ -294,9 +293,9 @@ static void crc7_be_benchmark(struct kunit *test) { crc_benchmark(test, crc7_be_wrapper); } +#endif /* CONFIG_CRC7 */ -/* crc16 */ - +#if IS_REACHABLE(CONFIG_CRC16) static u64 crc16_wrapper(u64 crc, const u8 *p, size_t len) { return crc16(crc, p, len); @@ -318,9 +317,9 @@ static void crc16_benchmark(struct kunit *test) { crc_benchmark(test, crc16_wrapper); } +#endif /* CONFIG_CRC16 */ -/* crc_t10dif */ - +#if IS_REACHABLE(CONFIG_CRC_T10DIF) static u64 crc_t10dif_wrapper(u64 crc, const u8 *p, size_t len) { return crc_t10dif_update(crc, p, len); @@ -342,6 +341,9 @@ static void crc_t10dif_benchmark(struct kunit *test) { crc_benchmark(test, crc_t10dif_wrapper); } +#endif /* CONFIG_CRC_T10DIF */ + +#if IS_REACHABLE(CONFIG_CRC32) /* crc32_le */ @@ -414,6 +416,9 @@ static void crc32c_benchmark(struct kunit *test) { crc_benchmark(test, crc32c_wrapper); } +#endif /* CONFIG_CRC32 */ + +#if IS_REACHABLE(CONFIG_CRC64) /* crc64_be */ @@ -463,24 +468,35 @@ static void crc64_nvme_benchmark(struct kunit *test) { crc_benchmark(test, crc64_nvme_wrapper); } +#endif /* CONFIG_CRC64 */ static struct kunit_case crc_test_cases[] = { +#if IS_REACHABLE(CONFIG_CRC7) KUNIT_CASE(crc7_be_test), KUNIT_CASE(crc7_be_benchmark), +#endif +#if IS_REACHABLE(CONFIG_CRC16) KUNIT_CASE(crc16_test), KUNIT_CASE(crc16_benchmark), +#endif +#if IS_REACHABLE(CONFIG_CRC_T10DIF) KUNIT_CASE(crc_t10dif_test), KUNIT_CASE(crc_t10dif_benchmark), +#endif +#if IS_REACHABLE(CONFIG_CRC32) KUNIT_CASE(crc32_le_test), KUNIT_CASE(crc32_le_benchmark), KUNIT_CASE(crc32_be_test), KUNIT_CASE(crc32_be_benchmark), KUNIT_CASE(crc32c_test), KUNIT_CASE(crc32c_benchmark), +#endif +#if IS_REACHABLE(CONFIG_CRC64) KUNIT_CASE(crc64_be_test), KUNIT_CASE(crc64_be_benchmark), KUNIT_CASE(crc64_nvme_test), KUNIT_CASE(crc64_nvme_benchmark), +#endif {}, }; diff --git a/lib/crypto/.kunitconfig b/lib/crypto/.kunitconfig index 6b2ce28ae509..60e0a77f9889 100644 --- a/lib/crypto/.kunitconfig +++ b/lib/crypto/.kunitconfig @@ -1,28 +1,15 @@ CONFIG_KUNIT=y -# These kconfig options select all the CONFIG_CRYPTO_LIB_* symbols that have a -# corresponding KUnit test. Those symbols cannot be directly enabled here, -# since they are hidden symbols. -CONFIG_CRYPTO=y -CONFIG_CRYPTO_ADIANTUM=y -CONFIG_CRYPTO_BLAKE2B=y -CONFIG_CRYPTO_CHACHA20POLY1305=y -CONFIG_CRYPTO_HCTR2=y -CONFIG_CRYPTO_MD5=y -CONFIG_CRYPTO_MLDSA=y -CONFIG_CRYPTO_SHA1=y -CONFIG_CRYPTO_SHA256=y -CONFIG_CRYPTO_SHA512=y -CONFIG_CRYPTO_SHA3=y -CONFIG_INET=y -CONFIG_IPV6=y -CONFIG_NET=y -CONFIG_NETDEVICES=y -CONFIG_WIREGUARD=y +CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=y +CONFIG_CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_AES_CCM_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_AES_GCM_KUNIT_TEST=y CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST=y CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_CHACHA20POLY1305_KUNIT_TEST=y CONFIG_CRYPTO_LIB_CURVE25519_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_GHASH_KUNIT_TEST=y CONFIG_CRYPTO_LIB_MD5_KUNIT_TEST=y CONFIG_CRYPTO_LIB_MLDSA_KUNIT_TEST=y CONFIG_CRYPTO_LIB_NH_KUNIT_TEST=y @@ -32,3 +19,4 @@ CONFIG_CRYPTO_LIB_SHA1_KUNIT_TEST=y CONFIG_CRYPTO_LIB_SHA256_KUNIT_TEST=y CONFIG_CRYPTO_LIB_SHA512_KUNIT_TEST=y CONFIG_CRYPTO_LIB_SHA3_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_SM3_KUNIT_TEST=y diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 032f9755f999..56f308dd0273 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -1,7 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -menu "Crypto library routines" - config CRYPTO_HASH_INFO bool @@ -10,6 +8,9 @@ config CRYPTO_LIB_UTILS config CRYPTO_LIB_AES tristate + # Select dependencies of modes that are part of libaes. + select CRYPTO_LIB_GF128HASH if CRYPTO_LIB_AES_GCM != n + select CRYPTO_LIB_UTILS config CRYPTO_LIB_AES_ARCH bool @@ -28,11 +29,52 @@ config CRYPTO_LIB_AESCFB select CRYPTO_LIB_AES select CRYPTO_LIB_UTILS -config CRYPTO_LIB_AESGCM +config CRYPTO_LIB_AES_CBC tristate select CRYPTO_LIB_AES - select CRYPTO_LIB_GF128MUL - select CRYPTO_LIB_UTILS + help + The AES-CBC and AES-CBC-CTS library functions. + +config CRYPTO_LIB_AES_CBC_MACS + tristate + select CRYPTO_LIB_AES + help + The AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC library functions. Select + this if your module uses any of the functions from + <crypto/aes-cbc-macs.h>. + +config CRYPTO_LIB_AES_CCM + tristate + select CRYPTO_LIB_AES + select CRYPTO_LIB_AES_CBC_MACS + select CRYPTO_LIB_AES_CTR + help + The AES-CCM library functions. + +config CRYPTO_LIB_AES_CTR + tristate + select CRYPTO_LIB_AES + help + The AES-CTR and AES-XCTR library functions. + +config CRYPTO_LIB_AES_ECB + tristate + select CRYPTO_LIB_AES + help + The AES-ECB library functions. + +config CRYPTO_LIB_AES_GCM + tristate + select CRYPTO_LIB_AES + select CRYPTO_LIB_AES_CTR + help + The AES-GCM library functions. + +config CRYPTO_LIB_AES_XTS + tristate + select CRYPTO_LIB_AES + help + The AES-XTS library functions. config CRYPTO_LIB_ARC4 tristate @@ -70,7 +112,7 @@ config CRYPTO_LIB_CHACHA_ARCH bool depends on CRYPTO_LIB_CHACHA && !UML && !KMSAN default y if ARM - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if MIPS && CPU_MIPS32_R2 default y if PPC64 && CPU_LITTLE_ENDIAN && VSX default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ @@ -100,19 +142,29 @@ config CRYPTO_LIB_CURVE25519_GENERIC config CRYPTO_LIB_DES tristate +config CRYPTO_LIB_GF128HASH + tristate + help + The GHASH and POLYVAL library functions. Select this if your module + uses any of the functions from <crypto/gf128hash.h>. + +config CRYPTO_LIB_GF128HASH_ARCH + bool + depends on CRYPTO_LIB_GF128HASH && !UML + default y if ARM && KERNEL_MODE_NEON + default y if ARM64 + default y if PPC64 && VSX + default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ + RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS + default y if S390 + default y if X86_64 + config CRYPTO_LIB_MD5 tristate help The MD5 and HMAC-MD5 library functions. Select this if your module uses any of the functions from <crypto/md5.h>. -config CRYPTO_LIB_MD5_ARCH - bool - depends on CRYPTO_LIB_MD5 && !UML - default y if MIPS && CPU_CAVIUM_OCTEON - default y if PPC - default y if SPARC64 - config CRYPTO_LIB_MLDSA tristate select CRYPTO_LIB_SHA3 @@ -130,7 +182,7 @@ config CRYPTO_LIB_NH_ARCH bool depends on CRYPTO_LIB_NH && !UML && !KMSAN default y if ARM && KERNEL_MODE_NEON - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if X86_64 config CRYPTO_LIB_POLY1305 @@ -143,7 +195,7 @@ config CRYPTO_LIB_POLY1305_ARCH bool depends on CRYPTO_LIB_POLY1305 && !UML && !KMSAN default y if ARM - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if MIPS # The PPC64 code needs to be fixed to work in softirq context. default y if PPC64 && CPU_LITTLE_ENDIAN && VSX && BROKEN @@ -168,18 +220,6 @@ config CRYPTO_LIB_POLY1305_RSIZE default 9 if ARM || ARM64 default 1 -config CRYPTO_LIB_POLYVAL - tristate - help - The POLYVAL library functions. Select this if your module uses any of - the functions from <crypto/polyval.h>. - -config CRYPTO_LIB_POLYVAL_ARCH - bool - depends on CRYPTO_LIB_POLYVAL && !UML - default y if ARM64 && KERNEL_MODE_NEON - default y if X86_64 - config CRYPTO_LIB_CHACHA20POLY1305 tristate select CRYPTO_LIB_CHACHA @@ -196,7 +236,7 @@ config CRYPTO_LIB_SHA1_ARCH bool depends on CRYPTO_LIB_SHA1 && !UML default y if ARM - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if MIPS && CPU_CAVIUM_OCTEON default y if PPC default y if S390 @@ -252,12 +292,19 @@ config CRYPTO_LIB_SHA3 config CRYPTO_LIB_SHA3_ARCH bool depends on CRYPTO_LIB_SHA3 && !UML - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if S390 config CRYPTO_LIB_SM3 tristate + help + The SM3 library functions. Select this if your module uses any of the + functions from <crypto/sm3.h>. -source "lib/crypto/tests/Kconfig" - -endmenu +config CRYPTO_LIB_SM3_ARCH + bool + depends on CRYPTO_LIB_SM3 && !UML + default y if ARM64 + default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ + RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS + default y if X86_64 diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index dc7a56f7287d..ca068df1f71f 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -8,6 +8,10 @@ quiet_cmd_perlasm = PERLASM $@ quiet_cmd_perlasm_with_args = PERLASM $@ cmd_perlasm_with_args = $(PERL) $(<) void $(@) +ppc64-perlasm-flavour-y := linux-ppc64 +ppc64-perlasm-flavour-$(CONFIG_PPC64_ELF_ABI_V2) := linux-ppc64-elfv2 +ppc64-perlasm-flavour-$(CONFIG_CPU_LITTLE_ENDIAN) := linux-ppc64le + obj-$(CONFIG_KUNIT) += tests/ obj-$(CONFIG_CRYPTO_HASH_INFO) += hash_info.o @@ -23,11 +27,10 @@ ifeq ($(CONFIG_CRYPTO_LIB_AES_ARCH),y) CFLAGS_aes.o += -I$(src)/$(SRCARCH) libaes-$(CONFIG_ARM) += arm/aes-cipher-core.o - -ifeq ($(CONFIG_ARM64),y) -libaes-y += arm64/aes-cipher-core.o -libaes-$(CONFIG_KERNEL_MODE_NEON) += arm64/aes-ce-core.o -endif +libaes-$(CONFIG_ARM64) += arm64/aes-cipher-core.o \ + arm64/aes-ce-core.o \ + arm64/aes-ce.o \ + arm64/aes-neon.o ifeq ($(CONFIG_PPC),y) ifeq ($(CONFIG_SPE),y) @@ -37,11 +40,8 @@ libaes-y += powerpc/aes-spe-core.o \ powerpc/aes-tab-4k.o else libaes-y += powerpc/aesp8-ppc.o -aes-perlasm-flavour-y := linux-ppc64 -aes-perlasm-flavour-$(CONFIG_PPC64_ELF_ABI_V2) := linux-ppc64-elfv2 -aes-perlasm-flavour-$(CONFIG_CPU_LITTLE_ENDIAN) := linux-ppc64le quiet_cmd_perlasm_aes = PERLASM $@ - cmd_perlasm_aes = $(PERL) $< $(aes-perlasm-flavour-y) $@ + cmd_perlasm_aes = $(PERL) $< $(ppc64-perlasm-flavour-y) $@ # Use if_changed instead of cmd, in case the flavour changed. $(obj)/powerpc/aesp8-ppc.S: $(src)/powerpc/aesp8-ppc.pl FORCE $(call if_changed,perlasm_aes) @@ -63,9 +63,6 @@ clean-files += powerpc/aesp8-ppc.S obj-$(CONFIG_CRYPTO_LIB_AESCFB) += libaescfb.o libaescfb-y := aescfb.o -obj-$(CONFIG_CRYPTO_LIB_AESGCM) += libaesgcm.o -libaesgcm-y := aesgcm.o - obj-$(CONFIG_CRYPTO_LIB_ARC4) += libarc4.o libarc4-y := arc4.o @@ -125,7 +122,6 @@ endif # CONFIG_CRYPTO_LIB_CHACHA_ARCH obj-$(CONFIG_CRYPTO_LIB_CHACHA20POLY1305) += libchacha20poly1305.o libchacha20poly1305-y += chacha20poly1305.o -libchacha20poly1305-$(CONFIG_CRYPTO_SELFTESTS) += chacha20poly1305-selftest.o ################################################################################ @@ -158,13 +154,36 @@ libdes-y := des.o ################################################################################ +obj-$(CONFIG_CRYPTO_LIB_GF128HASH) += libgf128hash.o +libgf128hash-y := gf128hash.o +ifeq ($(CONFIG_CRYPTO_LIB_GF128HASH_ARCH),y) +CFLAGS_gf128hash.o += -I$(src)/$(SRCARCH) +libgf128hash-$(CONFIG_ARM) += arm/ghash-neon-core.o +libgf128hash-$(CONFIG_ARM64) += arm64/ghash-neon-core.o \ + arm64/polyval-ce-core.o + +ifeq ($(CONFIG_PPC),y) +libgf128hash-y += powerpc/ghashp8-ppc.o +quiet_cmd_perlasm_ghash = PERLASM $@ + cmd_perlasm_ghash = $(PERL) $< $(ppc64-perlasm-flavour-y) $@ +$(obj)/powerpc/ghashp8-ppc.S: $(src)/powerpc/ghashp8-ppc.pl FORCE + $(call if_changed,perlasm_ghash) +targets += powerpc/ghashp8-ppc.S +OBJECT_FILES_NON_STANDARD_powerpc/ghashp8-ppc.o := y +endif + +libgf128hash-$(CONFIG_RISCV) += riscv/ghash-riscv64-zvkg.o +libgf128hash-$(CONFIG_X86) += x86/ghash-pclmul.o \ + x86/polyval-pclmul-avx.o +endif # CONFIG_CRYPTO_LIB_GF128HASH_ARCH + +# clean-files must be defined unconditionally +clean-files += powerpc/ghashp8-ppc.S + +################################################################################ + obj-$(CONFIG_CRYPTO_LIB_MD5) += libmd5.o libmd5-y := md5.o -ifeq ($(CONFIG_CRYPTO_LIB_MD5_ARCH),y) -CFLAGS_md5.o += -I$(src)/$(SRCARCH) -libmd5-$(CONFIG_PPC) += powerpc/md5-asm.o -libmd5-$(CONFIG_SPARC) += sparc/md5_asm.o -endif # CONFIG_CRYPTO_LIB_MD5_ARCH ################################################################################ @@ -255,16 +274,6 @@ clean-files += arm/poly1305-core.S \ ################################################################################ -obj-$(CONFIG_CRYPTO_LIB_POLYVAL) += libpolyval.o -libpolyval-y := polyval.o -ifeq ($(CONFIG_CRYPTO_LIB_POLYVAL_ARCH),y) -CFLAGS_polyval.o += -I$(src)/$(SRCARCH) -libpolyval-$(CONFIG_ARM64) += arm64/polyval-ce-core.o -libpolyval-$(CONFIG_X86) += x86/polyval-pclmul-avx.o -endif - -################################################################################ - obj-$(CONFIG_CRYPTO_LIB_SHA1) += libsha1.o libsha1-y := sha1.o ifeq ($(CONFIG_CRYPTO_LIB_SHA1_ARCH),y) @@ -300,10 +309,9 @@ AFLAGS_arm/sha256-core.o += $(aflags-thumb2-y) endif ifeq ($(CONFIG_ARM64),y) -libsha256-y += arm64/sha256-core.o +libsha256-y += arm64/sha256-ce.o arm64/sha256-core.o $(obj)/arm64/sha256-core.S: $(src)/arm64/sha2-armv8.pl $(call cmd,perlasm_with_args) -libsha256-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha256-ce.o endif libsha256-$(CONFIG_PPC) += powerpc/sha256-spe-asm.o @@ -330,10 +338,9 @@ AFLAGS_arm/sha512-core.o += $(aflags-thumb2-y) endif ifeq ($(CONFIG_ARM64),y) -libsha512-y += arm64/sha512-core.o +libsha512-y += arm64/sha512-ce-core.o arm64/sha512-core.o $(obj)/arm64/sha512-core.S: $(src)/arm64/sha2-armv8.pl $(call cmd,perlasm_with_args) -libsha512-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha512-ce-core.o endif libsha512-$(CONFIG_RISCV) += riscv/sha512-riscv64-zvknhb-zvkb.o @@ -355,13 +362,22 @@ endif # CONFIG_CRYPTO_LIB_SHA3_ARCH ################################################################################ +obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o +libsm3-y := sm3.o +ifeq ($(CONFIG_CRYPTO_LIB_SM3_ARCH),y) +CFLAGS_sm3.o += -I$(src)/$(SRCARCH) +libsm3-$(CONFIG_ARM64) += arm64/sm3-ce-core.o \ + arm64/sm3-neon-core.o +libsm3-$(CONFIG_RISCV) += riscv/sm3-riscv64-zvksh-zvkb.o +libsm3-$(CONFIG_X86) += x86/sm3-avx-asm_64.o +endif # CONFIG_CRYPTO_LIB_SM3_ARCH + +################################################################################ + obj-$(CONFIG_MPILIB) += mpi/ obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) += simd.o -obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o -libsm3-y := sm3.o - # clean-files must be defined unconditionally clean-files += arm/sha256-core.S arm/sha512-core.S clean-files += arm64/sha256-core.S arm64/sha512-core.S diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c index b73e19f1bb95..f1549839b3de 100644 --- a/lib/crypto/aes.c +++ b/lib/crypto/aes.c @@ -4,12 +4,22 @@ * Copyright 2026 Google LLC */ +#include <crypto/aes-cbc-macs.h> +#include <crypto/aes-cbc.h> +#include <crypto/aes-ccm.h> +#include <crypto/aes-ctr.h> +#include <crypto/aes-ecb.h> +#include <crypto/aes-gcm.h> +#include <crypto/aes-xts.h> #include <crypto/aes.h> +#include <crypto/gf128mul.h> +#include <crypto/utils.h> #include <linux/cache.h> #include <linux/crypto.h> #include <linux/export.h> #include <linux/module.h> #include <linux/unaligned.h> +#include "fips-aes.h" static const u8 ____cacheline_aligned aes_sbox[] = { 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, @@ -512,10 +522,1617 @@ void aes_decrypt(const struct aes_key *key, u8 out[AES_BLOCK_SIZE], } EXPORT_SYMBOL(aes_decrypt); -#ifdef aes_mod_init_arch +/* FIPS cryptographic algorithm self-test for "bare" AES */ +static void __init aes_fips_test(void) +{ + struct aes_key key; + u8 data[AES_BLOCK_SIZE]; + + if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0) + panic("aes: FIPS self-test failed (preparekey)\n"); + + aes_encrypt(&key, data, fips_test_data); + if (memcmp(fips_test_aes_ecb_ctext, data, sizeof(data)) != 0) + panic("aes: FIPS self-test failed (wrong ciphertext)\n"); + + aes_decrypt(&key, data, data); + if (memcmp(fips_test_data, data, sizeof(data)) != 0) + panic("aes: FIPS self-test failed (wrong plaintext)\n"); + + memzero_explicit(&key, sizeof(key)); +} + +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) + +#ifndef aes_cbcmac_blocks_arch +static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE], + const struct aes_enckey *key, const u8 *data, + size_t nblocks, bool enc_before, + bool enc_after) +{ + return false; +} +#endif + +/* This assumes nblocks >= 1. */ +static void aes_cbcmac_blocks(u8 h[AES_BLOCK_SIZE], + const struct aes_enckey *key, const u8 *data, + size_t nblocks, bool enc_before, bool enc_after) +{ + if (aes_cbcmac_blocks_arch(h, key, data, nblocks, enc_before, + enc_after)) + return; + + if (enc_before) + aes_encrypt(key, h, h); + for (; nblocks > 1; nblocks--) { + crypto_xor(h, data, AES_BLOCK_SIZE); + data += AES_BLOCK_SIZE; + aes_encrypt(key, h, h); + } + crypto_xor(h, data, AES_BLOCK_SIZE); + if (enc_after) + aes_encrypt(key, h, h); +} + +int aes_cmac_preparekey(struct aes_cmac_key *key, const u8 *in_key, + size_t key_len) +{ + u64 hi, lo, mask; + int err; + + /* Prepare the AES key. */ + err = aes_prepareenckey(&key->aes, in_key, key_len); + if (err) + return err; + + /* + * Prepare the subkeys K1 and K2 by encrypting the all-zeroes block, + * then multiplying by 'x' and 'x^2' (respectively) in GF(2^128). + * Reference: NIST SP 800-38B, Section 6.1 "Subkey Generation". + */ + memset(key->k_final[0].b, 0, AES_BLOCK_SIZE); + aes_encrypt(&key->aes, key->k_final[0].b, key->k_final[0].b); + hi = be64_to_cpu(key->k_final[0].w[0]); + lo = be64_to_cpu(key->k_final[0].w[1]); + for (int i = 0; i < 2; i++) { + mask = ((s64)hi >> 63) & 0x87; + hi = (hi << 1) ^ (lo >> 63); + lo = (lo << 1) ^ mask; + key->k_final[i].w[0] = cpu_to_be64(hi); + key->k_final[i].w[1] = cpu_to_be64(lo); + } + return 0; +} +EXPORT_SYMBOL_GPL(aes_cmac_preparekey); + +void aes_xcbcmac_preparekey(struct aes_cmac_key *key, + const u8 in_key[AES_KEYSIZE_128]) +{ + static const u8 constants[3][AES_BLOCK_SIZE] = { + { [0 ... AES_BLOCK_SIZE - 1] = 0x1 }, + { [0 ... AES_BLOCK_SIZE - 1] = 0x2 }, + { [0 ... AES_BLOCK_SIZE - 1] = 0x3 }, + }; + u8 new_aes_key[AES_BLOCK_SIZE]; + + static_assert(AES_BLOCK_SIZE == AES_KEYSIZE_128); + aes_prepareenckey(&key->aes, in_key, AES_BLOCK_SIZE); + aes_encrypt(&key->aes, new_aes_key, constants[0]); + aes_encrypt(&key->aes, key->k_final[0].b, constants[1]); + aes_encrypt(&key->aes, key->k_final[1].b, constants[2]); + aes_prepareenckey(&key->aes, new_aes_key, AES_BLOCK_SIZE); + memzero_explicit(new_aes_key, AES_BLOCK_SIZE); +} +EXPORT_SYMBOL_GPL(aes_xcbcmac_preparekey); + +void aes_cmac_update(struct aes_cmac_ctx *ctx, const u8 *data, size_t data_len) +{ + bool enc_before = false; + size_t nblocks; + + if (ctx->partial_len) { + /* XOR data into a pending block. */ + size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); + + crypto_xor(&ctx->h[ctx->partial_len], data, l); + data += l; + data_len -= l; + ctx->partial_len += l; + if (data_len == 0) { + /* + * Either the pending block hasn't been filled yet, or + * no more data was given so it's not yet known whether + * the block is the final block. + */ + return; + } + /* Pending block has been filled and isn't the final block. */ + enc_before = true; + } + + nblocks = data_len / AES_BLOCK_SIZE; + data_len %= AES_BLOCK_SIZE; + if (nblocks == 0) { + /* 0 additional full blocks, then optionally a partial block */ + if (enc_before) + aes_encrypt(&ctx->key->aes, ctx->h, ctx->h); + crypto_xor(ctx->h, data, data_len); + ctx->partial_len = data_len; + } else if (data_len != 0) { + /* 1 or more additional full blocks, then a partial block */ + aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, + enc_before, /* enc_after= */ true); + data += nblocks * AES_BLOCK_SIZE; + crypto_xor(ctx->h, data, data_len); + ctx->partial_len = data_len; + } else { + /* + * 1 or more additional full blocks only. Encryption of the + * last block is delayed until it's known whether it's the final + * block in the message or not. + */ + aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, + enc_before, /* enc_after= */ false); + ctx->partial_len = AES_BLOCK_SIZE; + } +} +EXPORT_SYMBOL_GPL(aes_cmac_update); + +void aes_cmac_final(struct aes_cmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) +{ + if (ctx->partial_len == AES_BLOCK_SIZE) { + /* Final block is a full block. Use k_final[0]. */ + crypto_xor(ctx->h, ctx->key->k_final[0].b, AES_BLOCK_SIZE); + } else { + /* Final block is a partial block. Pad, and use k_final[1]. */ + ctx->h[ctx->partial_len] ^= 0x80; + crypto_xor(ctx->h, ctx->key->k_final[1].b, AES_BLOCK_SIZE); + } + aes_encrypt(&ctx->key->aes, out, ctx->h); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(aes_cmac_final); + +void aes_cbcmac_update(struct aes_cbcmac_ctx *ctx, const u8 *data, + size_t data_len) +{ + bool enc_before = false; + size_t nblocks; + + if (ctx->partial_len) { + size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); + + crypto_xor(&ctx->h[ctx->partial_len], data, l); + data += l; + data_len -= l; + ctx->partial_len += l; + if (ctx->partial_len < AES_BLOCK_SIZE) + return; + enc_before = true; + } + + nblocks = data_len / AES_BLOCK_SIZE; + data_len %= AES_BLOCK_SIZE; + if (nblocks == 0) { + if (enc_before) + aes_encrypt(ctx->key, ctx->h, ctx->h); + } else { + aes_cbcmac_blocks(ctx->h, ctx->key, data, nblocks, enc_before, + /* enc_after= */ true); + data += nblocks * AES_BLOCK_SIZE; + } + crypto_xor(ctx->h, data, data_len); + ctx->partial_len = data_len; +} +EXPORT_SYMBOL_NS_GPL(aes_cbcmac_update, "CRYPTO_INTERNAL"); + +void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) +{ + if (ctx->partial_len) + aes_encrypt(ctx->key, out, ctx->h); + else + memcpy(out, ctx->h, AES_BLOCK_SIZE); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_NS_GPL(aes_cbcmac_final, "CRYPTO_INTERNAL"); + +/* FIPS cryptographic algorithm self-test for AES-CMAC */ +static void __init aes_cmac_fips_test(void) +{ + struct aes_cmac_key key __cleanup(aes_cmac_zeroize_key); + u8 mac[AES_BLOCK_SIZE]; + + if (aes_cmac_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != + 0) + panic("aes: CMAC FIPS self-test failed (preparekey)\n"); + aes_cmac(&key, fips_test_data, sizeof(fips_test_data), mac); + if (memcmp(fips_test_aes_cmac_value, mac, sizeof(mac)) != 0) + panic("aes: CMAC FIPS self-test failed (wrong MAC)\n"); +} +#else /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ +static inline void aes_cmac_fips_test(void) +{ +} +#endif /* !CONFIG_CRYPTO_LIB_AES_CBC_MACS */ + +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB) +/* + * Hooks for optimized AES-ECB implementations, overridable by the architecture. + * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. Returning false + * causes the fallback implementation to be used instead. + */ +#ifndef aes_ecb_encrypt_arch +static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len, + const struct aes_enckey *key) +{ + return false; +} +#endif +#ifndef aes_ecb_decrypt_arch +static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len, + const struct aes_key *key) +{ + return false; +} +#endif + +void aes_ecb_encrypt(u8 *dst, const u8 *src, size_t len, aes_encrypt_arg key) +{ + if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) + len = round_down(len, AES_BLOCK_SIZE); + + if (unlikely(len == 0)) + return; + + if (likely(aes_ecb_encrypt_arch(dst, src, len, key.enc_key))) + return; + + for (size_t i = 0; i < len; i += AES_BLOCK_SIZE) + aes_encrypt(key, &dst[i], &src[i]); +} +EXPORT_SYMBOL_GPL(aes_ecb_encrypt); + +void aes_ecb_decrypt(u8 *dst, const u8 *src, size_t len, + const struct aes_key *key) +{ + if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) + len = round_down(len, AES_BLOCK_SIZE); + + if (unlikely(len == 0)) + return; + + if (likely(aes_ecb_decrypt_arch(dst, src, len, key))) + return; + + for (size_t i = 0; i < len; i += AES_BLOCK_SIZE) + aes_decrypt(key, &dst[i], &src[i]); +} +EXPORT_SYMBOL_GPL(aes_ecb_decrypt); + +/* FIPS cryptographic algorithm self-test for AES-ECB */ +static void __init aes_ecb_fips_test(void) +{ + struct aes_key key; + u8 data[sizeof(fips_test_data)]; + + if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0) + panic("aes: ECB FIPS self-test failed (preparekey)\n"); + + aes_ecb_encrypt(data, fips_test_data, sizeof(data), &key); + if (memcmp(fips_test_aes_ecb_ctext, data, sizeof(data)) != 0) + panic("aes: ECB FIPS self-test failed (wrong ciphertext)\n"); + + aes_ecb_decrypt(data, data, sizeof(data), &key); + if (memcmp(fips_test_data, data, sizeof(data)) != 0) + panic("aes: ECB FIPS self-test failed (wrong plaintext)\n"); + + memzero_explicit(&key, sizeof(key)); +} +#else /* CONFIG_CRYPTO_LIB_AES_ECB */ +static inline void aes_ecb_fips_test(void) +{ +} +#endif /* !CONFIG_CRYPTO_LIB_AES_ECB */ + +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC) +/* + * Hooks for optimized AES-CBC implementations, overridable by the architecture. + * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. Returning false + * causes the fallback implementation to be used instead. + */ +#ifndef aes_cbc_encrypt_arch +static bool aes_cbc_encrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], + const struct aes_enckey *key) +{ + return false; +} +#endif +#ifndef aes_cbc_decrypt_arch +static bool aes_cbc_decrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], + const struct aes_key *key) +{ + return false; +} +#endif + +void aes_cbc_encrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_SIZE], + aes_encrypt_arg key) +{ + const u8 *prev = iv; + + if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) + len = round_down(len, AES_BLOCK_SIZE); + + if (unlikely(len == 0)) + return; + + if (likely(aes_cbc_encrypt_arch(dst, src, len, iv, key.enc_key))) + return; + + do { + crypto_xor_cpy(dst, src, prev, AES_BLOCK_SIZE); + aes_encrypt(key, dst, dst); + prev = dst; + dst += AES_BLOCK_SIZE; + src += AES_BLOCK_SIZE; + len -= AES_BLOCK_SIZE; + } while (len); + memcpy(iv, prev, AES_BLOCK_SIZE); +} +EXPORT_SYMBOL_GPL(aes_cbc_encrypt); + +void aes_cbc_decrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_SIZE], + const struct aes_key *key) +{ + u8 next_iv[AES_BLOCK_SIZE]; + + if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) + len = round_down(len, AES_BLOCK_SIZE); + + if (unlikely(len == 0)) + return; + + if (likely(aes_cbc_decrypt_arch(dst, src, len, iv, key))) + return; + + len -= AES_BLOCK_SIZE; + dst += len; + src += len; + memcpy(next_iv, src, AES_BLOCK_SIZE); + for (;;) { + aes_decrypt(key, dst, src); + if (len == 0) + break; + src -= AES_BLOCK_SIZE; + crypto_xor(dst, src, AES_BLOCK_SIZE); + dst -= AES_BLOCK_SIZE; + len -= AES_BLOCK_SIZE; + } + crypto_xor(dst, iv, AES_BLOCK_SIZE); + memcpy(iv, next_iv, AES_BLOCK_SIZE); +} +EXPORT_SYMBOL_GPL(aes_cbc_decrypt); + +/* + * Hooks for optimized AES-CBC-CTS implementations, overridable by the + * architecture. They are called with len > AES_BLOCK_SIZE. Returning false + * causes the fallback implementation to be used instead. The fallback + * implementation still uses the arch-optimized AES-CBC code if available, but + * direct implementation of AES-CBC-CTS is helpful on short messages. + */ +#ifndef aes_cbc_cts_encrypt_arch +static bool aes_cbc_cts_encrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], + const struct aes_enckey *key) +{ + return false; +} +#endif +#ifndef aes_cbc_cts_decrypt_arch +static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], + const struct aes_key *key) +{ + return false; +} +#endif + +void aes_cbc_cts_encrypt(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], aes_encrypt_arg key) +{ + /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ + size_t pn_offset = round_down(len - 1, AES_BLOCK_SIZE); + /* Length of P[n] and C[n], 1 <= pn_len <= AES_BLOCK_SIZE */ + size_t pn_len = len - pn_offset; + u8 tmp[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + u8 *pad; + + if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) + return; + + if (len == AES_BLOCK_SIZE) { + aes_cbc_encrypt(dst, src, len, iv, key); + return; + } + if (likely(aes_cbc_cts_encrypt_arch(dst, src, len, iv, key.enc_key))) + return; + + /* CBC-encrypt all blocks except the last. */ + aes_cbc_encrypt(dst, src, pn_offset, iv, key); + + /* + * Compute C[n] and C[n - 1]. + * + * Careful: src may equal dst (i.e., the encryption can be in-place), so + * src[pn_offset..] can't be read after dst[pn_offset..] is written. + */ + pad = &dst[pn_offset - AES_BLOCK_SIZE]; + memcpy(tmp, pad, AES_BLOCK_SIZE); + crypto_xor(tmp, &src[pn_offset], pn_len); + memcpy(&dst[pn_offset], pad, pn_len); /* C[n] */ + aes_encrypt(key, pad, tmp); /* C[n - 1] */ + + memzero_explicit(tmp, sizeof(tmp)); +} +EXPORT_SYMBOL_GPL(aes_cbc_cts_encrypt); + +void aes_cbc_cts_decrypt(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], const struct aes_key *key) +{ + /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ + size_t pn_offset = round_down(len - 1, AES_BLOCK_SIZE); + /* Length of P[n] and C[n], 1 <= pn_len <= AES_BLOCK_SIZE */ + size_t pn_len = len - pn_offset; + u8 *pad; + + if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) + return; + + if (len == AES_BLOCK_SIZE) { + aes_cbc_decrypt(dst, src, len, iv, key); + return; + } + if (likely(aes_cbc_cts_decrypt_arch(dst, src, len, iv, key))) + return; + + /* Compute P[0]..P[n - 2]. */ + aes_cbc_decrypt(dst, src, pn_offset - AES_BLOCK_SIZE, iv, key); + + /* + * Compute P[n] and P[n - 1]. + * + * Careful: src may equal dst (i.e., the decryption can be in-place), so + * src[pn_offset..] can't be read after dst[pn_offset..] is written. + * + * To avoid needing a temporary buffer, do a "redundant" XOR to recover + * src[pn_offset..] from dst[pn_offset..] after the latter is written. + */ + pad = &dst[pn_offset - AES_BLOCK_SIZE]; + aes_decrypt(key, pad, &src[pn_offset - AES_BLOCK_SIZE]); + crypto_xor_cpy(&dst[pn_offset], &src[pn_offset], pad, + pn_len); /* P[n] */ + crypto_xor(pad, &dst[pn_offset], pn_len); + aes_decrypt(key, pad, pad); + crypto_xor(pad, iv, AES_BLOCK_SIZE); /* P[n - 1] */ +} +EXPORT_SYMBOL_GPL(aes_cbc_cts_decrypt); + +/* FIPS cryptographic algorithm self-test for AES-CBC */ +static void __init aes_cbc_fips_test(void) +{ + struct aes_key key; + u8 iv[AES_BLOCK_SIZE]; + u8 data[sizeof(fips_test_data)]; + + if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0) + panic("aes: CBC FIPS self-test failed (preparekey)\n"); + + memcpy(iv, fips_test_iv, sizeof(iv)); + aes_cbc_encrypt(data, fips_test_data, sizeof(data), iv, &key); + if (memcmp(fips_test_aes_cbc_ctext, data, sizeof(data)) != 0) + panic("aes: CBC FIPS self-test failed (wrong ciphertext)\n"); + + memcpy(iv, fips_test_iv, sizeof(iv)); + aes_cbc_decrypt(data, data, sizeof(data), iv, &key); + if (memcmp(fips_test_data, data, sizeof(data)) != 0) + panic("aes: CBC FIPS self-test failed (wrong plaintext)\n"); + + memzero_explicit(&key, sizeof(key)); +} + +/* FIPS cryptographic algorithm self-test for AES-CBC-CTS */ +static void __init aes_cbc_cts_fips_test(void) +{ + struct aes_key key; + u8 iv[AES_BLOCK_SIZE]; + const size_t data_len = 2 * AES_BLOCK_SIZE; + u8 ptext[2 * AES_BLOCK_SIZE]; + u8 data[2 * AES_BLOCK_SIZE]; + + /* ptext = fips_test_data || fips_test_data */ + memcpy(ptext, fips_test_data, AES_BLOCK_SIZE); + memcpy(&ptext[AES_BLOCK_SIZE], ptext, AES_BLOCK_SIZE); + + if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0) + panic("aes: CBC-CTS FIPS self-test failed (preparekey)\n"); + + memcpy(iv, fips_test_iv, sizeof(iv)); + aes_cbc_cts_encrypt(data, ptext, data_len, iv, &key); + if (memcmp(fips_test_aes_cbc_cts_ctext, data, data_len) != 0) + panic("aes: CBC-CTS FIPS self-test failed (wrong ciphertext)\n"); + + memcpy(iv, fips_test_iv, sizeof(iv)); + aes_cbc_cts_decrypt(data, data, data_len, iv, &key); + if (memcmp(ptext, data, data_len) != 0) + panic("aes: CBC-CTS FIPS self-test failed (wrong plaintext)\n"); + + memzero_explicit(&key, sizeof(key)); +} +#else /* CONFIG_CRYPTO_LIB_AES_CBC */ +static inline void aes_cbc_fips_test(void) +{ +} +static inline void aes_cbc_cts_fips_test(void) +{ +} +#endif /* !CONFIG_CRYPTO_LIB_AES_CBC */ + +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR) +/* + * Hooks for optimized AES-CTR and AES-XCTR implementations, overridable by the + * architecture. They are called with any len >= 0. Returning false causes the + * fallback implementation to be used instead. + */ +#ifndef aes_ctr_arch +static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len, + u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key) +{ + return false; +} +#endif +#ifndef aes_xctr_arch +static bool aes_xctr_arch(u8 *dst, const u8 *src, size_t len, u64 *ctr, + const u8 iv[AES_BLOCK_SIZE], + const struct aes_enckey *key) +{ + return false; +} +#endif + +static __always_inline void inc_be128_ctr(u8 ctr[AES_BLOCK_SIZE]) +{ + /* + * 255 times out of 256 the first iteration is enough, so unroll the + * first iteration as a micro-optimization. + */ + if ((++ctr[AES_BLOCK_SIZE - 1]) != 0) + return; + for (int i = AES_BLOCK_SIZE - 2; i >= 0; i--) { + if (++ctr[i] != 0) + break; + } +} + +void aes_ctr(u8 *dst, const u8 *src, size_t len, u8 ctr[AES_BLOCK_SIZE], + aes_encrypt_arg key) +{ + u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + + if (likely(aes_ctr_arch(dst, src, len, ctr, key.enc_key))) + return; + + /* Handle the full blocks. */ + for (; len >= AES_BLOCK_SIZE; len -= AES_BLOCK_SIZE) { + aes_encrypt(key, keystream, ctr); + crypto_xor_cpy(dst, src, keystream, AES_BLOCK_SIZE); + inc_be128_ctr(ctr); + dst += AES_BLOCK_SIZE; + src += AES_BLOCK_SIZE; + } + /* Handle any partial block at the end. */ + if (len) { + aes_encrypt(key, keystream, ctr); + crypto_xor_cpy(dst, src, keystream, len); + /* Counter is incremented even with just a partial block. */ + inc_be128_ctr(ctr); + } + memzero_explicit(keystream, sizeof(keystream)); +} +EXPORT_SYMBOL_GPL(aes_ctr); + +void aes_xctr(u8 *dst, const u8 *src, size_t len, u64 *ctr, + const u8 iv[AES_BLOCK_SIZE], aes_encrypt_arg key) +{ + const __le64 iv0 = get_unaligned((const __le64 *)&iv[0]); + __le64 aes_input[2]; + u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + + if (likely(aes_xctr_arch(dst, src, len, ctr, iv, key.enc_key))) + return; + + aes_input[1] = get_unaligned((const __le64 *)&iv[8]); + /* Handle the full blocks. */ + for (; len >= AES_BLOCK_SIZE; len -= AES_BLOCK_SIZE) { + aes_input[0] = iv0 ^ cpu_to_le64((*ctr)++); + aes_encrypt(key, keystream, (const u8 *)aes_input); + crypto_xor_cpy(dst, src, keystream, AES_BLOCK_SIZE); + dst += AES_BLOCK_SIZE; + src += AES_BLOCK_SIZE; + } + /* Handle any partial block at the end. */ + if (len) { + /* Counter is incremented even with just a partial block. */ + aes_input[0] = iv0 ^ cpu_to_le64((*ctr)++); + aes_encrypt(key, keystream, (const u8 *)aes_input); + crypto_xor_cpy(dst, src, keystream, len); + } + memzero_explicit(keystream, sizeof(keystream)); + memzero_explicit(aes_input, sizeof(aes_input)); +} +EXPORT_SYMBOL_GPL(aes_xctr); + +/* FIPS cryptographic algorithm self-test for AES-CTR */ +static void __init aes_ctr_fips_test(void) +{ + struct aes_enckey key; + u8 ctr[AES_BLOCK_SIZE]; + u8 data[sizeof(fips_test_data)]; + + if (aes_prepareenckey(&key, fips_test_key, sizeof(fips_test_key)) != 0) + panic("aes: CTR FIPS self-test failed (preparekey)\n"); + + memcpy(ctr, fips_test_iv, sizeof(ctr)); + aes_ctr(data, fips_test_data, sizeof(data), ctr, &key); + if (memcmp(fips_test_aes_ctr_ctext, data, sizeof(data)) != 0) + panic("aes: CTR FIPS self-test failed (wrong ciphertext)\n"); + + memcpy(ctr, fips_test_iv, sizeof(ctr)); + aes_ctr(data, data, sizeof(data), ctr, &key); + if (memcmp(fips_test_data, data, sizeof(data)) != 0) + panic("aes: CTR FIPS self-test failed (wrong plaintext)\n"); + + memzero_explicit(&key, sizeof(key)); +} +#else /* CONFIG_CRYPTO_LIB_AES_CTR */ +static inline void aes_ctr_fips_test(void) +{ +} +#endif /* !CONFIG_CRYPTO_LIB_AES_CTR */ + +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS) +int aes_xts_preparekey(struct aes_xts_key *key, const u8 *in_key, + size_t key_len, int flags) +{ + int err; + + err = __xts_verify_key(in_key, key_len, flags); + if (unlikely(err)) + goto out_zeroize; + /* First half of XTS key is the main key */ + err = aes_preparekey(&key->main_key, in_key, key_len / 2); + if (unlikely(err)) + goto out_zeroize; + /* Second half of XTS key is the tweak key */ + err = aes_prepareenckey(&key->tweak_key, &in_key[key_len / 2], + key_len / 2); + if (unlikely(err)) + goto out_zeroize; + return 0; + +out_zeroize: + memzero_explicit(key, sizeof(*key)); + return err; +} +EXPORT_SYMBOL_GPL(aes_xts_preparekey); + +/* + * Hooks for optimized AES-XTS implementations, overridable by the architecture. + * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. In other words, + * they aren't expected to handle ciphertext stealing or empty inputs. + * Returning false causes the fallback implementation to be used instead. + * + * (Currently, all users of AES-XTS in the kernel seem to en/decrypt whole + * numbers of blocks anyway, with len >= 512. So there's no need to heavily + * optimize ciphertext stealing for short messages.) + */ +#ifndef aes_xts_encrypt_arch +static bool aes_xts_encrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, bool cont) +{ + return false; +} +#endif +#ifndef aes_xts_decrypt_arch +static bool aes_xts_decrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, bool cont) +{ + return false; +} +#endif + +static noinline void aes_xts_crypt_nocts_blockbyblock( + u8 *dst, const u8 *src, size_t len, u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, bool cont, bool enc) +{ + le128 t; + + if (cont) + memcpy(&t, tweak, sizeof(t)); + else + aes_encrypt(&key->tweak_key, (u8 *)&t, tweak); + do { + crypto_xor_cpy(dst, src, (const u8 *)&t, AES_BLOCK_SIZE); + if (enc) + aes_encrypt(&key->main_key, dst, dst); + else + aes_decrypt(&key->main_key, dst, dst); + crypto_xor(dst, (const u8 *)&t, AES_BLOCK_SIZE); + gf128mul_x_ble(&t, &t); + dst += AES_BLOCK_SIZE; + src += AES_BLOCK_SIZE; + len -= AES_BLOCK_SIZE; + } while (len); + memcpy(tweak, &t, sizeof(t)); + memzero_explicit(&t, sizeof(t)); +} + +/* Requires len > 0 && len % AES_BLOCK_SIZE == 0 */ +static __always_inline void aes_xts_encrypt_nocts(u8 *dst, const u8 *src, + size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, + bool cont) +{ + if (likely(aes_xts_encrypt_arch(dst, src, len, tweak, key, cont))) + return; + + /* + * For the fallback, just go block-by-block. It could be implemented on + * top of AES-ECB, which could be significantly faster than this if the + * arch has optimized AES-ECB code but not AES-XTS. However, AES-XTS + * performance is important enough that it needs to be (and has been) + * implemented directly by every non-obsolete arch anyway. + */ + aes_xts_crypt_nocts_blockbyblock(dst, src, len, tweak, key, cont, + /* enc= */ true); +} + +/* Requires len > 0 && len % AES_BLOCK_SIZE == 0 */ +static __always_inline void aes_xts_decrypt_nocts(u8 *dst, const u8 *src, + size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, + bool cont) +{ + if (likely(aes_xts_decrypt_arch(dst, src, len, tweak, key, cont))) + return; + + /* Just go block-by-block. See comment in aes_xts_encrypt_nocts(). */ + aes_xts_crypt_nocts_blockbyblock(dst, src, len, tweak, key, cont, + /* enc= */ false); +} + +static noinline void aes_xts_encrypt_cts(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, + bool cont) +{ + size_t partial_len = len % AES_BLOCK_SIZE; /* Length of partial block */ + size_t nocts_len = round_down(len, AES_BLOCK_SIZE); + u8 tmp_block[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + + /* Encrypt all full blocks. */ + aes_xts_encrypt_nocts(dst, src, nocts_len, tweak, key, cont); + dst += nocts_len - AES_BLOCK_SIZE; + src += nocts_len - AES_BLOCK_SIZE; + + /* + * Swap the partial block with the first 'partial_len' bytes of the + * encrypted last full block. Note that a temporary buffer is needed to + * support in-place encryption. + */ + memcpy(tmp_block, src + AES_BLOCK_SIZE, partial_len); + memcpy(dst + AES_BLOCK_SIZE, dst, partial_len); + memcpy(dst, tmp_block, partial_len); + + /* Encrypt the last full block again. */ + crypto_xor(dst, tweak, AES_BLOCK_SIZE); + aes_encrypt(&key->main_key, dst, dst); + crypto_xor(dst, tweak, AES_BLOCK_SIZE); + memzero_explicit(tmp_block, sizeof(tmp_block)); +} + +static noinline void aes_xts_decrypt_cts(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, + bool cont) +{ + size_t partial_len = len % AES_BLOCK_SIZE; /* Length of partial block */ + size_t nocts_len = round_down(len, AES_BLOCK_SIZE) - AES_BLOCK_SIZE; + union { + u8 block[AES_BLOCK_SIZE]; + le128 tweak; + } tmp __aligned(__alignof__(long)); + + /* + * Decrypt all blocks except the last full block and the partial block. + * The last full block has to be handled specially because decryption + * ciphertext stealing uses the last two tweaks in reverse order. + * + * nocts_len == 0 is possible here, which aes_xts_decrypt_nocts() + * doesn't handle (so that the length doesn't get checked redundantly in + * the fast path). So handle that case specially as well. + */ + if (nocts_len) + aes_xts_decrypt_nocts(dst, src, nocts_len, tweak, key, cont); + else if (!cont) + aes_encrypt(&key->tweak_key, tweak, tweak); + dst += nocts_len; + src += nocts_len; + + /* Copy the tweak, advance it again, then decrypt last full block. */ + memcpy(&tmp.tweak, tweak, AES_BLOCK_SIZE); + gf128mul_x_ble(&tmp.tweak, &tmp.tweak); + crypto_xor_cpy(dst, src, tmp.block, AES_BLOCK_SIZE); + aes_decrypt(&key->main_key, dst, dst); + crypto_xor(dst, tmp.block, AES_BLOCK_SIZE); + + /* + * Swap the partial block with the first 'partial_len' bytes of the + * decrypted last full block. Note that a temporary buffer is needed to + * support in-place decryption. + */ + memcpy(tmp.block, src + AES_BLOCK_SIZE, partial_len); + memcpy(dst + AES_BLOCK_SIZE, dst, partial_len); + memcpy(dst, tmp.block, partial_len); + + /* Decrypt the last full block again. */ + crypto_xor(dst, tweak, AES_BLOCK_SIZE); + aes_decrypt(&key->main_key, dst, dst); + crypto_xor(dst, tweak, AES_BLOCK_SIZE); + memzero_explicit(&tmp, sizeof(tmp)); +} + +void aes_xts_encrypt(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], const struct aes_xts_key *key, + bool cont) +{ + if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) + return; + + if (unlikely(len % AES_BLOCK_SIZE)) { + aes_xts_encrypt_cts(dst, src, len, tweak, key, cont); + return; + } + + aes_xts_encrypt_nocts(dst, src, len, tweak, key, cont); +} +EXPORT_SYMBOL_GPL(aes_xts_encrypt); + +void aes_xts_decrypt(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], const struct aes_xts_key *key, + bool cont) +{ + if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) + return; + + if (unlikely(len % AES_BLOCK_SIZE)) { + aes_xts_decrypt_cts(dst, src, len, tweak, key, cont); + return; + } + + aes_xts_decrypt_nocts(dst, src, len, tweak, key, cont); +} +EXPORT_SYMBOL_GPL(aes_xts_decrypt); + +/* FIPS cryptographic algorithm self-test for AES-XTS */ +static void __init aes_xts_fips_test(void) +{ + struct aes_xts_key *key __free(kfree_sensitive) = kmalloc_obj(*key); + u8 tweak[AES_BLOCK_SIZE]; + u8 data[sizeof(fips_test_data)]; + + if (key == NULL) + panic("aes: XTS FIPS self-test failed (kmalloc)\n"); + + if (aes_xts_preparekey(key, fips_test_xts_key, + sizeof(fips_test_xts_key), 0) != 0) + panic("aes: XTS FIPS self-test failed (preparekey)\n"); + + memcpy(tweak, fips_test_iv, sizeof(tweak)); + aes_xts_encrypt(data, fips_test_data, sizeof(data), tweak, key, false); + if (memcmp(fips_test_aes_xts_ctext, data, sizeof(data)) != 0) + panic("aes: XTS FIPS self-test failed (wrong ciphertext)\n"); + + memcpy(tweak, fips_test_iv, sizeof(tweak)); + aes_xts_decrypt(data, data, sizeof(data), tweak, key, false); + if (memcmp(fips_test_data, data, sizeof(data)) != 0) + panic("aes: XTS FIPS self-test failed (wrong plaintext)\n"); +} +#else /* CONFIG_CRYPTO_LIB_AES_XTS */ +static inline void aes_xts_fips_test(void) +{ +} +#endif /* !CONFIG_CRYPTO_LIB_AES_XTS */ + +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_GCM) +/* + * Hooks for optimized AES-GCM implementations, overridable by the architecture. + * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. I.e. they aren't + * expected to handle empty inputs or partial blocks, as those cases are handled + * by non-arch-specific code instead. + * + * The GHASH accumulator is provided in POLYVAL format. The counter is provided + * in big endian format, and it's read-only, as the caller handles updating it. + * + * Returning false causes the fallback implementation to be used instead. + * + * These hooks are used only for en/decrypted data. For the associated data the + * GHASH functions are called instead, so those should be implemented too. + */ +#ifndef aes_gcm_encrypt_update_arch +static bool aes_gcm_encrypt_update_arch(u8 *dst, const u8 *src, size_t len, + struct polyval_elem *ghash_acc, + const __be32 ctr32[4], + const struct aes_enckey *aes_key, + const struct ghash_key *ghash_key) +{ + return false; +} +#endif +#ifndef aes_gcm_decrypt_update_arch +static bool aes_gcm_decrypt_update_arch(u8 *dst, const u8 *src, size_t len, + struct polyval_elem *ghash_acc, + const __be32 ctr32[4], + const struct aes_enckey *aes_key, + const struct ghash_key *ghash_key) +{ + return false; +} +#endif + +int aes_gcm_preparekey(struct aes_gcm_key *key, const u8 *in_key, + size_t key_len, size_t authtag_len) +{ + u8 h[AES_BLOCK_SIZE] = { 0 }; + int err; + + err = crypto_gcm_check_authsize(authtag_len); + if (unlikely(err)) + return err; + + err = aes_prepareenckey(&key->aes, in_key, key_len); + if (unlikely(err)) + return err; + + aes_encrypt(&key->aes, h, h); + ghash_preparekey(&key->ghash, h); + + key->authtag_len = authtag_len; + + memzero_explicit(h, sizeof(h)); + return 0; +} +EXPORT_SYMBOL_GPL(aes_gcm_preparekey); + +void aes_gcm_init(struct aes_gcm_ctx *ctx, const u8 nonce[12], + const struct aes_gcm_key *key) +{ + ctx->key = key; + ctx->ad_len = 0; + ctx->data_len = 0; + ghash_init(&ctx->ghash, &key->ghash); + memset(ctx->keystream, 0, sizeof(ctx->keystream)); + + memcpy(ctx->ctr32, nonce, 12); + ctx->ctr32[3] = cpu_to_be32(1); + + aes_encrypt(&key->aes, ctx->j0_enc, ctx->ctr); + ctx->ctr32[3] = cpu_to_be32(2); +} +EXPORT_SYMBOL_GPL(aes_gcm_init); + +void aes_gcm_auth_update(struct aes_gcm_ctx *ctx, const u8 *ad, size_t len) +{ + WARN_ON_ONCE(ctx->data_len != 0); + if (len) { + ghash_update(&ctx->ghash, ad, len); + ctx->ad_len += len; + } +} +EXPORT_SYMBOL_GPL(aes_gcm_auth_update); + +static const u8 gcm_zeroes[AES_BLOCK_SIZE]; + +static __always_inline void ghash_pad(struct ghash_ctx *ghash, u64 len) +{ + if (len % AES_BLOCK_SIZE) + ghash_update(ghash, gcm_zeroes, -len % AES_BLOCK_SIZE); +} + +static __always_inline void aes_gcm_crypt_update(struct aes_gcm_ctx *ctx, + u8 *dst, const u8 *src, + size_t len, bool enc) +{ + size_t partial_len, n; + + if (unlikely(len == 0)) + return; + + partial_len = ctx->data_len % AES_BLOCK_SIZE; + if (ctx->data_len == 0) + ghash_pad(&ctx->ghash, ctx->ad_len); + ctx->data_len += len; + + if (unlikely(partial_len != 0)) { + /* + * The previous call ended on a non-block-aligned data_len, so + * continue using a previously-generated keystream block. + */ + n = min(len, AES_BLOCK_SIZE - partial_len); + if (enc) { + crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], + n); + ghash_update(&ctx->ghash, dst, n); + } else { + ghash_update(&ctx->ghash, src, n); + crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], + n); + } + dst += n; + src += n; + len -= n; + } + + if (len >= AES_BLOCK_SIZE) { + n = round_down(len, AES_BLOCK_SIZE); + if (enc) { + if (likely(aes_gcm_encrypt_update_arch( + dst, src, n, &ctx->ghash.acc, ctx->ctr32, + &ctx->key->aes, &ctx->key->ghash))) { + be32_add_cpu(&ctx->ctr32[3], + n / AES_BLOCK_SIZE); + } else { + aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); + ghash_update(&ctx->ghash, dst, n); + } + } else { + if (likely(aes_gcm_decrypt_update_arch( + dst, src, n, &ctx->ghash.acc, ctx->ctr32, + &ctx->key->aes, &ctx->key->ghash))) { + be32_add_cpu(&ctx->ctr32[3], + n / AES_BLOCK_SIZE); + } else { + ghash_update(&ctx->ghash, src, n); + aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); + } + } + dst += n; + src += n; + len -= n; + } + + if (len != 0) { + /* + * Ending on a non-block aligned data_len. Generate the next + * keystream block, use the needed portion of it, and leave it + * cached in ctx->keystream in case this isn't the final call. + */ + aes_encrypt(&ctx->key->aes, ctx->keystream, ctx->ctr); + be32_add_cpu(&ctx->ctr32[3], 1); + if (enc) { + crypto_xor_cpy(dst, src, ctx->keystream, len); + ghash_update(&ctx->ghash, dst, len); + } else { + ghash_update(&ctx->ghash, src, len); + crypto_xor_cpy(dst, src, ctx->keystream, len); + } + } +} + +void aes_gcm_encrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *src, + size_t len) +{ + aes_gcm_crypt_update(ctx, dst, src, len, /* enc= */ true); +} +EXPORT_SYMBOL_GPL(aes_gcm_encrypt_update); + +void aes_gcm_decrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *src, + size_t len) +{ + aes_gcm_crypt_update(ctx, dst, src, len, /* enc= */ false); +} +EXPORT_SYMBOL_GPL(aes_gcm_decrypt_update); + +/* Maximum AES-GCM associated data length in bytes */ +#define AES_GCM_MAX_AD_LEN ((1ULL << 61) - 1) +/* Maximum AES-GCM en/decrypted data length in bytes */ +#define AES_GCM_MAX_DATA_LEN ((1ULL << 36) - 32) + +void aes_gcm_encrypt_final(struct aes_gcm_ctx *ctx, u8 *authtag) +{ + __be64 tail[2]; + + WARN_ON_ONCE(ctx->ad_len > AES_GCM_MAX_AD_LEN); + WARN_ON_ONCE(ctx->data_len > AES_GCM_MAX_DATA_LEN); + + ghash_pad(&ctx->ghash, + ctx->data_len == 0 ? ctx->ad_len : ctx->data_len); + + tail[0] = cpu_to_be64(ctx->ad_len * 8); + tail[1] = cpu_to_be64(ctx->data_len * 8); + ghash_update(&ctx->ghash, (const u8 *)tail, 16); + ghash_final(&ctx->ghash, ctx->ctr); /* Use ctr as temp buffer */ + + crypto_xor_cpy(authtag, ctx->ctr, ctx->j0_enc, ctx->key->authtag_len); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(aes_gcm_encrypt_final); + +int aes_gcm_decrypt_final(struct aes_gcm_ctx *ctx, const u8 *authtag) +{ + __be64 tail[2]; + int err; + + if (WARN_ON_ONCE(ctx->ad_len > AES_GCM_MAX_AD_LEN) || + WARN_ON_ONCE(ctx->data_len > AES_GCM_MAX_DATA_LEN)) { + err = -EBADMSG; + goto out; + } + + ghash_pad(&ctx->ghash, + ctx->data_len == 0 ? ctx->ad_len : ctx->data_len); + + tail[0] = cpu_to_be64(ctx->ad_len * 8); + tail[1] = cpu_to_be64(ctx->data_len * 8); + ghash_update(&ctx->ghash, (const u8 *)tail, 16); + ghash_final(&ctx->ghash, ctx->ctr); /* Use ctr as temp buffer */ + crypto_xor(ctx->ctr, ctx->j0_enc, ctx->key->authtag_len); + err = crypto_memneq(ctx->ctr, authtag, ctx->key->authtag_len) ? + -EBADMSG : + 0; +out: + memzero_explicit(ctx, sizeof(*ctx)); + return err; +} +EXPORT_SYMBOL_GPL(aes_gcm_decrypt_final); + +void aes_gcm_encrypt(u8 *dst, const u8 *src, size_t data_len, u8 *authtag, + const u8 *ad, size_t ad_len, const u8 nonce[12], + const struct aes_gcm_key *key) +{ + struct aes_gcm_ctx ctx; + + aes_gcm_init(&ctx, nonce, key); + aes_gcm_auth_update(&ctx, ad, ad_len); + aes_gcm_encrypt_update(&ctx, dst, src, data_len); + aes_gcm_encrypt_final(&ctx, authtag); +} +EXPORT_SYMBOL_GPL(aes_gcm_encrypt); + +int aes_gcm_decrypt(u8 *dst, const u8 *src, size_t data_len, const u8 *authtag, + const u8 *ad, size_t ad_len, const u8 nonce[12], + const struct aes_gcm_key *key) +{ + struct aes_gcm_ctx ctx; + int err; + + aes_gcm_init(&ctx, nonce, key); + aes_gcm_auth_update(&ctx, ad, ad_len); + aes_gcm_decrypt_update(&ctx, dst, src, data_len); + err = aes_gcm_decrypt_final(&ctx, authtag); + if (unlikely(err) && data_len) { + /* + * Clear the inauthentic decrypted data so that callers won't + * receive it even if they fail to correctly handle errors. + */ + memset(dst, 0, data_len); + } + return err; +} +EXPORT_SYMBOL_GPL(aes_gcm_decrypt); + +/* FIPS cryptographic algorithm self-test for AES-GCM */ +static void __init aes_gcm_fips_test(void) +{ + const size_t data_len = sizeof(fips_test_data); + u8 buf[sizeof(fips_test_data) + AES_BLOCK_SIZE]; + struct aes_gcm_key key; + int err; + + if (aes_gcm_preparekey(&key, fips_test_key, sizeof(fips_test_key), + AES_BLOCK_SIZE) != 0) + panic("aes: GCM FIPS self-test failed (preparekey)\n"); + + aes_gcm_encrypt(buf, fips_test_data, data_len, &buf[data_len], + fips_test_ad, sizeof(fips_test_ad), fips_test_iv, &key); + if (memcmp(fips_test_aes_gcm_ctext_and_tag, buf, sizeof(buf)) != 0) + panic("aes: GCM FIPS self-test failed (wrong ciphertext and/or tag)\n"); + + err = aes_gcm_decrypt(buf, buf, data_len, &buf[data_len], fips_test_ad, + sizeof(fips_test_ad), fips_test_iv, &key); + if (err != 0) + panic("aes: GCM FIPS self-test failed (decryption failed)\n"); + if (memcmp(fips_test_data, buf, data_len) != 0) + panic("aes: GCM FIPS self-test failed (wrong plaintext)\n"); + + memzero_explicit(&key, sizeof(key)); +} +#else /* CONFIG_CRYPTO_LIB_AES_GCM */ +static inline void aes_gcm_fips_test(void) +{ +} +#endif /* !CONFIG_CRYPTO_LIB_AES_GCM */ + +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CCM) +int aes_ccm_preparekey(struct aes_ccm_key *key, const u8 *in_key, + size_t key_len, size_t authtag_len) +{ + int err; + + if (unlikely(authtag_len < 4 || authtag_len > 16 || authtag_len % 2)) + return -EINVAL; + + err = aes_prepareenckey(&key->aes, in_key, key_len); + if (unlikely(err)) + return err; + + key->authtag_len = authtag_len; + return 0; +} +EXPORT_SYMBOL_GPL(aes_ccm_preparekey); + +int aes_ccm_init(struct aes_ccm_ctx *ctx, u64 data_len, u64 ad_len, + const u8 *nonce, size_t nonce_len, + const struct aes_ccm_key *key) +{ + /* + * This is the value L defined in the CCM specification. It determines + * the maximum allowed message length, and it is itself determined by + * the nonce length. They are inversely related, i.e. the longer the + * nonce the smaller the maximum message length is. + */ + unsigned int l = 15 - nonce_len; + + if (unlikely(nonce_len < 7 || nonce_len > 13)) + return -EINVAL; + /* Thus 2 <= l <= 8. */ + + /* Check whether data_len can be represented in 'l' bytes. */ + if (unlikely(data_len > U64_MAX >> (64 - 8 * l))) + return -EOVERFLOW; + + ctx->key = key; + ctx->ad_remaining = ad_len; + ctx->data_remaining = data_len; + ctx->ad_padded = false; + + /* + * Initialize the zero-th counter block to: + * + * L - 1 || nonce || 0 + * + * ... and the zero-th CBC-MAC block to: + * + * Flags || nonce || data_len + */ + *(__be64 *)&ctx->ctr[8] = 0; + *(__be64 *)&ctx->mac[8] = cpu_to_be64(data_len); + ctx->ctr[0] = l - 1; + ctx->mac[0] = (ad_len ? 0x40 : 0) | + (((key->authtag_len - 2) / 2) << 3) | (l - 1); + memcpy(&ctx->ctr[1], nonce, nonce_len); /* Overlapping store */ + memcpy(&ctx->mac[1], nonce, nonce_len); /* Overlapping store */ + + /* + * Generate S_0 by encrypting the counter (this is used to encrypt the + * auth tag later), and encrypt the zero-th CBC-MAC block. + */ + aes_encrypt(&key->aes, ctx->s0, ctx->ctr); + aes_encrypt(&key->aes, ctx->mac, ctx->mac); + + /* Increment the counter from 0 to 1. */ + ctx->ctr[15] = 1; + + if (ad_len) { + /* + * Update CBC-MAC with the associated data length, represented + * using either 2, 6, or 10 bytes depending on the length. + */ + if (likely(ad_len < 0xff00)) { + *(__be16 *)&ctx->mac[0] ^= cpu_to_be16(ad_len); + ctx->partial_len = 2; + } else if (ad_len <= U32_MAX) { + __be32 *p = (__be32 *)&ctx->mac[2]; + + *(__be16 *)&ctx->mac[0] ^= cpu_to_be16(0xfffe); + put_unaligned(get_unaligned(p) ^ cpu_to_be32(ad_len), + p); + ctx->partial_len = 6; + } else { + __be64 *p = (__be64 *)&ctx->mac[2]; + + *(__be16 *)&ctx->mac[0] ^= cpu_to_be16(0xffff); + put_unaligned(get_unaligned(p) ^ cpu_to_be64(ad_len), + p); + ctx->partial_len = 10; + } + } else { + ctx->partial_len = 0; + } + return 0; +} +EXPORT_SYMBOL_GPL(aes_ccm_init); + +void aes_ccm_auth_update(struct aes_ccm_ctx *ctx, const u8 *ad, size_t len) +{ + size_t partial_len = ctx->partial_len; + bool enc_before = false; + size_t nblocks; + + WARN_ON_ONCE(ctx->ad_padded); + + /* + * We could warn on len > ad_remaining here, but underflow will be + * caught by the != 0 check at the end anyway. (It's a u64, so it isn't + * going to underflow all the way back to 0.) + */ + ctx->ad_remaining -= len; + + if (partial_len) { + size_t n = min(len, AES_BLOCK_SIZE - partial_len); + + crypto_xor(&ctx->mac[partial_len], ad, n); + ad += n; + len -= n; + partial_len += n; + if (partial_len < AES_BLOCK_SIZE) { + ctx->partial_len = partial_len; + return; + } + enc_before = true; + } + + nblocks = len / AES_BLOCK_SIZE; + len %= AES_BLOCK_SIZE; + if (nblocks == 0) { + if (enc_before) + aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); + } else { + aes_cbcmac_blocks(ctx->mac, &ctx->key->aes, ad, nblocks, + enc_before, /* enc_after= */ true); + ad += nblocks * AES_BLOCK_SIZE; + } + crypto_xor(ctx->mac, ad, len); + ctx->partial_len = len; +} +EXPORT_SYMBOL_GPL(aes_ccm_auth_update); + +static __always_inline void aes_ccm_crypt_update(struct aes_ccm_ctx *ctx, + u8 *dst, const u8 *src, + size_t len, bool enc) +{ + size_t partial_len = ctx->partial_len; + size_t n, nblocks; + + if (unlikely(len == 0)) + return; + + WARN_ON_ONCE(ctx->ad_remaining != 0); + + /* + * We could warn on len > data_remaining here, but underflow will be + * caught by the != 0 check at the end anyway. (It's a u64, so it isn't + * going to underflow all the way back to 0.) + */ + ctx->data_remaining -= len; + + if (!ctx->ad_padded) { + ctx->ad_padded = true; + if (partial_len) + aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); + } else if (partial_len) { + /* + * The previous call ended on a non-block-aligned data_len, so + * continue using a previously-generated keystream block. + */ + n = min(len, AES_BLOCK_SIZE - partial_len); + if (enc) + crypto_xor(&ctx->mac[partial_len], src, n); + crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], n); + if (!enc) + crypto_xor(&ctx->mac[partial_len], dst, n); + dst += n; + src += n; + len -= n; + partial_len += n; + if (partial_len < AES_BLOCK_SIZE) { + ctx->partial_len = partial_len; + return; + } + aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); + } + + if (len >= AES_BLOCK_SIZE) { + n = round_down(len, AES_BLOCK_SIZE); + nblocks = len / AES_BLOCK_SIZE; + if (enc) + aes_cbcmac_blocks(ctx->mac, &ctx->key->aes, src, + nblocks, /* enc_before= */ false, + /* enc_after= */ true); + aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); + if (!enc) + aes_cbcmac_blocks(ctx->mac, &ctx->key->aes, dst, + nblocks, /* enc_before= */ false, + /* enc_after= */ true); + dst += n; + src += n; + len -= n; + } + + if (len) { + /* + * Ending on a non-block aligned data_len. Generate the next + * keystream block, use the needed portion of it, and leave it + * cached in ctx->keystream in case this isn't the final call. + */ + aes_encrypt(&ctx->key->aes, ctx->keystream, ctx->ctr); + inc_be128_ctr(ctx->ctr); + if (enc) + crypto_xor(ctx->mac, src, len); + crypto_xor_cpy(dst, src, ctx->keystream, len); + if (!enc) + crypto_xor(ctx->mac, dst, len); + } + ctx->partial_len = len; +} + +void aes_ccm_encrypt_update(struct aes_ccm_ctx *ctx, u8 *dst, const u8 *src, + size_t len) +{ + aes_ccm_crypt_update(ctx, dst, src, len, /* enc= */ true); +} +EXPORT_SYMBOL_GPL(aes_ccm_encrypt_update); + +void aes_ccm_decrypt_update(struct aes_ccm_ctx *ctx, u8 *dst, const u8 *src, + size_t len) +{ + aes_ccm_crypt_update(ctx, dst, src, len, /* enc= */ false); +} +EXPORT_SYMBOL_GPL(aes_ccm_decrypt_update); + +void aes_ccm_encrypt_final(struct aes_ccm_ctx *ctx, u8 *authtag) +{ + WARN_ON_ONCE(ctx->ad_remaining != 0); + WARN_ON_ONCE(ctx->data_remaining != 0); + if (ctx->partial_len) + aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); + crypto_xor_cpy(authtag, ctx->mac, ctx->s0, ctx->key->authtag_len); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(aes_ccm_encrypt_final); + +int aes_ccm_decrypt_final(struct aes_ccm_ctx *ctx, const u8 *authtag) +{ + int err; + + if (WARN_ON_ONCE(ctx->ad_remaining != 0) || + WARN_ON_ONCE(ctx->data_remaining != 0)) { + err = -EBADMSG; + goto out; + } + + if (ctx->partial_len) + aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); + crypto_xor(ctx->mac, ctx->s0, ctx->key->authtag_len); + err = crypto_memneq(ctx->mac, authtag, ctx->key->authtag_len) ? + -EBADMSG : + 0; +out: + memzero_explicit(ctx, sizeof(*ctx)); + return err; +} +EXPORT_SYMBOL_GPL(aes_ccm_decrypt_final); + +int aes_ccm_encrypt(u8 *dst, const u8 *src, size_t data_len, u8 *authtag, + const u8 *ad, size_t ad_len, const u8 *nonce, + size_t nonce_len, const struct aes_ccm_key *key) +{ + struct aes_ccm_ctx ctx; + int err; + + err = aes_ccm_init(&ctx, data_len, ad_len, nonce, nonce_len, key); + if (unlikely(err)) + return err; + aes_ccm_auth_update(&ctx, ad, ad_len); + aes_ccm_encrypt_update(&ctx, dst, src, data_len); + aes_ccm_encrypt_final(&ctx, authtag); + return 0; +} +EXPORT_SYMBOL_GPL(aes_ccm_encrypt); + +int aes_ccm_decrypt(u8 *dst, const u8 *src, size_t data_len, const u8 *authtag, + const u8 *ad, size_t ad_len, const u8 *nonce, + size_t nonce_len, const struct aes_ccm_key *key) +{ + struct aes_ccm_ctx ctx; + int err; + + err = aes_ccm_init(&ctx, data_len, ad_len, nonce, nonce_len, key); + if (unlikely(err)) + return err; + aes_ccm_auth_update(&ctx, ad, ad_len); + aes_ccm_decrypt_update(&ctx, dst, src, data_len); + err = aes_ccm_decrypt_final(&ctx, authtag); + if (unlikely(err) && data_len) { + /* + * Clear the inauthentic decrypted data so that callers won't + * receive it even if they fail to correctly handle errors. + */ + memset(dst, 0, data_len); + } + return err; +} +EXPORT_SYMBOL_GPL(aes_ccm_decrypt); + +/* FIPS cryptographic algorithm self-test for AES-CCM */ +static void __init aes_ccm_fips_test(void) +{ + const size_t data_len = sizeof(fips_test_data); + const size_t nonce_len = 13; + u8 buf[sizeof(fips_test_data) + AES_BLOCK_SIZE]; + struct aes_ccm_key key; + int err; + + if (aes_ccm_preparekey(&key, fips_test_key, sizeof(fips_test_key), + AES_BLOCK_SIZE) != 0) + panic("aes: CCM FIPS self-test failed (preparekey)\n"); + + err = aes_ccm_encrypt(buf, fips_test_data, data_len, &buf[data_len], + fips_test_ad, sizeof(fips_test_ad), fips_test_iv, + nonce_len, &key); + if (err != 0) + panic("aes: CCM FIPS self-test failed (encryption failed)\n"); + if (memcmp(fips_test_aes_ccm_ctext_and_tag, buf, sizeof(buf)) != 0) + panic("aes: CCM FIPS self-test failed (wrong ciphertext and/or tag)\n"); + + err = aes_ccm_decrypt(buf, buf, data_len, &buf[data_len], fips_test_ad, + sizeof(fips_test_ad), fips_test_iv, nonce_len, + &key); + if (err != 0) + panic("aes: CCM FIPS self-test failed (decryption failed)\n"); + if (memcmp(fips_test_data, buf, data_len) != 0) + panic("aes: CCM FIPS self-test failed (wrong plaintext)\n"); + + memzero_explicit(&key, sizeof(key)); +} +#else /* CONFIG_CRYPTO_LIB_AES_CCM */ +static inline void aes_ccm_fips_test(void) +{ +} +#endif /* !CONFIG_CRYPTO_LIB_AES_CCM */ + static int __init aes_mod_init(void) { +#ifdef aes_mod_init_arch aes_mod_init_arch(); +#endif + if (fips_enabled) { + aes_fips_test(); + aes_cmac_fips_test(); + aes_ecb_fips_test(); + aes_cbc_fips_test(); + aes_cbc_cts_fips_test(); + aes_ctr_fips_test(); + aes_xts_fips_test(); + aes_gcm_fips_test(); + aes_ccm_fips_test(); + } return 0; } subsys_initcall(aes_mod_init); @@ -524,7 +2141,6 @@ static void __exit aes_mod_exit(void) { } module_exit(aes_mod_exit); -#endif MODULE_DESCRIPTION("AES block cipher"); MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); diff --git a/lib/crypto/aescfb.c b/lib/crypto/aescfb.c index 147e5211728f..82cd55436055 100644 --- a/lib/crypto/aescfb.c +++ b/lib/crypto/aescfb.c @@ -6,28 +6,9 @@ */ #include <crypto/aes.h> -#include <crypto/algapi.h> +#include <crypto/utils.h> #include <linux/export.h> #include <linux/module.h> -#include <asm/irqflags.h> - -static void aescfb_encrypt_block(const struct aes_enckey *key, void *dst, - const void *src) -{ - unsigned long flags; - - /* - * In AES-CFB, the AES encryption operates on known 'plaintext' (the IV - * and ciphertext), making it susceptible to timing attacks on the - * encryption key. The AES library already mitigates this risk to some - * extent by pulling the entire S-box into the caches before doing any - * substitutions, but this strategy is more effective when running with - * interrupts disabled. - */ - local_irq_save(flags); - aes_encrypt(key, dst, src); - local_irq_restore(flags); -} /** * aescfb_encrypt - Perform AES-CFB encryption on a block of data @@ -45,7 +26,7 @@ void aescfb_encrypt(const struct aes_enckey *key, u8 *dst, const u8 *src, const u8 *v = iv; while (len > 0) { - aescfb_encrypt_block(key, ks, v); + aes_encrypt(key, ks, v); crypto_xor_cpy(dst, src, ks, min(len, AES_BLOCK_SIZE)); v = dst; @@ -72,7 +53,7 @@ void aescfb_decrypt(const struct aes_enckey *key, u8 *dst, const u8 *src, { u8 ks[2][AES_BLOCK_SIZE]; - aescfb_encrypt_block(key, ks[0], iv); + aes_encrypt(key, ks[0], iv); for (int i = 0; len > 0; i ^= 1) { if (len > AES_BLOCK_SIZE) @@ -81,7 +62,7 @@ void aescfb_decrypt(const struct aes_enckey *key, u8 *dst, const u8 *src, * performing the XOR, as that may update in place and * overwrite the ciphertext. */ - aescfb_encrypt_block(key, ks[!i], src); + aes_encrypt(key, ks[!i], src); crypto_xor_cpy(dst, src, ks[i], min(len, AES_BLOCK_SIZE)); diff --git a/lib/crypto/aesgcm.c b/lib/crypto/aesgcm.c deleted file mode 100644 index 02f5b5f32c76..000000000000 --- a/lib/crypto/aesgcm.c +++ /dev/null @@ -1,739 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Minimal library implementation of GCM - * - * Copyright 2022 Google LLC - */ - -#include <crypto/algapi.h> -#include <crypto/gcm.h> -#include <crypto/ghash.h> -#include <linux/export.h> -#include <linux/module.h> -#include <asm/irqflags.h> - -static void aesgcm_encrypt_block(const struct aes_enckey *key, void *dst, - const void *src) -{ - unsigned long flags; - - /* - * In AES-GCM, both the GHASH key derivation and the CTR mode - * encryption operate on known plaintext, making them susceptible to - * timing attacks on the encryption key. The AES library already - * mitigates this risk to some extent by pulling the entire S-box into - * the caches before doing any substitutions, but this strategy is more - * effective when running with interrupts disabled. - */ - local_irq_save(flags); - aes_encrypt(key, dst, src); - local_irq_restore(flags); -} - -/** - * aesgcm_expandkey - Expands the AES and GHASH keys for the AES-GCM key - * schedule - * - * @ctx: The data structure that will hold the AES-GCM key schedule - * @key: The AES encryption input key - * @keysize: The length in bytes of the input key - * @authsize: The size in bytes of the GCM authentication tag - * - * Returns: 0 on success, or -EINVAL if @keysize or @authsize contain values - * that are not permitted by the GCM specification. - */ -int aesgcm_expandkey(struct aesgcm_ctx *ctx, const u8 *key, - unsigned int keysize, unsigned int authsize) -{ - u8 kin[AES_BLOCK_SIZE] = {}; - int ret; - - ret = crypto_gcm_check_authsize(authsize) ?: - aes_prepareenckey(&ctx->aes_key, key, keysize); - if (ret) - return ret; - - ctx->authsize = authsize; - aesgcm_encrypt_block(&ctx->aes_key, &ctx->ghash_key, kin); - - return 0; -} -EXPORT_SYMBOL(aesgcm_expandkey); - -static void aesgcm_ghash(be128 *ghash, const be128 *key, const void *src, - int len) -{ - while (len > 0) { - crypto_xor((u8 *)ghash, src, min(len, GHASH_BLOCK_SIZE)); - gf128mul_lle(ghash, key); - - src += GHASH_BLOCK_SIZE; - len -= GHASH_BLOCK_SIZE; - } -} - -/** - * aesgcm_mac - Generates the authentication tag using AES-GCM algorithm. - * @ctx: The data structure that will hold the AES-GCM key schedule - * @src: The input source data. - * @src_len: Length of the source data. - * @assoc: Points to the associated data. - * @assoc_len: Length of the associated data values. - * @ctr: Points to the counter value. - * @authtag: The output buffer for the authentication tag. - * - * It takes in the AES-GCM context, source data, associated data, counter value, - * and an output buffer for the authentication tag. - */ -static void aesgcm_mac(const struct aesgcm_ctx *ctx, const u8 *src, int src_len, - const u8 *assoc, int assoc_len, __be32 *ctr, u8 *authtag) -{ - be128 tail = { cpu_to_be64(assoc_len * 8), cpu_to_be64(src_len * 8) }; - u8 buf[AES_BLOCK_SIZE]; - be128 ghash = {}; - - aesgcm_ghash(&ghash, &ctx->ghash_key, assoc, assoc_len); - aesgcm_ghash(&ghash, &ctx->ghash_key, src, src_len); - aesgcm_ghash(&ghash, &ctx->ghash_key, &tail, sizeof(tail)); - - ctr[3] = cpu_to_be32(1); - aesgcm_encrypt_block(&ctx->aes_key, buf, ctr); - crypto_xor_cpy(authtag, buf, (u8 *)&ghash, ctx->authsize); - - memzero_explicit(&ghash, sizeof(ghash)); - memzero_explicit(buf, sizeof(buf)); -} - -static void aesgcm_crypt(const struct aesgcm_ctx *ctx, u8 *dst, const u8 *src, - int len, __be32 *ctr) -{ - u8 buf[AES_BLOCK_SIZE]; - unsigned int n = 2; - - while (len > 0) { - /* - * The counter increment below must not result in overflow or - * carry into the next 32-bit word, as this could result in - * inadvertent IV reuse, which must be avoided at all cost for - * stream ciphers such as AES-CTR. Given the range of 'int - * len', this cannot happen, so no explicit test is necessary. - */ - ctr[3] = cpu_to_be32(n++); - aesgcm_encrypt_block(&ctx->aes_key, buf, ctr); - crypto_xor_cpy(dst, src, buf, min(len, AES_BLOCK_SIZE)); - - dst += AES_BLOCK_SIZE; - src += AES_BLOCK_SIZE; - len -= AES_BLOCK_SIZE; - } - memzero_explicit(buf, sizeof(buf)); -} - -/** - * aesgcm_encrypt - Perform AES-GCM encryption on a block of data - * - * @ctx: The AES-GCM key schedule - * @dst: Pointer to the ciphertext output buffer - * @src: Pointer the plaintext (may equal @dst for encryption in place) - * @crypt_len: The size in bytes of the plaintext and ciphertext. - * @assoc: Pointer to the associated data, - * @assoc_len: The size in bytes of the associated data - * @iv: The initialization vector (IV) to use for this block of data - * (must be 12 bytes in size as per the GCM spec recommendation) - * @authtag: The address of the buffer in memory where the authentication - * tag should be stored. The buffer is assumed to have space for - * @ctx->authsize bytes. - */ -void aesgcm_encrypt(const struct aesgcm_ctx *ctx, u8 *dst, const u8 *src, - int crypt_len, const u8 *assoc, int assoc_len, - const u8 iv[GCM_AES_IV_SIZE], u8 *authtag) -{ - __be32 ctr[4]; - - memcpy(ctr, iv, GCM_AES_IV_SIZE); - - aesgcm_crypt(ctx, dst, src, crypt_len, ctr); - aesgcm_mac(ctx, dst, crypt_len, assoc, assoc_len, ctr, authtag); -} -EXPORT_SYMBOL(aesgcm_encrypt); - -/** - * aesgcm_decrypt - Perform AES-GCM decryption on a block of data - * - * @ctx: The AES-GCM key schedule - * @dst: Pointer to the plaintext output buffer - * @src: Pointer the ciphertext (may equal @dst for decryption in place) - * @crypt_len: The size in bytes of the plaintext and ciphertext. - * @assoc: Pointer to the associated data, - * @assoc_len: The size in bytes of the associated data - * @iv: The initialization vector (IV) to use for this block of data - * (must be 12 bytes in size as per the GCM spec recommendation) - * @authtag: The address of the buffer in memory where the authentication - * tag is stored. - * - * Returns: true on success, or false if the ciphertext failed authentication. - * On failure, no plaintext will be returned. - */ -bool __must_check aesgcm_decrypt(const struct aesgcm_ctx *ctx, u8 *dst, - const u8 *src, int crypt_len, const u8 *assoc, - int assoc_len, const u8 iv[GCM_AES_IV_SIZE], - const u8 *authtag) -{ - u8 tagbuf[AES_BLOCK_SIZE]; - __be32 ctr[4]; - - memcpy(ctr, iv, GCM_AES_IV_SIZE); - - aesgcm_mac(ctx, src, crypt_len, assoc, assoc_len, ctr, tagbuf); - if (crypto_memneq(authtag, tagbuf, ctx->authsize)) { - memzero_explicit(tagbuf, sizeof(tagbuf)); - return false; - } - aesgcm_crypt(ctx, dst, src, crypt_len, ctr); - return true; -} -EXPORT_SYMBOL(aesgcm_decrypt); - -MODULE_DESCRIPTION("Generic AES-GCM library"); -MODULE_AUTHOR("Ard Biesheuvel <ardb@kernel.org>"); -MODULE_LICENSE("GPL"); - -#ifdef CONFIG_CRYPTO_SELFTESTS - -/* - * Test code below. Vectors taken from crypto/testmgr.h - */ - -static const u8 __initconst ctext0[16] __nonstring = - "\x58\xe2\xfc\xce\xfa\x7e\x30\x61" - "\x36\x7f\x1d\x57\xa4\xe7\x45\x5a"; - -static const u8 __initconst ptext1[16]; - -static const u8 __initconst ctext1[32] __nonstring = - "\x03\x88\xda\xce\x60\xb6\xa3\x92" - "\xf3\x28\xc2\xb9\x71\xb2\xfe\x78" - "\xab\x6e\x47\xd4\x2c\xec\x13\xbd" - "\xf5\x3a\x67\xb2\x12\x57\xbd\xdf"; - -static const u8 __initconst ptext2[64] __nonstring = - "\xd9\x31\x32\x25\xf8\x84\x06\xe5" - "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" - "\x86\xa7\xa9\x53\x15\x34\xf7\xda" - "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" - "\x1c\x3c\x0c\x95\x95\x68\x09\x53" - "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" - "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" - "\xba\x63\x7b\x39\x1a\xaf\xd2\x55"; - -static const u8 __initconst ctext2[80] __nonstring = - "\x42\x83\x1e\xc2\x21\x77\x74\x24" - "\x4b\x72\x21\xb7\x84\xd0\xd4\x9c" - "\xe3\xaa\x21\x2f\x2c\x02\xa4\xe0" - "\x35\xc1\x7e\x23\x29\xac\xa1\x2e" - "\x21\xd5\x14\xb2\x54\x66\x93\x1c" - "\x7d\x8f\x6a\x5a\xac\x84\xaa\x05" - "\x1b\xa3\x0b\x39\x6a\x0a\xac\x97" - "\x3d\x58\xe0\x91\x47\x3f\x59\x85" - "\x4d\x5c\x2a\xf3\x27\xcd\x64\xa6" - "\x2c\xf3\x5a\xbd\x2b\xa6\xfa\xb4"; - -static const u8 __initconst ptext3[60] __nonstring = - "\xd9\x31\x32\x25\xf8\x84\x06\xe5" - "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" - "\x86\xa7\xa9\x53\x15\x34\xf7\xda" - "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" - "\x1c\x3c\x0c\x95\x95\x68\x09\x53" - "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" - "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" - "\xba\x63\x7b\x39"; - -static const u8 __initconst ctext3[76] __nonstring = - "\x42\x83\x1e\xc2\x21\x77\x74\x24" - "\x4b\x72\x21\xb7\x84\xd0\xd4\x9c" - "\xe3\xaa\x21\x2f\x2c\x02\xa4\xe0" - "\x35\xc1\x7e\x23\x29\xac\xa1\x2e" - "\x21\xd5\x14\xb2\x54\x66\x93\x1c" - "\x7d\x8f\x6a\x5a\xac\x84\xaa\x05" - "\x1b\xa3\x0b\x39\x6a\x0a\xac\x97" - "\x3d\x58\xe0\x91" - "\x5b\xc9\x4f\xbc\x32\x21\xa5\xdb" - "\x94\xfa\xe9\x5a\xe7\x12\x1a\x47"; - -static const u8 __initconst ctext4[16] __nonstring = - "\xcd\x33\xb2\x8a\xc7\x73\xf7\x4b" - "\xa0\x0e\xd1\xf3\x12\x57\x24\x35"; - -static const u8 __initconst ctext5[32] __nonstring = - "\x98\xe7\x24\x7c\x07\xf0\xfe\x41" - "\x1c\x26\x7e\x43\x84\xb0\xf6\x00" - "\x2f\xf5\x8d\x80\x03\x39\x27\xab" - "\x8e\xf4\xd4\x58\x75\x14\xf0\xfb"; - -static const u8 __initconst ptext6[64] __nonstring = - "\xd9\x31\x32\x25\xf8\x84\x06\xe5" - "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" - "\x86\xa7\xa9\x53\x15\x34\xf7\xda" - "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" - "\x1c\x3c\x0c\x95\x95\x68\x09\x53" - "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" - "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" - "\xba\x63\x7b\x39\x1a\xaf\xd2\x55"; - -static const u8 __initconst ctext6[80] __nonstring = - "\x39\x80\xca\x0b\x3c\x00\xe8\x41" - "\xeb\x06\xfa\xc4\x87\x2a\x27\x57" - "\x85\x9e\x1c\xea\xa6\xef\xd9\x84" - "\x62\x85\x93\xb4\x0c\xa1\xe1\x9c" - "\x7d\x77\x3d\x00\xc1\x44\xc5\x25" - "\xac\x61\x9d\x18\xc8\x4a\x3f\x47" - "\x18\xe2\x44\x8b\x2f\xe3\x24\xd9" - "\xcc\xda\x27\x10\xac\xad\xe2\x56" - "\x99\x24\xa7\xc8\x58\x73\x36\xbf" - "\xb1\x18\x02\x4d\xb8\x67\x4a\x14"; - -static const u8 __initconst ctext7[16] __nonstring = - "\x53\x0f\x8a\xfb\xc7\x45\x36\xb9" - "\xa9\x63\xb4\xf1\xc4\xcb\x73\x8b"; - -static const u8 __initconst ctext8[32] __nonstring = - "\xce\xa7\x40\x3d\x4d\x60\x6b\x6e" - "\x07\x4e\xc5\xd3\xba\xf3\x9d\x18" - "\xd0\xd1\xc8\xa7\x99\x99\x6b\xf0" - "\x26\x5b\x98\xb5\xd4\x8a\xb9\x19"; - -static const u8 __initconst ptext9[64] __nonstring = - "\xd9\x31\x32\x25\xf8\x84\x06\xe5" - "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" - "\x86\xa7\xa9\x53\x15\x34\xf7\xda" - "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" - "\x1c\x3c\x0c\x95\x95\x68\x09\x53" - "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" - "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" - "\xba\x63\x7b\x39\x1a\xaf\xd2\x55"; - -static const u8 __initconst ctext9[80] __nonstring = - "\x52\x2d\xc1\xf0\x99\x56\x7d\x07" - "\xf4\x7f\x37\xa3\x2a\x84\x42\x7d" - "\x64\x3a\x8c\xdc\xbf\xe5\xc0\xc9" - "\x75\x98\xa2\xbd\x25\x55\xd1\xaa" - "\x8c\xb0\x8e\x48\x59\x0d\xbb\x3d" - "\xa7\xb0\x8b\x10\x56\x82\x88\x38" - "\xc5\xf6\x1e\x63\x93\xba\x7a\x0a" - "\xbc\xc9\xf6\x62\x89\x80\x15\xad" - "\xb0\x94\xda\xc5\xd9\x34\x71\xbd" - "\xec\x1a\x50\x22\x70\xe3\xcc\x6c"; - -static const u8 __initconst ptext10[60] __nonstring = - "\xd9\x31\x32\x25\xf8\x84\x06\xe5" - "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" - "\x86\xa7\xa9\x53\x15\x34\xf7\xda" - "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" - "\x1c\x3c\x0c\x95\x95\x68\x09\x53" - "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" - "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" - "\xba\x63\x7b\x39"; - -static const u8 __initconst ctext10[76] __nonstring = - "\x52\x2d\xc1\xf0\x99\x56\x7d\x07" - "\xf4\x7f\x37\xa3\x2a\x84\x42\x7d" - "\x64\x3a\x8c\xdc\xbf\xe5\xc0\xc9" - "\x75\x98\xa2\xbd\x25\x55\xd1\xaa" - "\x8c\xb0\x8e\x48\x59\x0d\xbb\x3d" - "\xa7\xb0\x8b\x10\x56\x82\x88\x38" - "\xc5\xf6\x1e\x63\x93\xba\x7a\x0a" - "\xbc\xc9\xf6\x62" - "\x76\xfc\x6e\xce\x0f\x4e\x17\x68" - "\xcd\xdf\x88\x53\xbb\x2d\x55\x1b"; - -static const u8 __initconst ptext11[60] __nonstring = - "\xd9\x31\x32\x25\xf8\x84\x06\xe5" - "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" - "\x86\xa7\xa9\x53\x15\x34\xf7\xda" - "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" - "\x1c\x3c\x0c\x95\x95\x68\x09\x53" - "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" - "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" - "\xba\x63\x7b\x39"; - -static const u8 __initconst ctext11[76] __nonstring = - "\x39\x80\xca\x0b\x3c\x00\xe8\x41" - "\xeb\x06\xfa\xc4\x87\x2a\x27\x57" - "\x85\x9e\x1c\xea\xa6\xef\xd9\x84" - "\x62\x85\x93\xb4\x0c\xa1\xe1\x9c" - "\x7d\x77\x3d\x00\xc1\x44\xc5\x25" - "\xac\x61\x9d\x18\xc8\x4a\x3f\x47" - "\x18\xe2\x44\x8b\x2f\xe3\x24\xd9" - "\xcc\xda\x27\x10" - "\x25\x19\x49\x8e\x80\xf1\x47\x8f" - "\x37\xba\x55\xbd\x6d\x27\x61\x8c"; - -static const u8 __initconst ptext12[719] __nonstring = - "\x42\xc1\xcc\x08\x48\x6f\x41\x3f" - "\x2f\x11\x66\x8b\x2a\x16\xf0\xe0" - "\x58\x83\xf0\xc3\x70\x14\xc0\x5b" - "\x3f\xec\x1d\x25\x3c\x51\xd2\x03" - "\xcf\x59\x74\x1f\xb2\x85\xb4\x07" - "\xc6\x6a\x63\x39\x8a\x5b\xde\xcb" - "\xaf\x08\x44\xbd\x6f\x91\x15\xe1" - "\xf5\x7a\x6e\x18\xbd\xdd\x61\x50" - "\x59\xa9\x97\xab\xbb\x0e\x74\x5c" - "\x00\xa4\x43\x54\x04\x54\x9b\x3b" - "\x77\xec\xfd\x5c\xa6\xe8\x7b\x08" - "\xae\xe6\x10\x3f\x32\x65\xd1\xfc" - "\xa4\x1d\x2c\x31\xfb\x33\x7a\xb3" - "\x35\x23\xf4\x20\x41\xd4\xad\x82" - "\x8b\xa4\xad\x96\x1c\x20\x53\xbe" - "\x0e\xa6\xf4\xdc\x78\x49\x3e\x72" - "\xb1\xa9\xb5\x83\xcb\x08\x54\xb7" - "\xad\x49\x3a\xae\x98\xce\xa6\x66" - "\x10\x30\x90\x8c\x55\x83\xd7\x7c" - "\x8b\xe6\x53\xde\xd2\x6e\x18\x21" - "\x01\x52\xd1\x9f\x9d\xbb\x9c\x73" - "\x57\xcc\x89\x09\x75\x9b\x78\x70" - "\xed\x26\x97\x4d\xb4\xe4\x0c\xa5" - "\xfa\x70\x04\x70\xc6\x96\x1c\x7d" - "\x54\x41\x77\xa8\xe3\xb0\x7e\x96" - "\x82\xd9\xec\xa2\x87\x68\x55\xf9" - "\x8f\x9e\x73\x43\x47\x6a\x08\x36" - "\x93\x67\xa8\x2d\xde\xac\x41\xa9" - "\x5c\x4d\x73\x97\x0f\x70\x68\xfa" - "\x56\x4d\x00\xc2\x3b\x1f\xc8\xb9" - "\x78\x1f\x51\x07\xe3\x9a\x13\x4e" - "\xed\x2b\x2e\xa3\xf7\x44\xb2\xe7" - "\xab\x19\x37\xd9\xba\x76\x5e\xd2" - "\xf2\x53\x15\x17\x4c\x6b\x16\x9f" - "\x02\x66\x49\xca\x7c\x91\x05\xf2" - "\x45\x36\x1e\xf5\x77\xad\x1f\x46" - "\xa8\x13\xfb\x63\xb6\x08\x99\x63" - "\x82\xa2\xed\xb3\xac\xdf\x43\x19" - "\x45\xea\x78\x73\xd9\xb7\x39\x11" - "\xa3\x13\x7c\xf8\x3f\xf7\xad\x81" - "\x48\x2f\xa9\x5c\x5f\xa0\xf0\x79" - "\xa4\x47\x7d\x80\x20\x26\xfd\x63" - "\x0a\xc7\x7e\x6d\x75\x47\xff\x76" - "\x66\x2e\x8a\x6c\x81\x35\xaf\x0b" - "\x2e\x6a\x49\x60\xc1\x10\xe1\xe1" - "\x54\x03\xa4\x09\x0c\x37\x7a\x15" - "\x23\x27\x5b\x8b\x4b\xa5\x64\x97" - "\xae\x4a\x50\x73\x1f\x66\x1c\x5c" - "\x03\x25\x3c\x8d\x48\x58\x71\x34" - "\x0e\xec\x4e\x55\x1a\x03\x6a\xe5" - "\xb6\x19\x2b\x84\x2a\x20\xd1\xea" - "\x80\x6f\x96\x0e\x05\x62\xc7\x78" - "\x87\x79\x60\x38\x46\xb4\x25\x57" - "\x6e\x16\x63\xf8\xad\x6e\xd7\x42" - "\x69\xe1\x88\xef\x6e\xd5\xb4\x9a" - "\x3c\x78\x6c\x3b\xe5\xa0\x1d\x22" - "\x86\x5c\x74\x3a\xeb\x24\x26\xc7" - "\x09\xfc\x91\x96\x47\x87\x4f\x1a" - "\xd6\x6b\x2c\x18\x47\xc0\xb8\x24" - "\xa8\x5a\x4a\x9e\xcb\x03\xe7\x2a" - "\x09\xe6\x4d\x9c\x6d\x86\x60\xf5" - "\x2f\x48\x69\x37\x9f\xf2\xd2\xcb" - "\x0e\x5a\xdd\x6e\x8a\xfb\x6a\xfe" - "\x0b\x63\xde\x87\x42\x79\x8a\x68" - "\x51\x28\x9b\x7a\xeb\xaf\xb8\x2f" - "\x9d\xd1\xc7\x45\x90\x08\xc9\x83" - "\xe9\x83\x84\xcb\x28\x69\x09\x69" - "\xce\x99\x46\x00\x54\xcb\xd8\x38" - "\xf9\x53\x4a\xbf\x31\xce\x57\x15" - "\x33\xfa\x96\x04\x33\x42\xe3\xc0" - "\xb7\x54\x4a\x65\x7a\x7c\x02\xe6" - "\x19\x95\xd0\x0e\x82\x07\x63\xf9" - "\xe1\x2b\x2a\xfc\x55\x92\x52\xc9" - "\xb5\x9f\x23\x28\x60\xe7\x20\x51" - "\x10\xd3\xed\x6d\x9b\xab\xb8\xe2" - "\x5d\x9a\x34\xb3\xbe\x9c\x64\xcb" - "\x78\xc6\x91\x22\x40\x91\x80\xbe" - "\xd7\x78\x5c\x0e\x0a\xdc\x08\xe9" - "\x67\x10\xa4\x83\x98\x79\x23\xe7" - "\x92\xda\xa9\x22\x16\xb1\xe7\x78" - "\xa3\x1c\x6c\x8f\x35\x7c\x4d\x37" - "\x2f\x6e\x0b\x50\x5c\x34\xb9\xf9" - "\xe6\x3d\x91\x0d\x32\x95\xaa\x3d" - "\x48\x11\x06\xbb\x2d\xf2\x63\x88" - "\x3f\x73\x09\xe2\x45\x56\x31\x51" - "\xfa\x5e\x4e\x62\xf7\x90\xf9\xa9" - "\x7d\x7b\x1b\xb1\xc8\x26\x6e\x66" - "\xf6\x90\x9a\x7f\xf2\x57\xcc\x23" - "\x59\xfa\xfa\xaa\x44\x04\x01\xa7" - "\xa4\x78\xdb\x74\x3d\x8b\xb5"; - -static const u8 __initconst ctext12[735] __nonstring = - "\x84\x0b\xdb\xd5\xb7\xa8\xfe\x20" - "\xbb\xb1\x12\x7f\x41\xea\xb3\xc0" - "\xa2\xb4\x37\x19\x11\x58\xb6\x0b" - "\x4c\x1d\x38\x05\x54\xd1\x16\x73" - "\x8e\x1c\x20\x90\xa2\x9a\xb7\x74" - "\x47\xe6\xd8\xfc\x18\x3a\xb4\xea" - "\xd5\x16\x5a\x2c\x53\x01\x46\xb3" - "\x18\x33\x74\x6c\x50\xf2\xe8\xc0" - "\x73\xda\x60\x22\xeb\xe3\xe5\x9b" - "\x20\x93\x6c\x4b\x37\x99\xb8\x23" - "\x3b\x4e\xac\xe8\x5b\xe8\x0f\xb7" - "\xc3\x8f\xfb\x4a\x37\xd9\x39\x95" - "\x34\xf1\xdb\x8f\x71\xd9\xc7\x0b" - "\x02\xf1\x63\xfc\x9b\xfc\xc5\xab" - "\xb9\x14\x13\x21\xdf\xce\xaa\x88" - "\x44\x30\x1e\xce\x26\x01\x92\xf8" - "\x9f\x00\x4b\x0c\x4b\xf7\x5f\xe0" - "\x89\xca\x94\x66\x11\x21\x97\xca" - "\x3e\x83\x74\x2d\xdb\x4d\x11\xeb" - "\x97\xc2\x14\xff\x9e\x1e\xa0\x6b" - "\x08\xb4\x31\x2b\x85\xc6\x85\x6c" - "\x90\xec\x39\xc0\xec\xb3\xb5\x4e" - "\xf3\x9c\xe7\x83\x3a\x77\x0a\xf4" - "\x56\xfe\xce\x18\x33\x6d\x0b\x2d" - "\x33\xda\xc8\x05\x5c\xb4\x09\x2a" - "\xde\x6b\x52\x98\x01\xef\x36\x3d" - "\xbd\xf9\x8f\xa8\x3e\xaa\xcd\xd1" - "\x01\x2d\x42\x49\xc3\xb6\x84\xbb" - "\x48\x96\xe0\x90\x93\x6c\x48\x64" - "\xd4\xfa\x7f\x93\x2c\xa6\x21\xc8" - "\x7a\x23\x7b\xaa\x20\x56\x12\xae" - "\x16\x9d\x94\x0f\x54\xa1\xec\xca" - "\x51\x4e\xf2\x39\xf4\xf8\x5f\x04" - "\x5a\x0d\xbf\xf5\x83\xa1\x15\xe1" - "\xf5\x3c\xd8\x62\xa3\xed\x47\x89" - "\x85\x4c\xe5\xdb\xac\x9e\x17\x1d" - "\x0c\x09\xe3\x3e\x39\x5b\x4d\x74" - "\x0e\xf5\x34\xee\x70\x11\x4c\xfd" - "\xdb\x34\xb1\xb5\x10\x3f\x73\xb7" - "\xf5\xfa\xed\xb0\x1f\xa5\xcd\x3c" - "\x8d\x35\x83\xd4\x11\x44\x6e\x6c" - "\x5b\xe0\x0e\x69\xa5\x39\xe5\xbb" - "\xa9\x57\x24\x37\xe6\x1f\xdd\xcf" - "\x16\x2a\x13\xf9\x6a\x2d\x90\xa0" - "\x03\x60\x7a\xed\x69\xd5\x00\x8b" - "\x7e\x4f\xcb\xb9\xfa\x91\xb9\x37" - "\xc1\x26\xce\x90\x97\x22\x64\x64" - "\xc1\x72\x43\x1b\xf6\xac\xc1\x54" - "\x8a\x10\x9c\xdd\x8d\xd5\x8e\xb2" - "\xe4\x85\xda\xe0\x20\x5f\xf4\xb4" - "\x15\xb5\xa0\x8d\x12\x74\x49\x23" - "\x3a\xdf\x4a\xd3\xf0\x3b\x89\xeb" - "\xf8\xcc\x62\x7b\xfb\x93\x07\x41" - "\x61\x26\x94\x58\x70\xa6\x3c\xe4" - "\xff\x58\xc4\x13\x3d\xcb\x36\x6b" - "\x32\xe5\xb2\x6d\x03\x74\x6f\x76" - "\x93\x77\xde\x48\xc4\xfa\x30\x4a" - "\xda\x49\x80\x77\x0f\x1c\xbe\x11" - "\xc8\x48\xb1\xe5\xbb\xf2\x8a\xe1" - "\x96\x2f\x9f\xd1\x8e\x8a\x5c\xe2" - "\xf7\xd7\xd8\x54\xf3\x3f\xc4\x91" - "\xb8\xfb\x86\xdc\x46\x24\x91\x60" - "\x6c\x2f\xc9\x41\x37\x51\x49\x54" - "\x09\x81\x21\xf3\x03\x9f\x2b\xe3" - "\x1f\x39\x63\xaf\xf4\xd7\x53\x60" - "\xa7\xc7\x54\xf9\xee\xb1\xb1\x7d" - "\x75\x54\x65\x93\xfe\xb1\x68\x6b" - "\x57\x02\xf9\xbb\x0e\xf9\xf8\xbf" - "\x01\x12\x27\xb4\xfe\xe4\x79\x7a" - "\x40\x5b\x51\x4b\xdf\x38\xec\xb1" - "\x6a\x56\xff\x35\x4d\x42\x33\xaa" - "\x6f\x1b\xe4\xdc\xe0\xdb\x85\x35" - "\x62\x10\xd4\xec\xeb\xc5\x7e\x45" - "\x1c\x6f\x17\xca\x3b\x8e\x2d\x66" - "\x4f\x4b\x36\x56\xcd\x1b\x59\xaa" - "\xd2\x9b\x17\xb9\x58\xdf\x7b\x64" - "\x8a\xff\x3b\x9c\xa6\xb5\x48\x9e" - "\xaa\xe2\x5d\x09\x71\x32\x5f\xb6" - "\x29\xbe\xe7\xc7\x52\x7e\x91\x82" - "\x6b\x6d\x33\xe1\x34\x06\x36\x21" - "\x5e\xbe\x1e\x2f\x3e\xc1\xfb\xea" - "\x49\x2c\xb5\xca\xf7\xb0\x37\xea" - "\x1f\xed\x10\x04\xd9\x48\x0d\x1a" - "\x1c\xfb\xe7\x84\x0e\x83\x53\x74" - "\xc7\x65\xe2\x5c\xe5\xba\x73\x4c" - "\x0e\xe1\xb5\x11\x45\x61\x43\x46" - "\xaa\x25\x8f\xbd\x85\x08\xfa\x4c" - "\x15\xc1\xc0\xd8\xf5\xdc\x16\xbb" - "\x7b\x1d\xe3\x87\x57\xa7\x2a\x1d" - "\x38\x58\x9e\x8a\x43\xdc\x57" - "\xd1\x81\x7d\x2b\xe9\xff\x99\x3a" - "\x4b\x24\x52\x58\x55\xe1\x49\x14"; - -static struct { - const u8 *ptext; - const u8 *ctext; - - u8 key[AES_MAX_KEY_SIZE] __nonstring; - u8 iv[GCM_AES_IV_SIZE] __nonstring; - u8 assoc[20] __nonstring; - - int klen; - int clen; - int plen; - int alen; -} const aesgcm_tv[] __initconst = { - { /* From McGrew & Viega - http://citeseer.ist.psu.edu/656989.html */ - .klen = 16, - .ctext = ctext0, - .clen = sizeof(ctext0), - }, { - .klen = 16, - .ptext = ptext1, - .plen = sizeof(ptext1), - .ctext = ctext1, - .clen = sizeof(ctext1), - }, { - .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08", - .klen = 16, - .iv = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" - "\xde\xca\xf8\x88", - .ptext = ptext2, - .plen = sizeof(ptext2), - .ctext = ctext2, - .clen = sizeof(ctext2), - }, { - .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08", - .klen = 16, - .iv = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" - "\xde\xca\xf8\x88", - .ptext = ptext3, - .plen = sizeof(ptext3), - .assoc = "\xfe\xed\xfa\xce\xde\xad\xbe\xef" - "\xfe\xed\xfa\xce\xde\xad\xbe\xef" - "\xab\xad\xda\xd2", - .alen = 20, - .ctext = ctext3, - .clen = sizeof(ctext3), - }, { - .klen = 24, - .ctext = ctext4, - .clen = sizeof(ctext4), - }, { - .klen = 24, - .ptext = ptext1, - .plen = sizeof(ptext1), - .ctext = ctext5, - .clen = sizeof(ctext5), - }, { - .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\xfe\xff\xe9\x92\x86\x65\x73\x1c", - .klen = 24, - .iv = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" - "\xde\xca\xf8\x88", - .ptext = ptext6, - .plen = sizeof(ptext6), - .ctext = ctext6, - .clen = sizeof(ctext6), - }, { - .klen = 32, - .ctext = ctext7, - .clen = sizeof(ctext7), - }, { - .klen = 32, - .ptext = ptext1, - .plen = sizeof(ptext1), - .ctext = ctext8, - .clen = sizeof(ctext8), - }, { - .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08", - .klen = 32, - .iv = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" - "\xde\xca\xf8\x88", - .ptext = ptext9, - .plen = sizeof(ptext9), - .ctext = ctext9, - .clen = sizeof(ctext9), - }, { - .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08", - .klen = 32, - .iv = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" - "\xde\xca\xf8\x88", - .ptext = ptext10, - .plen = sizeof(ptext10), - .assoc = "\xfe\xed\xfa\xce\xde\xad\xbe\xef" - "\xfe\xed\xfa\xce\xde\xad\xbe\xef" - "\xab\xad\xda\xd2", - .alen = 20, - .ctext = ctext10, - .clen = sizeof(ctext10), - }, { - .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\xfe\xff\xe9\x92\x86\x65\x73\x1c", - .klen = 24, - .iv = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" - "\xde\xca\xf8\x88", - .ptext = ptext11, - .plen = sizeof(ptext11), - .assoc = "\xfe\xed\xfa\xce\xde\xad\xbe\xef" - "\xfe\xed\xfa\xce\xde\xad\xbe\xef" - "\xab\xad\xda\xd2", - .alen = 20, - .ctext = ctext11, - .clen = sizeof(ctext11), - }, { - .key = "\x62\x35\xf8\x95\xfc\xa5\xeb\xf6" - "\x0e\x92\x12\x04\xd3\xa1\x3f\x2e" - "\x8b\x32\xcf\xe7\x44\xed\x13\x59" - "\x04\x38\x77\xb0\xb9\xad\xb4\x38", - .klen = 32, - .iv = "\x00\xff\xff\xff\xff\x00\x00\xff" - "\xff\xff\x00\xff", - .ptext = ptext12, - .plen = sizeof(ptext12), - .ctext = ctext12, - .clen = sizeof(ctext12), - } -}; - -static int __init libaesgcm_init(void) -{ - for (int i = 0; i < ARRAY_SIZE(aesgcm_tv); i++) { - u8 tagbuf[AES_BLOCK_SIZE]; - int plen = aesgcm_tv[i].plen; - struct aesgcm_ctx ctx; - static u8 buf[sizeof(ptext12)]; - - if (aesgcm_expandkey(&ctx, aesgcm_tv[i].key, aesgcm_tv[i].klen, - aesgcm_tv[i].clen - plen)) { - pr_err("aesgcm_expandkey() failed on vector %d\n", i); - return -ENODEV; - } - - if (!aesgcm_decrypt(&ctx, buf, aesgcm_tv[i].ctext, plen, - aesgcm_tv[i].assoc, aesgcm_tv[i].alen, - aesgcm_tv[i].iv, aesgcm_tv[i].ctext + plen) - || memcmp(buf, aesgcm_tv[i].ptext, plen)) { - pr_err("aesgcm_decrypt() #1 failed on vector %d\n", i); - return -ENODEV; - } - - /* encrypt in place */ - aesgcm_encrypt(&ctx, buf, buf, plen, aesgcm_tv[i].assoc, - aesgcm_tv[i].alen, aesgcm_tv[i].iv, tagbuf); - if (memcmp(buf, aesgcm_tv[i].ctext, plen)) { - pr_err("aesgcm_encrypt() failed on vector %d\n", i); - return -ENODEV; - } - - /* decrypt in place */ - if (!aesgcm_decrypt(&ctx, buf, buf, plen, aesgcm_tv[i].assoc, - aesgcm_tv[i].alen, aesgcm_tv[i].iv, tagbuf) - || memcmp(buf, aesgcm_tv[i].ptext, plen)) { - pr_err("aesgcm_decrypt() #2 failed on vector %d\n", i); - return -ENODEV; - } - } - return 0; -} -module_init(libaesgcm_init); - -static void __exit libaesgcm_exit(void) -{ -} -module_exit(libaesgcm_exit); -#endif diff --git a/lib/crypto/arm/gf128hash.h b/lib/crypto/arm/gf128hash.h new file mode 100644 index 000000000000..c33c8cbe51fe --- /dev/null +++ b/lib/crypto/arm/gf128hash.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * GHASH, arm optimized + * + * Copyright 2026 Google LLC + */ + +#include <asm/hwcap.h> +#include <asm/neon.h> +#include <asm/simd.h> + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); + +void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg, + const u8 *src, const struct polyval_elem *h); + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_neon) && may_use_simd()) { + do { + /* Allow rescheduling every 4 KiB. */ + size_t n = + min_t(size_t, nblocks, 4096 / GHASH_BLOCK_SIZE); + + scoped_ksimd() + pmull_ghash_update_p8(n, acc, data, &key->h); + data += n * GHASH_BLOCK_SIZE; + nblocks -= n; + } while (nblocks); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (elf_hwcap & HWCAP_NEON) + static_branch_enable(&have_neon); +} diff --git a/lib/crypto/arm/ghash-neon-core.S b/lib/crypto/arm/ghash-neon-core.S new file mode 100644 index 000000000000..eeffd12504a9 --- /dev/null +++ b/lib/crypto/arm/ghash-neon-core.S @@ -0,0 +1,209 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Accelerated GHASH implementation with NEON vmull.p8 instructions. + * + * Copyright (C) 2015 - 2017 Linaro Ltd. + * Copyright (C) 2023 Google LLC. <ardb@google.com> + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> + + .fpu neon + + SHASH .req q0 + T1 .req q1 + XL .req q2 + XM .req q3 + XH .req q4 + IN1 .req q4 + + SHASH_L .req d0 + SHASH_H .req d1 + T1_L .req d2 + T1_H .req d3 + XL_L .req d4 + XL_H .req d5 + XM_L .req d6 + XM_H .req d7 + XH_L .req d8 + + t0l .req d10 + t0h .req d11 + t1l .req d12 + t1h .req d13 + t2l .req d14 + t2h .req d15 + t3l .req d16 + t3h .req d17 + t4l .req d18 + t4h .req d19 + + t0q .req q5 + t1q .req q6 + t2q .req q7 + t3q .req q8 + t4q .req q9 + + s1l .req d20 + s1h .req d21 + s2l .req d22 + s2h .req d23 + s3l .req d24 + s3h .req d25 + s4l .req d26 + s4h .req d27 + + SHASH2_p8 .req d28 + + k16 .req d29 + k32 .req d30 + k48 .req d31 + + T2 .req q7 + + .text + + /* + * This implementation of 64x64 -> 128 bit polynomial multiplication + * using vmull.p8 instructions (8x8 -> 16) is taken from the paper + * "Fast Software Polynomial Multiplication on ARM Processors Using + * the NEON Engine" by Danilo Camara, Conrado Gouvea, Julio Lopez and + * Ricardo Dahab (https://hal.inria.fr/hal-01506572) + * + * It has been slightly tweaked for in-order performance, and to allow + * 'rq' to overlap with 'ad' or 'bd'. + */ + .macro __pmull_p8, rq, ad, bd, b1=t4l, b2=t3l, b3=t4l, b4=t3l + vext.8 t0l, \ad, \ad, #1 @ A1 + .ifc \b1, t4l + vext.8 t4l, \bd, \bd, #1 @ B1 + .endif + vmull.p8 t0q, t0l, \bd @ F = A1*B + vext.8 t1l, \ad, \ad, #2 @ A2 + vmull.p8 t4q, \ad, \b1 @ E = A*B1 + .ifc \b2, t3l + vext.8 t3l, \bd, \bd, #2 @ B2 + .endif + vmull.p8 t1q, t1l, \bd @ H = A2*B + vext.8 t2l, \ad, \ad, #3 @ A3 + vmull.p8 t3q, \ad, \b2 @ G = A*B2 + veor t0q, t0q, t4q @ L = E + F + .ifc \b3, t4l + vext.8 t4l, \bd, \bd, #3 @ B3 + .endif + vmull.p8 t2q, t2l, \bd @ J = A3*B + veor t0l, t0l, t0h @ t0 = (L) (P0 + P1) << 8 + veor t1q, t1q, t3q @ M = G + H + .ifc \b4, t3l + vext.8 t3l, \bd, \bd, #4 @ B4 + .endif + vmull.p8 t4q, \ad, \b3 @ I = A*B3 + veor t1l, t1l, t1h @ t1 = (M) (P2 + P3) << 16 + vmull.p8 t3q, \ad, \b4 @ K = A*B4 + vand t0h, t0h, k48 + vand t1h, t1h, k32 + veor t2q, t2q, t4q @ N = I + J + veor t0l, t0l, t0h + veor t1l, t1l, t1h + veor t2l, t2l, t2h @ t2 = (N) (P4 + P5) << 24 + vand t2h, t2h, k16 + veor t3l, t3l, t3h @ t3 = (K) (P6 + P7) << 32 + vmov.i64 t3h, #0 + vext.8 t0q, t0q, t0q, #15 + veor t2l, t2l, t2h + vext.8 t1q, t1q, t1q, #14 + vmull.p8 \rq, \ad, \bd @ D = A*B + vext.8 t2q, t2q, t2q, #13 + vext.8 t3q, t3q, t3q, #12 + veor t0q, t0q, t1q + veor t2q, t2q, t3q + veor \rq, \rq, t0q + veor \rq, \rq, t2q + .endm + + .macro __pmull_reduce_p8 + veor XL_H, XL_H, XM_L + veor XH_L, XH_L, XM_H + + vshl.i64 T1, XL, #57 + vshl.i64 T2, XL, #62 + veor T1, T1, T2 + vshl.i64 T2, XL, #63 + veor T1, T1, T2 + veor XL_H, XL_H, T1_L + veor XH_L, XH_L, T1_H + + vshr.u64 T1, XL, #1 + veor XH, XH, XL + veor XL, XL, T1 + vshr.u64 T1, T1, #6 + vshr.u64 XL, XL, #1 + .endm + + .macro vrev64_if_be a +#ifdef CONFIG_CPU_BIG_ENDIAN + vrev64.8 \a, \a +#endif + .endm + + .macro ghash_update + vld1.64 {XL}, [r1] + vrev64_if_be XL + +0: + vld1.8 {T1}, [r2]! + subs r0, r0, #1 + + /* multiply XL by SHASH in GF(2^128) */ + vrev64.8 T1, T1 + + vext.8 IN1, T1, T1, #8 + veor T1_L, T1_L, XL_H + veor XL, XL, IN1 + + __pmull_p8 XH, XL_H, SHASH_H, s1h, s2h, s3h, s4h @ a1 * b1 + veor T1, T1, XL + __pmull_p8 XL, XL_L, SHASH_L, s1l, s2l, s3l, s4l @ a0 * b0 + __pmull_p8 XM, T1_L, SHASH2_p8 @ (a1+a0)(b1+b0) + + veor T1, XL, XH + veor XM, XM, T1 + + __pmull_reduce_p8 + + veor T1, T1, XH + veor XL, XL, T1 + + bne 0b + .endm + + /* + * void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg, + * const u8 *src, + * const struct polyval_elem *h) + */ +ENTRY(pmull_ghash_update_p8) + vld1.64 {SHASH}, [r3] + vrev64_if_be SHASH + veor SHASH2_p8, SHASH_L, SHASH_H + + vext.8 s1l, SHASH_L, SHASH_L, #1 + vext.8 s2l, SHASH_L, SHASH_L, #2 + vext.8 s3l, SHASH_L, SHASH_L, #3 + vext.8 s4l, SHASH_L, SHASH_L, #4 + vext.8 s1h, SHASH_H, SHASH_H, #1 + vext.8 s2h, SHASH_H, SHASH_H, #2 + vext.8 s3h, SHASH_H, SHASH_H, #3 + vext.8 s4h, SHASH_H, SHASH_H, #4 + + vmov.i64 k16, #0xffff + vmov.i64 k32, #0xffffffff + vmov.i64 k48, #0xffffffffffff + + ghash_update + vrev64_if_be XL + vst1.64 {XL}, [r1] + + bx lr +ENDPROC(pmull_ghash_update_p8) diff --git a/lib/crypto/arm64/aes-ce.S b/lib/crypto/arm64/aes-ce.S new file mode 100644 index 000000000000..b853e02f7b1e --- /dev/null +++ b/lib/crypto/arm64/aes-ce.S @@ -0,0 +1,145 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * AES cipher for ARMv8 with Crypto Extensions + * + * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> + +#define AES_FUNC_START(func) SYM_FUNC_START(ce_ ## func) +#define AES_FUNC_END(func) SYM_FUNC_END(ce_ ## func) + + .arch armv8-a+crypto + + xtsmask .req v16 + cbciv .req v16 + vctr .req v16 + + .macro xts_reload_mask, tmp + .endm + + .macro xts_cts_skip_tw, reg, lbl + .endm + + /* preload all round keys */ + .macro load_round_keys, rk, nr, tmp + add \tmp, \rk, \nr, sxtw #4 + sub \tmp, \tmp, #160 + ld1 {v17.4s-v20.4s}, [\rk] + ld1 {v21.4s-v24.4s}, [\tmp], #64 + ld1 {v25.4s-v28.4s}, [\tmp], #64 + ld1 {v29.4s-v31.4s}, [\tmp] + .endm + + /* prepare for encryption with key in rk[] */ + .macro enc_prepare, rounds, rk, temp + load_round_keys \rk, \rounds, \temp + .endm + + /* prepare for encryption (again) but with new key in rk[] */ + .macro enc_switch_key, rounds, rk, temp + load_round_keys \rk, \rounds, \temp + .endm + + /* prepare for decryption with key in rk[] */ + .macro dec_prepare, rounds, rk, temp + load_round_keys \rk, \rounds, \temp + .endm + + .macro do_enc_Nx, de, mc, k, i0, i1, i2, i3, i4 + aes\de \i0\().16b, \k\().16b + aes\mc \i0\().16b, \i0\().16b + .ifnb \i1 + aes\de \i1\().16b, \k\().16b + aes\mc \i1\().16b, \i1\().16b + .ifnb \i3 + aes\de \i2\().16b, \k\().16b + aes\mc \i2\().16b, \i2\().16b + aes\de \i3\().16b, \k\().16b + aes\mc \i3\().16b, \i3\().16b + .ifnb \i4 + aes\de \i4\().16b, \k\().16b + aes\mc \i4\().16b, \i4\().16b + .endif + .endif + .endif + .endm + + /* up to 5 interleaved encryption rounds with the same round key */ + .macro round_Nx, enc, k, i0, i1, i2, i3, i4 + .ifc \enc, e + do_enc_Nx e, mc, \k, \i0, \i1, \i2, \i3, \i4 + .else + do_enc_Nx d, imc, \k, \i0, \i1, \i2, \i3, \i4 + .endif + .endm + + /* up to 5 interleaved final rounds */ + .macro fin_round_Nx, de, k, k2, i0, i1, i2, i3, i4 + aes\de \i0\().16b, \k\().16b + .ifnb \i1 + aes\de \i1\().16b, \k\().16b + .ifnb \i3 + aes\de \i2\().16b, \k\().16b + aes\de \i3\().16b, \k\().16b + .ifnb \i4 + aes\de \i4\().16b, \k\().16b + .endif + .endif + .endif + eor \i0\().16b, \i0\().16b, \k2\().16b + .ifnb \i1 + eor \i1\().16b, \i1\().16b, \k2\().16b + .ifnb \i3 + eor \i2\().16b, \i2\().16b, \k2\().16b + eor \i3\().16b, \i3\().16b, \k2\().16b + .ifnb \i4 + eor \i4\().16b, \i4\().16b, \k2\().16b + .endif + .endif + .endif + .endm + + /* up to 5 interleaved blocks */ + .macro do_block_Nx, enc, rounds, i0, i1, i2, i3, i4 + tbz \rounds, #2, .L\@ /* 128 bits */ + round_Nx \enc, v17, \i0, \i1, \i2, \i3, \i4 + round_Nx \enc, v18, \i0, \i1, \i2, \i3, \i4 + tbz \rounds, #1, .L\@ /* 192 bits */ + round_Nx \enc, v19, \i0, \i1, \i2, \i3, \i4 + round_Nx \enc, v20, \i0, \i1, \i2, \i3, \i4 +.L\@: .irp key, v21, v22, v23, v24, v25, v26, v27, v28, v29 + round_Nx \enc, \key, \i0, \i1, \i2, \i3, \i4 + .endr + fin_round_Nx \enc, v30, v31, \i0, \i1, \i2, \i3, \i4 + .endm + + .macro encrypt_block, in, rounds, t0, t1, t2 + do_block_Nx e, \rounds, \in + .endm + + .macro encrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2 + do_block_Nx e, \rounds, \i0, \i1, \i2, \i3 + .endm + + .macro encrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2 + do_block_Nx e, \rounds, \i0, \i1, \i2, \i3, \i4 + .endm + + .macro decrypt_block, in, rounds, t0, t1, t2 + do_block_Nx d, \rounds, \in + .endm + + .macro decrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2 + do_block_Nx d, \rounds, \i0, \i1, \i2, \i3 + .endm + + .macro decrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2 + do_block_Nx d, \rounds, \i0, \i1, \i2, \i3, \i4 + .endm + +#define MAX_STRIDE 5 + +#include "aes-modes.S" diff --git a/lib/crypto/arm64/aes-cipher-core.S b/lib/crypto/arm64/aes-cipher-core.S index 651f701c56a8..0b05ec4be65f 100644 --- a/lib/crypto/arm64/aes-cipher-core.S +++ b/lib/crypto/arm64/aes-cipher-core.S @@ -87,11 +87,6 @@ ldp w8, w9, [rk], #16 ldp w10, w11, [rk, #-8] -CPU_BE( rev w4, w4 ) -CPU_BE( rev w5, w5 ) -CPU_BE( rev w6, w6 ) -CPU_BE( rev w7, w7 ) - eor w4, w4, w8 eor w5, w5, w9 eor w6, w6, w10 @@ -112,11 +107,6 @@ CPU_BE( rev w7, w7 ) 3: adr_l tt, \ltab \round w4, w5, w6, w7, w8, w9, w10, w11, \bsz, b -CPU_BE( rev w4, w4 ) -CPU_BE( rev w5, w5 ) -CPU_BE( rev w6, w6 ) -CPU_BE( rev w7, w7 ) - stp w4, w5, [out] stp w6, w7, [out, #8] ret diff --git a/lib/crypto/arm64/aes-modes.S b/lib/crypto/arm64/aes-modes.S new file mode 100644 index 000000000000..10e537317eaf --- /dev/null +++ b/lib/crypto/arm64/aes-modes.S @@ -0,0 +1,867 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Chaining mode wrappers for AES + * + * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> + */ + +/* included by aes-ce.S and aes-neon.S */ + + .text + .align 4 + +#ifndef MAX_STRIDE +#define MAX_STRIDE 4 +#endif + +#if MAX_STRIDE == 4 +#define ST4(x...) x +#define ST5(x...) +#else +#define ST4(x...) +#define ST5(x...) x +#endif + +SYM_FUNC_START_LOCAL(aes_encrypt_block4x) + encrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7 + ret +SYM_FUNC_END(aes_encrypt_block4x) + +SYM_FUNC_START_LOCAL(aes_decrypt_block4x) + decrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7 + ret +SYM_FUNC_END(aes_decrypt_block4x) + +#if MAX_STRIDE == 5 +SYM_FUNC_START_LOCAL(aes_encrypt_block5x) + encrypt_block5x v0, v1, v2, v3, v4, w3, x2, x8, w7 + ret +SYM_FUNC_END(aes_encrypt_block5x) + +SYM_FUNC_START_LOCAL(aes_decrypt_block5x) + decrypt_block5x v0, v1, v2, v3, v4, w3, x2, x8, w7 + ret +SYM_FUNC_END(aes_decrypt_block5x) +#endif + + /* + * aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, + * int blocks) + * aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, + * int blocks) + */ + +AES_FUNC_START(aes_ecb_encrypt) + frame_push 0 + + enc_prepare w3, x2, x5 + +.LecbencloopNx: + subs w4, w4, #MAX_STRIDE + bmi .Lecbenc1x + ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */ +ST4( bl aes_encrypt_block4x ) +ST5( ld1 {v4.16b}, [x1], #16 ) +ST5( bl aes_encrypt_block5x ) + st1 {v0.16b-v3.16b}, [x0], #64 +ST5( st1 {v4.16b}, [x0], #16 ) + b .LecbencloopNx +.Lecbenc1x: + adds w4, w4, #MAX_STRIDE + beq .Lecbencout +.Lecbencloop: + ld1 {v0.16b}, [x1], #16 /* get next pt block */ + encrypt_block v0, w3, x2, x5, w6 + st1 {v0.16b}, [x0], #16 + subs w4, w4, #1 + bne .Lecbencloop +.Lecbencout: + frame_pop + ret +AES_FUNC_END(aes_ecb_encrypt) + + +AES_FUNC_START(aes_ecb_decrypt) + frame_push 0 + + dec_prepare w3, x2, x5 + +.LecbdecloopNx: + subs w4, w4, #MAX_STRIDE + bmi .Lecbdec1x + ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */ +ST4( bl aes_decrypt_block4x ) +ST5( ld1 {v4.16b}, [x1], #16 ) +ST5( bl aes_decrypt_block5x ) + st1 {v0.16b-v3.16b}, [x0], #64 +ST5( st1 {v4.16b}, [x0], #16 ) + b .LecbdecloopNx +.Lecbdec1x: + adds w4, w4, #MAX_STRIDE + beq .Lecbdecout +.Lecbdecloop: + ld1 {v0.16b}, [x1], #16 /* get next ct block */ + decrypt_block v0, w3, x2, x5, w6 + st1 {v0.16b}, [x0], #16 + subs w4, w4, #1 + bne .Lecbdecloop +.Lecbdecout: + frame_pop + ret +AES_FUNC_END(aes_ecb_decrypt) + + + /* + * aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, + * int blocks, u8 iv[]) + * aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, + * int blocks, u8 iv[]) + * aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[], + * int rounds, int blocks, u8 iv[], + * u32 const rk2[]); + * aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[], + * int rounds, int blocks, u8 iv[], + * u32 const rk2[]); + */ + +AES_FUNC_START(aes_essiv_cbc_encrypt) + ld1 {v4.16b}, [x5] /* get iv */ + + mov w8, #14 /* AES-256: 14 rounds */ + enc_prepare w8, x6, x7 + encrypt_block v4, w8, x6, x7, w9 + enc_switch_key w3, x2, x6 + b .Lcbcencloop4x + +AES_FUNC_START(aes_cbc_encrypt) + ld1 {v4.16b}, [x5] /* get iv */ + enc_prepare w3, x2, x6 + +.Lcbcencloop4x: + subs w4, w4, #4 + bmi .Lcbcenc1x + ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */ + eor v0.16b, v0.16b, v4.16b /* ..and xor with iv */ + encrypt_block v0, w3, x2, x6, w7 + eor v1.16b, v1.16b, v0.16b + encrypt_block v1, w3, x2, x6, w7 + eor v2.16b, v2.16b, v1.16b + encrypt_block v2, w3, x2, x6, w7 + eor v3.16b, v3.16b, v2.16b + encrypt_block v3, w3, x2, x6, w7 + st1 {v0.16b-v3.16b}, [x0], #64 + mov v4.16b, v3.16b + b .Lcbcencloop4x +.Lcbcenc1x: + adds w4, w4, #4 + beq .Lcbcencout +.Lcbcencloop: + ld1 {v0.16b}, [x1], #16 /* get next pt block */ + eor v4.16b, v4.16b, v0.16b /* ..and xor with iv */ + encrypt_block v4, w3, x2, x6, w7 + st1 {v4.16b}, [x0], #16 + subs w4, w4, #1 + bne .Lcbcencloop +.Lcbcencout: + st1 {v4.16b}, [x5] /* return iv */ + ret +AES_FUNC_END(aes_cbc_encrypt) +AES_FUNC_END(aes_essiv_cbc_encrypt) + +AES_FUNC_START(aes_essiv_cbc_decrypt) + ld1 {cbciv.16b}, [x5] /* get iv */ + + mov w8, #14 /* AES-256: 14 rounds */ + enc_prepare w8, x6, x7 + encrypt_block cbciv, w8, x6, x7, w9 + b .Lessivcbcdecstart + +AES_FUNC_START(aes_cbc_decrypt) + ld1 {cbciv.16b}, [x5] /* get iv */ +.Lessivcbcdecstart: + frame_push 0 + dec_prepare w3, x2, x6 + +.LcbcdecloopNx: + subs w4, w4, #MAX_STRIDE + bmi .Lcbcdec1x + ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */ +#if MAX_STRIDE == 5 + ld1 {v4.16b}, [x1], #16 /* get 1 ct block */ + mov v5.16b, v0.16b + mov v6.16b, v1.16b + mov v7.16b, v2.16b + bl aes_decrypt_block5x + sub x1, x1, #32 + eor v0.16b, v0.16b, cbciv.16b + eor v1.16b, v1.16b, v5.16b + ld1 {v5.16b}, [x1], #16 /* reload 1 ct block */ + ld1 {cbciv.16b}, [x1], #16 /* reload 1 ct block */ + eor v2.16b, v2.16b, v6.16b + eor v3.16b, v3.16b, v7.16b + eor v4.16b, v4.16b, v5.16b +#else + mov v4.16b, v0.16b + mov v5.16b, v1.16b + mov v6.16b, v2.16b + bl aes_decrypt_block4x + sub x1, x1, #16 + eor v0.16b, v0.16b, cbciv.16b + eor v1.16b, v1.16b, v4.16b + ld1 {cbciv.16b}, [x1], #16 /* reload 1 ct block */ + eor v2.16b, v2.16b, v5.16b + eor v3.16b, v3.16b, v6.16b +#endif + st1 {v0.16b-v3.16b}, [x0], #64 +ST5( st1 {v4.16b}, [x0], #16 ) + b .LcbcdecloopNx +.Lcbcdec1x: + adds w4, w4, #MAX_STRIDE + beq .Lcbcdecout +.Lcbcdecloop: + ld1 {v1.16b}, [x1], #16 /* get next ct block */ + mov v0.16b, v1.16b /* ...and copy to v0 */ + decrypt_block v0, w3, x2, x6, w7 + eor v0.16b, v0.16b, cbciv.16b /* xor with iv => pt */ + mov cbciv.16b, v1.16b /* ct is next iv */ + st1 {v0.16b}, [x0], #16 + subs w4, w4, #1 + bne .Lcbcdecloop +.Lcbcdecout: + st1 {cbciv.16b}, [x5] /* return iv */ + frame_pop + ret +AES_FUNC_END(aes_cbc_decrypt) +AES_FUNC_END(aes_essiv_cbc_decrypt) + + + /* + * aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[], + * int rounds, int bytes, u8 const iv[]) + * aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[], + * int rounds, int bytes, u8 const iv[]) + */ + +AES_FUNC_START(aes_cbc_cts_encrypt) + adr_l x8, .Lcts_permute_table + sub x4, x4, #16 + add x9, x8, #32 + add x8, x8, x4 + sub x9, x9, x4 + ld1 {v3.16b}, [x8] + ld1 {v4.16b}, [x9] + + ld1 {v0.16b}, [x1], x4 /* overlapping loads */ + ld1 {v1.16b}, [x1] + + ld1 {v5.16b}, [x5] /* get iv */ + enc_prepare w3, x2, x6 + + eor v0.16b, v0.16b, v5.16b /* xor with iv */ + tbl v1.16b, {v1.16b}, v4.16b + encrypt_block v0, w3, x2, x6, w7 + + eor v1.16b, v1.16b, v0.16b + tbl v0.16b, {v0.16b}, v3.16b + encrypt_block v1, w3, x2, x6, w7 + + add x4, x0, x4 + st1 {v0.16b}, [x4] /* overlapping stores */ + st1 {v1.16b}, [x0] + ret +AES_FUNC_END(aes_cbc_cts_encrypt) + +AES_FUNC_START(aes_cbc_cts_decrypt) + adr_l x8, .Lcts_permute_table + sub x4, x4, #16 + add x9, x8, #32 + add x8, x8, x4 + sub x9, x9, x4 + ld1 {v3.16b}, [x8] + ld1 {v4.16b}, [x9] + + ld1 {v0.16b}, [x1], x4 /* overlapping loads */ + ld1 {v1.16b}, [x1] + + ld1 {v5.16b}, [x5] /* get iv */ + dec_prepare w3, x2, x6 + + decrypt_block v0, w3, x2, x6, w7 + tbl v2.16b, {v0.16b}, v3.16b + eor v2.16b, v2.16b, v1.16b + + tbx v0.16b, {v1.16b}, v4.16b + decrypt_block v0, w3, x2, x6, w7 + eor v0.16b, v0.16b, v5.16b /* xor with iv */ + + add x4, x0, x4 + st1 {v2.16b}, [x4] /* overlapping stores */ + st1 {v0.16b}, [x0] + ret +AES_FUNC_END(aes_cbc_cts_decrypt) + + .section ".rodata", "a" + .align 6 +.Lcts_permute_table: + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7 + .byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .previous + + /* + * This macro generates the code for CTR and XCTR mode. + */ +.macro ctr_encrypt xctr + // Arguments + OUT .req x0 + IN .req x1 + KEY .req x2 + ROUNDS_W .req w3 + BYTES_W .req w4 + IV .req x5 + BYTE_CTR_W .req w6 // XCTR only + // Intermediate values + CTR_W .req w11 // XCTR only + CTR .req x11 // XCTR only + IV_PART .req x12 + BLOCKS .req x13 + BLOCKS_W .req w13 + + frame_push 0 + + enc_prepare ROUNDS_W, KEY, IV_PART + ld1 {vctr.16b}, [IV] + + /* + * Keep 64 bits of the IV in a register. For CTR mode this lets us + * easily increment the IV. For XCTR mode this lets us efficiently XOR + * the 64-bit counter with the IV. + */ + .if \xctr + umov IV_PART, vctr.d[0] + lsr CTR_W, BYTE_CTR_W, #4 + .else + umov IV_PART, vctr.d[1] + rev IV_PART, IV_PART + .endif + +.LctrloopNx\xctr: + add BLOCKS_W, BYTES_W, #15 + sub BYTES_W, BYTES_W, #MAX_STRIDE << 4 + lsr BLOCKS_W, BLOCKS_W, #4 + mov w8, #MAX_STRIDE + cmp BLOCKS_W, w8 + csel BLOCKS_W, BLOCKS_W, w8, lt + + /* + * Set up the counter values in v0-v{MAX_STRIDE-1}. + * + * If we are encrypting less than MAX_STRIDE blocks, the tail block + * handling code expects the last keystream block to be in + * v{MAX_STRIDE-1}. For example: if encrypting two blocks with + * MAX_STRIDE=5, then v3 and v4 should have the next two counter blocks. + */ + .if \xctr + add CTR, CTR, BLOCKS + .else + adds IV_PART, IV_PART, BLOCKS + .endif + mov v0.16b, vctr.16b + mov v1.16b, vctr.16b + mov v2.16b, vctr.16b + mov v3.16b, vctr.16b +ST5( mov v4.16b, vctr.16b ) + .if \xctr + sub x6, CTR, #MAX_STRIDE - 1 + sub x7, CTR, #MAX_STRIDE - 2 + sub x8, CTR, #MAX_STRIDE - 3 + sub x9, CTR, #MAX_STRIDE - 4 +ST5( sub x10, CTR, #MAX_STRIDE - 5 ) + eor x6, x6, IV_PART + eor x7, x7, IV_PART + eor x8, x8, IV_PART + eor x9, x9, IV_PART +ST5( eor x10, x10, IV_PART ) + mov v0.d[0], x6 + mov v1.d[0], x7 + mov v2.d[0], x8 + mov v3.d[0], x9 +ST5( mov v4.d[0], x10 ) + .else + bcs 0f + .subsection 1 + /* + * This subsection handles carries. + * + * Conditional branching here is allowed with respect to time + * invariance since the branches are dependent on the IV instead + * of the plaintext or key. This code is rarely executed in + * practice anyway. + */ + + /* Apply carry to outgoing counter. */ +0: umov x8, vctr.d[0] + rev x8, x8 + add x8, x8, #1 + rev x8, x8 + ins vctr.d[0], x8 + + /* + * Apply carry to counter blocks if needed. + * + * Since the carry flag was set, we know 0 <= IV_PART < + * MAX_STRIDE. Using the value of IV_PART we can determine how + * many counter blocks need to be updated. + */ + cbz IV_PART, 2f + adr x16, 1f + sub x16, x16, IV_PART, lsl #3 + br x16 + bti c + mov v0.d[0], vctr.d[0] + bti c + mov v1.d[0], vctr.d[0] + bti c + mov v2.d[0], vctr.d[0] + bti c + mov v3.d[0], vctr.d[0] +ST5( bti c ) +ST5( mov v4.d[0], vctr.d[0] ) +1: b 2f + .previous + +2: rev x7, IV_PART + ins vctr.d[1], x7 + sub x7, IV_PART, #MAX_STRIDE - 1 + sub x8, IV_PART, #MAX_STRIDE - 2 + sub x9, IV_PART, #MAX_STRIDE - 3 + rev x7, x7 + rev x8, x8 + mov v1.d[1], x7 + rev x9, x9 +ST5( sub x10, IV_PART, #MAX_STRIDE - 4 ) + mov v2.d[1], x8 +ST5( rev x10, x10 ) + mov v3.d[1], x9 +ST5( mov v4.d[1], x10 ) + .endif + + /* + * If there are at least MAX_STRIDE blocks left, XOR the data with + * keystream and store. Otherwise jump to tail handling. + */ + tbnz BYTES_W, #31, .Lctrtail\xctr + ld1 {v5.16b-v7.16b}, [IN], #48 +ST4( bl aes_encrypt_block4x ) +ST5( bl aes_encrypt_block5x ) + eor v0.16b, v5.16b, v0.16b +ST4( ld1 {v5.16b}, [IN], #16 ) + eor v1.16b, v6.16b, v1.16b +ST5( ld1 {v5.16b-v6.16b}, [IN], #32 ) + eor v2.16b, v7.16b, v2.16b + eor v3.16b, v5.16b, v3.16b +ST5( eor v4.16b, v6.16b, v4.16b ) + st1 {v0.16b-v3.16b}, [OUT], #64 +ST5( st1 {v4.16b}, [OUT], #16 ) + cbz BYTES_W, .Lctrout\xctr + b .LctrloopNx\xctr + +.Lctrout\xctr: + .if !\xctr + st1 {vctr.16b}, [IV] /* return next CTR value */ + .endif + frame_pop + ret + +.Lctrtail\xctr: + /* + * Handle up to MAX_STRIDE * 16 - 1 bytes of plaintext + * + * This code expects the last keystream block to be in v{MAX_STRIDE-1}. + * For example: if encrypting two blocks with MAX_STRIDE=5, then v3 and + * v4 should have the next two counter blocks. + * + * This allows us to store the ciphertext by writing to overlapping + * regions of memory. Any invalid ciphertext blocks get overwritten by + * correctly computed blocks. This approach greatly simplifies the + * logic for storing the ciphertext. + */ + mov x16, #16 + ands w7, BYTES_W, #0xf + csel x13, x7, x16, ne + +ST5( cmp BYTES_W, #64 - (MAX_STRIDE << 4)) +ST5( csel x14, x16, xzr, gt ) + cmp BYTES_W, #48 - (MAX_STRIDE << 4) + csel x15, x16, xzr, gt + cmp BYTES_W, #32 - (MAX_STRIDE << 4) + csel x16, x16, xzr, gt + cmp BYTES_W, #16 - (MAX_STRIDE << 4) + + adr_l x9, .Lcts_permute_table + add x9, x9, x13 + ble .Lctrtail1x\xctr + +ST5( ld1 {v5.16b}, [IN], x14 ) + ld1 {v6.16b}, [IN], x15 + ld1 {v7.16b}, [IN], x16 + +ST4( bl aes_encrypt_block4x ) +ST5( bl aes_encrypt_block5x ) + + ld1 {v8.16b}, [IN], x13 + ld1 {v9.16b}, [IN] + ld1 {v10.16b}, [x9] + +ST4( eor v6.16b, v6.16b, v0.16b ) +ST4( eor v7.16b, v7.16b, v1.16b ) +ST4( tbl v3.16b, {v3.16b}, v10.16b ) +ST4( eor v8.16b, v8.16b, v2.16b ) +ST4( eor v9.16b, v9.16b, v3.16b ) + +ST5( eor v5.16b, v5.16b, v0.16b ) +ST5( eor v6.16b, v6.16b, v1.16b ) +ST5( tbl v4.16b, {v4.16b}, v10.16b ) +ST5( eor v7.16b, v7.16b, v2.16b ) +ST5( eor v8.16b, v8.16b, v3.16b ) +ST5( eor v9.16b, v9.16b, v4.16b ) + +ST5( st1 {v5.16b}, [OUT], x14 ) + st1 {v6.16b}, [OUT], x15 + st1 {v7.16b}, [OUT], x16 + add x13, x13, OUT + st1 {v9.16b}, [x13] // overlapping stores + st1 {v8.16b}, [OUT] + b .Lctrout\xctr + +.Lctrtail1x\xctr: + /* + * Handle <= 16 bytes of plaintext + * + * This code always reads and writes 16 bytes. To avoid out of bounds + * accesses, XCTR and CTR modes must use a temporary buffer when + * encrypting/decrypting less than 16 bytes. + * + * This code is unusual in that it loads the input and stores the output + * relative to the end of the buffers rather than relative to the start. + * This causes unusual behaviour when encrypting/decrypting less than 16 + * bytes; the end of the data is expected to be at the end of the + * temporary buffer rather than the start of the data being at the start + * of the temporary buffer. + */ + sub x8, x7, #16 + csel x7, x7, x8, eq + add IN, IN, x7 + add OUT, OUT, x7 + ld1 {v5.16b}, [IN] + ld1 {v6.16b}, [OUT] +ST5( mov v3.16b, v4.16b ) + encrypt_block v3, ROUNDS_W, KEY, x8, w7 + ld1 {v10.16b-v11.16b}, [x9] + tbl v3.16b, {v3.16b}, v10.16b + sshr v11.16b, v11.16b, #7 + eor v5.16b, v5.16b, v3.16b + bif v5.16b, v6.16b, v11.16b + st1 {v5.16b}, [OUT] + b .Lctrout\xctr + + // Arguments + .unreq OUT + .unreq IN + .unreq KEY + .unreq ROUNDS_W + .unreq BYTES_W + .unreq IV + .unreq BYTE_CTR_W // XCTR only + // Intermediate values + .unreq CTR_W // XCTR only + .unreq CTR // XCTR only + .unreq IV_PART + .unreq BLOCKS + .unreq BLOCKS_W +.endm + + /* + * aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, + * int bytes, u8 ctr[]) + * + * The input and output buffers must always be at least 16 bytes even if + * encrypting/decrypting less than 16 bytes. Otherwise out of bounds + * accesses will occur. The data to be encrypted/decrypted is expected + * to be at the end of this 16-byte temporary buffer rather than the + * start. + */ + +AES_FUNC_START(aes_ctr_encrypt) + ctr_encrypt 0 +AES_FUNC_END(aes_ctr_encrypt) + + /* + * aes_xctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, + * int bytes, u8 const iv[], int byte_ctr) + * + * The input and output buffers must always be at least 16 bytes even if + * encrypting/decrypting less than 16 bytes. Otherwise out of bounds + * accesses will occur. The data to be encrypted/decrypted is expected + * to be at the end of this 16-byte temporary buffer rather than the + * start. + */ + +AES_FUNC_START(aes_xctr_encrypt) + ctr_encrypt 1 +AES_FUNC_END(aes_xctr_encrypt) + + + /* + * aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds, + * int bytes, u8 const rk2[], u8 iv[], int first) + * aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds, + * int bytes, u8 const rk2[], u8 iv[], int first) + */ + + .macro next_tweak, out, in, tmp + sshr \tmp\().2d, \in\().2d, #63 + and \tmp\().16b, \tmp\().16b, xtsmask.16b + add \out\().2d, \in\().2d, \in\().2d + ext \tmp\().16b, \tmp\().16b, \tmp\().16b, #8 + eor \out\().16b, \out\().16b, \tmp\().16b + .endm + + .macro xts_load_mask, tmp + movi xtsmask.2s, #0x1 + movi \tmp\().2s, #0x87 + uzp1 xtsmask.4s, xtsmask.4s, \tmp\().4s + .endm + +AES_FUNC_START(aes_xts_encrypt) + frame_push 0 + + ld1 {v4.16b}, [x6] + xts_load_mask v8 + cbz w7, .Lxtsencnotfirst + + enc_prepare w3, x5, x8 + xts_cts_skip_tw w7, .LxtsencNx + encrypt_block v4, w3, x5, x8, w7 /* first tweak */ + enc_switch_key w3, x2, x8 + b .LxtsencNx + +.Lxtsencnotfirst: + enc_prepare w3, x2, x8 +.LxtsencloopNx: + next_tweak v4, v4, v8 +.LxtsencNx: + subs w4, w4, #64 + bmi .Lxtsenc1x + ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */ + next_tweak v5, v4, v8 + eor v0.16b, v0.16b, v4.16b + next_tweak v6, v5, v8 + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + next_tweak v7, v6, v8 + eor v3.16b, v3.16b, v7.16b + bl aes_encrypt_block4x + eor v3.16b, v3.16b, v7.16b + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + st1 {v0.16b-v3.16b}, [x0], #64 + mov v4.16b, v7.16b + cbz w4, .Lxtsencret + xts_reload_mask v8 + b .LxtsencloopNx +.Lxtsenc1x: + adds w4, w4, #64 + beq .Lxtsencout + subs w4, w4, #16 + bmi .LxtsencctsNx +.Lxtsencloop: + ld1 {v0.16b}, [x1], #16 +.Lxtsencctsout: + eor v0.16b, v0.16b, v4.16b + encrypt_block v0, w3, x2, x8, w7 + eor v0.16b, v0.16b, v4.16b + cbz w4, .Lxtsencout + subs w4, w4, #16 + next_tweak v4, v4, v8 + bmi .Lxtsenccts + st1 {v0.16b}, [x0], #16 + b .Lxtsencloop +.Lxtsencout: + st1 {v0.16b}, [x0] +.Lxtsencret: + st1 {v4.16b}, [x6] + frame_pop + ret + +.LxtsencctsNx: + mov v0.16b, v3.16b + sub x0, x0, #16 +.Lxtsenccts: + adr_l x8, .Lcts_permute_table + + add x1, x1, w4, sxtw /* rewind input pointer */ + add w4, w4, #16 /* # bytes in final block */ + add x9, x8, #32 + add x8, x8, x4 + sub x9, x9, x4 + add x4, x0, x4 /* output address of final block */ + + ld1 {v1.16b}, [x1] /* load final block */ + ld1 {v2.16b}, [x8] + ld1 {v3.16b}, [x9] + + tbl v2.16b, {v0.16b}, v2.16b + tbx v0.16b, {v1.16b}, v3.16b + st1 {v2.16b}, [x4] /* overlapping stores */ + mov w4, wzr + b .Lxtsencctsout +AES_FUNC_END(aes_xts_encrypt) + +AES_FUNC_START(aes_xts_decrypt) + frame_push 0 + + /* subtract 16 bytes if we are doing CTS */ + sub w8, w4, #0x10 + tst w4, #0xf + csel w4, w4, w8, eq + + ld1 {v4.16b}, [x6] + xts_load_mask v8 + xts_cts_skip_tw w7, .Lxtsdecskiptw + cbz w7, .Lxtsdecnotfirst + + enc_prepare w3, x5, x8 + encrypt_block v4, w3, x5, x8, w7 /* first tweak */ +.Lxtsdecskiptw: + dec_prepare w3, x2, x8 + b .LxtsdecNx + +.Lxtsdecnotfirst: + dec_prepare w3, x2, x8 +.LxtsdecloopNx: + next_tweak v4, v4, v8 +.LxtsdecNx: + subs w4, w4, #64 + bmi .Lxtsdec1x + ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */ + next_tweak v5, v4, v8 + eor v0.16b, v0.16b, v4.16b + next_tweak v6, v5, v8 + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + next_tweak v7, v6, v8 + eor v3.16b, v3.16b, v7.16b + bl aes_decrypt_block4x + eor v3.16b, v3.16b, v7.16b + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + st1 {v0.16b-v3.16b}, [x0], #64 + mov v4.16b, v7.16b + cbz w4, .Lxtsdecout + xts_reload_mask v8 + b .LxtsdecloopNx +.Lxtsdec1x: + adds w4, w4, #64 + beq .Lxtsdecout + subs w4, w4, #16 +.Lxtsdecloop: + ld1 {v0.16b}, [x1], #16 + bmi .Lxtsdeccts +.Lxtsdecctsout: + eor v0.16b, v0.16b, v4.16b + decrypt_block v0, w3, x2, x8, w7 + eor v0.16b, v0.16b, v4.16b + st1 {v0.16b}, [x0], #16 + cbz w4, .Lxtsdecout + subs w4, w4, #16 + next_tweak v4, v4, v8 + b .Lxtsdecloop +.Lxtsdecout: + st1 {v4.16b}, [x6] + frame_pop + ret + +.Lxtsdeccts: + adr_l x8, .Lcts_permute_table + + add x1, x1, w4, sxtw /* rewind input pointer */ + add w4, w4, #16 /* # bytes in final block */ + add x9, x8, #32 + add x8, x8, x4 + sub x9, x9, x4 + add x4, x0, x4 /* output address of final block */ + + next_tweak v5, v4, v8 + + ld1 {v1.16b}, [x1] /* load final block */ + ld1 {v2.16b}, [x8] + ld1 {v3.16b}, [x9] + + eor v0.16b, v0.16b, v5.16b + decrypt_block v0, w3, x2, x8, w7 + eor v0.16b, v0.16b, v5.16b + + tbl v2.16b, {v0.16b}, v2.16b + tbx v0.16b, {v1.16b}, v3.16b + + st1 {v2.16b}, [x4] /* overlapping stores */ + mov w4, wzr + b .Lxtsdecctsout +AES_FUNC_END(aes_xts_decrypt) + +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) + /* + * void aes_mac_update(u8 const in[], u32 const rk[], int rounds, + * size_t blocks, u8 dg[], int enc_before, + * int enc_after); + */ +AES_FUNC_START(aes_mac_update) + ld1 {v0.16b}, [x4] /* get dg */ + enc_prepare w2, x1, x7 + cbz w5, .Lmacloop4x + + encrypt_block v0, w2, x1, x7, w8 + +.Lmacloop4x: + subs x3, x3, #4 + bmi .Lmac1x + ld1 {v1.16b-v4.16b}, [x0], #64 /* get next pt block */ + eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */ + encrypt_block v0, w2, x1, x7, w8 + eor v0.16b, v0.16b, v2.16b + encrypt_block v0, w2, x1, x7, w8 + eor v0.16b, v0.16b, v3.16b + encrypt_block v0, w2, x1, x7, w8 + eor v0.16b, v0.16b, v4.16b + cmp x3, xzr + csinv w5, w6, wzr, eq + cbz w5, .Lmacout + encrypt_block v0, w2, x1, x7, w8 + st1 {v0.16b}, [x4] /* return dg */ + b .Lmacloop4x +.Lmac1x: + add x3, x3, #4 +.Lmacloop: + cbz x3, .Lmacout + ld1 {v1.16b}, [x0], #16 /* get next pt block */ + eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */ + + subs x3, x3, #1 + csinv w5, w6, wzr, eq + cbz w5, .Lmacout + +.Lmacenc: + encrypt_block v0, w2, x1, x7, w8 + b .Lmacloop + +.Lmacout: + st1 {v0.16b}, [x4] /* return dg */ + ret +AES_FUNC_END(aes_mac_update) +#endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ diff --git a/lib/crypto/arm64/aes-neon.S b/lib/crypto/arm64/aes-neon.S new file mode 100644 index 000000000000..f37b1dbd887f --- /dev/null +++ b/lib/crypto/arm64/aes-neon.S @@ -0,0 +1,250 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * AES cipher for ARMv8 NEON + * + * Copyright (C) 2013 - 2017 Linaro Ltd. <ard.biesheuvel@linaro.org> + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> + +#define AES_FUNC_START(func) SYM_FUNC_START(neon_ ## func) +#define AES_FUNC_END(func) SYM_FUNC_END(neon_ ## func) + + xtsmask .req v7 + cbciv .req v7 + vctr .req v4 + + .macro xts_reload_mask, tmp + xts_load_mask \tmp + .endm + + /* special case for the neon-bs driver calling into this one for CTS */ + .macro xts_cts_skip_tw, reg, lbl + tbnz \reg, #1, \lbl + .endm + + /* multiply by polynomial 'x' in GF(2^8) */ + .macro mul_by_x, out, in, temp, const + sshr \temp, \in, #7 + shl \out, \in, #1 + and \temp, \temp, \const + eor \out, \out, \temp + .endm + + /* multiply by polynomial 'x^2' in GF(2^8) */ + .macro mul_by_x2, out, in, temp, const + ushr \temp, \in, #6 + shl \out, \in, #2 + pmul \temp, \temp, \const + eor \out, \out, \temp + .endm + + /* preload the entire Sbox */ + .macro prepare, sbox, shiftrows, temp + movi v12.16b, #0x1b + ldr_l q13, \shiftrows, \temp + ldr_l q14, .Lror32by8, \temp + adr_l \temp, \sbox + ld1 {v16.16b-v19.16b}, [\temp], #64 + ld1 {v20.16b-v23.16b}, [\temp], #64 + ld1 {v24.16b-v27.16b}, [\temp], #64 + ld1 {v28.16b-v31.16b}, [\temp] + .endm + + /* do preload for encryption */ + .macro enc_prepare, ignore0, ignore1, temp + prepare crypto_aes_sbox, .LForward_ShiftRows, \temp + .endm + + .macro enc_switch_key, ignore0, ignore1, temp + /* do nothing */ + .endm + + /* do preload for decryption */ + .macro dec_prepare, ignore0, ignore1, temp + prepare crypto_aes_inv_sbox, .LReverse_ShiftRows, \temp + .endm + + /* apply SubBytes transformation using the preloaded Sbox */ + .macro sub_bytes, in + sub v9.16b, \in\().16b, v15.16b + tbl \in\().16b, {v16.16b-v19.16b}, \in\().16b + sub v10.16b, v9.16b, v15.16b + tbx \in\().16b, {v20.16b-v23.16b}, v9.16b + sub v11.16b, v10.16b, v15.16b + tbx \in\().16b, {v24.16b-v27.16b}, v10.16b + tbx \in\().16b, {v28.16b-v31.16b}, v11.16b + .endm + + /* apply MixColumns transformation */ + .macro mix_columns, in, enc + .if \enc == 0 + /* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */ + mul_by_x2 v8.16b, \in\().16b, v9.16b, v12.16b + eor \in\().16b, \in\().16b, v8.16b + rev32 v8.8h, v8.8h + eor \in\().16b, \in\().16b, v8.16b + .endif + + mul_by_x v9.16b, \in\().16b, v8.16b, v12.16b + rev32 v8.8h, \in\().8h + eor v8.16b, v8.16b, v9.16b + eor \in\().16b, \in\().16b, v8.16b + tbl \in\().16b, {\in\().16b}, v14.16b + eor \in\().16b, \in\().16b, v8.16b + .endm + + .macro do_block, enc, in, rounds, rk, rkp, i + ld1 {v15.4s}, [\rk] + add \rkp, \rk, #16 + mov \i, \rounds +.La\@: eor \in\().16b, \in\().16b, v15.16b /* ^round key */ + movi v15.16b, #0x40 + tbl \in\().16b, {\in\().16b}, v13.16b /* ShiftRows */ + sub_bytes \in + sub \i, \i, #1 + ld1 {v15.4s}, [\rkp], #16 + cbz \i, .Lb\@ + mix_columns \in, \enc + b .La\@ +.Lb\@: eor \in\().16b, \in\().16b, v15.16b /* ^round key */ + .endm + + .macro encrypt_block, in, rounds, rk, rkp, i + do_block 1, \in, \rounds, \rk, \rkp, \i + .endm + + .macro decrypt_block, in, rounds, rk, rkp, i + do_block 0, \in, \rounds, \rk, \rkp, \i + .endm + + /* + * Interleaved versions: functionally equivalent to the + * ones above, but applied to AES states in parallel. + */ + + .macro sub_bytes_4x, in0, in1, in2, in3 + sub v8.16b, \in0\().16b, v15.16b + tbl \in0\().16b, {v16.16b-v19.16b}, \in0\().16b + sub v9.16b, \in1\().16b, v15.16b + tbl \in1\().16b, {v16.16b-v19.16b}, \in1\().16b + sub v10.16b, \in2\().16b, v15.16b + tbl \in2\().16b, {v16.16b-v19.16b}, \in2\().16b + sub v11.16b, \in3\().16b, v15.16b + tbl \in3\().16b, {v16.16b-v19.16b}, \in3\().16b + tbx \in0\().16b, {v20.16b-v23.16b}, v8.16b + tbx \in1\().16b, {v20.16b-v23.16b}, v9.16b + sub v8.16b, v8.16b, v15.16b + tbx \in2\().16b, {v20.16b-v23.16b}, v10.16b + sub v9.16b, v9.16b, v15.16b + tbx \in3\().16b, {v20.16b-v23.16b}, v11.16b + sub v10.16b, v10.16b, v15.16b + tbx \in0\().16b, {v24.16b-v27.16b}, v8.16b + sub v11.16b, v11.16b, v15.16b + tbx \in1\().16b, {v24.16b-v27.16b}, v9.16b + sub v8.16b, v8.16b, v15.16b + tbx \in2\().16b, {v24.16b-v27.16b}, v10.16b + sub v9.16b, v9.16b, v15.16b + tbx \in3\().16b, {v24.16b-v27.16b}, v11.16b + sub v10.16b, v10.16b, v15.16b + tbx \in0\().16b, {v28.16b-v31.16b}, v8.16b + sub v11.16b, v11.16b, v15.16b + tbx \in1\().16b, {v28.16b-v31.16b}, v9.16b + tbx \in2\().16b, {v28.16b-v31.16b}, v10.16b + tbx \in3\().16b, {v28.16b-v31.16b}, v11.16b + .endm + + .macro mul_by_x_2x, out0, out1, in0, in1, tmp0, tmp1, const + sshr \tmp0\().16b, \in0\().16b, #7 + shl \out0\().16b, \in0\().16b, #1 + sshr \tmp1\().16b, \in1\().16b, #7 + and \tmp0\().16b, \tmp0\().16b, \const\().16b + shl \out1\().16b, \in1\().16b, #1 + and \tmp1\().16b, \tmp1\().16b, \const\().16b + eor \out0\().16b, \out0\().16b, \tmp0\().16b + eor \out1\().16b, \out1\().16b, \tmp1\().16b + .endm + + .macro mul_by_x2_2x, out0, out1, in0, in1, tmp0, tmp1, const + ushr \tmp0\().16b, \in0\().16b, #6 + shl \out0\().16b, \in0\().16b, #2 + ushr \tmp1\().16b, \in1\().16b, #6 + pmul \tmp0\().16b, \tmp0\().16b, \const\().16b + shl \out1\().16b, \in1\().16b, #2 + pmul \tmp1\().16b, \tmp1\().16b, \const\().16b + eor \out0\().16b, \out0\().16b, \tmp0\().16b + eor \out1\().16b, \out1\().16b, \tmp1\().16b + .endm + + .macro mix_columns_2x, in0, in1, enc + .if \enc == 0 + /* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */ + mul_by_x2_2x v8, v9, \in0, \in1, v10, v11, v12 + eor \in0\().16b, \in0\().16b, v8.16b + rev32 v8.8h, v8.8h + eor \in1\().16b, \in1\().16b, v9.16b + rev32 v9.8h, v9.8h + eor \in0\().16b, \in0\().16b, v8.16b + eor \in1\().16b, \in1\().16b, v9.16b + .endif + + mul_by_x_2x v8, v9, \in0, \in1, v10, v11, v12 + rev32 v10.8h, \in0\().8h + rev32 v11.8h, \in1\().8h + eor v10.16b, v10.16b, v8.16b + eor v11.16b, v11.16b, v9.16b + eor \in0\().16b, \in0\().16b, v10.16b + eor \in1\().16b, \in1\().16b, v11.16b + tbl \in0\().16b, {\in0\().16b}, v14.16b + tbl \in1\().16b, {\in1\().16b}, v14.16b + eor \in0\().16b, \in0\().16b, v10.16b + eor \in1\().16b, \in1\().16b, v11.16b + .endm + + .macro do_block_4x, enc, in0, in1, in2, in3, rounds, rk, rkp, i + ld1 {v15.4s}, [\rk] + add \rkp, \rk, #16 + mov \i, \rounds +.La\@: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */ + eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */ + eor \in2\().16b, \in2\().16b, v15.16b /* ^round key */ + eor \in3\().16b, \in3\().16b, v15.16b /* ^round key */ + movi v15.16b, #0x40 + tbl \in0\().16b, {\in0\().16b}, v13.16b /* ShiftRows */ + tbl \in1\().16b, {\in1\().16b}, v13.16b /* ShiftRows */ + tbl \in2\().16b, {\in2\().16b}, v13.16b /* ShiftRows */ + tbl \in3\().16b, {\in3\().16b}, v13.16b /* ShiftRows */ + sub_bytes_4x \in0, \in1, \in2, \in3 + sub \i, \i, #1 + ld1 {v15.4s}, [\rkp], #16 + cbz \i, .Lb\@ + mix_columns_2x \in0, \in1, \enc + mix_columns_2x \in2, \in3, \enc + b .La\@ +.Lb\@: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */ + eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */ + eor \in2\().16b, \in2\().16b, v15.16b /* ^round key */ + eor \in3\().16b, \in3\().16b, v15.16b /* ^round key */ + .endm + + .macro encrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i + do_block_4x 1, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i + .endm + + .macro decrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i + do_block_4x 0, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i + .endm + +#include "aes-modes.S" + + .section ".rodata", "a" + .align 4 +.LForward_ShiftRows: + .octa 0x0b06010c07020d08030e09040f0a0500 + +.LReverse_ShiftRows: + .octa 0x0306090c0f0205080b0e0104070a0d00 + +.Lror32by8: + .octa 0x0c0f0e0d080b0a090407060500030201 diff --git a/lib/crypto/arm64/aes.h b/lib/crypto/arm64/aes.h index 63eea6271ef9..9e9e45a6f787 100644 --- a/lib/crypto/arm64/aes.h +++ b/lib/crypto/arm64/aes.h @@ -11,6 +11,7 @@ #include <linux/unaligned.h> #include <linux/cpufeature.h> +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aes); struct aes_block { @@ -28,6 +29,9 @@ asmlinkage void __aes_ce_decrypt(const u32 inv_rk[], u8 out[AES_BLOCK_SIZE], asmlinkage u32 __aes_ce_sub(u32 l); asmlinkage void __aes_ce_invert(struct aes_block *out, const struct aes_block *in); +asmlinkage void neon_aes_mac_update(u8 const in[], u32 const rk[], int rounds, + size_t blocks, u8 dg[], int enc_before, + int enc_after); /* * Expand an AES key using the crypto extensions if supported and usable or @@ -48,8 +52,7 @@ static void aes_expandkey_arm64(u32 rndkeys[], u32 *inv_rndkeys, struct aes_block *key_enc, *key_dec; int i, j; - if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || - !static_branch_likely(&have_aes) || unlikely(!may_use_simd())) { + if (!static_branch_likely(&have_aes) || unlikely(!may_use_simd())) { aes_expandkey_generic(rndkeys, inv_rndkeys, in_key, key_len); return; } @@ -126,12 +129,40 @@ int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, } EXPORT_SYMBOL(ce_aes_expandkey); +EXPORT_SYMBOL_NS_GPL(neon_aes_ecb_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_ecb_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_cts_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_cts_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_ctr_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_xctr_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_xts_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_xts_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL"); + +EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_cts_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_cts_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_ctr_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_xctr_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_xts_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_xts_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL"); +#if IS_MODULE(CONFIG_CRYPTO_AES_ARM64_CE_CCM) +EXPORT_SYMBOL_NS_GPL(ce_aes_mac_update, "CRYPTO_INTERNAL"); +#endif + static void aes_encrypt_arch(const struct aes_enckey *key, u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_aes) && likely(may_use_simd())) { + if (static_branch_likely(&have_aes) && likely(may_use_simd())) { scoped_ksimd() __aes_ce_encrypt(key->k.rndkeys, out, in, key->nrounds); } else { @@ -143,8 +174,7 @@ static void aes_decrypt_arch(const struct aes_key *key, u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_aes) && likely(may_use_simd())) { + if (static_branch_likely(&have_aes) && likely(may_use_simd())) { scoped_ksimd() __aes_ce_decrypt(key->inv_k.inv_rndkeys, out, in, key->nrounds); @@ -154,11 +184,36 @@ static void aes_decrypt_arch(const struct aes_key *key, } } -#ifdef CONFIG_KERNEL_MODE_NEON +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) +#define aes_cbcmac_blocks_arch aes_cbcmac_blocks_arch +static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE], + const struct aes_enckey *key, const u8 *data, + size_t nblocks, bool enc_before, + bool enc_after) +{ + if (static_branch_likely(&have_neon) && likely(may_use_simd())) { + scoped_ksimd() { + if (static_branch_likely(&have_aes)) + ce_aes_mac_update(data, key->k.rndkeys, + key->nrounds, nblocks, h, + enc_before, enc_after); + else + neon_aes_mac_update(data, key->k.rndkeys, + key->nrounds, nblocks, h, + enc_before, enc_after); + } + return true; + } + return false; +} +#endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ + #define aes_mod_init_arch aes_mod_init_arch static void aes_mod_init_arch(void) { - if (cpu_have_named_feature(AES)) - static_branch_enable(&have_aes); + if (cpu_have_named_feature(ASIMD)) { + static_branch_enable(&have_neon); + if (cpu_have_named_feature(AES)) + static_branch_enable(&have_aes); + } } -#endif /* CONFIG_KERNEL_MODE_NEON */ diff --git a/lib/crypto/arm64/chacha-neon-core.S b/lib/crypto/arm64/chacha-neon-core.S index 80079586ecc7..cb18eec968bd 100644 --- a/lib/crypto/arm64/chacha-neon-core.S +++ b/lib/crypto/arm64/chacha-neon-core.S @@ -531,10 +531,6 @@ SYM_FUNC_START(chacha_4block_xor_neon) add v3.4s, v3.4s, v19.4s add a2, a2, w8 add a3, a3, w9 -CPU_BE( rev a0, a0 ) -CPU_BE( rev a1, a1 ) -CPU_BE( rev a2, a2 ) -CPU_BE( rev a3, a3 ) ld4r {v24.4s-v27.4s}, [x0], #16 ld4r {v28.4s-v31.4s}, [x0] @@ -555,10 +551,6 @@ CPU_BE( rev a3, a3 ) add v7.4s, v7.4s, v23.4s add a6, a6, w8 add a7, a7, w9 -CPU_BE( rev a4, a4 ) -CPU_BE( rev a5, a5 ) -CPU_BE( rev a6, a6 ) -CPU_BE( rev a7, a7 ) // x8[0-3] += s2[0] // x9[0-3] += s2[1] @@ -576,10 +568,6 @@ CPU_BE( rev a7, a7 ) add v11.4s, v11.4s, v27.4s add a10, a10, w8 add a11, a11, w9 -CPU_BE( rev a8, a8 ) -CPU_BE( rev a9, a9 ) -CPU_BE( rev a10, a10 ) -CPU_BE( rev a11, a11 ) // x12[0-3] += s3[0] // x13[0-3] += s3[1] @@ -597,10 +585,6 @@ CPU_BE( rev a11, a11 ) add v15.4s, v15.4s, v31.4s add a14, a14, w8 add a15, a15, w9 -CPU_BE( rev a12, a12 ) -CPU_BE( rev a13, a13 ) -CPU_BE( rev a14, a14 ) -CPU_BE( rev a15, a15 ) // interleave 32-bit words in state n, n+1 ldp w6, w7, [x2], #64 diff --git a/lib/crypto/arm64/chacha.h b/lib/crypto/arm64/chacha.h index ca8c6a8b0578..c6f8ddf98e2d 100644 --- a/lib/crypto/arm64/chacha.h +++ b/lib/crypto/arm64/chacha.h @@ -36,9 +36,9 @@ asmlinkage void hchacha_block_neon(const struct chacha_state *state, static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); static void chacha_doneon(struct chacha_state *state, u8 *dst, const u8 *src, - int bytes, int nrounds) + unsigned int bytes, int nrounds) { - while (bytes > 0) { + while (bytes) { int l = min(bytes, CHACHA_BLOCK_SIZE * 5); if (l <= CHACHA_BLOCK_SIZE) { @@ -76,16 +76,8 @@ static void chacha_crypt_arch(struct chacha_state *state, u8 *dst, !crypto_simd_usable()) return chacha_crypt_generic(state, dst, src, bytes, nrounds); - do { - unsigned int todo = min_t(unsigned int, bytes, SZ_4K); - - scoped_ksimd() - chacha_doneon(state, dst, src, todo, nrounds); - - bytes -= todo; - src += todo; - dst += todo; - } while (bytes); + scoped_ksimd() + chacha_doneon(state, dst, src, bytes, nrounds); } #define chacha_mod_init_arch chacha_mod_init_arch diff --git a/lib/crypto/arm64/gf128hash.h b/lib/crypto/arm64/gf128hash.h new file mode 100644 index 000000000000..1d1179f87f8d --- /dev/null +++ b/lib/crypto/arm64/gf128hash.h @@ -0,0 +1,121 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * GHASH and POLYVAL, arm64 optimized + * + * Copyright 2025 Google LLC + */ +#include <asm/simd.h> +#include <linux/cpufeature.h> + +#define NUM_H_POWERS 8 + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_asimd); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); + +asmlinkage void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg, + const u8 *src, + const struct polyval_elem *h); +asmlinkage void polyval_mul_pmull(struct polyval_elem *a, + const struct polyval_elem *b); +asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc, + const struct polyval_key *key, + const u8 *data, size_t nblocks); + +#define polyval_preparekey_arch polyval_preparekey_arch +static void polyval_preparekey_arch(struct polyval_key *key, + const u8 raw_key[POLYVAL_BLOCK_SIZE]) +{ + static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS); + memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE); + if (static_branch_likely(&have_pmull) && may_use_simd()) { + scoped_ksimd() { + for (int i = NUM_H_POWERS - 2; i >= 0; i--) { + key->h_powers[i] = key->h_powers[i + 1]; + polyval_mul_pmull( + &key->h_powers[i], + &key->h_powers[NUM_H_POWERS - 1]); + } + } + } else { + for (int i = NUM_H_POWERS - 2; i >= 0; i--) { + key->h_powers[i] = key->h_powers[i + 1]; + polyval_mul_generic(&key->h_powers[i], + &key->h_powers[NUM_H_POWERS - 1]); + } + } +} + +static void polyval_mul_arm64(struct polyval_elem *a, + const struct polyval_elem *b) +{ + if (static_branch_likely(&have_asimd) && may_use_simd()) { + static const u8 zeroes[GHASH_BLOCK_SIZE]; + + scoped_ksimd() { + if (static_branch_likely(&have_pmull)) { + polyval_mul_pmull(a, b); + } else { + /* + * Note that this is indeed equivalent to a + * POLYVAL multiplication, since it takes the + * accumulator and key in POLYVAL format, and + * byte-swapping a block of zeroes is a no-op. + */ + pmull_ghash_update_p8(1, a, zeroes, b); + } + } + } else { + polyval_mul_generic(a, b); + } +} + +#define ghash_mul_arch ghash_mul_arch +static void ghash_mul_arch(struct polyval_elem *acc, + const struct ghash_key *key) +{ + polyval_mul_arm64(acc, &key->h); +} + +#define polyval_mul_arch polyval_mul_arch +static void polyval_mul_arch(struct polyval_elem *acc, + const struct polyval_key *key) +{ + polyval_mul_arm64(acc, &key->h_powers[NUM_H_POWERS - 1]); +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_asimd) && may_use_simd()) { + scoped_ksimd() + pmull_ghash_update_p8(nblocks, acc, data, &key->h); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define polyval_blocks_arch polyval_blocks_arch +static void polyval_blocks_arch(struct polyval_elem *acc, + const struct polyval_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_pmull) && may_use_simd()) { + scoped_ksimd() + polyval_blocks_pmull(acc, key, data, nblocks); + } else { + polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1], + data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (cpu_have_named_feature(ASIMD)) { + static_branch_enable(&have_asimd); + if (cpu_have_named_feature(PMULL)) + static_branch_enable(&have_pmull); + } +} diff --git a/lib/crypto/arm64/ghash-neon-core.S b/lib/crypto/arm64/ghash-neon-core.S new file mode 100644 index 000000000000..4c5799172b49 --- /dev/null +++ b/lib/crypto/arm64/ghash-neon-core.S @@ -0,0 +1,220 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Accelerated GHASH implementation with ARMv8 ASIMD instructions. + * + * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org> + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> + + SHASH .req v0 + SHASH2 .req v1 + T1 .req v2 + T2 .req v3 + XM .req v5 + XL .req v6 + XH .req v7 + IN1 .req v7 + + k00_16 .req v8 + k32_48 .req v9 + + t3 .req v10 + t4 .req v11 + t5 .req v12 + t6 .req v13 + t7 .req v14 + t8 .req v15 + t9 .req v16 + + perm1 .req v17 + perm2 .req v18 + perm3 .req v19 + + sh1 .req v20 + sh2 .req v21 + sh3 .req v22 + sh4 .req v23 + + ss1 .req v24 + ss2 .req v25 + ss3 .req v26 + ss4 .req v27 + + .text + + .macro __pmull_p8, rq, ad, bd + ext t3.8b, \ad\().8b, \ad\().8b, #1 // A1 + ext t5.8b, \ad\().8b, \ad\().8b, #2 // A2 + ext t7.8b, \ad\().8b, \ad\().8b, #3 // A3 + + __pmull_p8_\bd \rq, \ad + .endm + + .macro __pmull2_p8, rq, ad, bd + tbl t3.16b, {\ad\().16b}, perm1.16b // A1 + tbl t5.16b, {\ad\().16b}, perm2.16b // A2 + tbl t7.16b, {\ad\().16b}, perm3.16b // A3 + + __pmull2_p8_\bd \rq, \ad + .endm + + .macro __pmull_p8_SHASH, rq, ad + __pmull_p8_tail \rq, \ad\().8b, SHASH.8b, 8b,, sh1, sh2, sh3, sh4 + .endm + + .macro __pmull_p8_SHASH2, rq, ad + __pmull_p8_tail \rq, \ad\().8b, SHASH2.8b, 8b,, ss1, ss2, ss3, ss4 + .endm + + .macro __pmull2_p8_SHASH, rq, ad + __pmull_p8_tail \rq, \ad\().16b, SHASH.16b, 16b, 2, sh1, sh2, sh3, sh4 + .endm + + .macro __pmull_p8_tail, rq, ad, bd, nb, t, b1, b2, b3, b4 + pmull\t t3.8h, t3.\nb, \bd // F = A1*B + pmull\t t4.8h, \ad, \b1\().\nb // E = A*B1 + pmull\t t5.8h, t5.\nb, \bd // H = A2*B + pmull\t t6.8h, \ad, \b2\().\nb // G = A*B2 + pmull\t t7.8h, t7.\nb, \bd // J = A3*B + pmull\t t8.8h, \ad, \b3\().\nb // I = A*B3 + pmull\t t9.8h, \ad, \b4\().\nb // K = A*B4 + pmull\t \rq\().8h, \ad, \bd // D = A*B + + eor t3.16b, t3.16b, t4.16b // L = E + F + eor t5.16b, t5.16b, t6.16b // M = G + H + eor t7.16b, t7.16b, t8.16b // N = I + J + + uzp1 t4.2d, t3.2d, t5.2d + uzp2 t3.2d, t3.2d, t5.2d + uzp1 t6.2d, t7.2d, t9.2d + uzp2 t7.2d, t7.2d, t9.2d + + // t3 = (L) (P0 + P1) << 8 + // t5 = (M) (P2 + P3) << 16 + eor t4.16b, t4.16b, t3.16b + and t3.16b, t3.16b, k32_48.16b + + // t7 = (N) (P4 + P5) << 24 + // t9 = (K) (P6 + P7) << 32 + eor t6.16b, t6.16b, t7.16b + and t7.16b, t7.16b, k00_16.16b + + eor t4.16b, t4.16b, t3.16b + eor t6.16b, t6.16b, t7.16b + + zip2 t5.2d, t4.2d, t3.2d + zip1 t3.2d, t4.2d, t3.2d + zip2 t9.2d, t6.2d, t7.2d + zip1 t7.2d, t6.2d, t7.2d + + ext t3.16b, t3.16b, t3.16b, #15 + ext t5.16b, t5.16b, t5.16b, #14 + ext t7.16b, t7.16b, t7.16b, #13 + ext t9.16b, t9.16b, t9.16b, #12 + + eor t3.16b, t3.16b, t5.16b + eor t7.16b, t7.16b, t9.16b + eor \rq\().16b, \rq\().16b, t3.16b + eor \rq\().16b, \rq\().16b, t7.16b + .endm + + .macro __pmull_pre_p8 + ext SHASH2.16b, SHASH.16b, SHASH.16b, #8 + eor SHASH2.16b, SHASH2.16b, SHASH.16b + + // k00_16 := 0x0000000000000000_000000000000ffff + // k32_48 := 0x00000000ffffffff_0000ffffffffffff + movi k32_48.2d, #0xffffffff + mov k32_48.h[2], k32_48.h[0] + ushr k00_16.2d, k32_48.2d, #32 + + // prepare the permutation vectors + mov_q x5, 0x080f0e0d0c0b0a09 + movi T1.8b, #8 + dup perm1.2d, x5 + eor perm1.16b, perm1.16b, T1.16b + ushr perm2.2d, perm1.2d, #8 + ushr perm3.2d, perm1.2d, #16 + ushr T1.2d, perm1.2d, #24 + sli perm2.2d, perm1.2d, #56 + sli perm3.2d, perm1.2d, #48 + sli T1.2d, perm1.2d, #40 + + // precompute loop invariants + tbl sh1.16b, {SHASH.16b}, perm1.16b + tbl sh2.16b, {SHASH.16b}, perm2.16b + tbl sh3.16b, {SHASH.16b}, perm3.16b + tbl sh4.16b, {SHASH.16b}, T1.16b + ext ss1.8b, SHASH2.8b, SHASH2.8b, #1 + ext ss2.8b, SHASH2.8b, SHASH2.8b, #2 + ext ss3.8b, SHASH2.8b, SHASH2.8b, #3 + ext ss4.8b, SHASH2.8b, SHASH2.8b, #4 + .endm + + .macro __pmull_reduce_p8 + eor XM.16b, XM.16b, T1.16b + + mov XL.d[1], XM.d[0] + mov XH.d[0], XM.d[1] + + shl T1.2d, XL.2d, #57 + shl T2.2d, XL.2d, #62 + eor T2.16b, T2.16b, T1.16b + shl T1.2d, XL.2d, #63 + eor T2.16b, T2.16b, T1.16b + ext T1.16b, XL.16b, XH.16b, #8 + eor T2.16b, T2.16b, T1.16b + + mov XL.d[1], T2.d[0] + mov XH.d[0], T2.d[1] + + ushr T2.2d, XL.2d, #1 + eor XH.16b, XH.16b, XL.16b + eor XL.16b, XL.16b, T2.16b + ushr T2.2d, T2.2d, #6 + ushr XL.2d, XL.2d, #1 + .endm + + /* + * void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg, + * const u8 *src, + * const struct polyval_elem *h) + */ +SYM_FUNC_START(pmull_ghash_update_p8) + ld1 {SHASH.2d}, [x3] + ld1 {XL.2d}, [x1] + + __pmull_pre_p8 + +0: ld1 {T1.2d}, [x2], #16 + sub x0, x0, #1 + + /* multiply XL by SHASH in GF(2^128) */ + rev64 T1.16b, T1.16b + + ext T2.16b, XL.16b, XL.16b, #8 + ext IN1.16b, T1.16b, T1.16b, #8 + eor T1.16b, T1.16b, T2.16b + eor XL.16b, XL.16b, IN1.16b + + __pmull2_p8 XH, XL, SHASH // a1 * b1 + eor T1.16b, T1.16b, XL.16b + __pmull_p8 XL, XL, SHASH // a0 * b0 + __pmull_p8 XM, T1, SHASH2 // (a1 + a0)(b1 + b0) + + eor T2.16b, XL.16b, XH.16b + ext T1.16b, XL.16b, XH.16b, #8 + eor XM.16b, XM.16b, T2.16b + + __pmull_reduce_p8 + + eor T2.16b, T2.16b, XH.16b + eor XL.16b, XL.16b, T2.16b + + cbnz x0, 0b + + st1 {XL.2d}, [x1] + ret +SYM_FUNC_END(pmull_ghash_update_p8) diff --git a/lib/crypto/arm64/poly1305.h b/lib/crypto/arm64/poly1305.h index b77669767cd6..3d4bde857699 100644 --- a/lib/crypto/arm64/poly1305.h +++ b/lib/crypto/arm64/poly1305.h @@ -27,17 +27,11 @@ static void poly1305_blocks(struct poly1305_block_state *state, const u8 *src, unsigned int len, u32 padbit) { if (static_branch_likely(&have_neon) && likely(may_use_simd())) { - do { - unsigned int todo = min_t(unsigned int, len, SZ_4K); - - scoped_ksimd() - poly1305_blocks_neon(state, src, todo, padbit); - - len -= todo; - src += todo; - } while (len); - } else + scoped_ksimd() + poly1305_blocks_neon(state, src, len, padbit); + } else { poly1305_blocks_arm64(state, src, len, padbit); + } } #define poly1305_mod_init_arch poly1305_mod_init_arch diff --git a/lib/crypto/arm64/polyval.h b/lib/crypto/arm64/polyval.h deleted file mode 100644 index a39763395e9b..000000000000 --- a/lib/crypto/arm64/polyval.h +++ /dev/null @@ -1,80 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * POLYVAL library functions, arm64 optimized - * - * Copyright 2025 Google LLC - */ -#include <asm/simd.h> -#include <linux/cpufeature.h> - -#define NUM_H_POWERS 8 - -static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); - -asmlinkage void polyval_mul_pmull(struct polyval_elem *a, - const struct polyval_elem *b); -asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc, - const struct polyval_key *key, - const u8 *data, size_t nblocks); - -static void polyval_preparekey_arch(struct polyval_key *key, - const u8 raw_key[POLYVAL_BLOCK_SIZE]) -{ - static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS); - memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE); - if (static_branch_likely(&have_pmull) && may_use_simd()) { - scoped_ksimd() { - for (int i = NUM_H_POWERS - 2; i >= 0; i--) { - key->h_powers[i] = key->h_powers[i + 1]; - polyval_mul_pmull( - &key->h_powers[i], - &key->h_powers[NUM_H_POWERS - 1]); - } - } - } else { - for (int i = NUM_H_POWERS - 2; i >= 0; i--) { - key->h_powers[i] = key->h_powers[i + 1]; - polyval_mul_generic(&key->h_powers[i], - &key->h_powers[NUM_H_POWERS - 1]); - } - } -} - -static void polyval_mul_arch(struct polyval_elem *acc, - const struct polyval_key *key) -{ - if (static_branch_likely(&have_pmull) && may_use_simd()) { - scoped_ksimd() - polyval_mul_pmull(acc, &key->h_powers[NUM_H_POWERS - 1]); - } else { - polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]); - } -} - -static void polyval_blocks_arch(struct polyval_elem *acc, - const struct polyval_key *key, - const u8 *data, size_t nblocks) -{ - if (static_branch_likely(&have_pmull) && may_use_simd()) { - do { - /* Allow rescheduling every 4 KiB. */ - size_t n = min_t(size_t, nblocks, - 4096 / POLYVAL_BLOCK_SIZE); - - scoped_ksimd() - polyval_blocks_pmull(acc, key, data, n); - data += n * POLYVAL_BLOCK_SIZE; - nblocks -= n; - } while (nblocks); - } else { - polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1], - data, nblocks); - } -} - -#define polyval_mod_init_arch polyval_mod_init_arch -static void polyval_mod_init_arch(void) -{ - if (cpu_have_named_feature(PMULL)) - static_branch_enable(&have_pmull); -} diff --git a/lib/crypto/arm64/sha1-ce-core.S b/lib/crypto/arm64/sha1-ce-core.S index 8fbd4767f0f0..40f2a6c8d0c5 100644 --- a/lib/crypto/arm64/sha1-ce-core.S +++ b/lib/crypto/arm64/sha1-ce-core.S @@ -62,10 +62,10 @@ .endm /* - * size_t __sha1_ce_transform(struct sha1_block_state *state, - * const u8 *data, size_t nblocks); + * void sha1_ce_transform(struct sha1_block_state *state, + * const u8 *data, size_t nblocks); */ -SYM_FUNC_START(__sha1_ce_transform) +SYM_FUNC_START(sha1_ce_transform) /* load round constants */ loadrc k0.4s, 0x5a827999, w6 loadrc k1.4s, 0x6ed9eba1, w6 @@ -80,10 +80,10 @@ SYM_FUNC_START(__sha1_ce_transform) 0: ld1 {v8.4s-v11.4s}, [x1], #64 sub x2, x2, #1 -CPU_LE( rev32 v8.16b, v8.16b ) -CPU_LE( rev32 v9.16b, v9.16b ) -CPU_LE( rev32 v10.16b, v10.16b ) -CPU_LE( rev32 v11.16b, v11.16b ) + rev32 v8.16b, v8.16b + rev32 v9.16b, v9.16b + rev32 v10.16b, v10.16b + rev32 v11.16b, v11.16b add t0.4s, v8.4s, k0.4s mov dg0v.16b, dgav.16b @@ -116,15 +116,11 @@ CPU_LE( rev32 v11.16b, v11.16b ) add dgbv.2s, dgbv.2s, dg1v.2s add dgav.4s, dgav.4s, dg0v.4s - /* return early if voluntary preemption is needed */ - cond_yield 1f, x5, x6 - /* handled all input blocks? */ cbnz x2, 0b /* store new state */ -1: st1 {dgav.4s}, [x0] + st1 {dgav.4s}, [x0] str dgb, [x0, #16] - mov x0, x2 ret -SYM_FUNC_END(__sha1_ce_transform) +SYM_FUNC_END(sha1_ce_transform) diff --git a/lib/crypto/arm64/sha1.h b/lib/crypto/arm64/sha1.h index bc7071f1be09..112c5d443c56 100644 --- a/lib/crypto/arm64/sha1.h +++ b/lib/crypto/arm64/sha1.h @@ -9,22 +9,15 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); -asmlinkage size_t __sha1_ce_transform(struct sha1_block_state *state, - const u8 *data, size_t nblocks); +asmlinkage void sha1_ce_transform(struct sha1_block_state *state, + const u8 *data, size_t nblocks); static void sha1_blocks(struct sha1_block_state *state, const u8 *data, size_t nblocks) { if (static_branch_likely(&have_ce) && likely(may_use_simd())) { - do { - size_t rem; - - scoped_ksimd() - rem = __sha1_ce_transform(state, data, nblocks); - - data += (nblocks - rem) * SHA1_BLOCK_SIZE; - nblocks = rem; - } while (nblocks); + scoped_ksimd() + sha1_ce_transform(state, data, nblocks); } else { sha1_blocks_generic(state, data, nblocks); } diff --git a/lib/crypto/arm64/sha256-ce.S b/lib/crypto/arm64/sha256-ce.S index e4bfe42a61a9..8fdbf0a9ff89 100644 --- a/lib/crypto/arm64/sha256-ce.S +++ b/lib/crypto/arm64/sha256-ce.S @@ -79,11 +79,11 @@ .endm /* - * size_t __sha256_ce_transform(struct sha256_block_state *state, - * const u8 *data, size_t nblocks); + * void sha256_ce_transform(struct sha256_block_state *state, + * const u8 *data, size_t nblocks); */ .text -SYM_FUNC_START(__sha256_ce_transform) +SYM_FUNC_START(sha256_ce_transform) load_round_constants x8 @@ -94,10 +94,10 @@ SYM_FUNC_START(__sha256_ce_transform) 0: ld1 {v16.4s-v19.4s}, [x1], #64 sub x2, x2, #1 -CPU_LE( rev32 v16.16b, v16.16b ) -CPU_LE( rev32 v17.16b, v17.16b ) -CPU_LE( rev32 v18.16b, v18.16b ) -CPU_LE( rev32 v19.16b, v19.16b ) + rev32 v16.16b, v16.16b + rev32 v17.16b, v17.16b + rev32 v18.16b, v18.16b + rev32 v19.16b, v19.16b add t0.4s, v16.4s, v0.4s mov dg0v.16b, dgav.16b @@ -127,17 +127,13 @@ CPU_LE( rev32 v19.16b, v19.16b ) add dgav.4s, dgav.4s, dg0v.4s add dgbv.4s, dgbv.4s, dg1v.4s - /* return early if voluntary preemption is needed */ - cond_yield 1f, x5, x6 - /* handled all input blocks? */ cbnz x2, 0b /* store new state */ -1: st1 {dgav.4s, dgbv.4s}, [x0] - mov x0, x2 + st1 {dgav.4s, dgbv.4s}, [x0] ret -SYM_FUNC_END(__sha256_ce_transform) +SYM_FUNC_END(sha256_ce_transform) .unreq dga .unreq dgav @@ -293,14 +289,14 @@ SYM_FUNC_START(sha256_ce_finup2x) ld1 {v20.4s-v23.4s}, [data2], #64 .Lfinup2x_loop_have_data: // Convert the words of the data blocks from big endian. -CPU_LE( rev32 v16.16b, v16.16b ) -CPU_LE( rev32 v17.16b, v17.16b ) -CPU_LE( rev32 v18.16b, v18.16b ) -CPU_LE( rev32 v19.16b, v19.16b ) -CPU_LE( rev32 v20.16b, v20.16b ) -CPU_LE( rev32 v21.16b, v21.16b ) -CPU_LE( rev32 v22.16b, v22.16b ) -CPU_LE( rev32 v23.16b, v23.16b ) + rev32 v16.16b, v16.16b + rev32 v17.16b, v17.16b + rev32 v18.16b, v18.16b + rev32 v19.16b, v19.16b + rev32 v20.16b, v20.16b + rev32 v21.16b, v21.16b + rev32 v22.16b, v22.16b + rev32 v23.16b, v23.16b .Lfinup2x_loop_have_bswapped_data: // Save the original state for each block. @@ -340,11 +336,8 @@ CPU_LE( rev32 v23.16b, v23.16b ) sub w8, len, #64 // w8 = len - 64 add data1, data1, w8, sxtw // data1 += len - 64 add data2, data2, w8, sxtw // data2 += len - 64 -CPU_LE( mov x9, #0x80 ) -CPU_LE( fmov d16, x9 ) -CPU_BE( movi v16.16b, #0 ) -CPU_BE( mov x9, #0x8000000000000000 ) -CPU_BE( mov v16.d[1], x9 ) + mov x9, #0x80 + fmov d16, x9 movi v17.16b, #0 stp q16, q17, [sp, #64] stp q17, q17, [sp, #96] @@ -352,7 +345,7 @@ CPU_BE( mov v16.d[1], x9 ) cmp len, #56 b.ge 1f // will count spill into its own block? lsl count, count, #3 -CPU_LE( rev count, count ) + rev count, count str count, [x9, #56] mov final_step, #2 // won't need count-only block b 2f @@ -397,10 +390,10 @@ CPU_LE( rev count, count ) .Lfinup2x_done: // Write the two digests with all bytes in the correct order. -CPU_LE( rev32 state0_a.16b, state0_a.16b ) -CPU_LE( rev32 state1_a.16b, state1_a.16b ) -CPU_LE( rev32 state0_b.16b, state0_b.16b ) -CPU_LE( rev32 state1_b.16b, state1_b.16b ) + rev32 state0_a.16b, state0_a.16b + rev32 state1_a.16b, state1_a.16b + rev32 state0_b.16b, state0_b.16b + rev32 state1_b.16b, state1_b.16b st1 {state0_a.4s-state1_a.4s}, [out1] st1 {state0_b.4s-state1_b.4s}, [out2] add sp, sp, #128 diff --git a/lib/crypto/arm64/sha256.h b/lib/crypto/arm64/sha256.h index 568dff0f276a..b4353d3c4dd0 100644 --- a/lib/crypto/arm64/sha256.h +++ b/lib/crypto/arm64/sha256.h @@ -14,27 +14,17 @@ asmlinkage void sha256_block_data_order(struct sha256_block_state *state, const u8 *data, size_t nblocks); asmlinkage void sha256_block_neon(struct sha256_block_state *state, const u8 *data, size_t nblocks); -asmlinkage size_t __sha256_ce_transform(struct sha256_block_state *state, - const u8 *data, size_t nblocks); +asmlinkage void sha256_ce_transform(struct sha256_block_state *state, + const u8 *data, size_t nblocks); static void sha256_blocks(struct sha256_block_state *state, const u8 *data, size_t nblocks) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_neon) && likely(may_use_simd())) { - if (static_branch_likely(&have_ce)) { - do { - size_t rem; - - scoped_ksimd() - rem = __sha256_ce_transform(state, data, - nblocks); - - data += (nblocks - rem) * SHA256_BLOCK_SIZE; - nblocks = rem; - } while (nblocks); - } else { - scoped_ksimd() + if (static_branch_likely(&have_neon) && likely(may_use_simd())) { + scoped_ksimd() { + if (static_branch_likely(&have_ce)) + sha256_ce_transform(state, data, nblocks); + else sha256_block_neon(state, data, nblocks); } } else { @@ -56,14 +46,9 @@ static bool sha256_finup_2x_arch(const struct __sha256_ctx *ctx, u8 out1[SHA256_DIGEST_SIZE], u8 out2[SHA256_DIGEST_SIZE]) { - /* - * The assembly requires len >= SHA256_BLOCK_SIZE && len <= INT_MAX. - * Further limit len to 65536 to avoid spending too long with preemption - * disabled. (Of course, in practice len is nearly always 4096 anyway.) - */ - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_ce) && len >= SHA256_BLOCK_SIZE && - len <= 65536 && likely(may_use_simd())) { + /* The assembly requires len >= SHA256_BLOCK_SIZE && len <= INT_MAX. */ + if (static_branch_likely(&have_ce) && len >= SHA256_BLOCK_SIZE && + len <= INT_MAX && likely(may_use_simd())) { scoped_ksimd() sha256_ce_finup2x(ctx, data1, data2, len, out1, out2); kmsan_unpoison_memory(out1, SHA256_DIGEST_SIZE); @@ -78,7 +63,6 @@ static bool sha256_finup_2x_is_optimized_arch(void) return static_key_enabled(&have_ce); } -#ifdef CONFIG_KERNEL_MODE_NEON #define sha256_mod_init_arch sha256_mod_init_arch static void sha256_mod_init_arch(void) { @@ -88,4 +72,3 @@ static void sha256_mod_init_arch(void) static_branch_enable(&have_ce); } } -#endif /* CONFIG_KERNEL_MODE_NEON */ diff --git a/lib/crypto/arm64/sha3-ce-core.S b/lib/crypto/arm64/sha3-ce-core.S index ace90b506490..b8ab01987ae0 100644 --- a/lib/crypto/arm64/sha3-ce-core.S +++ b/lib/crypto/arm64/sha3-ce-core.S @@ -37,8 +37,8 @@ .endm /* - * size_t sha3_ce_transform(struct sha3_state *state, const u8 *data, - * size_t nblocks, size_t block_size) + * void sha3_ce_transform(struct sha3_state *state, const u8 *data, + * size_t nblocks, size_t block_size) * * block_size is assumed to be one of 72 (SHA3-512), 104 (SHA3-384), 136 * (SHA3-256 and SHAKE256), 144 (SHA3-224), or 168 (SHAKE128). @@ -185,18 +185,16 @@ SYM_FUNC_START(sha3_ce_transform) eor v0.16b, v0.16b, v31.16b cbnz w8, 3b - cond_yield 4f, x8, x9 cbnz x2, 0b /* save state */ -4: st1 { v0.1d- v3.1d}, [x0], #32 + st1 { v0.1d- v3.1d}, [x0], #32 st1 { v4.1d- v7.1d}, [x0], #32 st1 { v8.1d-v11.1d}, [x0], #32 st1 {v12.1d-v15.1d}, [x0], #32 st1 {v16.1d-v19.1d}, [x0], #32 st1 {v20.1d-v23.1d}, [x0], #32 st1 {v24.1d}, [x0] - mov x0, x2 ret SYM_FUNC_END(sha3_ce_transform) diff --git a/lib/crypto/arm64/sha3.h b/lib/crypto/arm64/sha3.h index b602f1b3b282..eaaba3224acc 100644 --- a/lib/crypto/arm64/sha3.h +++ b/lib/crypto/arm64/sha3.h @@ -12,22 +12,15 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3); -asmlinkage size_t sha3_ce_transform(struct sha3_state *state, const u8 *data, - size_t nblocks, size_t block_size); +asmlinkage void sha3_ce_transform(struct sha3_state *state, const u8 *data, + size_t nblocks, size_t block_size); static void sha3_absorb_blocks(struct sha3_state *state, const u8 *data, size_t nblocks, size_t block_size) { if (static_branch_likely(&have_sha3) && likely(may_use_simd())) { - do { - size_t rem; - - scoped_ksimd() - rem = sha3_ce_transform(state, data, nblocks, - block_size); - data += (nblocks - rem) * block_size; - nblocks = rem; - } while (nblocks); + scoped_ksimd() + sha3_ce_transform(state, data, nblocks, block_size); } else { sha3_absorb_blocks_generic(state, data, nblocks, block_size); } diff --git a/lib/crypto/arm64/sha512-ce-core.S b/lib/crypto/arm64/sha512-ce-core.S index ffd51acfd1ee..daa2c63a20c0 100644 --- a/lib/crypto/arm64/sha512-ce-core.S +++ b/lib/crypto/arm64/sha512-ce-core.S @@ -93,11 +93,11 @@ .endm /* - * size_t __sha512_ce_transform(struct sha512_block_state *state, - * const u8 *data, size_t nblocks); + * void sha512_ce_transform(struct sha512_block_state *state, + * const u8 *data, size_t nblocks); */ .text -SYM_FUNC_START(__sha512_ce_transform) +SYM_FUNC_START(sha512_ce_transform) /* load state */ ld1 {v8.2d-v11.2d}, [x0] @@ -110,14 +110,14 @@ SYM_FUNC_START(__sha512_ce_transform) ld1 {v16.2d-v19.2d}, [x1], #64 sub x2, x2, #1 -CPU_LE( rev64 v12.16b, v12.16b ) -CPU_LE( rev64 v13.16b, v13.16b ) -CPU_LE( rev64 v14.16b, v14.16b ) -CPU_LE( rev64 v15.16b, v15.16b ) -CPU_LE( rev64 v16.16b, v16.16b ) -CPU_LE( rev64 v17.16b, v17.16b ) -CPU_LE( rev64 v18.16b, v18.16b ) -CPU_LE( rev64 v19.16b, v19.16b ) + rev64 v12.16b, v12.16b + rev64 v13.16b, v13.16b + rev64 v14.16b, v14.16b + rev64 v15.16b, v15.16b + rev64 v16.16b, v16.16b + rev64 v17.16b, v17.16b + rev64 v18.16b, v18.16b + rev64 v19.16b, v19.16b mov x4, x3 // rc pointer @@ -186,12 +186,10 @@ CPU_LE( rev64 v19.16b, v19.16b ) add v10.2d, v10.2d, v2.2d add v11.2d, v11.2d, v3.2d - cond_yield 3f, x4, x5 /* handled all input blocks? */ cbnz x2, 0b /* store new state */ -3: st1 {v8.2d-v11.2d}, [x0] - mov x0, x2 + st1 {v8.2d-v11.2d}, [x0] ret -SYM_FUNC_END(__sha512_ce_transform) +SYM_FUNC_END(sha512_ce_transform) diff --git a/lib/crypto/arm64/sha512.h b/lib/crypto/arm64/sha512.h index 7eb7ef04d268..5da27e6e23ea 100644 --- a/lib/crypto/arm64/sha512.h +++ b/lib/crypto/arm64/sha512.h @@ -12,34 +12,24 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha512_insns); asmlinkage void sha512_block_data_order(struct sha512_block_state *state, const u8 *data, size_t nblocks); -asmlinkage size_t __sha512_ce_transform(struct sha512_block_state *state, - const u8 *data, size_t nblocks); +asmlinkage void sha512_ce_transform(struct sha512_block_state *state, + const u8 *data, size_t nblocks); static void sha512_blocks(struct sha512_block_state *state, const u8 *data, size_t nblocks) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_sha512_insns) && + if (static_branch_likely(&have_sha512_insns) && likely(may_use_simd())) { - do { - size_t rem; - - scoped_ksimd() - rem = __sha512_ce_transform(state, data, nblocks); - - data += (nblocks - rem) * SHA512_BLOCK_SIZE; - nblocks = rem; - } while (nblocks); + scoped_ksimd() + sha512_ce_transform(state, data, nblocks); } else { sha512_block_data_order(state, data, nblocks); } } -#ifdef CONFIG_KERNEL_MODE_NEON #define sha512_mod_init_arch sha512_mod_init_arch static void sha512_mod_init_arch(void) { if (cpu_have_named_feature(SHA512)) static_branch_enable(&have_sha512_insns); } -#endif /* CONFIG_KERNEL_MODE_NEON */ diff --git a/lib/crypto/arm64/sm3-ce-core.S b/lib/crypto/arm64/sm3-ce-core.S new file mode 100644 index 000000000000..ee7f900d7cff --- /dev/null +++ b/lib/crypto/arm64/sm3-ce-core.S @@ -0,0 +1,138 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * sm3-ce-core.S - SM3 secure hash using ARMv8.2 Crypto Extensions + * + * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org> + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> + + .irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 + .set .Lv\b\().4s, \b + .endr + + .macro sm3partw1, rd, rn, rm + .inst 0xce60c000 | .L\rd | (.L\rn << 5) | (.L\rm << 16) + .endm + + .macro sm3partw2, rd, rn, rm + .inst 0xce60c400 | .L\rd | (.L\rn << 5) | (.L\rm << 16) + .endm + + .macro sm3ss1, rd, rn, rm, ra + .inst 0xce400000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16) + .endm + + .macro sm3tt1a, rd, rn, rm, imm2 + .inst 0xce408000 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16) + .endm + + .macro sm3tt1b, rd, rn, rm, imm2 + .inst 0xce408400 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16) + .endm + + .macro sm3tt2a, rd, rn, rm, imm2 + .inst 0xce408800 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16) + .endm + + .macro sm3tt2b, rd, rn, rm, imm2 + .inst 0xce408c00 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16) + .endm + + .macro round, ab, s0, t0, t1, i + sm3ss1 v5.4s, v8.4s, \t0\().4s, v9.4s + shl \t1\().4s, \t0\().4s, #1 + sri \t1\().4s, \t0\().4s, #31 + sm3tt1\ab v8.4s, v5.4s, v10.4s, \i + sm3tt2\ab v9.4s, v5.4s, \s0\().4s, \i + .endm + + .macro qround, ab, s0, s1, s2, s3, s4 + .ifnb \s4 + ext \s4\().16b, \s1\().16b, \s2\().16b, #12 + ext v6.16b, \s0\().16b, \s1\().16b, #12 + ext v7.16b, \s2\().16b, \s3\().16b, #8 + sm3partw1 \s4\().4s, \s0\().4s, \s3\().4s + .endif + + eor v10.16b, \s0\().16b, \s1\().16b + + round \ab, \s0, v11, v12, 0 + round \ab, \s0, v12, v11, 1 + round \ab, \s0, v11, v12, 2 + round \ab, \s0, v12, v11, 3 + + .ifnb \s4 + sm3partw2 \s4\().4s, v7.4s, v6.4s + .endif + .endm + + /* + * void sm3_ce_transform(struct sm3_block_state *state, + * const u8 *data, size_t nblocks) + */ + .text +SYM_FUNC_START(sm3_ce_transform) + /* load state */ + ld1 {v8.4s-v9.4s}, [x0] + rev64 v8.4s, v8.4s + rev64 v9.4s, v9.4s + ext v8.16b, v8.16b, v8.16b, #8 + ext v9.16b, v9.16b, v9.16b, #8 + + adr_l x8, .Lt + ldp s13, s14, [x8] + + /* load input */ +0: ld1 {v0.16b-v3.16b}, [x1], #64 + sub x2, x2, #1 + + mov v15.16b, v8.16b + mov v16.16b, v9.16b + + rev32 v0.16b, v0.16b + rev32 v1.16b, v1.16b + rev32 v2.16b, v2.16b + rev32 v3.16b, v3.16b + + ext v11.16b, v13.16b, v13.16b, #4 + + qround a, v0, v1, v2, v3, v4 + qround a, v1, v2, v3, v4, v0 + qround a, v2, v3, v4, v0, v1 + qround a, v3, v4, v0, v1, v2 + + ext v11.16b, v14.16b, v14.16b, #4 + + qround b, v4, v0, v1, v2, v3 + qround b, v0, v1, v2, v3, v4 + qround b, v1, v2, v3, v4, v0 + qround b, v2, v3, v4, v0, v1 + qround b, v3, v4, v0, v1, v2 + qround b, v4, v0, v1, v2, v3 + qround b, v0, v1, v2, v3, v4 + qround b, v1, v2, v3, v4, v0 + qround b, v2, v3, v4, v0, v1 + qround b, v3, v4 + qround b, v4, v0 + qround b, v0, v1 + + eor v8.16b, v8.16b, v15.16b + eor v9.16b, v9.16b, v16.16b + + /* handled all input blocks? */ + cbnz x2, 0b + + /* save state */ + rev64 v8.4s, v8.4s + rev64 v9.4s, v9.4s + ext v8.16b, v8.16b, v8.16b, #8 + ext v9.16b, v9.16b, v9.16b, #8 + st1 {v8.4s-v9.4s}, [x0] + ret +SYM_FUNC_END(sm3_ce_transform) + + .section ".rodata", "a" + .align 3 +.Lt: .word 0x79cc4519, 0x9d8a7a87 diff --git a/lib/crypto/arm64/sm3-neon-core.S b/lib/crypto/arm64/sm3-neon-core.S new file mode 100644 index 000000000000..ad874af13802 --- /dev/null +++ b/lib/crypto/arm64/sm3-neon-core.S @@ -0,0 +1,600 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * sm3-neon-core.S - SM3 secure hash using NEON instructions + * + * Linux/arm64 port of the libgcrypt SM3 implementation for AArch64 + * + * Copyright (C) 2021 Jussi Kivilinna <jussi.kivilinna@iki.fi> + * Copyright (c) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> + +/* Context structure */ + +#define state_h0 0 +#define state_h1 4 +#define state_h2 8 +#define state_h3 12 +#define state_h4 16 +#define state_h5 20 +#define state_h6 24 +#define state_h7 28 + +/* Stack structure */ + +#define STACK_W_SIZE (32 * 2 * 3) + +#define STACK_W (0) +#define STACK_SIZE (STACK_W + STACK_W_SIZE) + +/* Register macros */ + +#define RSTATE x0 +#define RDATA x1 +#define RNBLKS x2 +#define RKPTR x28 +#define RFRAME x29 + +#define ra w3 +#define rb w4 +#define rc w5 +#define rd w6 +#define re w7 +#define rf w8 +#define rg w9 +#define rh w10 + +#define t0 w11 +#define t1 w12 +#define t2 w13 +#define t3 w14 +#define t4 w15 +#define t5 w16 +#define t6 w17 + +#define k_even w19 +#define k_odd w20 + +#define addr0 x21 +#define addr1 x22 + +#define s0 w23 +#define s1 w24 +#define s2 w25 +#define s3 w26 + +#define W0 v0 +#define W1 v1 +#define W2 v2 +#define W3 v3 +#define W4 v4 +#define W5 v5 + +#define XTMP0 v6 +#define XTMP1 v7 +#define XTMP2 v16 +#define XTMP3 v17 +#define XTMP4 v18 +#define XTMP5 v19 +#define XTMP6 v20 + +/* Helper macros. */ + +#define _(...) /*_*/ + +#define clear_vec(x) \ + movi x.8h, #0; + +#define rolw(o, a, n) \ + ror o, a, #(32 - n); + +/* Round function macros. */ + +#define GG1_1(x, y, z, o, t) \ + eor o, x, y; +#define GG1_2(x, y, z, o, t) \ + eor o, o, z; +#define GG1_3(x, y, z, o, t) + +#define FF1_1(x, y, z, o, t) GG1_1(x, y, z, o, t) +#define FF1_2(x, y, z, o, t) +#define FF1_3(x, y, z, o, t) GG1_2(x, y, z, o, t) + +#define GG2_1(x, y, z, o, t) \ + bic o, z, x; +#define GG2_2(x, y, z, o, t) \ + and t, y, x; +#define GG2_3(x, y, z, o, t) \ + eor o, o, t; + +#define FF2_1(x, y, z, o, t) \ + eor o, x, y; +#define FF2_2(x, y, z, o, t) \ + and t, x, y; \ + and o, o, z; +#define FF2_3(x, y, z, o, t) \ + eor o, o, t; + +#define R(i, a, b, c, d, e, f, g, h, k, K_LOAD, round, widx, wtype, IOP, iop_param) \ + K_LOAD(round); \ + ldr t5, [sp, #(wtype##_W1_ADDR(round, widx))]; \ + rolw(t0, a, 12); /* rol(a, 12) => t0 */ \ + IOP(1, iop_param); \ + FF##i##_1(a, b, c, t1, t2); \ + ldr t6, [sp, #(wtype##_W1W2_ADDR(round, widx))]; \ + add k, k, e; \ + IOP(2, iop_param); \ + GG##i##_1(e, f, g, t3, t4); \ + FF##i##_2(a, b, c, t1, t2); \ + IOP(3, iop_param); \ + add k, k, t0; \ + add h, h, t5; \ + add d, d, t6; /* w1w2 + d => d */ \ + IOP(4, iop_param); \ + rolw(k, k, 7); /* rol (t0 + e + t), 7) => k */ \ + GG##i##_2(e, f, g, t3, t4); \ + add h, h, k; /* h + w1 + k => h */ \ + IOP(5, iop_param); \ + FF##i##_3(a, b, c, t1, t2); \ + eor t0, t0, k; /* k ^ t0 => t0 */ \ + GG##i##_3(e, f, g, t3, t4); \ + add d, d, t1; /* FF(a,b,c) + d => d */ \ + IOP(6, iop_param); \ + add t3, t3, h; /* GG(e,f,g) + h => t3 */ \ + rolw(b, b, 9); /* rol(b, 9) => b */ \ + eor h, t3, t3, ror #(32-9); \ + IOP(7, iop_param); \ + add d, d, t0; /* t0 + d => d */ \ + rolw(f, f, 19); /* rol(f, 19) => f */ \ + IOP(8, iop_param); \ + eor h, h, t3, ror #(32-17); /* P0(t3) => h */ + +#define R1(a, b, c, d, e, f, g, h, k, K_LOAD, round, widx, wtype, IOP, iop_param) \ + R(1, ##a, ##b, ##c, ##d, ##e, ##f, ##g, ##h, ##k, K_LOAD, round, widx, wtype, IOP, iop_param) + +#define R2(a, b, c, d, e, f, g, h, k, K_LOAD, round, widx, wtype, IOP, iop_param) \ + R(2, ##a, ##b, ##c, ##d, ##e, ##f, ##g, ##h, ##k, K_LOAD, round, widx, wtype, IOP, iop_param) + +#define KL(round) \ + ldp k_even, k_odd, [RKPTR, #(4*(round))]; + +/* Input expansion macros. */ + +/* Byte-swapped input address. */ +#define IW_W_ADDR(round, widx, offs) \ + (STACK_W + ((round) / 4) * 64 + (offs) + ((widx) * 4)) + +/* Expanded input address. */ +#define XW_W_ADDR(round, widx, offs) \ + (STACK_W + ((((round) / 3) - 4) % 2) * 64 + (offs) + ((widx) * 4)) + +/* Rounds 1-12, byte-swapped input block addresses. */ +#define IW_W1_ADDR(round, widx) IW_W_ADDR(round, widx, 32) +#define IW_W1W2_ADDR(round, widx) IW_W_ADDR(round, widx, 48) + +/* Rounds 1-12, expanded input block addresses. */ +#define XW_W1_ADDR(round, widx) XW_W_ADDR(round, widx, 0) +#define XW_W1W2_ADDR(round, widx) XW_W_ADDR(round, widx, 16) + +/* Input block loading. + * Interleaving within round function needed for in-order CPUs. */ +#define LOAD_W_VEC_1_1() \ + add addr0, sp, #IW_W1_ADDR(0, 0); +#define LOAD_W_VEC_1_2() \ + add addr1, sp, #IW_W1_ADDR(4, 0); +#define LOAD_W_VEC_1_3() \ + ld1 {W0.16b}, [RDATA], #16; +#define LOAD_W_VEC_1_4() \ + ld1 {W1.16b}, [RDATA], #16; +#define LOAD_W_VEC_1_5() \ + ld1 {W2.16b}, [RDATA], #16; +#define LOAD_W_VEC_1_6() \ + ld1 {W3.16b}, [RDATA], #16; +#define LOAD_W_VEC_1_7() \ + rev32 XTMP0.16b, W0.16b; +#define LOAD_W_VEC_1_8() \ + rev32 XTMP1.16b, W1.16b; +#define LOAD_W_VEC_2_1() \ + rev32 XTMP2.16b, W2.16b; +#define LOAD_W_VEC_2_2() \ + rev32 XTMP3.16b, W3.16b; +#define LOAD_W_VEC_2_3() \ + eor XTMP4.16b, XTMP1.16b, XTMP0.16b; +#define LOAD_W_VEC_2_4() \ + eor XTMP5.16b, XTMP2.16b, XTMP1.16b; +#define LOAD_W_VEC_2_5() \ + st1 {XTMP0.16b}, [addr0], #16; +#define LOAD_W_VEC_2_6() \ + st1 {XTMP4.16b}, [addr0]; \ + add addr0, sp, #IW_W1_ADDR(8, 0); +#define LOAD_W_VEC_2_7() \ + eor XTMP6.16b, XTMP3.16b, XTMP2.16b; +#define LOAD_W_VEC_2_8() \ + ext W0.16b, XTMP0.16b, XTMP0.16b, #8; /* W0: xx, w0, xx, xx */ +#define LOAD_W_VEC_3_1() \ + mov W2.16b, XTMP1.16b; /* W2: xx, w6, w5, w4 */ +#define LOAD_W_VEC_3_2() \ + st1 {XTMP1.16b}, [addr1], #16; +#define LOAD_W_VEC_3_3() \ + st1 {XTMP5.16b}, [addr1]; \ + ext W1.16b, XTMP0.16b, XTMP0.16b, #4; /* W1: xx, w3, w2, w1 */ +#define LOAD_W_VEC_3_4() \ + ext W3.16b, XTMP1.16b, XTMP2.16b, #12; /* W3: xx, w9, w8, w7 */ +#define LOAD_W_VEC_3_5() \ + ext W4.16b, XTMP2.16b, XTMP3.16b, #8; /* W4: xx, w12, w11, w10 */ +#define LOAD_W_VEC_3_6() \ + st1 {XTMP2.16b}, [addr0], #16; +#define LOAD_W_VEC_3_7() \ + st1 {XTMP6.16b}, [addr0]; +#define LOAD_W_VEC_3_8() \ + ext W5.16b, XTMP3.16b, XTMP3.16b, #4; /* W5: xx, w15, w14, w13 */ + +#define LOAD_W_VEC_1(iop_num, ...) \ + LOAD_W_VEC_1_##iop_num() +#define LOAD_W_VEC_2(iop_num, ...) \ + LOAD_W_VEC_2_##iop_num() +#define LOAD_W_VEC_3(iop_num, ...) \ + LOAD_W_VEC_3_##iop_num() + +/* Message scheduling. Note: 3 words per vector register. + * Interleaving within round function needed for in-order CPUs. */ +#define SCHED_W_1_1(round, w0, w1, w2, w3, w4, w5) \ + /* Load (w[i - 16]) => XTMP0 */ \ + /* Load (w[i - 13]) => XTMP5 */ \ + ext XTMP0.16b, w0.16b, w0.16b, #12; /* XTMP0: w0, xx, xx, xx */ +#define SCHED_W_1_2(round, w0, w1, w2, w3, w4, w5) \ + ext XTMP5.16b, w1.16b, w1.16b, #12; +#define SCHED_W_1_3(round, w0, w1, w2, w3, w4, w5) \ + ext XTMP0.16b, XTMP0.16b, w1.16b, #12; /* XTMP0: xx, w2, w1, w0 */ +#define SCHED_W_1_4(round, w0, w1, w2, w3, w4, w5) \ + ext XTMP5.16b, XTMP5.16b, w2.16b, #12; +#define SCHED_W_1_5(round, w0, w1, w2, w3, w4, w5) \ + /* w[i - 9] == w3 */ \ + /* W3 ^ XTMP0 => XTMP0 */ \ + eor XTMP0.16b, XTMP0.16b, w3.16b; +#define SCHED_W_1_6(round, w0, w1, w2, w3, w4, w5) \ + /* w[i - 3] == w5 */ \ + /* rol(XMM5, 15) ^ XTMP0 => XTMP0 */ \ + /* rol(XTMP5, 7) => XTMP1 */ \ + add addr0, sp, #XW_W1_ADDR((round), 0); \ + shl XTMP2.4s, w5.4s, #15; +#define SCHED_W_1_7(round, w0, w1, w2, w3, w4, w5) \ + shl XTMP1.4s, XTMP5.4s, #7; +#define SCHED_W_1_8(round, w0, w1, w2, w3, w4, w5) \ + sri XTMP2.4s, w5.4s, #(32-15); +#define SCHED_W_2_1(round, w0, w1, w2, w3, w4, w5) \ + sri XTMP1.4s, XTMP5.4s, #(32-7); +#define SCHED_W_2_2(round, w0, w1, w2, w3, w4, w5) \ + eor XTMP0.16b, XTMP0.16b, XTMP2.16b; +#define SCHED_W_2_3(round, w0, w1, w2, w3, w4, w5) \ + /* w[i - 6] == W4 */ \ + /* W4 ^ XTMP1 => XTMP1 */ \ + eor XTMP1.16b, XTMP1.16b, w4.16b; +#define SCHED_W_2_4(round, w0, w1, w2, w3, w4, w5) \ + /* P1(XTMP0) ^ XTMP1 => W0 */ \ + shl XTMP3.4s, XTMP0.4s, #15; +#define SCHED_W_2_5(round, w0, w1, w2, w3, w4, w5) \ + shl XTMP4.4s, XTMP0.4s, #23; +#define SCHED_W_2_6(round, w0, w1, w2, w3, w4, w5) \ + eor w0.16b, XTMP1.16b, XTMP0.16b; +#define SCHED_W_2_7(round, w0, w1, w2, w3, w4, w5) \ + sri XTMP3.4s, XTMP0.4s, #(32-15); +#define SCHED_W_2_8(round, w0, w1, w2, w3, w4, w5) \ + sri XTMP4.4s, XTMP0.4s, #(32-23); +#define SCHED_W_3_1(round, w0, w1, w2, w3, w4, w5) \ + eor w0.16b, w0.16b, XTMP3.16b; +#define SCHED_W_3_2(round, w0, w1, w2, w3, w4, w5) \ + /* Load (w[i - 3]) => XTMP2 */ \ + ext XTMP2.16b, w4.16b, w4.16b, #12; +#define SCHED_W_3_3(round, w0, w1, w2, w3, w4, w5) \ + eor w0.16b, w0.16b, XTMP4.16b; +#define SCHED_W_3_4(round, w0, w1, w2, w3, w4, w5) \ + ext XTMP2.16b, XTMP2.16b, w5.16b, #12; +#define SCHED_W_3_5(round, w0, w1, w2, w3, w4, w5) \ + /* W1 ^ W2 => XTMP3 */ \ + eor XTMP3.16b, XTMP2.16b, w0.16b; +#define SCHED_W_3_6(round, w0, w1, w2, w3, w4, w5) +#define SCHED_W_3_7(round, w0, w1, w2, w3, w4, w5) \ + st1 {XTMP2.16b-XTMP3.16b}, [addr0]; +#define SCHED_W_3_8(round, w0, w1, w2, w3, w4, w5) + +#define SCHED_W_W0W1W2W3W4W5_1(iop_num, round) \ + SCHED_W_1_##iop_num(round, W0, W1, W2, W3, W4, W5) +#define SCHED_W_W0W1W2W3W4W5_2(iop_num, round) \ + SCHED_W_2_##iop_num(round, W0, W1, W2, W3, W4, W5) +#define SCHED_W_W0W1W2W3W4W5_3(iop_num, round) \ + SCHED_W_3_##iop_num(round, W0, W1, W2, W3, W4, W5) + +#define SCHED_W_W1W2W3W4W5W0_1(iop_num, round) \ + SCHED_W_1_##iop_num(round, W1, W2, W3, W4, W5, W0) +#define SCHED_W_W1W2W3W4W5W0_2(iop_num, round) \ + SCHED_W_2_##iop_num(round, W1, W2, W3, W4, W5, W0) +#define SCHED_W_W1W2W3W4W5W0_3(iop_num, round) \ + SCHED_W_3_##iop_num(round, W1, W2, W3, W4, W5, W0) + +#define SCHED_W_W2W3W4W5W0W1_1(iop_num, round) \ + SCHED_W_1_##iop_num(round, W2, W3, W4, W5, W0, W1) +#define SCHED_W_W2W3W4W5W0W1_2(iop_num, round) \ + SCHED_W_2_##iop_num(round, W2, W3, W4, W5, W0, W1) +#define SCHED_W_W2W3W4W5W0W1_3(iop_num, round) \ + SCHED_W_3_##iop_num(round, W2, W3, W4, W5, W0, W1) + +#define SCHED_W_W3W4W5W0W1W2_1(iop_num, round) \ + SCHED_W_1_##iop_num(round, W3, W4, W5, W0, W1, W2) +#define SCHED_W_W3W4W5W0W1W2_2(iop_num, round) \ + SCHED_W_2_##iop_num(round, W3, W4, W5, W0, W1, W2) +#define SCHED_W_W3W4W5W0W1W2_3(iop_num, round) \ + SCHED_W_3_##iop_num(round, W3, W4, W5, W0, W1, W2) + +#define SCHED_W_W4W5W0W1W2W3_1(iop_num, round) \ + SCHED_W_1_##iop_num(round, W4, W5, W0, W1, W2, W3) +#define SCHED_W_W4W5W0W1W2W3_2(iop_num, round) \ + SCHED_W_2_##iop_num(round, W4, W5, W0, W1, W2, W3) +#define SCHED_W_W4W5W0W1W2W3_3(iop_num, round) \ + SCHED_W_3_##iop_num(round, W4, W5, W0, W1, W2, W3) + +#define SCHED_W_W5W0W1W2W3W4_1(iop_num, round) \ + SCHED_W_1_##iop_num(round, W5, W0, W1, W2, W3, W4) +#define SCHED_W_W5W0W1W2W3W4_2(iop_num, round) \ + SCHED_W_2_##iop_num(round, W5, W0, W1, W2, W3, W4) +#define SCHED_W_W5W0W1W2W3W4_3(iop_num, round) \ + SCHED_W_3_##iop_num(round, W5, W0, W1, W2, W3, W4) + + + /* + * Transform nblocks*64 bytes (nblocks*16 32-bit words) at 'data'. + * + * void sm3_neon_transform(struct sm3_block_state *state, + * const u8 *data, size_t nblocks) + */ + .text +.align 3 +SYM_FUNC_START(sm3_neon_transform) + ldp ra, rb, [RSTATE, #0] + ldp rc, rd, [RSTATE, #8] + ldp re, rf, [RSTATE, #16] + ldp rg, rh, [RSTATE, #24] + + stp x28, x29, [sp, #-16]! + stp x19, x20, [sp, #-16]! + stp x21, x22, [sp, #-16]! + stp x23, x24, [sp, #-16]! + stp x25, x26, [sp, #-16]! + mov RFRAME, sp + + sub addr0, sp, #STACK_SIZE + adr_l RKPTR, .LKtable + and sp, addr0, #(~63) + + /* Preload first block. */ + LOAD_W_VEC_1(1, 0) + LOAD_W_VEC_1(2, 0) + LOAD_W_VEC_1(3, 0) + LOAD_W_VEC_1(4, 0) + LOAD_W_VEC_1(5, 0) + LOAD_W_VEC_1(6, 0) + LOAD_W_VEC_1(7, 0) + LOAD_W_VEC_1(8, 0) + LOAD_W_VEC_2(1, 0) + LOAD_W_VEC_2(2, 0) + LOAD_W_VEC_2(3, 0) + LOAD_W_VEC_2(4, 0) + LOAD_W_VEC_2(5, 0) + LOAD_W_VEC_2(6, 0) + LOAD_W_VEC_2(7, 0) + LOAD_W_VEC_2(8, 0) + LOAD_W_VEC_3(1, 0) + LOAD_W_VEC_3(2, 0) + LOAD_W_VEC_3(3, 0) + LOAD_W_VEC_3(4, 0) + LOAD_W_VEC_3(5, 0) + LOAD_W_VEC_3(6, 0) + LOAD_W_VEC_3(7, 0) + LOAD_W_VEC_3(8, 0) + +.balign 16 +.Loop: + /* Transform 0-3 */ + R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 0, 0, IW, _, 0) + R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 1, 1, IW, _, 0) + R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 2, 2, IW, _, 0) + R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 3, 3, IW, _, 0) + + /* Transform 4-7 + Precalc 12-14 */ + R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 4, 0, IW, _, 0) + R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 5, 1, IW, _, 0) + R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 6, 2, IW, SCHED_W_W0W1W2W3W4W5_1, 12) + R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 7, 3, IW, SCHED_W_W0W1W2W3W4W5_2, 12) + + /* Transform 8-11 + Precalc 12-17 */ + R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 8, 0, IW, SCHED_W_W0W1W2W3W4W5_3, 12) + R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 9, 1, IW, SCHED_W_W1W2W3W4W5W0_1, 15) + R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 10, 2, IW, SCHED_W_W1W2W3W4W5W0_2, 15) + R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 11, 3, IW, SCHED_W_W1W2W3W4W5W0_3, 15) + + /* Transform 12-14 + Precalc 18-20 */ + R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 12, 0, XW, SCHED_W_W2W3W4W5W0W1_1, 18) + R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 13, 1, XW, SCHED_W_W2W3W4W5W0W1_2, 18) + R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 14, 2, XW, SCHED_W_W2W3W4W5W0W1_3, 18) + + /* Transform 15-17 + Precalc 21-23 */ + R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 15, 0, XW, SCHED_W_W3W4W5W0W1W2_1, 21) + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 16, 1, XW, SCHED_W_W3W4W5W0W1W2_2, 21) + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 17, 2, XW, SCHED_W_W3W4W5W0W1W2_3, 21) + + /* Transform 18-20 + Precalc 24-26 */ + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 18, 0, XW, SCHED_W_W4W5W0W1W2W3_1, 24) + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 19, 1, XW, SCHED_W_W4W5W0W1W2W3_2, 24) + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 20, 2, XW, SCHED_W_W4W5W0W1W2W3_3, 24) + + /* Transform 21-23 + Precalc 27-29 */ + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 21, 0, XW, SCHED_W_W5W0W1W2W3W4_1, 27) + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 22, 1, XW, SCHED_W_W5W0W1W2W3W4_2, 27) + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 23, 2, XW, SCHED_W_W5W0W1W2W3W4_3, 27) + + /* Transform 24-26 + Precalc 30-32 */ + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 24, 0, XW, SCHED_W_W0W1W2W3W4W5_1, 30) + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 25, 1, XW, SCHED_W_W0W1W2W3W4W5_2, 30) + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 26, 2, XW, SCHED_W_W0W1W2W3W4W5_3, 30) + + /* Transform 27-29 + Precalc 33-35 */ + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 27, 0, XW, SCHED_W_W1W2W3W4W5W0_1, 33) + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 28, 1, XW, SCHED_W_W1W2W3W4W5W0_2, 33) + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 29, 2, XW, SCHED_W_W1W2W3W4W5W0_3, 33) + + /* Transform 30-32 + Precalc 36-38 */ + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 30, 0, XW, SCHED_W_W2W3W4W5W0W1_1, 36) + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 31, 1, XW, SCHED_W_W2W3W4W5W0W1_2, 36) + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 32, 2, XW, SCHED_W_W2W3W4W5W0W1_3, 36) + + /* Transform 33-35 + Precalc 39-41 */ + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 33, 0, XW, SCHED_W_W3W4W5W0W1W2_1, 39) + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 34, 1, XW, SCHED_W_W3W4W5W0W1W2_2, 39) + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 35, 2, XW, SCHED_W_W3W4W5W0W1W2_3, 39) + + /* Transform 36-38 + Precalc 42-44 */ + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 36, 0, XW, SCHED_W_W4W5W0W1W2W3_1, 42) + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 37, 1, XW, SCHED_W_W4W5W0W1W2W3_2, 42) + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 38, 2, XW, SCHED_W_W4W5W0W1W2W3_3, 42) + + /* Transform 39-41 + Precalc 45-47 */ + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 39, 0, XW, SCHED_W_W5W0W1W2W3W4_1, 45) + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 40, 1, XW, SCHED_W_W5W0W1W2W3W4_2, 45) + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 41, 2, XW, SCHED_W_W5W0W1W2W3W4_3, 45) + + /* Transform 42-44 + Precalc 48-50 */ + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 42, 0, XW, SCHED_W_W0W1W2W3W4W5_1, 48) + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 43, 1, XW, SCHED_W_W0W1W2W3W4W5_2, 48) + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 44, 2, XW, SCHED_W_W0W1W2W3W4W5_3, 48) + + /* Transform 45-47 + Precalc 51-53 */ + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 45, 0, XW, SCHED_W_W1W2W3W4W5W0_1, 51) + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 46, 1, XW, SCHED_W_W1W2W3W4W5W0_2, 51) + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 47, 2, XW, SCHED_W_W1W2W3W4W5W0_3, 51) + + /* Transform 48-50 + Precalc 54-56 */ + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 48, 0, XW, SCHED_W_W2W3W4W5W0W1_1, 54) + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 49, 1, XW, SCHED_W_W2W3W4W5W0W1_2, 54) + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 50, 2, XW, SCHED_W_W2W3W4W5W0W1_3, 54) + + /* Transform 51-53 + Precalc 57-59 */ + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 51, 0, XW, SCHED_W_W3W4W5W0W1W2_1, 57) + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 52, 1, XW, SCHED_W_W3W4W5W0W1W2_2, 57) + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 53, 2, XW, SCHED_W_W3W4W5W0W1W2_3, 57) + + /* Transform 54-56 + Precalc 60-62 */ + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 54, 0, XW, SCHED_W_W4W5W0W1W2W3_1, 60) + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 55, 1, XW, SCHED_W_W4W5W0W1W2W3_2, 60) + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 56, 2, XW, SCHED_W_W4W5W0W1W2W3_3, 60) + + /* Transform 57-59 + Precalc 63 */ + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 57, 0, XW, SCHED_W_W5W0W1W2W3W4_1, 63) + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 58, 1, XW, SCHED_W_W5W0W1W2W3W4_2, 63) + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 59, 2, XW, SCHED_W_W5W0W1W2W3W4_3, 63) + + /* Transform 60 */ + R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 60, 0, XW, _, _) + subs RNBLKS, RNBLKS, #1 + b.eq .Lend + + /* Transform 61-63 + Preload next block */ + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 61, 1, XW, LOAD_W_VEC_1, _) + ldp s0, s1, [RSTATE, #0] + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 62, 2, XW, LOAD_W_VEC_2, _) + ldp s2, s3, [RSTATE, #8] + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 63, 0, XW, LOAD_W_VEC_3, _) + + /* Update the chaining variables. */ + eor ra, ra, s0 + eor rb, rb, s1 + ldp s0, s1, [RSTATE, #16] + eor rc, rc, s2 + ldp k_even, k_odd, [RSTATE, #24] + eor rd, rd, s3 + eor re, re, s0 + stp ra, rb, [RSTATE, #0] + eor rf, rf, s1 + stp rc, rd, [RSTATE, #8] + eor rg, rg, k_even + stp re, rf, [RSTATE, #16] + eor rh, rh, k_odd + stp rg, rh, [RSTATE, #24] + b .Loop + +.Lend: + /* Transform 61-63 */ + R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 61, 1, XW, _, _) + ldp s0, s1, [RSTATE, #0] + R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 62, 2, XW, _, _) + ldp s2, s3, [RSTATE, #8] + R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 63, 0, XW, _, _) + + /* Update the chaining variables. */ + eor ra, ra, s0 + clear_vec(W0) + eor rb, rb, s1 + clear_vec(W1) + ldp s0, s1, [RSTATE, #16] + clear_vec(W2) + eor rc, rc, s2 + clear_vec(W3) + ldp k_even, k_odd, [RSTATE, #24] + clear_vec(W4) + eor rd, rd, s3 + clear_vec(W5) + eor re, re, s0 + clear_vec(XTMP0) + stp ra, rb, [RSTATE, #0] + clear_vec(XTMP1) + eor rf, rf, s1 + clear_vec(XTMP2) + stp rc, rd, [RSTATE, #8] + clear_vec(XTMP3) + eor rg, rg, k_even + clear_vec(XTMP4) + stp re, rf, [RSTATE, #16] + clear_vec(XTMP5) + eor rh, rh, k_odd + clear_vec(XTMP6) + stp rg, rh, [RSTATE, #24] + + /* Clear message expansion area */ + add addr0, sp, #STACK_W + st1 {W0.16b-W3.16b}, [addr0], #64 + st1 {W0.16b-W3.16b}, [addr0], #64 + st1 {W0.16b-W3.16b}, [addr0] + + mov sp, RFRAME + + ldp x25, x26, [sp], #16 + ldp x23, x24, [sp], #16 + ldp x21, x22, [sp], #16 + ldp x19, x20, [sp], #16 + ldp x28, x29, [sp], #16 + + ret +SYM_FUNC_END(sm3_neon_transform) + + + .section ".rodata", "a" + + .align 4 +.LKtable: + .long 0x79cc4519, 0xf3988a32, 0xe7311465, 0xce6228cb + .long 0x9cc45197, 0x3988a32f, 0x7311465e, 0xe6228cbc + .long 0xcc451979, 0x988a32f3, 0x311465e7, 0x6228cbce + .long 0xc451979c, 0x88a32f39, 0x11465e73, 0x228cbce6 + .long 0x9d8a7a87, 0x3b14f50f, 0x7629ea1e, 0xec53d43c + .long 0xd8a7a879, 0xb14f50f3, 0x629ea1e7, 0xc53d43ce + .long 0x8a7a879d, 0x14f50f3b, 0x29ea1e76, 0x53d43cec + .long 0xa7a879d8, 0x4f50f3b1, 0x9ea1e762, 0x3d43cec5 + .long 0x7a879d8a, 0xf50f3b14, 0xea1e7629, 0xd43cec53 + .long 0xa879d8a7, 0x50f3b14f, 0xa1e7629e, 0x43cec53d + .long 0x879d8a7a, 0x0f3b14f5, 0x1e7629ea, 0x3cec53d4 + .long 0x79d8a7a8, 0xf3b14f50, 0xe7629ea1, 0xcec53d43 + .long 0x9d8a7a87, 0x3b14f50f, 0x7629ea1e, 0xec53d43c + .long 0xd8a7a879, 0xb14f50f3, 0x629ea1e7, 0xc53d43ce + .long 0x8a7a879d, 0x14f50f3b, 0x29ea1e76, 0x53d43cec + .long 0xa7a879d8, 0x4f50f3b1, 0x9ea1e762, 0x3d43cec5 diff --git a/lib/crypto/arm64/sm3.h b/lib/crypto/arm64/sm3.h new file mode 100644 index 000000000000..beb9cd82bb7d --- /dev/null +++ b/lib/crypto/arm64/sm3.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SM3 optimized for ARM64 + * + * Copyright 2026 Google LLC + */ +#include <asm/simd.h> +#include <linux/cpufeature.h> + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); + +asmlinkage void sm3_neon_transform(struct sm3_block_state *state, + const u8 *data, size_t nblocks); +asmlinkage void sm3_ce_transform(struct sm3_block_state *state, + const u8 *data, size_t nblocks); + +static void sm3_blocks(struct sm3_block_state *state, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_neon) && likely(may_use_simd())) { + scoped_ksimd() { + if (static_branch_likely(&have_ce)) + sm3_ce_transform(state, data, nblocks); + else + sm3_neon_transform(state, data, nblocks); + } + } else { + sm3_blocks_generic(state, data, nblocks); + } +} + +#define sm3_mod_init_arch sm3_mod_init_arch +static void sm3_mod_init_arch(void) +{ + if (cpu_have_named_feature(ASIMD)) { + static_branch_enable(&have_neon); + if (cpu_have_named_feature(SM3)) + static_branch_enable(&have_ce); + } +} diff --git a/lib/crypto/chacha-block-generic.c b/lib/crypto/chacha-block-generic.c index 77f68de71066..4a6d627580cb 100644 --- a/lib/crypto/chacha-block-generic.c +++ b/lib/crypto/chacha-block-generic.c @@ -87,6 +87,8 @@ void chacha_block_generic(struct chacha_state *state, &out[i * sizeof(u32)]); state->x[12]++; + + chacha_zeroize_state(&permuted_state); } EXPORT_SYMBOL(chacha_block_generic); @@ -110,5 +112,7 @@ void hchacha_block_generic(const struct chacha_state *state, memcpy(&out[0], &permuted_state.x[0], 16); memcpy(&out[4], &permuted_state.x[12], 16); + + chacha_zeroize_state(&permuted_state); } EXPORT_SYMBOL(hchacha_block_generic); diff --git a/lib/crypto/chacha.c b/lib/crypto/chacha.c index e0c7cb4af318..86e5d382a4e0 100644 --- a/lib/crypto/chacha.c +++ b/lib/crypto/chacha.c @@ -5,8 +5,8 @@ * Copyright (C) 2015 Martin Willi */ -#include <crypto/algapi.h> // for crypto_xor_cpy #include <crypto/chacha.h> +#include <crypto/utils.h> #include <linux/export.h> #include <linux/kernel.h> #include <linux/module.h> diff --git a/lib/crypto/chacha20poly1305.c b/lib/crypto/chacha20poly1305.c index 212ce33562af..ea42a28f4ff7 100644 --- a/lib/crypto/chacha20poly1305.c +++ b/lib/crypto/chacha20poly1305.c @@ -356,20 +356,6 @@ bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len } EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace); -static int __init chacha20poly1305_init(void) -{ - if (IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) && - WARN_ON(!chacha20poly1305_selftest())) - return -ENODEV; - return 0; -} - -static void __exit chacha20poly1305_exit(void) -{ -} - -module_init(chacha20poly1305_init); -module_exit(chacha20poly1305_exit); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction"); MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); diff --git a/lib/crypto/fips-aes.h b/lib/crypto/fips-aes.h new file mode 100644 index 000000000000..2a1746606533 --- /dev/null +++ b/lib/crypto/fips-aes.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* This file was generated by: gen-fips-testvecs.py */ +/* clang-format off */ + +#include <linux/fips.h> + +static const u8 fips_test_data[] __initconst __maybe_unused = { + 0x66, 0x69, 0x70, 0x73, 0x20, 0x74, 0x65, 0x73, + 0x74, 0x20, 0x64, 0x61, 0x74, 0x61, 0x00, 0x00, +}; + +static const u8 fips_test_ad[] __initconst __maybe_unused = { + 0x66, 0x69, 0x70, 0x73, 0x20, 0x74, 0x65, 0x73, + 0x74, 0x20, 0x61, 0x64, 0x00, 0x00, 0x00, 0x00, +}; + +static const u8 fips_test_iv[] __initconst __maybe_unused = { + 0x66, 0x69, 0x70, 0x73, 0x20, 0x74, 0x65, 0x73, + 0x74, 0x20, 0x69, 0x76, 0x00, 0x00, 0x00, 0x00, +}; + +static const u8 fips_test_key[] __initconst __maybe_unused = { + 0x66, 0x69, 0x70, 0x73, 0x20, 0x74, 0x65, 0x73, + 0x74, 0x20, 0x6b, 0x65, 0x79, 0x00, 0x00, 0x00, +}; + +static const u8 fips_test_xts_key[] __initconst __maybe_unused = { + 0x6b, 0x65, 0x79, 0x31, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x6b, 0x65, 0x79, 0x32, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +static const u8 fips_test_aes_cmac_value[] __initconst __maybe_unused = { + 0xc5, 0x88, 0x28, 0x55, 0xd7, 0x2c, 0x00, 0xb6, + 0x6a, 0xa7, 0xfc, 0x82, 0x90, 0x81, 0xcf, 0x18, +}; + +static const u8 fips_test_aes_ecb_ctext[] __initconst __maybe_unused = { + 0x47, 0x76, 0x48, 0xaf, 0x1b, 0xd8, 0x4c, 0xe6, + 0xb5, 0xa7, 0x20, 0x8d, 0x64, 0x88, 0xbc, 0x3f, +}; + +static const u8 fips_test_aes_cbc_ctext[] __initconst __maybe_unused = { + 0xc8, 0x7d, 0x7c, 0x25, 0xba, 0x15, 0xf7, 0xe1, + 0x08, 0xa0, 0xd0, 0x7a, 0x20, 0x37, 0xaf, 0x5e, +}; + +static const u8 fips_test_aes_cbc_cts_ctext[] __initconst __maybe_unused = { + 0x36, 0x8e, 0x37, 0xb4, 0x78, 0xe2, 0x88, 0x59, + 0xd5, 0xe8, 0x17, 0x65, 0x5c, 0xa1, 0x25, 0xe6, + 0xc8, 0x7d, 0x7c, 0x25, 0xba, 0x15, 0xf7, 0xe1, + 0x08, 0xa0, 0xd0, 0x7a, 0x20, 0x37, 0xaf, 0x5e, +}; + +static const u8 fips_test_aes_ctr_ctext[] __initconst __maybe_unused = { + 0x95, 0xf4, 0xf4, 0x7a, 0xc8, 0xa2, 0x53, 0x73, + 0x53, 0x8f, 0x95, 0xfc, 0x18, 0xfe, 0x58, 0x2f, +}; + +static const u8 fips_test_aes_xts_ctext[] __initconst __maybe_unused = { + 0xd4, 0x51, 0x7f, 0x01, 0x14, 0x91, 0x16, 0x29, + 0x26, 0xbe, 0xec, 0x9b, 0x90, 0xed, 0x59, 0x30, +}; + +static const u8 fips_test_aes_gcm_ctext_and_tag[] __initconst __maybe_unused = { + 0x12, 0x0c, 0x5d, 0x03, 0x32, 0x93, 0x13, 0x44, + 0x06, 0x35, 0x26, 0x9d, 0xe0, 0xea, 0xbc, 0xe2, + 0x30, 0xa9, 0xa4, 0x15, 0xc5, 0x3d, 0xb3, 0xf9, + 0x30, 0x82, 0xdf, 0x9c, 0xd8, 0xc4, 0x3f, 0x2f, +}; + +static const u8 fips_test_aes_ccm_ctext_and_tag[] __initconst __maybe_unused = { + 0x11, 0x8e, 0x01, 0xcb, 0xb5, 0x22, 0x6d, 0xb4, + 0x66, 0x98, 0x97, 0x1d, 0x35, 0x53, 0x78, 0xdd, + 0xd1, 0xc5, 0xff, 0xb6, 0x90, 0xcf, 0xb1, 0xf2, + 0x87, 0x99, 0xd6, 0x1e, 0xd5, 0xd1, 0xed, 0x63, +}; diff --git a/lib/crypto/fips.h b/lib/crypto/fips-sha.h index 023410c2e0db..68af7e14e09c 100644 --- a/lib/crypto/fips.h +++ b/lib/crypto/fips-sha.h @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* This file was generated by: gen-fips-testvecs.py */ +/* clang-format off */ #include <linux/fips.h> diff --git a/lib/crypto/polyval.c b/lib/crypto/gf128hash.c index 5796275f574a..8dcdf5ec98be 100644 --- a/lib/crypto/polyval.c +++ b/lib/crypto/gf128hash.c @@ -1,34 +1,41 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * POLYVAL library functions + * GF(2^128) polynomial hashing: GHASH and POLYVAL * * Copyright 2025 Google LLC */ -#include <crypto/polyval.h> +#include <crypto/gf128hash.h> #include <linux/export.h> #include <linux/module.h> #include <linux/string.h> #include <linux/unaligned.h> /* - * POLYVAL is an almost-XOR-universal hash function. Similar to GHASH, POLYVAL - * interprets the message as the coefficients of a polynomial in GF(2^128) and - * evaluates that polynomial at a secret point. POLYVAL has a simple - * mathematical relationship with GHASH, but it uses a better field convention - * which makes it easier and faster to implement. + * GHASH and POLYVAL are almost-XOR-universal hash functions. They interpret + * the message as the coefficients of a polynomial in the finite field GF(2^128) + * and evaluate that polynomial at a secret point. * - * POLYVAL is not a cryptographic hash function, and it should be used only by - * algorithms that are specifically designed to use it. + * Neither GHASH nor POLYVAL is a cryptographic hash function. They should be + * used only by algorithms that are specifically designed to use them. * - * POLYVAL is specified by "AES-GCM-SIV: Nonce Misuse-Resistant Authenticated - * Encryption" (https://datatracker.ietf.org/doc/html/rfc8452) + * GHASH is the older variant, defined as part of GCM in NIST SP 800-38D + * (https://nvlpubs.nist.gov/nistpubs/legacy/sp/nistspecialpublication800-38d.pdf). + * GHASH is hard to implement directly, due to its backwards mapping between + * bits and polynomial coefficients. GHASH implementations typically pre and + * post-process the inputs and outputs (mainly by byte-swapping) to convert the + * GHASH computation into an equivalent computation over a different, + * easier-to-use representation of GF(2^128). * - * POLYVAL is also used by HCTR2. See "Length-preserving encryption with HCTR2" - * (https://eprint.iacr.org/2021/1441.pdf). + * POLYVAL is a newer GF(2^128) polynomial hash, originally defined as part of + * AES-GCM-SIV (https://datatracker.ietf.org/doc/html/rfc8452) and also used by + * HCTR2 (https://eprint.iacr.org/2021/1441.pdf). It uses that easier-to-use + * field representation directly, eliminating the data conversion steps. * - * This file provides a library API for POLYVAL. This API can delegate to - * either a generic implementation or an architecture-optimized implementation. + * This file provides library APIs for GHASH and POLYVAL. These APIs can + * delegate to either a generic implementation or an architecture-optimized + * implementation. Due to the mathematical relationship between GHASH and + * POLYVAL, in some cases code for one is reused with the other. * * For the generic implementation, we don't use the traditional table approach * to GF(2^128) multiplication. That approach is not constant-time and requires @@ -102,7 +109,7 @@ static void clmul64(u64 a, u64 b, u64 *out_lo, u64 *out_hi) #else /* CONFIG_ARCH_SUPPORTS_INT128 */ /* Do a 32 x 32 => 64 bit carryless multiplication. */ -static u64 clmul32(u32 a, u32 b) +static noinline_for_stack u64 clmul32(u32 a, u32 b) { /* * With 32-bit multiplicands and one term every 4 bits, there are up to @@ -205,6 +212,19 @@ polyval_mul_generic(struct polyval_elem *a, const struct polyval_elem *b) a->hi = cpu_to_le64(c3); } +static void __maybe_unused ghash_blocks_generic(struct polyval_elem *acc, + const struct polyval_elem *key, + const u8 *data, size_t nblocks) +{ + do { + acc->lo ^= + cpu_to_le64(get_unaligned_be64((__be64 *)(data + 8))); + acc->hi ^= cpu_to_le64(get_unaligned_be64((__be64 *)data)); + polyval_mul_generic(acc, key); + data += GHASH_BLOCK_SIZE; + } while (--nblocks); +} + static void __maybe_unused polyval_blocks_generic(struct polyval_elem *acc, const struct polyval_elem *key, const u8 *data, size_t nblocks) @@ -217,16 +237,118 @@ polyval_blocks_generic(struct polyval_elem *acc, const struct polyval_elem *key, } while (--nblocks); } -/* Include the arch-optimized implementation of POLYVAL, if one is available. */ -#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH -#include "polyval.h" /* $(SRCARCH)/polyval.h */ +/* Convert the key from GHASH format to POLYVAL format. */ +static void __maybe_unused ghash_key_to_polyval(const u8 in[GHASH_BLOCK_SIZE], + struct polyval_elem *out) +{ + u64 hi = get_unaligned_be64(&in[0]); + u64 lo = get_unaligned_be64(&in[8]); + u64 mask = (s64)hi >> 63; + + hi = (hi << 1) ^ (lo >> 63) ^ (mask & ((u64)0xc2 << 56)); + lo = (lo << 1) ^ (mask & 1); + out->lo = cpu_to_le64(lo); + out->hi = cpu_to_le64(hi); +} + +/* Convert the accumulator from POLYVAL format to GHASH format. */ +static void polyval_acc_to_ghash(const struct polyval_elem *in, + u8 out[GHASH_BLOCK_SIZE]) +{ + put_unaligned_be64(le64_to_cpu(in->hi), &out[0]); + put_unaligned_be64(le64_to_cpu(in->lo), &out[8]); +} + +/* Convert the accumulator from GHASH format to POLYVAL format. */ +static void __maybe_unused ghash_acc_to_polyval(const u8 in[GHASH_BLOCK_SIZE], + struct polyval_elem *out) +{ + out->lo = cpu_to_le64(get_unaligned_be64(&in[8])); + out->hi = cpu_to_le64(get_unaligned_be64(&in[0])); +} + +#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH +#include "gf128hash.h" /* $(SRCARCH)/gf128hash.h */ +#endif + +void ghash_preparekey(struct ghash_key *key, const u8 raw_key[GHASH_BLOCK_SIZE]) +{ +#ifdef ghash_preparekey_arch + ghash_preparekey_arch(key, raw_key); +#else + ghash_key_to_polyval(raw_key, &key->h); +#endif +} +EXPORT_SYMBOL_GPL(ghash_preparekey); + +static void ghash_mul(struct ghash_ctx *ctx) +{ +#ifdef ghash_mul_arch + ghash_mul_arch(&ctx->acc, ctx->key); +#elif defined(ghash_blocks_arch) + static const u8 zeroes[GHASH_BLOCK_SIZE]; + + ghash_blocks_arch(&ctx->acc, ctx->key, zeroes, 1); +#else + polyval_mul_generic(&ctx->acc, &ctx->key->h); +#endif +} + +/* nblocks is always >= 1. */ +static void ghash_blocks(struct ghash_ctx *ctx, const u8 *data, size_t nblocks) +{ +#ifdef ghash_blocks_arch + ghash_blocks_arch(&ctx->acc, ctx->key, data, nblocks); +#else + ghash_blocks_generic(&ctx->acc, &ctx->key->h, data, nblocks); +#endif +} + +void ghash_update(struct ghash_ctx *ctx, const u8 *data, size_t len) +{ + if (unlikely(ctx->partial)) { + size_t n = min(len, GHASH_BLOCK_SIZE - ctx->partial); + + len -= n; + while (n--) + ctx->acc.bytes[GHASH_BLOCK_SIZE - 1 - ctx->partial++] ^= + *data++; + if (ctx->partial < GHASH_BLOCK_SIZE) + return; + ghash_mul(ctx); + } + if (len >= GHASH_BLOCK_SIZE) { + size_t nblocks = len / GHASH_BLOCK_SIZE; + + ghash_blocks(ctx, data, nblocks); + data += len & ~(GHASH_BLOCK_SIZE - 1); + len &= GHASH_BLOCK_SIZE - 1; + } + for (size_t i = 0; i < len; i++) + ctx->acc.bytes[GHASH_BLOCK_SIZE - 1 - i] ^= data[i]; + ctx->partial = len; +} +EXPORT_SYMBOL_GPL(ghash_update); + +void ghash_final(struct ghash_ctx *ctx, u8 out[GHASH_BLOCK_SIZE]) +{ + if (unlikely(ctx->partial)) + ghash_mul(ctx); + polyval_acc_to_ghash(&ctx->acc, out); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(ghash_final); + void polyval_preparekey(struct polyval_key *key, const u8 raw_key[POLYVAL_BLOCK_SIZE]) { +#ifdef polyval_preparekey_arch polyval_preparekey_arch(key, raw_key); +#else + memcpy(key->h.bytes, raw_key, POLYVAL_BLOCK_SIZE); +#endif } EXPORT_SYMBOL_GPL(polyval_preparekey); -#endif /* Else, polyval_preparekey() is an inline function. */ /* * polyval_mul_generic() and polyval_blocks_generic() take the key as a @@ -238,17 +360,22 @@ EXPORT_SYMBOL_GPL(polyval_preparekey); static void polyval_mul(struct polyval_ctx *ctx) { -#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH +#ifdef polyval_mul_arch polyval_mul_arch(&ctx->acc, ctx->key); +#elif defined(polyval_blocks_arch) + static const u8 zeroes[POLYVAL_BLOCK_SIZE]; + + polyval_blocks_arch(&ctx->acc, ctx->key, zeroes, 1); #else polyval_mul_generic(&ctx->acc, &ctx->key->h); #endif } +/* nblocks is always >= 1. */ static void polyval_blocks(struct polyval_ctx *ctx, const u8 *data, size_t nblocks) { -#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH +#ifdef polyval_blocks_arch polyval_blocks_arch(&ctx->acc, ctx->key, data, nblocks); #else polyval_blocks_generic(&ctx->acc, &ctx->key->h, data, nblocks); @@ -289,19 +416,19 @@ void polyval_final(struct polyval_ctx *ctx, u8 out[POLYVAL_BLOCK_SIZE]) } EXPORT_SYMBOL_GPL(polyval_final); -#ifdef polyval_mod_init_arch -static int __init polyval_mod_init(void) +#ifdef gf128hash_mod_init_arch +static int __init gf128hash_mod_init(void) { - polyval_mod_init_arch(); + gf128hash_mod_init_arch(); return 0; } -subsys_initcall(polyval_mod_init); +subsys_initcall(gf128hash_mod_init); -static void __exit polyval_mod_exit(void) +static void __exit gf128hash_mod_exit(void) { } -module_exit(polyval_mod_exit); +module_exit(gf128hash_mod_exit); #endif -MODULE_DESCRIPTION("POLYVAL almost-XOR-universal hash function"); +MODULE_DESCRIPTION("GF(2^128) polynomial hashing: GHASH and POLYVAL"); MODULE_LICENSE("GPL"); diff --git a/lib/crypto/gf128mul.c b/lib/crypto/gf128mul.c index e5a727b15f07..7ebf07ce1168 100644 --- a/lib/crypto/gf128mul.c +++ b/lib/crypto/gf128mul.c @@ -127,27 +127,9 @@ (i & 0x02 ? 0x0384 : 0) ^ (i & 0x01 ? 0x01c2 : 0) \ ) -static const u16 gf128mul_table_le[256] = gf128mul_dat(xda_le); static const u16 gf128mul_table_be[256] = gf128mul_dat(xda_be); -/* - * The following functions multiply a field element by x^8 in - * the polynomial field representation. They use 64-bit word operations - * to gain speed but compensate for machine endianness and hence work - * correctly on both styles of machine. - */ - -static void gf128mul_x8_lle(be128 *x) -{ - u64 a = be64_to_cpu(x->a); - u64 b = be64_to_cpu(x->b); - u64 _tt = gf128mul_table_le[b & 0xff]; - - x->b = cpu_to_be64((b >> 8) | (a << 56)); - x->a = cpu_to_be64((a >> 8) ^ (_tt << 48)); -} - -/* time invariant version of gf128mul_x8_lle */ +/* A table-less implementation of multiplying by x^8 */ static void gf128mul_x8_lle_ti(be128 *x) { u64 a = be64_to_cpu(x->a); @@ -305,58 +287,5 @@ void gf128mul_64k_bbe(be128 *a, const struct gf128mul_64k *t) } EXPORT_SYMBOL(gf128mul_64k_bbe); -/* This version uses 4k bytes of table space. - A 16 byte buffer has to be multiplied by a 16 byte key - value in GF(2^128). If we consider a GF(2^128) value in a - single byte, we can construct a table of the 256 16 byte - values that result from the 256 values of this byte. - This requires 4096 bytes. If we take the highest byte in - the buffer and use this table to get the result, we then - have to multiply by x^120 to get the final value. For the - next highest byte the result has to be multiplied by x^112 - and so on. But we can do this by accumulating the result - in an accumulator starting with the result for the top - byte. We repeatedly multiply the accumulator value by - x^8 and then add in (i.e. xor) the 16 bytes of the next - lower byte in the buffer, stopping when we reach the - lowest byte. This requires a 4096 byte table. -*/ -struct gf128mul_4k *gf128mul_init_4k_lle(const be128 *g) -{ - struct gf128mul_4k *t; - int j, k; - - t = kzalloc_obj(*t); - if (!t) - goto out; - - t->t[128] = *g; - for (j = 64; j > 0; j >>= 1) - gf128mul_x_lle(&t->t[j], &t->t[j+j]); - - for (j = 2; j < 256; j += j) - for (k = 1; k < j; ++k) - be128_xor(&t->t[j + k], &t->t[j], &t->t[k]); - -out: - return t; -} -EXPORT_SYMBOL(gf128mul_init_4k_lle); - -void gf128mul_4k_lle(be128 *a, const struct gf128mul_4k *t) -{ - u8 *ap = (u8 *)a; - be128 r[1]; - int i = 15; - - *r = t->t[ap[15]]; - while (i--) { - gf128mul_x8_lle(r); - be128_xor(r, r, &t->t[ap[i]]); - } - *a = *r; -} -EXPORT_SYMBOL(gf128mul_4k_lle); - MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Functions for multiplying elements of GF(2^128)"); diff --git a/lib/crypto/md5.c b/lib/crypto/md5.c index c4af57db0ea8..3d2b017a0525 100644 --- a/lib/crypto/md5.c +++ b/lib/crypto/md5.c @@ -2,8 +2,8 @@ /* * MD5 and HMAC-MD5 library functions * - * md5_block_generic() is derived from cryptoapi implementation, originally - * based on the public domain implementation written by Colin Plumb in 1993. + * md5_block() is derived from cryptoapi implementation, originally based on the + * public domain implementation written by Colin Plumb in 1993. * * Copyright (c) Cryptoapi developers. * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> @@ -31,8 +31,8 @@ static const struct md5_block_state md5_iv = { #define MD5STEP(f, w, x, y, z, in, s) \ (w += f(x, y, z) + in, w = rol32(w, s) + x) -static void md5_block_generic(struct md5_block_state *state, - const u8 data[MD5_BLOCK_SIZE]) +static void md5_block(struct md5_block_state *state, + const u8 data[MD5_BLOCK_SIZE]) { u32 in[MD5_BLOCK_WORDS]; u32 a, b, c, d; @@ -119,21 +119,15 @@ static void md5_block_generic(struct md5_block_state *state, state->h[3] += d; } -static void __maybe_unused md5_blocks_generic(struct md5_block_state *state, - const u8 *data, size_t nblocks) +static void md5_blocks(struct md5_block_state *state, + const u8 *data, size_t nblocks) { do { - md5_block_generic(state, data); + md5_block(state, data); data += MD5_BLOCK_SIZE; } while (--nblocks); } -#ifdef CONFIG_CRYPTO_LIB_MD5_ARCH -#include "md5.h" /* $(SRCARCH)/md5.h */ -#else -#define md5_blocks md5_blocks_generic -#endif - void md5_init(struct md5_ctx *ctx) { ctx->state = md5_iv; @@ -304,19 +298,5 @@ void hmac_md5_usingrawkey(const u8 *raw_key, size_t raw_key_len, } EXPORT_SYMBOL_GPL(hmac_md5_usingrawkey); -#ifdef md5_mod_init_arch -static int __init md5_mod_init(void) -{ - md5_mod_init_arch(); - return 0; -} -subsys_initcall(md5_mod_init); - -static void __exit md5_mod_exit(void) -{ -} -module_exit(md5_mod_exit); -#endif - MODULE_DESCRIPTION("MD5 and HMAC-MD5 library functions"); MODULE_LICENSE("GPL"); diff --git a/lib/crypto/memneq.c b/lib/crypto/memneq.c index 44daacb8cb51..08924acd44bc 100644 --- a/lib/crypto/memneq.c +++ b/lib/crypto/memneq.c @@ -59,7 +59,7 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include <crypto/algapi.h> +#include <crypto/utils.h> #include <linux/export.h> #include <linux/module.h> #include <linux/unaligned.h> @@ -159,7 +159,7 @@ static inline unsigned long __crypto_memneq_16(const void *a, const void *b) /* Compare two areas of memory without leaking timing information, * and with special optimizations for common sizes. Users should * not call this function directly, but should instead use - * crypto_memneq defined in crypto/algapi.h. + * crypto_memneq defined in crypto/utils.h. */ noinline unsigned long __crypto_memneq(const void *a, const void *b, size_t size) diff --git a/lib/crypto/mips/md5.h b/lib/crypto/mips/md5.h deleted file mode 100644 index e08e28aeffa4..000000000000 --- a/lib/crypto/mips/md5.h +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Cryptographic API. - * - * MD5 Message Digest Algorithm (RFC1321). - * - * Adapted for OCTEON by Aaro Koskinen <aaro.koskinen@iki.fi>. - * - * Based on crypto/md5.c, which is: - * - * Derived from cryptoapi implementation, originally based on the - * public domain implementation written by Colin Plumb in 1993. - * - * Copyright (c) Cryptoapi developers. - * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) - * any later version. - */ - -#include <asm/octeon/crypto.h> -#include <asm/octeon/octeon.h> - -/* - * We pass everything as 64-bit. OCTEON can handle misaligned data. - */ - -static void md5_blocks(struct md5_block_state *state, - const u8 *data, size_t nblocks) -{ - struct octeon_cop2_state cop2_state; - u64 *state64 = (u64 *)state; - unsigned long flags; - - if (!octeon_has_crypto()) - return md5_blocks_generic(state, data, nblocks); - - cpu_to_le32_array(state->h, ARRAY_SIZE(state->h)); - - flags = octeon_crypto_enable(&cop2_state); - write_octeon_64bit_hash_dword(state64[0], 0); - write_octeon_64bit_hash_dword(state64[1], 1); - - do { - const u64 *block = (const u64 *)data; - - write_octeon_64bit_block_dword(block[0], 0); - write_octeon_64bit_block_dword(block[1], 1); - write_octeon_64bit_block_dword(block[2], 2); - write_octeon_64bit_block_dword(block[3], 3); - write_octeon_64bit_block_dword(block[4], 4); - write_octeon_64bit_block_dword(block[5], 5); - write_octeon_64bit_block_dword(block[6], 6); - octeon_md5_start(block[7]); - - data += MD5_BLOCK_SIZE; - } while (--nblocks); - - state64[0] = read_octeon_64bit_hash_dword(0); - state64[1] = read_octeon_64bit_hash_dword(1); - octeon_crypto_disable(&cop2_state, flags); - - le32_to_cpu_array(state->h, ARRAY_SIZE(state->h)); -} diff --git a/lib/crypto/mpi/longlong.h b/lib/crypto/mpi/longlong.h index b6fa1d08fb55..6d31c3a729f1 100644 --- a/lib/crypto/mpi/longlong.h +++ b/lib/crypto/mpi/longlong.h @@ -66,12 +66,12 @@ * denominator). Like udiv_qrnnd but the numbers are signed. The quotient * is rounded towards 0. * - * 5) count_leading_zeros(count, x) counts the number of zero-bits from the + * 5) count_leading_zeros(x) counts the number of zero-bits from the * msb to the first non-zero bit in the UWtype X. This is the number of - * steps X needs to be shifted left to set the msb. Undefined for X == 0, - * unless the symbol COUNT_LEADING_ZEROS_0 is defined to some value. + * steps X needs to be shifted left to set the msb. + * count_leading_zeros(0) == BITS_PER_LONG * - * 6) count_trailing_zeros(count, x) like count_leading_zeros, but counts + * 6) count_trailing_zeros() like count_leading_zeros(), but counts * from the least significant end. * * 7) add_ssaaaa(high_sum, low_sum, high_addend_1, low_addend_1, diff --git a/lib/crypto/mpi/mpicoder.c b/lib/crypto/mpi/mpicoder.c index bf716a03c704..9359a58c29ec 100644 --- a/lib/crypto/mpi/mpicoder.c +++ b/lib/crypto/mpi/mpicoder.c @@ -347,7 +347,7 @@ MPI mpi_read_raw_from_sgl(struct scatterlist *sgl, unsigned int nbytes) lzeros = 0; len = 0; while (nbytes > 0) { - while (len && !*buff) { + while (len && !*buff && lzeros < nbytes) { lzeros++; len--; buff++; diff --git a/lib/crypto/powerpc/.gitignore b/lib/crypto/powerpc/.gitignore index 598ca7aff6b1..7aa71d83f739 100644 --- a/lib/crypto/powerpc/.gitignore +++ b/lib/crypto/powerpc/.gitignore @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only aesp8-ppc.S +ghashp8-ppc.S diff --git a/lib/crypto/powerpc/gf128hash.h b/lib/crypto/powerpc/gf128hash.h new file mode 100644 index 000000000000..629cd325d0c7 --- /dev/null +++ b/lib/crypto/powerpc/gf128hash.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * GHASH routines supporting VMX instructions on the Power 8 + * + * Copyright (C) 2015, 2019 International Business Machines Inc. + * Copyright (C) 2014 - 2018 Linaro Ltd. + * Copyright 2026 Google LLC + */ + +#include <asm/simd.h> +#include <asm/switch_to.h> +#include <linux/cpufeature.h> +#include <linux/jump_label.h> +#include <linux/preempt.h> +#include <linux/uaccess.h> + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto); + +void gcm_init_p8(u64 htable[4][2], const u8 h[16]); +void gcm_gmult_p8(u8 Xi[16], const u64 htable[4][2]); +void gcm_ghash_p8(u8 Xi[16], const u64 htable[4][2], const u8 *in, size_t len); + +#define ghash_preparekey_arch ghash_preparekey_arch +static void ghash_preparekey_arch(struct ghash_key *key, + const u8 raw_key[GHASH_BLOCK_SIZE]) +{ + ghash_key_to_polyval(raw_key, &key->h); + + if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) { + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + gcm_init_p8(key->htable, raw_key); + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + } else { + /* This reproduces gcm_init_p8() on both LE and BE systems. */ + key->htable[0][0] = 0; + key->htable[0][1] = 0xc200000000000000; + + key->htable[1][0] = 0; + key->htable[1][1] = le64_to_cpu(key->h.lo); + + key->htable[2][0] = le64_to_cpu(key->h.lo); + key->htable[2][1] = le64_to_cpu(key->h.hi); + + key->htable[3][0] = le64_to_cpu(key->h.hi); + key->htable[3][1] = 0; + } +} + +#define ghash_mul_arch ghash_mul_arch +static void ghash_mul_arch(struct polyval_elem *acc, + const struct ghash_key *key) +{ + if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) { + u8 ghash_acc[GHASH_BLOCK_SIZE]; + + polyval_acc_to_ghash(acc, ghash_acc); + + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + gcm_gmult_p8(ghash_acc, key->htable); + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + + ghash_acc_to_polyval(ghash_acc, acc); + memzero_explicit(ghash_acc, sizeof(ghash_acc)); + } else { + polyval_mul_generic(acc, &key->h); + } +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) { + u8 ghash_acc[GHASH_BLOCK_SIZE]; + + polyval_acc_to_ghash(acc, ghash_acc); + + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + gcm_ghash_p8(ghash_acc, key->htable, data, + nblocks * GHASH_BLOCK_SIZE); + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + + ghash_acc_to_polyval(ghash_acc, acc); + memzero_explicit(ghash_acc, sizeof(ghash_acc)); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (cpu_has_feature(CPU_FTR_ARCH_207S) && + (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO)) + static_branch_enable(&have_vec_crypto); +} diff --git a/lib/crypto/powerpc/ghashp8-ppc.pl b/lib/crypto/powerpc/ghashp8-ppc.pl new file mode 100644 index 000000000000..7c38eedc02cc --- /dev/null +++ b/lib/crypto/powerpc/ghashp8-ppc.pl @@ -0,0 +1,244 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 + +# This code is taken from the OpenSSL project but the author (Andy Polyakov) +# has relicensed it under the GPLv2. Therefore this program is free software; +# you can redistribute it and/or modify it under the terms of the GNU General +# Public License version 2 as published by the Free Software Foundation. +# +# The original headers, including the original license headers, are +# included below for completeness. + +# ==================================================================== +# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see https://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# GHASH for PowerISA v2.07. +# +# July 2014 +# +# Accurate performance measurements are problematic, because it's +# always virtualized setup with possibly throttled processor. +# Relative comparison is therefore more informative. This initial +# version is ~2.1x slower than hardware-assisted AES-128-CTR, ~12x +# faster than "4-bit" integer-only compiler-generated 64-bit code. +# "Initial version" means that there is room for futher improvement. + +$flavour=shift; +$output =shift; + +if ($flavour =~ /64/) { + $SIZE_T=8; + $LRSAVE=2*$SIZE_T; + $STU="stdu"; + $POP="ld"; + $PUSH="std"; +} elsif ($flavour =~ /32/) { + $SIZE_T=4; + $LRSAVE=$SIZE_T; + $STU="stwu"; + $POP="lwz"; + $PUSH="stw"; +} else { die "nonsense $flavour"; } + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or +( $xlate="${dir}../../../arch/powerpc/crypto/ppc-xlate.pl" and -f $xlate) or +die "can't locate ppc-xlate.pl"; + +open STDOUT,"| $^X $xlate $flavour $output" || die "can't call $xlate: $!"; + +my ($Xip,$Htbl,$inp,$len)=map("r$_",(3..6)); # argument block + +my ($Xl,$Xm,$Xh,$IN)=map("v$_",(0..3)); +my ($zero,$t0,$t1,$t2,$xC2,$H,$Hh,$Hl,$lemask)=map("v$_",(4..12)); +my $vrsave="r12"; + +$code=<<___; +.machine "any" + +.text + +.globl .gcm_init_p8 + lis r0,0xfff0 + li r8,0x10 + mfspr $vrsave,256 + li r9,0x20 + mtspr 256,r0 + li r10,0x30 + lvx_u $H,0,r4 # load H + le?xor r7,r7,r7 + le?addi r7,r7,0x8 # need a vperm start with 08 + le?lvsr 5,0,r7 + le?vspltisb 6,0x0f + le?vxor 5,5,6 # set a b-endian mask + le?vperm $H,$H,$H,5 + + vspltisb $xC2,-16 # 0xf0 + vspltisb $t0,1 # one + vaddubm $xC2,$xC2,$xC2 # 0xe0 + vxor $zero,$zero,$zero + vor $xC2,$xC2,$t0 # 0xe1 + vsldoi $xC2,$xC2,$zero,15 # 0xe1... + vsldoi $t1,$zero,$t0,1 # ...1 + vaddubm $xC2,$xC2,$xC2 # 0xc2... + vspltisb $t2,7 + vor $xC2,$xC2,$t1 # 0xc2....01 + vspltb $t1,$H,0 # most significant byte + vsl $H,$H,$t0 # H<<=1 + vsrab $t1,$t1,$t2 # broadcast carry bit + vand $t1,$t1,$xC2 + vxor $H,$H,$t1 # twisted H + + vsldoi $H,$H,$H,8 # twist even more ... + vsldoi $xC2,$zero,$xC2,8 # 0xc2.0 + vsldoi $Hl,$zero,$H,8 # ... and split + vsldoi $Hh,$H,$zero,8 + + stvx_u $xC2,0,r3 # save pre-computed table + stvx_u $Hl,r8,r3 + stvx_u $H, r9,r3 + stvx_u $Hh,r10,r3 + + mtspr 256,$vrsave + blr + .long 0 + .byte 0,12,0x14,0,0,0,2,0 + .long 0 +.size .gcm_init_p8,.-.gcm_init_p8 + +.globl .gcm_gmult_p8 + lis r0,0xfff8 + li r8,0x10 + mfspr $vrsave,256 + li r9,0x20 + mtspr 256,r0 + li r10,0x30 + lvx_u $IN,0,$Xip # load Xi + + lvx_u $Hl,r8,$Htbl # load pre-computed table + le?lvsl $lemask,r0,r0 + lvx_u $H, r9,$Htbl + le?vspltisb $t0,0x07 + lvx_u $Hh,r10,$Htbl + le?vxor $lemask,$lemask,$t0 + lvx_u $xC2,0,$Htbl + le?vperm $IN,$IN,$IN,$lemask + vxor $zero,$zero,$zero + + vpmsumd $Xl,$IN,$Hl # H.lo·Xi.lo + vpmsumd $Xm,$IN,$H # H.hi·Xi.lo+H.lo·Xi.hi + vpmsumd $Xh,$IN,$Hh # H.hi·Xi.hi + + vpmsumd $t2,$Xl,$xC2 # 1st phase + + vsldoi $t0,$Xm,$zero,8 + vsldoi $t1,$zero,$Xm,8 + vxor $Xl,$Xl,$t0 + vxor $Xh,$Xh,$t1 + + vsldoi $Xl,$Xl,$Xl,8 + vxor $Xl,$Xl,$t2 + + vsldoi $t1,$Xl,$Xl,8 # 2nd phase + vpmsumd $Xl,$Xl,$xC2 + vxor $t1,$t1,$Xh + vxor $Xl,$Xl,$t1 + + le?vperm $Xl,$Xl,$Xl,$lemask + stvx_u $Xl,0,$Xip # write out Xi + + mtspr 256,$vrsave + blr + .long 0 + .byte 0,12,0x14,0,0,0,2,0 + .long 0 +.size .gcm_gmult_p8,.-.gcm_gmult_p8 + +.globl .gcm_ghash_p8 + lis r0,0xfff8 + li r8,0x10 + mfspr $vrsave,256 + li r9,0x20 + mtspr 256,r0 + li r10,0x30 + lvx_u $Xl,0,$Xip # load Xi + + lvx_u $Hl,r8,$Htbl # load pre-computed table + le?lvsl $lemask,r0,r0 + lvx_u $H, r9,$Htbl + le?vspltisb $t0,0x07 + lvx_u $Hh,r10,$Htbl + le?vxor $lemask,$lemask,$t0 + lvx_u $xC2,0,$Htbl + le?vperm $Xl,$Xl,$Xl,$lemask + vxor $zero,$zero,$zero + + lvx_u $IN,0,$inp + addi $inp,$inp,16 + subi $len,$len,16 + le?vperm $IN,$IN,$IN,$lemask + vxor $IN,$IN,$Xl + b Loop + +.align 5 +Loop: + subic $len,$len,16 + vpmsumd $Xl,$IN,$Hl # H.lo·Xi.lo + subfe. r0,r0,r0 # borrow?-1:0 + vpmsumd $Xm,$IN,$H # H.hi·Xi.lo+H.lo·Xi.hi + and r0,r0,$len + vpmsumd $Xh,$IN,$Hh # H.hi·Xi.hi + add $inp,$inp,r0 + + vpmsumd $t2,$Xl,$xC2 # 1st phase + + vsldoi $t0,$Xm,$zero,8 + vsldoi $t1,$zero,$Xm,8 + vxor $Xl,$Xl,$t0 + vxor $Xh,$Xh,$t1 + + vsldoi $Xl,$Xl,$Xl,8 + vxor $Xl,$Xl,$t2 + lvx_u $IN,0,$inp + addi $inp,$inp,16 + + vsldoi $t1,$Xl,$Xl,8 # 2nd phase + vpmsumd $Xl,$Xl,$xC2 + le?vperm $IN,$IN,$IN,$lemask + vxor $t1,$t1,$Xh + vxor $IN,$IN,$t1 + vxor $IN,$IN,$Xl + beq Loop # did $len-=16 borrow? + + vxor $Xl,$Xl,$t1 + le?vperm $Xl,$Xl,$Xl,$lemask + stvx_u $Xl,0,$Xip # write out Xi + + mtspr 256,$vrsave + blr + .long 0 + .byte 0,12,0x14,0,0,0,4,0 + .long 0 +.size .gcm_ghash_p8,.-.gcm_ghash_p8 + +.asciz "GHASH for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>" +.align 2 +___ + +foreach (split("\n",$code)) { + if ($flavour =~ /le$/o) { # little-endian + s/le\?//o or + s/be\?/#be#/o; + } else { + s/le\?/#le#/o or + s/be\?//o; + } + print $_,"\n"; +} + +close STDOUT; # enforce flush diff --git a/lib/crypto/powerpc/md5-asm.S b/lib/crypto/powerpc/md5-asm.S deleted file mode 100644 index fa6bc440cf4a..000000000000 --- a/lib/crypto/powerpc/md5-asm.S +++ /dev/null @@ -1,235 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Fast MD5 implementation for PPC - * - * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> - */ -#include <asm/ppc_asm.h> -#include <asm/asm-offsets.h> -#include <asm/asm-compat.h> - -#define rHP r3 -#define rWP r4 - -#define rH0 r0 -#define rH1 r6 -#define rH2 r7 -#define rH3 r5 - -#define rW00 r8 -#define rW01 r9 -#define rW02 r10 -#define rW03 r11 -#define rW04 r12 -#define rW05 r14 -#define rW06 r15 -#define rW07 r16 -#define rW08 r17 -#define rW09 r18 -#define rW10 r19 -#define rW11 r20 -#define rW12 r21 -#define rW13 r22 -#define rW14 r23 -#define rW15 r24 - -#define rT0 r25 -#define rT1 r26 - -#define INITIALIZE \ - PPC_STLU r1,-INT_FRAME_SIZE(r1); \ - SAVE_GPRS(14, 26, r1) /* push registers onto stack */ - -#define FINALIZE \ - REST_GPRS(14, 26, r1); /* pop registers from stack */ \ - addi r1,r1,INT_FRAME_SIZE - -#ifdef __BIG_ENDIAN__ -#define LOAD_DATA(reg, off) \ - lwbrx reg,0,rWP; /* load data */ -#define INC_PTR \ - addi rWP,rWP,4; /* increment per word */ -#define NEXT_BLOCK /* nothing to do */ -#else -#define LOAD_DATA(reg, off) \ - lwz reg,off(rWP); /* load data */ -#define INC_PTR /* nothing to do */ -#define NEXT_BLOCK \ - addi rWP,rWP,64; /* increment per block */ -#endif - -#define R_00_15(a, b, c, d, w0, w1, p, q, off, k0h, k0l, k1h, k1l) \ - LOAD_DATA(w0, off) /* W */ \ - and rT0,b,c; /* 1: f = b and c */ \ - INC_PTR /* ptr++ */ \ - andc rT1,d,b; /* 1: f' = ~b and d */ \ - LOAD_DATA(w1, off+4) /* W */ \ - or rT0,rT0,rT1; /* 1: f = f or f' */ \ - addi w0,w0,k0l; /* 1: wk = w + k */ \ - add a,a,rT0; /* 1: a = a + f */ \ - addis w0,w0,k0h; /* 1: wk = w + k' */ \ - addis w1,w1,k1h; /* 2: wk = w + k */ \ - add a,a,w0; /* 1: a = a + wk */ \ - addi w1,w1,k1l; /* 2: wk = w + k' */ \ - rotrwi a,a,p; /* 1: a = a rotl x */ \ - add d,d,w1; /* 2: a = a + wk */ \ - add a,a,b; /* 1: a = a + b */ \ - and rT0,a,b; /* 2: f = b and c */ \ - andc rT1,c,a; /* 2: f' = ~b and d */ \ - or rT0,rT0,rT1; /* 2: f = f or f' */ \ - add d,d,rT0; /* 2: a = a + f */ \ - INC_PTR /* ptr++ */ \ - rotrwi d,d,q; /* 2: a = a rotl x */ \ - add d,d,a; /* 2: a = a + b */ - -#define R_16_31(a, b, c, d, w0, w1, p, q, k0h, k0l, k1h, k1l) \ - andc rT0,c,d; /* 1: f = c and ~d */ \ - and rT1,b,d; /* 1: f' = b and d */ \ - addi w0,w0,k0l; /* 1: wk = w + k */ \ - or rT0,rT0,rT1; /* 1: f = f or f' */ \ - addis w0,w0,k0h; /* 1: wk = w + k' */ \ - add a,a,rT0; /* 1: a = a + f */ \ - addi w1,w1,k1l; /* 2: wk = w + k */ \ - add a,a,w0; /* 1: a = a + wk */ \ - addis w1,w1,k1h; /* 2: wk = w + k' */ \ - andc rT0,b,c; /* 2: f = c and ~d */ \ - rotrwi a,a,p; /* 1: a = a rotl x */ \ - add a,a,b; /* 1: a = a + b */ \ - add d,d,w1; /* 2: a = a + wk */ \ - and rT1,a,c; /* 2: f' = b and d */ \ - or rT0,rT0,rT1; /* 2: f = f or f' */ \ - add d,d,rT0; /* 2: a = a + f */ \ - rotrwi d,d,q; /* 2: a = a rotl x */ \ - add d,d,a; /* 2: a = a +b */ - -#define R_32_47(a, b, c, d, w0, w1, p, q, k0h, k0l, k1h, k1l) \ - xor rT0,b,c; /* 1: f' = b xor c */ \ - addi w0,w0,k0l; /* 1: wk = w + k */ \ - xor rT1,rT0,d; /* 1: f = f xor f' */ \ - addis w0,w0,k0h; /* 1: wk = w + k' */ \ - add a,a,rT1; /* 1: a = a + f */ \ - addi w1,w1,k1l; /* 2: wk = w + k */ \ - add a,a,w0; /* 1: a = a + wk */ \ - addis w1,w1,k1h; /* 2: wk = w + k' */ \ - rotrwi a,a,p; /* 1: a = a rotl x */ \ - add d,d,w1; /* 2: a = a + wk */ \ - add a,a,b; /* 1: a = a + b */ \ - xor rT1,rT0,a; /* 2: f = b xor f' */ \ - add d,d,rT1; /* 2: a = a + f */ \ - rotrwi d,d,q; /* 2: a = a rotl x */ \ - add d,d,a; /* 2: a = a + b */ - -#define R_48_63(a, b, c, d, w0, w1, p, q, k0h, k0l, k1h, k1l) \ - addi w0,w0,k0l; /* 1: w = w + k */ \ - orc rT0,b,d; /* 1: f = b or ~d */ \ - addis w0,w0,k0h; /* 1: w = w + k' */ \ - xor rT0,rT0,c; /* 1: f = f xor c */ \ - add a,a,w0; /* 1: a = a + wk */ \ - addi w1,w1,k1l; /* 2: w = w + k */ \ - add a,a,rT0; /* 1: a = a + f */ \ - addis w1,w1,k1h; /* 2: w = w + k' */ \ - rotrwi a,a,p; /* 1: a = a rotl x */ \ - add a,a,b; /* 1: a = a + b */ \ - orc rT0,a,c; /* 2: f = b or ~d */ \ - add d,d,w1; /* 2: a = a + wk */ \ - xor rT0,rT0,b; /* 2: f = f xor c */ \ - add d,d,rT0; /* 2: a = a + f */ \ - rotrwi d,d,q; /* 2: a = a rotl x */ \ - add d,d,a; /* 2: a = a + b */ - -_GLOBAL(ppc_md5_transform) - INITIALIZE - - mtctr r5 - lwz rH0,0(rHP) - lwz rH1,4(rHP) - lwz rH2,8(rHP) - lwz rH3,12(rHP) - -ppc_md5_main: - R_00_15(rH0, rH1, rH2, rH3, rW00, rW01, 25, 20, 0, - 0xd76b, -23432, 0xe8c8, -18602) - R_00_15(rH2, rH3, rH0, rH1, rW02, rW03, 15, 10, 8, - 0x2420, 0x70db, 0xc1be, -12562) - R_00_15(rH0, rH1, rH2, rH3, rW04, rW05, 25, 20, 16, - 0xf57c, 0x0faf, 0x4788, -14806) - R_00_15(rH2, rH3, rH0, rH1, rW06, rW07, 15, 10, 24, - 0xa830, 0x4613, 0xfd47, -27391) - R_00_15(rH0, rH1, rH2, rH3, rW08, rW09, 25, 20, 32, - 0x6981, -26408, 0x8b45, -2129) - R_00_15(rH2, rH3, rH0, rH1, rW10, rW11, 15, 10, 40, - 0xffff, 0x5bb1, 0x895d, -10306) - R_00_15(rH0, rH1, rH2, rH3, rW12, rW13, 25, 20, 48, - 0x6b90, 0x1122, 0xfd98, 0x7193) - R_00_15(rH2, rH3, rH0, rH1, rW14, rW15, 15, 10, 56, - 0xa679, 0x438e, 0x49b4, 0x0821) - - R_16_31(rH0, rH1, rH2, rH3, rW01, rW06, 27, 23, - 0x0d56, 0x6e0c, 0x1810, 0x6d2d) - R_16_31(rH2, rH3, rH0, rH1, rW11, rW00, 18, 12, - 0x9d02, -32109, 0x124c, 0x2332) - R_16_31(rH0, rH1, rH2, rH3, rW05, rW10, 27, 23, - 0x8ea7, 0x4a33, 0x0245, -18270) - R_16_31(rH2, rH3, rH0, rH1, rW15, rW04, 18, 12, - 0x8eee, -8608, 0xf258, -5095) - R_16_31(rH0, rH1, rH2, rH3, rW09, rW14, 27, 23, - 0x969d, -10697, 0x1cbe, -15288) - R_16_31(rH2, rH3, rH0, rH1, rW03, rW08, 18, 12, - 0x3317, 0x3e99, 0xdbd9, 0x7c15) - R_16_31(rH0, rH1, rH2, rH3, rW13, rW02, 27, 23, - 0xac4b, 0x7772, 0xd8cf, 0x331d) - R_16_31(rH2, rH3, rH0, rH1, rW07, rW12, 18, 12, - 0x6a28, 0x6dd8, 0x219a, 0x3b68) - - R_32_47(rH0, rH1, rH2, rH3, rW05, rW08, 28, 21, - 0x29cb, 0x28e5, 0x4218, -7788) - R_32_47(rH2, rH3, rH0, rH1, rW11, rW14, 16, 9, - 0x473f, 0x06d1, 0x3aae, 0x3036) - R_32_47(rH0, rH1, rH2, rH3, rW01, rW04, 28, 21, - 0xaea1, -15134, 0x640b, -11295) - R_32_47(rH2, rH3, rH0, rH1, rW07, rW10, 16, 9, - 0x8f4c, 0x4887, 0xbc7c, -22499) - R_32_47(rH0, rH1, rH2, rH3, rW13, rW00, 28, 21, - 0x7eb8, -27199, 0x00ea, 0x6050) - R_32_47(rH2, rH3, rH0, rH1, rW03, rW06, 16, 9, - 0xe01a, 0x22fe, 0x4447, 0x69c5) - R_32_47(rH0, rH1, rH2, rH3, rW09, rW12, 28, 21, - 0xb7f3, 0x0253, 0x59b1, 0x4d5b) - R_32_47(rH2, rH3, rH0, rH1, rW15, rW02, 16, 9, - 0x4701, -27017, 0xc7bd, -19859) - - R_48_63(rH0, rH1, rH2, rH3, rW00, rW07, 26, 22, - 0x0988, -1462, 0x4c70, -19401) - R_48_63(rH2, rH3, rH0, rH1, rW14, rW05, 17, 11, - 0xadaf, -5221, 0xfc99, 0x66f7) - R_48_63(rH0, rH1, rH2, rH3, rW12, rW03, 26, 22, - 0x7e80, -16418, 0xba1e, -25587) - R_48_63(rH2, rH3, rH0, rH1, rW10, rW01, 17, 11, - 0x4130, 0x380d, 0xe0c5, 0x738d) - lwz rW00,0(rHP) - R_48_63(rH0, rH1, rH2, rH3, rW08, rW15, 26, 22, - 0xe837, -30770, 0xde8a, 0x69e8) - lwz rW14,4(rHP) - R_48_63(rH2, rH3, rH0, rH1, rW06, rW13, 17, 11, - 0x9e79, 0x260f, 0x256d, -27941) - lwz rW12,8(rHP) - R_48_63(rH0, rH1, rH2, rH3, rW04, rW11, 26, 22, - 0xab75, -20775, 0x4f9e, -28397) - lwz rW10,12(rHP) - R_48_63(rH2, rH3, rH0, rH1, rW02, rW09, 17, 11, - 0x662b, 0x7c56, 0x11b2, 0x0358) - - add rH0,rH0,rW00 - stw rH0,0(rHP) - add rH1,rH1,rW14 - stw rH1,4(rHP) - add rH2,rH2,rW12 - stw rH2,8(rHP) - add rH3,rH3,rW10 - stw rH3,12(rHP) - NEXT_BLOCK - - bdnz ppc_md5_main - - FINALIZE - blr diff --git a/lib/crypto/powerpc/md5.h b/lib/crypto/powerpc/md5.h deleted file mode 100644 index 540b08e34d1d..000000000000 --- a/lib/crypto/powerpc/md5.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * MD5 optimized for PowerPC - */ - -void ppc_md5_transform(u32 *state, const u8 *data, size_t nblocks); - -static void md5_blocks(struct md5_block_state *state, - const u8 *data, size_t nblocks) -{ - ppc_md5_transform(state->h, data, nblocks); -} diff --git a/lib/crypto/riscv/gf128hash.h b/lib/crypto/riscv/gf128hash.h new file mode 100644 index 000000000000..4301a0384f60 --- /dev/null +++ b/lib/crypto/riscv/gf128hash.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * GHASH, RISC-V optimized + * + * Copyright (C) 2023 VRULL GmbH + * Copyright (C) 2023 SiFive, Inc. + * Copyright 2026 Google LLC + */ + +#include <asm/simd.h> +#include <asm/vector.h> + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkg); + +asmlinkage void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE], + const u8 key[GHASH_BLOCK_SIZE], + const u8 *data, size_t nblocks); + +#define ghash_preparekey_arch ghash_preparekey_arch +static void ghash_preparekey_arch(struct ghash_key *key, + const u8 raw_key[GHASH_BLOCK_SIZE]) +{ + /* Save key in POLYVAL format for fallback */ + ghash_key_to_polyval(raw_key, &key->h); + + /* Save key in GHASH format for zvkg */ + memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE); +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_zvkg) && likely(may_use_simd())) { + u8 ghash_acc[GHASH_BLOCK_SIZE]; + + polyval_acc_to_ghash(acc, ghash_acc); + + kernel_vector_begin(); + ghash_zvkg(ghash_acc, key->h_raw, data, nblocks); + kernel_vector_end(); + + ghash_acc_to_polyval(ghash_acc, acc); + memzero_explicit(ghash_acc, sizeof(ghash_acc)); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (riscv_isa_extension_available(NULL, ZVKG) && + riscv_vector_vlen() >= 128) + static_branch_enable(&have_zvkg); +} diff --git a/lib/crypto/riscv/ghash-riscv64-zvkg.S b/lib/crypto/riscv/ghash-riscv64-zvkg.S new file mode 100644 index 000000000000..6a2a2f2bc7c8 --- /dev/null +++ b/lib/crypto/riscv/ghash-riscv64-zvkg.S @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */ +// +// This file is dual-licensed, meaning that you can use it under your +// choice of either of the following two licenses: +// +// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. +// +// Licensed under the Apache License 2.0 (the "License"). You can obtain +// a copy in the file LICENSE in the source distribution or at +// https://www.openssl.org/source/license.html +// +// or +// +// Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu> +// Copyright (c) 2023, Jerry Shih <jerry.shih@sifive.com> +// Copyright 2024 Google LLC +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// The generated code of this file depends on the following RISC-V extensions: +// - RV64I +// - RISC-V Vector ('V') with VLEN >= 128 +// - RISC-V Vector GCM/GMAC extension ('Zvkg') + +#include <linux/linkage.h> + +.text +.option arch, +zvkg + +#define ACCUMULATOR a0 +#define KEY a1 +#define DATA a2 +#define NBLOCKS a3 + +// void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE], +// const u8 key[GHASH_BLOCK_SIZE], +// const u8 *data, size_t nblocks); +// +// |nblocks| must be nonzero. +SYM_FUNC_START(ghash_zvkg) + vsetivli zero, 4, e32, m1, ta, ma + vle32.v v1, (ACCUMULATOR) + vle32.v v2, (KEY) +.Lnext_block: + vle32.v v3, (DATA) + vghsh.vv v1, v2, v3 + addi DATA, DATA, 16 + addi NBLOCKS, NBLOCKS, -1 + bnez NBLOCKS, .Lnext_block + + vse32.v v1, (ACCUMULATOR) + ret +SYM_FUNC_END(ghash_zvkg) diff --git a/lib/crypto/riscv/sm3-riscv64-zvksh-zvkb.S b/lib/crypto/riscv/sm3-riscv64-zvksh-zvkb.S new file mode 100644 index 000000000000..a1d4468b0485 --- /dev/null +++ b/lib/crypto/riscv/sm3-riscv64-zvksh-zvkb.S @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */ +// +// This file is dual-licensed, meaning that you can use it under your +// choice of either of the following two licenses: +// +// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. +// +// Licensed under the Apache License 2.0 (the "License"). You can obtain +// a copy in the file LICENSE in the source distribution or at +// https://www.openssl.org/source/license.html +// +// or +// +// Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu> +// Copyright (c) 2023, Jerry Shih <jerry.shih@sifive.com> +// Copyright 2024 Google LLC +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// The generated code of this file depends on the following RISC-V extensions: +// - RV64I +// - RISC-V Vector ('V') with VLEN >= 128 +// - RISC-V Vector SM3 Secure Hash extension ('Zvksh') +// - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb') + +#include <linux/linkage.h> + +.text +.option arch, +zvksh, +zvkb + +#define STATEP a0 +#define DATA a1 +#define NUM_BLOCKS a2 + +#define STATE v0 // LMUL=2 +#define PREV_STATE v2 // LMUL=2 +#define W0 v4 // LMUL=2 +#define W1 v6 // LMUL=2 +#define VTMP v8 // LMUL=2 + +.macro sm3_8rounds i, w0, w1 + // Do 4 rounds using W_{0+i}..W_{7+i}. + vsm3c.vi STATE, \w0, \i + 0 + vslidedown.vi VTMP, \w0, 2 + vsm3c.vi STATE, VTMP, \i + 1 + + // Compute W_{4+i}..W_{11+i}. + vslidedown.vi VTMP, \w0, 4 + vslideup.vi VTMP, \w1, 4 + + // Do 4 rounds using W_{4+i}..W_{11+i}. + vsm3c.vi STATE, VTMP, \i + 2 + vslidedown.vi VTMP, VTMP, 2 + vsm3c.vi STATE, VTMP, \i + 3 + +.if \i < 28 + // Compute W_{16+i}..W_{23+i}. + vsm3me.vv \w0, \w1, \w0 +.endif + // For the next 8 rounds, w0 and w1 are swapped. +.endm + +// void sm3_transform_zvksh_zvkb(struct sm3_block_state *state, +// const u8 *data, size_t nblocks); +SYM_FUNC_START(sm3_transform_zvksh_zvkb) + + // Load the state and endian-swap each 32-bit word. + vsetivli zero, 8, e32, m2, ta, ma + vle32.v STATE, (STATEP) + vrev8.v STATE, STATE + +.Lnext_block: + addi NUM_BLOCKS, NUM_BLOCKS, -1 + + // Save the previous state, as it's needed later. + vmv.v.v PREV_STATE, STATE + + // Load the next 512-bit message block into W0-W1. + vle32.v W0, (DATA) + addi DATA, DATA, 32 + vle32.v W1, (DATA) + addi DATA, DATA, 32 + + // Do the 64 rounds of SM3. + sm3_8rounds 0, W0, W1 + sm3_8rounds 4, W1, W0 + sm3_8rounds 8, W0, W1 + sm3_8rounds 12, W1, W0 + sm3_8rounds 16, W0, W1 + sm3_8rounds 20, W1, W0 + sm3_8rounds 24, W0, W1 + sm3_8rounds 28, W1, W0 + + // XOR in the previous state. + vxor.vv STATE, STATE, PREV_STATE + + // Repeat if more blocks remain. + bnez NUM_BLOCKS, .Lnext_block + + // Store the new state and return. + vrev8.v STATE, STATE + vse32.v STATE, (STATEP) + ret +SYM_FUNC_END(sm3_transform_zvksh_zvkb) diff --git a/lib/crypto/riscv/sm3.h b/lib/crypto/riscv/sm3.h new file mode 100644 index 000000000000..c1fbee7094e6 --- /dev/null +++ b/lib/crypto/riscv/sm3.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SM3 using the RISC-V vector crypto extensions + * + * Copyright (C) 2023 VRULL GmbH + * Author: Heiko Stuebner <heiko.stuebner@vrull.eu> + * + * Copyright (C) 2023 SiFive, Inc. + * Author: Jerry Shih <jerry.shih@sifive.com> + */ + +#include <asm/simd.h> +#include <asm/vector.h> + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions); + +asmlinkage void sm3_transform_zvksh_zvkb(struct sm3_block_state *state, + const u8 *data, size_t nblocks); + +static void sm3_blocks(struct sm3_block_state *state, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_extensions) && likely(may_use_simd())) { + kernel_vector_begin(); + sm3_transform_zvksh_zvkb(state, data, nblocks); + kernel_vector_end(); + } else { + sm3_blocks_generic(state, data, nblocks); + } +} + +#define sm3_mod_init_arch sm3_mod_init_arch +static void sm3_mod_init_arch(void) +{ + if (riscv_isa_extension_available(NULL, ZVKSH) && + riscv_isa_extension_available(NULL, ZVKB) && + riscv_vector_vlen() >= 128) + static_branch_enable(&have_extensions); +} diff --git a/lib/crypto/s390/gf128hash.h b/lib/crypto/s390/gf128hash.h new file mode 100644 index 000000000000..1e46ce4bca40 --- /dev/null +++ b/lib/crypto/s390/gf128hash.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * GHASH optimized using the CP Assist for Cryptographic Functions (CPACF) + * + * Copyright 2026 Google LLC + */ +#include <asm/cpacf.h> +#include <linux/cpufeature.h> + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_ghash); + +#define ghash_preparekey_arch ghash_preparekey_arch +static void ghash_preparekey_arch(struct ghash_key *key, + const u8 raw_key[GHASH_BLOCK_SIZE]) +{ + /* Save key in POLYVAL format for fallback */ + ghash_key_to_polyval(raw_key, &key->h); + + /* Save key in GHASH format for CPACF_KIMD_GHASH */ + memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE); +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_cpacf_ghash)) { + /* + * CPACF_KIMD_GHASH requires the accumulator and key in a single + * buffer, each using the GHASH convention. + */ + u8 ctx[2][GHASH_BLOCK_SIZE] __aligned(8); + + polyval_acc_to_ghash(acc, ctx[0]); + memcpy(ctx[1], key->h_raw, GHASH_BLOCK_SIZE); + + cpacf_kimd(CPACF_KIMD_GHASH, ctx, data, + nblocks * GHASH_BLOCK_SIZE); + + ghash_acc_to_polyval(ctx[0], acc); + memzero_explicit(ctx, sizeof(ctx)); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (cpu_have_feature(S390_CPU_FEATURE_MSA) && + cpacf_query_func(CPACF_KIMD, CPACF_KIMD_GHASH)) + static_branch_enable(&have_cpacf_ghash); +} diff --git a/lib/crypto/sha1.c b/lib/crypto/sha1.c index daf18c862fdf..b687b89d97cb 100644 --- a/lib/crypto/sha1.c +++ b/lib/crypto/sha1.c @@ -12,7 +12,7 @@ #include <linux/string.h> #include <linux/unaligned.h> #include <linux/wordpart.h> -#include "fips.h" +#include "fips-sha.h" static const struct sha1_block_state sha1_iv = { .h = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 5d6b77e7e141..e8c346f563c5 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -17,7 +17,7 @@ #include <linux/string.h> #include <linux/unaligned.h> #include <linux/wordpart.h> -#include "fips.h" +#include "fips-sha.h" static const struct sha256_block_state sha224_iv = { .h = { diff --git a/lib/crypto/sha3.c b/lib/crypto/sha3.c index 32b7074de792..286a2373c156 100644 --- a/lib/crypto/sha3.c +++ b/lib/crypto/sha3.c @@ -17,7 +17,7 @@ #include <linux/kernel.h> #include <linux/module.h> #include <linux/unaligned.h> -#include "fips.h" +#include "fips-sha.h" /* * On some 32-bit architectures, such as h8300, GCC ends up using over 1 KB of diff --git a/lib/crypto/sha512.c b/lib/crypto/sha512.c index 605eab51aabd..0dd6fb4ca15b 100644 --- a/lib/crypto/sha512.c +++ b/lib/crypto/sha512.c @@ -17,7 +17,7 @@ #include <linux/string.h> #include <linux/unaligned.h> #include <linux/wordpart.h> -#include "fips.h" +#include "fips-sha.h" static const struct sha512_block_state sha384_iv = { .h = { diff --git a/lib/crypto/sm3.c b/lib/crypto/sm3.c index c6b9ad8a3ac6..b02b8a247adf 100644 --- a/lib/crypto/sm3.c +++ b/lib/crypto/sm3.c @@ -15,6 +15,13 @@ #include <linux/string.h> #include <linux/unaligned.h> +static const struct sm3_block_state sm3_iv = { + .h = { + SM3_IVA, SM3_IVB, SM3_IVC, SM3_IVD, + SM3_IVE, SM3_IVF, SM3_IVG, SM3_IVH, + }, +}; + static const u32 ____cacheline_aligned K[64] = { 0x79cc4519, 0xf3988a32, 0xe7311465, 0xce6228cb, 0x9cc45197, 0x3988a32f, 0x7311465e, 0xe6228cbc, @@ -72,18 +79,19 @@ static const u32 ____cacheline_aligned K[64] = { ^ rol32(W[(i-13) & 0x0f], 7) \ ^ W[(i-6) & 0x0f]) -static void sm3_transform(struct sm3_state *sctx, u8 const *data, u32 W[16]) +static void sm3_block_generic(struct sm3_block_state *state, + const u8 data[SM3_BLOCK_SIZE], u32 W[16]) { u32 a, b, c, d, e, f, g, h, ss1, ss2; - a = sctx->state[0]; - b = sctx->state[1]; - c = sctx->state[2]; - d = sctx->state[3]; - e = sctx->state[4]; - f = sctx->state[5]; - g = sctx->state[6]; - h = sctx->state[7]; + a = state->h[0]; + b = state->h[1]; + c = state->h[2]; + d = state->h[3]; + e = state->h[4]; + f = state->h[5]; + g = state->h[6]; + h = state->h[7]; R1(a, b, c, d, e, f, g, h, K[0], I(0), I(4)); R1(d, a, b, c, h, e, f, g, K[1], I(1), I(5)); @@ -153,14 +161,14 @@ static void sm3_transform(struct sm3_state *sctx, u8 const *data, u32 W[16]) R2(c, d, a, b, g, h, e, f, K[62], W1(62), W2(66)); R2(b, c, d, a, f, g, h, e, K[63], W1(63), W2(67)); - sctx->state[0] ^= a; - sctx->state[1] ^= b; - sctx->state[2] ^= c; - sctx->state[3] ^= d; - sctx->state[4] ^= e; - sctx->state[5] ^= f; - sctx->state[6] ^= g; - sctx->state[7] ^= h; + state->h[0] ^= a; + state->h[1] ^= b; + state->h[2] ^= c; + state->h[3] ^= d; + state->h[4] ^= e; + state->h[5] ^= f; + state->h[6] ^= g; + state->h[7] ^= h; } #undef R #undef R1 @@ -169,18 +177,114 @@ static void sm3_transform(struct sm3_state *sctx, u8 const *data, u32 W[16]) #undef W1 #undef W2 -void sm3_block_generic(struct sm3_state *sctx, u8 const *data, int blocks) +static void __maybe_unused sm3_blocks_generic(struct sm3_block_state *state, + const u8 *data, size_t nblocks) { u32 W[16]; do { - sm3_transform(sctx, data, W); + sm3_block_generic(state, data, W); data += SM3_BLOCK_SIZE; - } while (--blocks); + } while (--nblocks); memzero_explicit(W, sizeof(W)); } -EXPORT_SYMBOL_GPL(sm3_block_generic); -MODULE_DESCRIPTION("Generic SM3 library"); +#ifdef CONFIG_CRYPTO_LIB_SM3_ARCH +#include "sm3.h" /* $(SRCARCH)/sm3.h */ +#else +#define sm3_blocks sm3_blocks_generic +#endif + +void sm3_init(struct sm3_ctx *ctx) +{ + ctx->state = sm3_iv; + ctx->bytecount = 0; +} +EXPORT_SYMBOL_GPL(sm3_init); + +void sm3_update(struct sm3_ctx *ctx, const u8 *data, size_t len) +{ + size_t partial = ctx->bytecount % SM3_BLOCK_SIZE; + + ctx->bytecount += len; + + if (partial + len >= SM3_BLOCK_SIZE) { + size_t nblocks; + + if (partial) { + size_t l = SM3_BLOCK_SIZE - partial; + + memcpy(&ctx->buf[partial], data, l); + data += l; + len -= l; + + sm3_blocks(&ctx->state, ctx->buf, 1); + } + + nblocks = len / SM3_BLOCK_SIZE; + len %= SM3_BLOCK_SIZE; + + if (nblocks) { + sm3_blocks(&ctx->state, data, nblocks); + data += nblocks * SM3_BLOCK_SIZE; + } + partial = 0; + } + if (len) + memcpy(&ctx->buf[partial], data, len); +} +EXPORT_SYMBOL_GPL(sm3_update); + +static void __sm3_final(struct sm3_ctx *ctx, u8 out[SM3_DIGEST_SIZE]) +{ + u64 bitcount = ctx->bytecount << 3; + size_t partial = ctx->bytecount % SM3_BLOCK_SIZE; + + ctx->buf[partial++] = 0x80; + if (partial > SM3_BLOCK_SIZE - 8) { + memset(&ctx->buf[partial], 0, SM3_BLOCK_SIZE - partial); + sm3_blocks(&ctx->state, ctx->buf, 1); + partial = 0; + } + memset(&ctx->buf[partial], 0, SM3_BLOCK_SIZE - 8 - partial); + *(__be64 *)&ctx->buf[SM3_BLOCK_SIZE - 8] = cpu_to_be64(bitcount); + sm3_blocks(&ctx->state, ctx->buf, 1); + + for (size_t i = 0; i < SM3_DIGEST_SIZE; i += 4) + put_unaligned_be32(ctx->state.h[i / 4], out + i); +} + +void sm3_final(struct sm3_ctx *ctx, u8 out[SM3_DIGEST_SIZE]) +{ + __sm3_final(ctx, out); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(sm3_final); + +void sm3(const u8 *data, size_t len, u8 out[SM3_DIGEST_SIZE]) +{ + struct sm3_ctx ctx; + + sm3_init(&ctx); + sm3_update(&ctx, data, len); + sm3_final(&ctx, out); +} +EXPORT_SYMBOL_GPL(sm3); + +#ifdef sm3_mod_init_arch +static int __init sm3_mod_init(void) +{ + sm3_mod_init_arch(); + return 0; +} +subsys_initcall(sm3_mod_init); + +static void __exit sm3_mod_exit(void) +{ +} +module_exit(sm3_mod_exit); +#endif + +MODULE_DESCRIPTION("SM3 library functions"); MODULE_LICENSE("GPL v2"); diff --git a/lib/crypto/sparc/md5.h b/lib/crypto/sparc/md5.h deleted file mode 100644 index 3995f3e075eb..000000000000 --- a/lib/crypto/sparc/md5.h +++ /dev/null @@ -1,48 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * MD5 accelerated using the sparc64 crypto opcodes - * - * Copyright (c) Alan Smithee. - * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> - * Copyright (c) Jean-Francois Dive <jef@linuxbe.org> - * Copyright (c) Mathias Krause <minipli@googlemail.com> - * Copyright (c) Cryptoapi developers. - * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> - */ - -#include <asm/elf.h> -#include <asm/opcodes.h> -#include <asm/pstate.h> - -static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_md5_opcodes); - -asmlinkage void md5_sparc64_transform(struct md5_block_state *state, - const u8 *data, size_t nblocks); - -static void md5_blocks(struct md5_block_state *state, - const u8 *data, size_t nblocks) -{ - if (static_branch_likely(&have_md5_opcodes)) { - cpu_to_le32_array(state->h, ARRAY_SIZE(state->h)); - md5_sparc64_transform(state, data, nblocks); - le32_to_cpu_array(state->h, ARRAY_SIZE(state->h)); - } else { - md5_blocks_generic(state, data, nblocks); - } -} - -#define md5_mod_init_arch md5_mod_init_arch -static void md5_mod_init_arch(void) -{ - unsigned long cfr; - - if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) - return; - - __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); - if (!(cfr & CFR_MD5)) - return; - - static_branch_enable(&have_md5_opcodes); - pr_info("Using sparc64 md5 opcode optimized MD5 implementation\n"); -} diff --git a/lib/crypto/sparc/md5_asm.S b/lib/crypto/sparc/md5_asm.S deleted file mode 100644 index 60b544e4d205..000000000000 --- a/lib/crypto/sparc/md5_asm.S +++ /dev/null @@ -1,70 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#include <linux/linkage.h> -#include <asm/opcodes.h> -#include <asm/visasm.h> - -ENTRY(md5_sparc64_transform) - /* %o0 = digest, %o1 = data, %o2 = rounds */ - VISEntryHalf - ld [%o0 + 0x00], %f0 - ld [%o0 + 0x04], %f1 - andcc %o1, 0x7, %g0 - ld [%o0 + 0x08], %f2 - bne,pn %xcc, 10f - ld [%o0 + 0x0c], %f3 - -1: - ldd [%o1 + 0x00], %f8 - ldd [%o1 + 0x08], %f10 - ldd [%o1 + 0x10], %f12 - ldd [%o1 + 0x18], %f14 - ldd [%o1 + 0x20], %f16 - ldd [%o1 + 0x28], %f18 - ldd [%o1 + 0x30], %f20 - ldd [%o1 + 0x38], %f22 - - MD5 - - subcc %o2, 1, %o2 - bne,pt %xcc, 1b - add %o1, 0x40, %o1 - -5: - st %f0, [%o0 + 0x00] - st %f1, [%o0 + 0x04] - st %f2, [%o0 + 0x08] - st %f3, [%o0 + 0x0c] - retl - VISExitHalf -10: - alignaddr %o1, %g0, %o1 - - ldd [%o1 + 0x00], %f10 -1: - ldd [%o1 + 0x08], %f12 - ldd [%o1 + 0x10], %f14 - ldd [%o1 + 0x18], %f16 - ldd [%o1 + 0x20], %f18 - ldd [%o1 + 0x28], %f20 - ldd [%o1 + 0x30], %f22 - ldd [%o1 + 0x38], %f24 - ldd [%o1 + 0x40], %f26 - - faligndata %f10, %f12, %f8 - faligndata %f12, %f14, %f10 - faligndata %f14, %f16, %f12 - faligndata %f16, %f18, %f14 - faligndata %f18, %f20, %f16 - faligndata %f20, %f22, %f18 - faligndata %f22, %f24, %f20 - faligndata %f24, %f26, %f22 - - MD5 - - subcc %o2, 1, %o2 - fsrc2 %f26, %f10 - bne,pt %xcc, 1b - add %o1, 0x40, %o1 - - ba,a,pt %xcc, 5b -ENDPROC(md5_sparc64_transform) diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig index 0de289b429a9..e121114624da 100644 --- a/lib/crypto/tests/Kconfig +++ b/lib/crypto/tests/Kconfig @@ -1,9 +1,36 @@ # SPDX-License-Identifier: GPL-2.0-or-later +config CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST + tristate "KUnit tests for AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC" if !KUNIT_ALL_TESTS + depends on KUNIT && CRYPTO_LIB_AES_CBC_MACS + default KUNIT_ALL_TESTS + select CRYPTO_LIB_BENCHMARK_VISIBLE + help + KUnit tests for the AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC message + authentication codes. + +config CRYPTO_LIB_AES_CCM_KUNIT_TEST + tristate "KUnit tests for AES-CCM" if !KUNIT_ALL_TESTS + depends on KUNIT && CRYPTO_LIB_AES_CCM + default KUNIT_ALL_TESTS + select CRYPTO_LIB_BENCHMARK_VISIBLE + # This test uses BLAKE2s, but that's always built-in. + help + KUnit tests for the AES-CCM authenticated encryption algorithm. + +config CRYPTO_LIB_AES_GCM_KUNIT_TEST + tristate "KUnit tests for AES-GCM" if !KUNIT_ALL_TESTS + depends on KUNIT && CRYPTO_LIB_AES_GCM + default KUNIT_ALL_TESTS + select CRYPTO_LIB_BENCHMARK_VISIBLE + # This test uses BLAKE2s, but that's always built-in. + help + KUnit tests for the AES-GCM authenticated encryption algorithm. + config CRYPTO_LIB_BLAKE2B_KUNIT_TEST tristate "KUnit tests for BLAKE2b" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_BLAKE2B - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the BLAKE2b cryptographic hash function. @@ -11,25 +38,42 @@ config CRYPTO_LIB_BLAKE2B_KUNIT_TEST config CRYPTO_LIB_BLAKE2S_KUNIT_TEST tristate "KUnit tests for BLAKE2s" if !KUNIT_ALL_TESTS depends on KUNIT - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE # No need to depend on CRYPTO_LIB_BLAKE2S here, as that option doesn't # exist; the BLAKE2s code is always built-in for the /dev/random driver. help KUnit tests for the BLAKE2s cryptographic hash function. +config CRYPTO_LIB_CHACHA20POLY1305_KUNIT_TEST + tristate "KUnit tests for ChaCha20Poly1305" if !KUNIT_ALL_TESTS + depends on KUNIT && CRYPTO_LIB_CHACHA20POLY1305 + default KUNIT_ALL_TESTS + select CRYPTO_LIB_BENCHMARK_VISIBLE + help + KUnit tests for the ChaCha20Poly1305 authenticated encryption + algorithm. + config CRYPTO_LIB_CURVE25519_KUNIT_TEST tristate "KUnit tests for Curve25519" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_CURVE25519 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the Curve25519 Diffie-Hellman function. +config CRYPTO_LIB_GHASH_KUNIT_TEST + tristate "KUnit tests for GHASH" if !KUNIT_ALL_TESTS + depends on KUNIT && CRYPTO_LIB_GF128HASH + default KUNIT_ALL_TESTS + select CRYPTO_LIB_BENCHMARK_VISIBLE + help + KUnit tests for the GHASH library functions. + config CRYPTO_LIB_MD5_KUNIT_TEST tristate "KUnit tests for MD5" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_MD5 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the MD5 cryptographic hash function and its @@ -38,7 +82,7 @@ config CRYPTO_LIB_MD5_KUNIT_TEST config CRYPTO_LIB_MLDSA_KUNIT_TEST tristate "KUnit tests for ML-DSA" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_MLDSA - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the ML-DSA digital signature algorithm. @@ -46,22 +90,22 @@ config CRYPTO_LIB_MLDSA_KUNIT_TEST config CRYPTO_LIB_NH_KUNIT_TEST tristate "KUnit tests for NH" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_NH - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS help KUnit tests for the NH almost-universal hash function. config CRYPTO_LIB_POLY1305_KUNIT_TEST tristate "KUnit tests for Poly1305" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_POLY1305 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the Poly1305 library functions. config CRYPTO_LIB_POLYVAL_KUNIT_TEST tristate "KUnit tests for POLYVAL" if !KUNIT_ALL_TESTS - depends on KUNIT && CRYPTO_LIB_POLYVAL - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + depends on KUNIT && CRYPTO_LIB_GF128HASH + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the POLYVAL library functions. @@ -69,7 +113,7 @@ config CRYPTO_LIB_POLYVAL_KUNIT_TEST config CRYPTO_LIB_SHA1_KUNIT_TEST tristate "KUnit tests for SHA-1" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_SHA1 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the SHA-1 cryptographic hash function and its @@ -80,7 +124,7 @@ config CRYPTO_LIB_SHA1_KUNIT_TEST config CRYPTO_LIB_SHA256_KUNIT_TEST tristate "KUnit tests for SHA-224 and SHA-256" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_SHA256 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the SHA-224 and SHA-256 cryptographic hash functions @@ -91,7 +135,7 @@ config CRYPTO_LIB_SHA256_KUNIT_TEST config CRYPTO_LIB_SHA512_KUNIT_TEST tristate "KUnit tests for SHA-384 and SHA-512" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_SHA512 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the SHA-384 and SHA-512 cryptographic hash functions @@ -100,13 +144,53 @@ config CRYPTO_LIB_SHA512_KUNIT_TEST config CRYPTO_LIB_SHA3_KUNIT_TEST tristate "KUnit tests for SHA-3" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_SHA3 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the SHA3 cryptographic hash and XOF functions, including SHA3-224, SHA3-256, SHA3-384, SHA3-512, SHAKE128 and SHAKE256. +config CRYPTO_LIB_SM3_KUNIT_TEST + tristate "KUnit tests for SM3" if !KUNIT_ALL_TESTS + depends on KUNIT && CRYPTO_LIB_SM3 + default KUNIT_ALL_TESTS + select CRYPTO_LIB_BENCHMARK_VISIBLE + help + KUnit tests for the SM3 cryptographic hash function. + +config CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT + tristate "Enable all crypto library code for KUnit tests" + depends on KUNIT + select CRYPTO_LIB_AES_CBC + select CRYPTO_LIB_AES_CBC_MACS + select CRYPTO_LIB_AES_CCM + select CRYPTO_LIB_AES_CTR + select CRYPTO_LIB_AES_ECB + select CRYPTO_LIB_AES_GCM + select CRYPTO_LIB_AES_XTS + select CRYPTO_LIB_BLAKE2B + select CRYPTO_LIB_CHACHA20POLY1305 + select CRYPTO_LIB_CURVE25519 + select CRYPTO_LIB_GF128HASH + select CRYPTO_LIB_MD5 + select CRYPTO_LIB_MLDSA + select CRYPTO_LIB_NH + select CRYPTO_LIB_POLY1305 + select CRYPTO_LIB_SHA1 + select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_SHA512 + select CRYPTO_LIB_SHA3 + select CRYPTO_LIB_SM3 + help + Enable all the crypto library code that has KUnit tests. + + Enable this only if you'd like to test all the crypto library code, + even code that wouldn't otherwise need to be built. + + You'll still need to enable the tests themselves, either individually + or using KUNIT_ALL_TESTS. + config CRYPTO_LIB_BENCHMARK_VISIBLE bool diff --git a/lib/crypto/tests/Makefile b/lib/crypto/tests/Makefile index f4262379f56c..c97c1d78784a 100644 --- a/lib/crypto/tests/Makefile +++ b/lib/crypto/tests/Makefile @@ -1,8 +1,13 @@ # SPDX-License-Identifier: GPL-2.0-or-later +obj-$(CONFIG_CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST) += aes_cbc_macs_kunit.o +obj-$(CONFIG_CRYPTO_LIB_AES_CCM_KUNIT_TEST) += aes_ccm_kunit.o +obj-$(CONFIG_CRYPTO_LIB_AES_GCM_KUNIT_TEST) += aes_gcm_kunit.o obj-$(CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST) += blake2b_kunit.o obj-$(CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST) += blake2s_kunit.o +obj-$(CONFIG_CRYPTO_LIB_CHACHA20POLY1305_KUNIT_TEST) += chacha20poly1305_kunit.o obj-$(CONFIG_CRYPTO_LIB_CURVE25519_KUNIT_TEST) += curve25519_kunit.o +obj-$(CONFIG_CRYPTO_LIB_GHASH_KUNIT_TEST) += ghash_kunit.o obj-$(CONFIG_CRYPTO_LIB_MD5_KUNIT_TEST) += md5_kunit.o obj-$(CONFIG_CRYPTO_LIB_MLDSA_KUNIT_TEST) += mldsa_kunit.o obj-$(CONFIG_CRYPTO_LIB_NH_KUNIT_TEST) += nh_kunit.o @@ -12,3 +17,4 @@ obj-$(CONFIG_CRYPTO_LIB_SHA1_KUNIT_TEST) += sha1_kunit.o obj-$(CONFIG_CRYPTO_LIB_SHA256_KUNIT_TEST) += sha224_kunit.o sha256_kunit.o obj-$(CONFIG_CRYPTO_LIB_SHA512_KUNIT_TEST) += sha384_kunit.o sha512_kunit.o obj-$(CONFIG_CRYPTO_LIB_SHA3_KUNIT_TEST) += sha3_kunit.o +obj-$(CONFIG_CRYPTO_LIB_SM3_KUNIT_TEST) += sm3_kunit.o diff --git a/lib/crypto/tests/aead-test-template.h b/lib/crypto/tests/aead-test-template.h new file mode 100644 index 000000000000..0c0138d2fa8a --- /dev/null +++ b/lib/crypto/tests/aead-test-template.h @@ -0,0 +1,1039 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Shared KUnit test cases for AEAD algorithms, including a benchmark + * + * Copyright 2026 Google LLC + */ + +/* + * This file implements KUnit test cases shared by the different KUnit test + * suites for Authenticated Encryption with Associated Data (AEAD) algorithms. + * + * Test suites including this file must #define the following: + * + * Data structs: + * - AEAD_KEY: name of key struct + * - AEAD_CTX: name of context for incremental computation + * + * Constants: + * - AEAD_VALID_KEY_LENS: array of all valid key lengths in bytes + * - AEAD_VALID_NONCE_LENS: array of all valid nonce lengths in bytes + * - AEAD_VALID_TAG_LENS: array of all valid authtag lengths in bytes + * - AEAD_MAX_KEY_LEN: max key length in bytes (assumed to fit on stack) + * - AEAD_MAX_NONCE_LEN: max nonce length in bytes (assumed to fit on stack) + * - AEAD_MAX_TAG_LEN: max authtag length in bytes (assumed to fit on stack) + * - AEAD_MONTE_CARLO_CHECKSUM: checksum of a deterministically generated series + * of (ciphertext, authtag) pairs (see test_aead_monte_carlo()) + * + * Functions: + * - AEAD_PREPAREKEY: key preparation + * - AEAD_ENCRYPT and AEAD_DECRYPT: one-shot encryption and decryption + * - AEAD_INIT, AEAD_AUTH_UPDATE, AEAD_ENCRYPT_UPDATE, AEAD_ENCRYPT_FINAL, + * AEAD_DECRYPT_UPDATE, AEAD_DECRYPT_FINAL: functions for incremental + * encryption and decryption + * + * Function prototypes and their behavior must match the AES-CCM API. + */ + +#include <crypto/blake2s.h> +#include <kunit/run-in-irq-context.h> +#include <kunit/test.h> +#include <linux/ktime.h> +#include <linux/preempt.h> +#include "test-utils.h" + +/* + * Allocate a KUnit-managed struct AEAD_KEY and prepare it with a random key, + * using a random key length and random authentication tag length. + */ +static struct AEAD_KEY *aead_alloc_random_key(struct kunit *test, + size_t *tag_len_ret) +{ + size_t key_len = + AEAD_VALID_KEY_LENS[rand32() % ARRAY_SIZE(AEAD_VALID_KEY_LENS)]; + size_t tag_len = + AEAD_VALID_TAG_LENS[rand32() % ARRAY_SIZE(AEAD_VALID_TAG_LENS)]; + u8 raw_key[AEAD_MAX_KEY_LEN]; + struct AEAD_KEY *key = alloc_buf(test, sizeof(*key)); + int err; + + rand_bytes(raw_key, key_len); + err = AEAD_PREPAREKEY(key, raw_key, key_len, tag_len); + KUNIT_ASSERT_EQ(test, 0, err); + *tag_len_ret = tag_len; + return key; +} + +/* + * Allocate a KUnit-managed slab buffer of length @len bytes and initialize it + * with random data. + */ +static u8 *aead_alloc_random_data(struct kunit *test, size_t len) +{ + u8 *buf = alloc_buf(test, len); + + rand_bytes(buf, len); + return buf; +} + +/* + * Allocate a KUnit-managed guarded buffer of length @len bytes and initialize + * it with random data. + */ +static u8 *aead_alloc_random_data_guarded(struct kunit *test, size_t len) +{ + u8 *buf = alloc_guarded_buf(test, len); + + rand_bytes(buf, len); + return buf; +} + +/* Process the given associated data using a random incremental strategy. */ +static size_t aead_auth_incrementally(struct AEAD_CTX *ctx, const u8 *ad, + size_t ad_len) +{ + size_t num_parts = 0; + size_t pos = 0; + + while (rand_bool()) { + size_t part_len = rand_length(ad_len - pos); + + AEAD_AUTH_UPDATE(ctx, &ad[pos], part_len); + pos += part_len; + num_parts++; + } + if (pos < ad_len || rand_bool()) { + AEAD_AUTH_UPDATE(ctx, &ad[pos], ad_len - pos); + num_parts++; + } + return num_parts; +} + +/* Process the given en/decrypted data using a random incremental strategy. */ +static size_t aead_crypt_incrementally(struct AEAD_CTX *ctx, u8 *dst, + const u8 *src, size_t data_len, bool enc) +{ + size_t num_parts = 0; + size_t pos = 0; + + while (rand_bool()) { + size_t part_len = rand_length(data_len - pos); + + if (enc) + AEAD_ENCRYPT_UPDATE(ctx, &dst[pos], &src[pos], + part_len); + else + AEAD_DECRYPT_UPDATE(ctx, &dst[pos], &src[pos], + part_len); + pos += part_len; + num_parts++; + } + if (pos < data_len || rand_bool()) { + if (enc) + AEAD_ENCRYPT_UPDATE(ctx, &dst[pos], &src[pos], + data_len - pos); + else + AEAD_DECRYPT_UPDATE(ctx, &dst[pos], &src[pos], + data_len - pos); + num_parts++; + } + return num_parts; +} + +struct aead_incremental_info { + size_t num_data_parts; + size_t num_ad_parts; +}; + +static const char *aead_incr_info_str(struct kunit *test, + const struct aead_incremental_info *info) +{ + const size_t max_str_len = 64; + char *str = alloc_buf(test, max_str_len); + + snprintf(str, max_str_len, "num_data_parts=%zu num_ad_parts=%zu", + info->num_data_parts, info->num_ad_parts); + return str; +} + +/* + * Encrypt data using a random incremental strategy. + * Return information about the incremental strategy used. + */ +static struct aead_incremental_info +aead_encrypt_incrementally(struct kunit *test, struct AEAD_CTX *ctx, u8 *dst, + const u8 *src, size_t data_len, u8 *tag, + const u8 *ad, size_t ad_len, const u8 *nonce, + size_t nonce_len, const struct AEAD_KEY *key) +{ + struct aead_incremental_info info; + int err; + + err = AEAD_INIT(ctx, data_len, ad_len, nonce, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + info.num_ad_parts = aead_auth_incrementally(ctx, ad, ad_len); + info.num_data_parts = aead_crypt_incrementally(ctx, dst, src, data_len, + /* enc= */ true); + AEAD_ENCRYPT_FINAL(ctx, tag); + KUNIT_ASSERT_TRUE_MSG(test, mem_is_zero(ctx, sizeof(*ctx)), + "encrypt_final didn't zeroize context"); + return info; +} + +/* + * Decrypt authentic data using a random incremental strategy. + * Return information about the incremental strategy used. + */ +static struct aead_incremental_info +aead_decrypt_incrementally(struct kunit *test, struct AEAD_CTX *ctx, u8 *dst, + const u8 *src, size_t data_len, const u8 *tag, + const u8 *ad, size_t ad_len, const u8 *nonce, + size_t nonce_len, const struct AEAD_KEY *key) +{ + struct aead_incremental_info info; + int err; + + err = AEAD_INIT(ctx, data_len, ad_len, nonce, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + info.num_ad_parts = aead_auth_incrementally(ctx, ad, ad_len); + info.num_data_parts = aead_crypt_incrementally(ctx, dst, src, data_len, + /* enc= */ false); + err = AEAD_DECRYPT_FINAL(ctx, tag); + KUNIT_ASSERT_EQ(test, 0, err); + KUNIT_ASSERT_TRUE_MSG(test, mem_is_zero(ctx, sizeof(*ctx)), + "decrypt_final didn't zeroize context"); + return info; +} + +/* Return true if key_len is declared to be a valid key length. */ +static bool aead_is_key_len_expected_valid(size_t key_len) +{ + for (size_t i = 0; i < ARRAY_SIZE(AEAD_VALID_KEY_LENS); i++) { + if (AEAD_VALID_KEY_LENS[i] == key_len) + return true; + } + return false; +} + +/* Return true if nonce_len is declared to be a valid nonce length. */ +static bool aead_is_nonce_len_expected_valid(size_t nonce_len) +{ + for (size_t i = 0; i < ARRAY_SIZE(AEAD_VALID_NONCE_LENS); i++) { + if (AEAD_VALID_NONCE_LENS[i] == nonce_len) + return true; + } + return false; +} + +/* Return true if tag_len is declared to be a valid tag length. */ +static bool aead_is_tag_len_expected_valid(size_t tag_len) +{ + for (size_t i = 0; i < ARRAY_SIZE(AEAD_VALID_TAG_LENS); i++) { + if (AEAD_VALID_TAG_LENS[i] == tag_len) + return true; + } + return false; +} + +struct aead_basic_validation_test_ctx { + struct AEAD_KEY key; + struct AEAD_CTX ctx; + u8 *raw_key_buf_end; + u8 *nonce_buf_end; + u8 *tag_buf_end; + u8 pt[64]; /* plaintext */ + u8 ct[64]; /* ciphertext */ + u8 decrypted[64]; + u8 ad[16]; /* associated data */ + u8 *unused_buf; + size_t data_len; + size_t ad_len; +}; + +static struct aead_basic_validation_test_ctx * +aead_alloc_basic_validation_test_ctx(struct kunit *test) +{ + struct aead_basic_validation_test_ctx *ctx = + alloc_buf(test, sizeof(*ctx)); + + memset(ctx, 0, sizeof(*ctx)); + ctx->raw_key_buf_end = + aead_alloc_random_data_guarded(test, AEAD_MAX_KEY_LEN) + + AEAD_MAX_KEY_LEN; + ctx->nonce_buf_end = + aead_alloc_random_data_guarded(test, AEAD_MAX_NONCE_LEN) + + AEAD_MAX_NONCE_LEN; + ctx->tag_buf_end = + aead_alloc_random_data_guarded(test, AEAD_MAX_TAG_LEN) + + AEAD_MAX_TAG_LEN; + + /* + * A pointer to this buffer is passed when passing a length that is + * expected to be invalid. It should never actually be accessed. + */ + ctx->unused_buf = + alloc_buf(test, max3(AEAD_MAX_KEY_LEN, AEAD_MAX_NONCE_LEN, + AEAD_MAX_TAG_LEN)); + + ctx->data_len = sizeof(ctx->pt); + ctx->ad_len = sizeof(ctx->ad); + return ctx; +} + +/* + * Given an expected-valid key_len, nonce_len, and tag_len, verify round-trip + * encryption and decryption with them. Use guarded buffers for each of the raw + * key, nonce, and tag to detect any buffer overruns in them. Also, verify that + * every byte of the tag is actually checked. + */ +static void aead_do_basic_checks(struct kunit *test, + struct aead_basic_validation_test_ctx *ctx, + size_t key_len, size_t nonce_len, + size_t tag_len) +{ + /* Set up exact-size guarded buffers for (raw_key, nonce, tag). */ + const u8 *raw_key = ctx->raw_key_buf_end - key_len; + const u8 *nonce = ctx->nonce_buf_end - nonce_len; + u8 *tag = ctx->tag_buf_end - tag_len; + int err; + + /* Key preparation should succeed. */ + err = AEAD_PREPAREKEY(&ctx->key, raw_key, key_len, tag_len); + KUNIT_ASSERT_EQ_MSG(test, 0, err, + "key_len=%zu, tag_len=%zu wasn't accepted", key_len, + tag_len); + + /* Encryption should succeed. */ + err = AEAD_ENCRYPT(ctx->ct, ctx->pt, ctx->data_len, tag, ctx->ad, + ctx->ad_len, nonce, nonce_len, &ctx->key); + KUNIT_ASSERT_EQ_MSG( + test, 0, err, + "Encryption failed with key_len=%zu, nonce_len=%zu, tag_len=%zu", + key_len, nonce_len, tag_len); + + /* Decryption should succeed and give the original data. */ + err = AEAD_DECRYPT(ctx->decrypted, ctx->ct, ctx->data_len, tag, ctx->ad, + ctx->ad_len, nonce, nonce_len, &ctx->key); + KUNIT_ASSERT_EQ_MSG( + test, 0, err, + "Decryption failed with key_len=%zu, nonce_len=%zu, tag_len=%zu", + key_len, nonce_len, tag_len); + KUNIT_ASSERT_MEMEQ_MSG( + test, ctx->pt, ctx->decrypted, ctx->data_len, + "Decryption gave wrong output with key_len=%zu, nonce_len=%zu, tag_len=%zu", + key_len, nonce_len, tag_len); + + /* + * Every byte of the tag should actually be checked. + * And on authentication failure, the dst buffer should be cleared. + */ + for (size_t i = 0; i < tag_len; i++) { + memset(ctx->decrypted, 0xff, ctx->data_len); + tag[i] ^= 1; + err = AEAD_DECRYPT(ctx->decrypted, ctx->ct, ctx->data_len, tag, + ctx->ad, ctx->ad_len, nonce, nonce_len, + &ctx->key); + KUNIT_ASSERT_EQ_MSG( + test, -EBADMSG, err, + "Decryption with bad auth tag with key_len=%zu, nonce_len=%zu, tag_len=%zu didn't fail with -EBADMSG", + key_len, nonce_len, tag_len); + KUNIT_ASSERT_TRUE_MSG( + test, mem_is_zero(ctx->decrypted, ctx->data_len), + "dst wasn't cleared on authentication failure"); + tag[i] ^= 1; + } +} + +/* Verify that the given expected-invalid key_len is actually rejected. */ +static void +aead_verify_invalid_key_len(struct kunit *test, + struct aead_basic_validation_test_ctx *ctx, + size_t key_len) +{ + int err; + + /* + * The preparekey function should reject the key_len. It should do so + * before writing to the key struct. + */ + memset(&ctx->key, 0, sizeof(ctx->key)); + err = AEAD_PREPAREKEY(&ctx->key, ctx->unused_buf, key_len, + AEAD_MAX_TAG_LEN); + KUNIT_ASSERT_EQ_MSG(test, -EINVAL, err, + "key_len=%zu wasn't rejected with -EINVAL", + key_len); + KUNIT_ASSERT_TRUE_MSG( + test, mem_is_zero(&ctx->key, sizeof(ctx->key)), + "Key struct was written to before length validation"); +} + +/* + * Test that every valid key length is accepted and basic checks pass with it, + * and test that invalid key lengths are rejected. + */ +static void test_aead_all_key_lens(struct kunit *test) +{ + struct aead_basic_validation_test_ctx *ctx = + aead_alloc_basic_validation_test_ctx(test); + + for (size_t key_len = 0; key_len <= AEAD_MAX_KEY_LEN; key_len++) { + if (aead_is_key_len_expected_valid(key_len)) + aead_do_basic_checks(test, ctx, key_len, + AEAD_MAX_NONCE_LEN, + AEAD_MAX_TAG_LEN); + else + aead_verify_invalid_key_len(test, ctx, key_len); + } + aead_verify_invalid_key_len(test, ctx, AEAD_MAX_KEY_LEN + 1); + aead_verify_invalid_key_len(test, ctx, AEAD_MAX_KEY_LEN * 2); + aead_verify_invalid_key_len(test, ctx, U32_MAX); + aead_verify_invalid_key_len(test, ctx, SIZE_MAX); +} + +/* Verify that the given expected-invalid nonce_len is actually rejected. */ +static void +aead_verify_invalid_nonce_len(struct kunit *test, + struct aead_basic_validation_test_ctx *ctx, + size_t nonce_len) +{ + static const u8 raw_key[AEAD_MAX_KEY_LEN]; + int err; + + /* Key preparation should succeed, as nonce_len isn't given yet. */ + err = AEAD_PREPAREKEY(&ctx->key, raw_key, sizeof(raw_key), + AEAD_MAX_TAG_LEN); + KUNIT_ASSERT_EQ(test, 0, err); + + /* The init function should reject the nonce_len. */ + memset(&ctx->ctx, 0, sizeof(ctx->ctx)); + err = AEAD_INIT(&ctx->ctx, ctx->data_len, ctx->ad_len, ctx->unused_buf, + nonce_len, &ctx->key); + KUNIT_ASSERT_EQ_MSG(test, -EINVAL, err, + "nonce_len=%zu wasn't rejected with -EINVAL (init)", + nonce_len); + KUNIT_ASSERT_TRUE_MSG( + test, mem_is_zero(&ctx->ctx, sizeof(ctx->ctx)), + "Context struct was written to before length validation"); + + /* The encrypt function should reject the nonce_len. */ + err = AEAD_ENCRYPT(ctx->ct, ctx->pt, ctx->data_len, ctx->unused_buf, + ctx->ad, ctx->ad_len, ctx->unused_buf, nonce_len, + &ctx->key); + KUNIT_ASSERT_EQ_MSG( + test, -EINVAL, err, + "nonce_len=%zu wasn't rejected with -EINVAL (encrypt)", + nonce_len); + + /* The decrypt function should reject the nonce_len. */ + err = AEAD_DECRYPT(ctx->pt, ctx->ct, ctx->data_len, ctx->unused_buf, + ctx->ad, ctx->ad_len, ctx->unused_buf, nonce_len, + &ctx->key); + KUNIT_ASSERT_EQ_MSG( + test, -EINVAL, err, + "nonce_len=%zu wasn't rejected with -EINVAL (decrypt)", + nonce_len); +} + +/* + * Test that every valid nonce length is accepted and basic checks pass with it, + * and test that invalid nonce lengths are rejected. + */ +static void test_aead_all_nonce_lens(struct kunit *test) +{ + struct aead_basic_validation_test_ctx *ctx = + aead_alloc_basic_validation_test_ctx(test); + + for (size_t nonce_len = 0; nonce_len <= AEAD_MAX_NONCE_LEN; + nonce_len++) { + if (aead_is_nonce_len_expected_valid(nonce_len)) + aead_do_basic_checks(test, ctx, AEAD_MAX_KEY_LEN, + nonce_len, AEAD_MAX_TAG_LEN); + else + aead_verify_invalid_nonce_len(test, ctx, nonce_len); + } + aead_verify_invalid_nonce_len(test, ctx, AEAD_MAX_NONCE_LEN + 1); + aead_verify_invalid_nonce_len(test, ctx, AEAD_MAX_NONCE_LEN * 2); + aead_verify_invalid_nonce_len(test, ctx, U32_MAX); + aead_verify_invalid_nonce_len(test, ctx, SIZE_MAX); +} + +/* Verify that the given expected-invalid tag_len is actually rejected. */ +static void +aead_verify_invalid_tag_len(struct kunit *test, + struct aead_basic_validation_test_ctx *ctx, + size_t tag_len) +{ + static const u8 raw_key[AEAD_MAX_KEY_LEN]; + int err; + + /* + * The preparekey function should reject the tag_len. It should do so + * before writing to the key struct. + */ + memset(&ctx->key, 0, sizeof(ctx->key)); + err = AEAD_PREPAREKEY(&ctx->key, raw_key, sizeof(raw_key), tag_len); + KUNIT_ASSERT_EQ_MSG(test, -EINVAL, err, + "tag_len=%zu wasn't rejected with -EINVAL", + tag_len); + KUNIT_ASSERT_TRUE_MSG( + test, mem_is_zero(&ctx->key, sizeof(ctx->key)), + "Key struct was written to before length validation"); +} + +/* + * Test that every valid authentication tag length is accepted and basic checks + * pass with it, and test that invalid authentication tag lengths are rejected. + */ +static void test_aead_all_tag_lens(struct kunit *test) +{ + struct aead_basic_validation_test_ctx *ctx = + aead_alloc_basic_validation_test_ctx(test); + + for (size_t tag_len = 0; tag_len <= AEAD_MAX_TAG_LEN; tag_len++) { + if (aead_is_tag_len_expected_valid(tag_len)) + aead_do_basic_checks(test, ctx, AEAD_MAX_KEY_LEN, + AEAD_MAX_NONCE_LEN, tag_len); + else + aead_verify_invalid_tag_len(test, ctx, tag_len); + } + aead_verify_invalid_tag_len(test, ctx, AEAD_MAX_TAG_LEN + 1); + aead_verify_invalid_tag_len(test, ctx, AEAD_MAX_TAG_LEN * 2); + aead_verify_invalid_tag_len(test, ctx, U32_MAX); + aead_verify_invalid_tag_len(test, ctx, SIZE_MAX); +} + +/* + * Test that one-shot encryption and decryption are consistent with each other + * and with incremental encryption and decryption. + */ +static void test_aead_incremental_updates(struct kunit *test) +{ + const size_t max_data_len = 1024; + const size_t max_ad_len = 512; + const size_t nonce_len = AEAD_MAX_NONCE_LEN; + size_t tag_len; + struct AEAD_KEY *key = aead_alloc_random_key(test, &tag_len); + struct AEAD_CTX *ctx = alloc_buf(test, sizeof(*ctx)); + u8 *pt = aead_alloc_random_data(test, max_data_len); + u8 *ad = aead_alloc_random_data(test, max_ad_len); + u8 *nonce = aead_alloc_random_data(test, nonce_len); + u8 *ct = alloc_buf(test, max_data_len); + u8 *ct2 = alloc_buf(test, max_data_len); + u8 *decrypted = alloc_buf(test, max_data_len); + u8 *tag = alloc_buf(test, tag_len); + u8 *tag2 = alloc_buf(test, tag_len); + int err; + + for (int i = 0; i < 500; i++) { + /* Select the lengths to test. */ + const size_t data_len = rand_length(max_data_len); + const size_t ad_len = rand_length(max_ad_len); + struct aead_incremental_info incr_info; + + /* Try one-shot encryption and decryption. */ + err = AEAD_ENCRYPT(ct, pt, data_len, tag, ad, ad_len, nonce, + nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + err = AEAD_DECRYPT(decrypted, ct, data_len, tag, ad, ad_len, + nonce, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + KUNIT_ASSERT_MEMEQ_MSG( + test, pt, decrypted, data_len, + "Decryption didn't invert encryption; data_len=%zu, ad_len=%zu", + data_len, ad_len); + + /* Try incremental encryption and decryption. */ + incr_info = aead_encrypt_incrementally(test, ctx, ct2, pt, + data_len, tag2, ad, + ad_len, nonce, nonce_len, + key); + KUNIT_ASSERT_MEMEQ_MSG( + test, ct, ct2, data_len, + "One-shot and incremental encryption gave different ciphertexts; data_len=%zu ad_len=%zu %s", + data_len, ad_len, aead_incr_info_str(test, &incr_info)); + KUNIT_ASSERT_MEMEQ_MSG( + test, tag, tag2, tag_len, + "One-shot and incremental encryption gave different auth tags; data_len=%zu ad_len=%zu %s", + data_len, ad_len, aead_incr_info_str(test, &incr_info)); + incr_info = aead_decrypt_incrementally(test, ctx, decrypted, + ct2, data_len, tag2, ad, + ad_len, nonce, nonce_len, + key); + KUNIT_ASSERT_MEMEQ_MSG( + test, pt, decrypted, data_len, + "One-shot and incremental decryption gave different plaintexts; data_len=%zu ad_len=%zu %s", + data_len, ad_len, aead_incr_info_str(test, &incr_info)); + } +} + +/* + * Test using guarded buffers for the plaintext, ciphertext, and associated + * data. This detects out-of-bounds accesses, even in assembly code. + * + * Note: other test cases cover overrun of raw_key, nonce, and tag. + */ +static void test_aead_data_buffer_overruns(struct kunit *test) +{ + const size_t max_data_len = 1024; + const size_t max_ad_len = 512; + const size_t nonce_len = AEAD_MAX_NONCE_LEN; + size_t tag_len; + struct AEAD_KEY *key = aead_alloc_random_key(test, &tag_len); + const u8 *nonce = aead_alloc_random_data(test, nonce_len); + const u8 *pt_end = aead_alloc_random_data_guarded(test, max_data_len) + + max_data_len; + const u8 *ad_end = + aead_alloc_random_data_guarded(test, max_ad_len) + max_ad_len; + u8 *ct_end = alloc_guarded_buf(test, max_data_len) + max_data_len; + u8 *decrypted_end = + alloc_guarded_buf(test, max_data_len) + max_data_len; + u8 *tag = alloc_buf(test, tag_len); + + for (int i = 0; i < 200; i++) { + /* Select the lengths to test. */ + const size_t data_len = rand_length(max_data_len); + const size_t ad_len = rand_length(max_ad_len); + /* Set up exact-size guarded buffers. */ + const u8 *pt = pt_end - data_len; + const u8 *ad = ad_end - ad_len; + u8 *ct = ct_end - data_len; + u8 *decrypted = decrypted_end - data_len; + int err; + + /* Encrypt and decrypt. */ + err = AEAD_ENCRYPT(ct, pt, data_len, tag, ad, ad_len, nonce, + nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + err = AEAD_DECRYPT(decrypted, ct, data_len, tag, ad, ad_len, + nonce, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + KUNIT_ASSERT_MEMEQ_MSG( + test, pt, decrypted, data_len, + "Decryption didn't invert encryption; data_len=%zu, ad_len=%zu", + data_len, ad_len); + } +} + +/* + * Test that encryption and decryption produce the same results regardless of + * how the buffers are aligned in memory. + */ +static void test_aead_alignment_consistency(struct kunit *test) +{ + const size_t max_data_len = 4096; + const size_t max_ad_len = 4096; + const size_t max_offset = 128; + const size_t nonce_len = AEAD_MAX_NONCE_LEN; + const size_t key_len = AEAD_MAX_KEY_LEN; + const size_t tag_len = AEAD_MAX_TAG_LEN; + u8 *raw_key1_buf = alloc_buf(test, key_len + max_offset); + u8 *raw_key2_buf = alloc_buf(test, key_len + max_offset); + u8 *nonce1_buf = alloc_buf(test, nonce_len + max_offset); + u8 *nonce2_buf = alloc_buf(test, nonce_len + max_offset); + u8 *pt1_buf = alloc_buf(test, max_data_len); + u8 *pt2_buf = alloc_buf(test, max_data_len); + u8 *ct1_buf = alloc_buf(test, max_data_len); + u8 *ct2_buf = alloc_buf(test, max_data_len); + u8 *ad1_buf = alloc_buf(test, max_ad_len); + u8 *ad2_buf = alloc_buf(test, max_ad_len); + u8 *tag1_buf = alloc_buf(test, tag_len + max_offset); + u8 *tag2_buf = alloc_buf(test, tag_len + max_offset); + struct AEAD_KEY *key = alloc_buf(test, sizeof(*key)); + int err; + + for (int i = 0; i < 100; i++) { + /* Generate lengths. */ + size_t data_len = rand_length(max_data_len); + size_t ad_len = rand_length(max_ad_len); + + /* Generate two sets of alignments. */ + u8 *raw_key1 = raw_key1_buf + rand_offset(max_offset); + u8 *raw_key2 = raw_key2_buf + rand_offset(max_offset); + u8 *nonce1 = nonce1_buf + rand_offset(max_offset); + u8 *nonce2 = nonce2_buf + rand_offset(max_offset); + u8 *pt1 = pt1_buf + rand_offset(max_data_len - data_len); + u8 *pt2 = pt2_buf + rand_offset(max_data_len - data_len); + u8 *ct1 = ct1_buf + rand_offset(max_data_len - data_len); + u8 *ct2 = ct2_buf + rand_offset(max_data_len - data_len); + u8 *ad1 = ad1_buf + rand_offset(max_ad_len - ad_len); + u8 *ad2 = ad2_buf + rand_offset(max_ad_len - ad_len); + u8 *tag1 = tag1_buf + rand_offset(max_offset); + u8 *tag2 = tag2_buf + rand_offset(max_offset); + + /* + * Generate inputs in the first set of buffers using the first + * set of alignments. + */ + rand_bytes(raw_key1, key_len); + rand_bytes(nonce1, nonce_len); + rand_bytes(pt1, data_len); + rand_bytes(ad1, ad_len); + + /* + * Copy the inputs to the second set of buffers using the second + * set of alignments. + */ + memcpy(raw_key2, raw_key1, key_len); + memcpy(nonce2, nonce1, nonce_len); + memcpy(pt2, pt1, data_len); + memcpy(ad2, ad1, ad_len); + + /* Verify encryption consistency. */ + + err = AEAD_PREPAREKEY(key, raw_key1, key_len, tag_len); + KUNIT_ASSERT_EQ(test, 0, err); + err = AEAD_ENCRYPT(ct1, pt1, data_len, tag1, ad1, ad_len, + nonce1, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + + err = AEAD_PREPAREKEY(key, raw_key2, key_len, tag_len); + KUNIT_ASSERT_EQ(test, 0, err); + err = AEAD_ENCRYPT(ct2, pt2, data_len, tag2, ad2, ad_len, + nonce2, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + + KUNIT_ASSERT_MEMEQ(test, ct1, ct2, data_len); + KUNIT_ASSERT_MEMEQ(test, tag1, tag2, tag_len); + + /* Verify decryption consistency. */ + + err = AEAD_PREPAREKEY(key, raw_key1, key_len, tag_len); + KUNIT_ASSERT_EQ(test, 0, err); + err = AEAD_DECRYPT(pt1, ct1, data_len, tag1, ad1, ad_len, + nonce1, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + + err = AEAD_PREPAREKEY(key, raw_key2, key_len, tag_len); + KUNIT_ASSERT_EQ(test, 0, err); + err = AEAD_DECRYPT(pt2, ct2, data_len, tag2, ad2, ad_len, + nonce2, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + + KUNIT_ASSERT_MEMEQ(test, pt1, pt2, data_len); + } +} + +static void test_aead_inplace(struct kunit *test) +{ + const size_t max_data_len = 1024; + const size_t max_ad_len = 512; + const size_t nonce_len = AEAD_MAX_NONCE_LEN; + size_t tag_len; + struct AEAD_KEY *key = aead_alloc_random_key(test, &tag_len); + u8 *data = aead_alloc_random_data(test, max_data_len + tag_len); + u8 *data2 = alloc_buf(test, max_data_len + tag_len); + u8 *ad = aead_alloc_random_data(test, max_ad_len); + const u8 *nonce = aead_alloc_random_data(test, nonce_len); + + for (int i = 0; i < 100; i++) { + size_t data_len = rand_length(max_data_len); + size_t ad_len = rand_length(max_ad_len); + int err; + + /* Encrypt out-of-place. */ + err = AEAD_ENCRYPT(data2, data, data_len, data2 + data_len, ad, + ad_len, nonce, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + + /* Encrypt in-place. */ + err = AEAD_ENCRYPT(data, data, data_len, data + data_len, ad, + ad_len, nonce, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + + /* Compare the results. */ + KUNIT_ASSERT_MEMEQ(test, data2, data, data_len + tag_len); + + /* Decrypt out-of-place. */ + err = AEAD_DECRYPT(data2, data, data_len, data + data_len, ad, + ad_len, nonce, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + + /* Decrypt in-place. */ + err = AEAD_DECRYPT(data, data, data_len, data + data_len, ad, + ad_len, nonce, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + + /* Compare the results. */ + KUNIT_ASSERT_MEMEQ(test, data2, data, data_len); + } +} + +/* + * Monte-Carlo test for AEAD algorithms. This deterministically generates + * random AEAD inputs, encrypts them, verifies decryption, and computes and + * verifies the checksum of all computed (ciphertext, tag) pairs. + */ +static void test_aead_monte_carlo(struct kunit *test) +{ + const size_t max_data_len = 1024; + const size_t max_ad_len = 293; + u8 raw_key[AEAD_MAX_KEY_LEN]; + u8 nonce[AEAD_MAX_NONCE_LEN]; + u8 tag[AEAD_MAX_TAG_LEN]; + u8 *pt = alloc_buf(test, max_data_len); + u8 *ct = alloc_buf(test, max_data_len); + u8 *decrypted = alloc_buf(test, max_data_len); + u8 *ad = alloc_buf(test, max_ad_len); + struct AEAD_KEY *key = alloc_buf(test, sizeof(*key)); + struct blake2s_ctx checksum_ctx; + u8 actual_checksum[BLAKE2S_HASH_SIZE]; + int err; + + blake2s_init(&checksum_ctx, BLAKE2S_HASH_SIZE); + + for (size_t data_len = 0; data_len <= max_data_len; data_len++) { + size_t ad_len = data_len % max_ad_len; + size_t key_len = + AEAD_VALID_KEY_LENS[data_len % + ARRAY_SIZE(AEAD_VALID_KEY_LENS)]; + size_t nonce_len = + AEAD_VALID_NONCE_LENS[data_len % + ARRAY_SIZE(AEAD_VALID_NONCE_LENS)]; + size_t tag_len = + AEAD_VALID_TAG_LENS[data_len % + ARRAY_SIZE(AEAD_VALID_TAG_LENS)]; + + rand_bytes_seeded_from_len(pt, data_len); + rand_bytes_seeded_from_len(ad, ad_len); + rand_bytes_seeded_from_len(raw_key, key_len); + rand_bytes_seeded_from_len(nonce, nonce_len); + + err = AEAD_PREPAREKEY(key, raw_key, key_len, tag_len); + KUNIT_ASSERT_EQ(test, 0, err); + + err = AEAD_ENCRYPT(ct, pt, data_len, tag, ad, ad_len, nonce, + nonce_len, key); + KUNIT_ASSERT_EQ_MSG( + test, 0, err, + "Encryption failed with data_len=%zu, ad_len=%zu", + data_len, ad_len); + err = AEAD_DECRYPT(decrypted, ct, data_len, tag, ad, ad_len, + nonce, nonce_len, key); + KUNIT_ASSERT_EQ_MSG( + test, 0, err, + "Decryption failed with data_len=%zu, ad_len=%zu", + data_len, ad_len); + KUNIT_ASSERT_MEMEQ_MSG( + test, pt, decrypted, data_len, + "Decryption didn't invert encryption; data_len=%zu, ad_len=%zu", + data_len, ad_len); + + blake2s_update(&checksum_ctx, ct, data_len); + blake2s_update(&checksum_ctx, tag, tag_len); + } + + blake2s_final(&checksum_ctx, actual_checksum); + KUNIT_EXPECT_MEMEQ_MSG(test, actual_checksum, AEAD_MONTE_CARLO_CHECKSUM, + BLAKE2S_HASH_SIZE, + "Monte-Carlo checksum mismatch"); +} + +#define IRQ_TEST_DATA_LEN 256 +#define IRQ_TEST_NUM_BUFFERS 3 /* matches max concurrency level */ + +struct aead_irq_test_slot { + /* Fields written only at test case initialization time */ + u8 raw_key[AEAD_MAX_KEY_LEN]; + u8 nonce[AEAD_MAX_NONCE_LEN]; + u8 pt[IRQ_TEST_DATA_LEN]; + u8 ct[IRQ_TEST_DATA_LEN + AEAD_MAX_TAG_LEN]; + u8 ad[IRQ_TEST_DATA_LEN]; + + /* Fields written throughout the test case */ + struct AEAD_KEY key; + u8 scratch_buf[IRQ_TEST_DATA_LEN + AEAD_MAX_TAG_LEN]; + int phase; + atomic_t in_use; +}; + +struct aead_irq_test_state { + struct aead_irq_test_slot slots[IRQ_TEST_NUM_BUFFERS]; +}; + +static bool aead_irq_test_func(void *state_) +{ + struct aead_irq_test_state *state = state_; + struct aead_irq_test_slot *slot; + size_t data_len; + bool ok = true; + + /* + * Find a free slot. This should always succeed, since the number of + * slots is equal to the max concurrency level of kunit_run_irq_test(). + */ + for (slot = &state->slots[0]; + slot < &state->slots[ARRAY_SIZE(state->slots)]; slot++) { + if (atomic_cmpxchg(&slot->in_use, 0, 1) == 0) + break; + } + if (WARN_ON_ONCE(slot == &state->slots[ARRAY_SIZE(state->slots)])) + return false; + /* + * This execution context now has exclusive access to 'slot'. + * Next, execute the next operation that the slot is set to perform. + */ + + data_len = sizeof(slot->pt); + if (slot->phase == 0) { + /* Phase 0: Prepare slot's key in current context. */ + ok = ok && AEAD_PREPAREKEY(&slot->key, slot->raw_key, + sizeof(slot->raw_key), + AEAD_MAX_TAG_LEN) == 0; + } else if (slot->phase == 1) { + /* + * Phase 1: Encrypt plaintext using key that may have been + * prepared in a different context. + */ + ok = ok && AEAD_ENCRYPT(slot->scratch_buf, slot->pt, data_len, + &slot->scratch_buf[data_len], slot->ad, + sizeof(slot->ad), slot->nonce, + sizeof(slot->nonce), &slot->key) == 0; + /* Verify the ciphertext (with concatenated auth tag) matches */ + ok = ok && + memcmp(slot->scratch_buf, slot->ct, sizeof(slot->ct)) == 0; + } else { + /* + * Phase 2: Decrypt ciphertext using key that may have been + * prepared in a different context. + */ + ok = ok && AEAD_DECRYPT(slot->scratch_buf, slot->ct, data_len, + &slot->ct[data_len], slot->ad, + sizeof(slot->ad), slot->nonce, + sizeof(slot->nonce), &slot->key) == 0; + /* Verify the plaintext matches. */ + ok = ok && memcmp(slot->scratch_buf, slot->pt, data_len) == 0; + } + slot->phase = (slot->phase + 1) % 3; + atomic_set_release(&slot->in_use, 0); + return ok; +} + +/* + * Test that encryption and decryption produce the correct results in task, + * softirq, and hardirq contexts running concurrently -- including with keys + * prepared in other contexts. This is needed to cover fallback code paths that + * execute in contexts where FPU or vector registers cannot be used. + */ +static void test_aead_interrupt_context(struct kunit *test) +{ + struct aead_irq_test_state *state = alloc_buf(test, sizeof(*state)); + + memset(state, 0, sizeof(*state)); + + /* + * For each slot, generate a set of AEAD inputs: a key, a nonce, a + * plaintext, and some associated data. Then generate the corresponding + * ciphertext with concatenated auth tag. + */ + for (int i = 0; i < IRQ_TEST_NUM_BUFFERS; i++) { + struct aead_irq_test_slot *slot = &state->slots[i]; + int err; + + rand_bytes(slot->raw_key, sizeof(slot->raw_key)); + rand_bytes(slot->nonce, sizeof(slot->nonce)); + rand_bytes(slot->pt, sizeof(slot->pt)); + rand_bytes(slot->ad, sizeof(slot->ad)); + err = AEAD_PREPAREKEY(&slot->key, slot->raw_key, + sizeof(slot->raw_key), AEAD_MAX_TAG_LEN); + KUNIT_ASSERT_EQ(test, 0, err); + err = AEAD_ENCRYPT(slot->ct, slot->pt, sizeof(slot->pt), + &slot->ct[sizeof(slot->pt)], slot->ad, + sizeof(slot->ad), slot->nonce, + sizeof(slot->nonce), &slot->key); + KUNIT_ASSERT_EQ(test, 0, err); + } + + kunit_run_irq_test(test, aead_irq_test_func, 100000, state); +} + +/* Benchmark AEAD encryption and decryption on various data lengths. */ +static void benchmark_aead(struct kunit *test) +{ + static const size_t data_lens_to_test[] = { + 16, 64, 128, 256, 512, 1024, 1420, 4096, 16384, + }; + const size_t max_data_len = 16384; + const size_t ad_len = 16; + const size_t key_len = AEAD_MAX_KEY_LEN; + const size_t nonce_len = AEAD_MAX_NONCE_LEN; + const size_t tag_len = AEAD_MAX_TAG_LEN; + const u8 *raw_key, *nonce, *ad; + u8 *pt, *ct, *tag; + struct AEAD_KEY *key; + int err; + + if (!IS_ENABLED(CONFIG_CRYPTO_LIB_BENCHMARK)) + kunit_skip(test, "not enabled"); + + raw_key = aead_alloc_random_data(test, key_len); + nonce = aead_alloc_random_data(test, nonce_len); + ad = aead_alloc_random_data(test, ad_len); + pt = aead_alloc_random_data(test, max_data_len); + ct = alloc_buf(test, max_data_len); + tag = alloc_buf(test, tag_len); + + key = alloc_buf(test, sizeof(*key)); + err = AEAD_PREPAREKEY(key, raw_key, key_len, tag_len); + KUNIT_ASSERT_EQ(test, 0, err); + + /* Warm-up */ + for (size_t i = 0; i < 10000000; i += max_data_len) { + err = AEAD_ENCRYPT(ct, pt, max_data_len, tag, ad, ad_len, nonce, + nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + err = AEAD_DECRYPT(pt, ct, max_data_len, tag, ad, ad_len, nonce, + nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + } + + for (size_t i = 0; i < ARRAY_SIZE(data_lens_to_test); i++) { + size_t data_len = data_lens_to_test[i]; + size_t num_iters = 10000000 / (data_len + 128); + u64 t_enc, t_dec; + bool ok = true; + + KUNIT_ASSERT_LE(test, data_len, max_data_len); + + preempt_disable(); + + t_enc = ktime_get_ns(); + for (size_t j = 0; j < num_iters; j++) { + err = AEAD_ENCRYPT(ct, pt, data_len, tag, ad, ad_len, + nonce, nonce_len, key); + ok &= (err == 0); + } + t_enc = ktime_get_ns() - t_enc; + + t_dec = ktime_get_ns(); + for (size_t j = 0; j < num_iters; j++) { + err = AEAD_DECRYPT(pt, ct, data_len, tag, ad, ad_len, + nonce, nonce_len, key); + ok &= (err == 0); + } + t_dec = ktime_get_ns() - t_dec; + + preempt_enable(); + + KUNIT_ASSERT_TRUE_MSG(test, ok, "data_len=%zu", data_len); + + kunit_info(test, "data_len=%zu: enc %llu MB/s, dec %llu MB/s", + data_len, + div64_u64((u64)data_len * num_iters * 1000, + t_enc ?: 1), + div64_u64((u64)data_len * num_iters * 1000, + t_dec ?: 1)); + } +} + +/* clang-format off */ +#define AEAD_KUNIT_CASES \ + KUNIT_CASE(test_aead_all_key_lens), \ + KUNIT_CASE(test_aead_all_nonce_lens), \ + KUNIT_CASE(test_aead_all_tag_lens), \ + KUNIT_CASE(test_aead_incremental_updates), \ + KUNIT_CASE(test_aead_data_buffer_overruns), \ + KUNIT_CASE(test_aead_alignment_consistency), \ + KUNIT_CASE(test_aead_inplace), \ + KUNIT_CASE(test_aead_monte_carlo), \ + KUNIT_CASE(test_aead_interrupt_context), \ + KUNIT_CASE(benchmark_aead) diff --git a/lib/crypto/tests/aes-cmac-testvecs.h b/lib/crypto/tests/aes-cmac-testvecs.h new file mode 100644 index 000000000000..8177862ba5a3 --- /dev/null +++ b/lib/crypto/tests/aes-cmac-testvecs.h @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* This file was generated by: ./scripts/crypto/gen-hash-testvecs.py aes-cmac */ + +static const struct { + size_t data_len; + u8 digest[AES_BLOCK_SIZE]; +} hash_testvecs[] = { + { + .data_len = 0, + .digest = { + 0x9a, 0xeb, 0x94, 0xc1, 0xe9, 0xc1, 0x57, 0x49, + 0x7e, 0xba, 0x66, 0x47, 0x9f, 0x03, 0x2c, 0x5b, + }, + }, + { + .data_len = 1, + .digest = { + 0x52, 0xef, 0x65, 0xda, 0x7b, 0x92, 0x0c, 0x0f, + 0xdd, 0xd6, 0xb9, 0x68, 0x3f, 0xcc, 0x5f, 0xea, + }, + }, + { + .data_len = 2, + .digest = { + 0xc3, 0x95, 0x15, 0xea, 0x16, 0x33, 0xbe, 0x5a, + 0xd4, 0x2c, 0x25, 0x06, 0x15, 0xc6, 0x10, 0x19, + }, + }, + { + .data_len = 3, + .digest = { + 0x82, 0x41, 0x41, 0xd5, 0x33, 0x26, 0x0b, 0xb6, + 0xc8, 0xf7, 0x8d, 0x76, 0x8a, 0xfc, 0x0e, 0xe4, + }, + }, + { + .data_len = 16, + .digest = { + 0x94, 0x09, 0x80, 0x07, 0xba, 0x7c, 0xed, 0xd2, + 0x74, 0x72, 0x30, 0x26, 0xb5, 0x11, 0x64, 0xc1, + }, + }, + { + .data_len = 32, + .digest = { + 0xeb, 0xcf, 0x1e, 0x67, 0x21, 0x64, 0x93, 0xa0, + 0xea, 0xc4, 0xb9, 0x2d, 0x55, 0xc8, 0xac, 0x99, + }, + }, + { + .data_len = 48, + .digest = { + 0xd0, 0xd6, 0xdb, 0xe2, 0x45, 0x98, 0x6a, 0x7a, + 0x5f, 0xd6, 0xcd, 0x9d, 0x12, 0x26, 0x20, 0x87, + }, + }, + { + .data_len = 49, + .digest = { + 0x63, 0x25, 0x3c, 0xe2, 0x2a, 0xfa, 0xe3, 0x1e, + 0x54, 0x10, 0x18, 0x28, 0xc6, 0xb8, 0xcb, 0x58, + }, + }, + { + .data_len = 63, + .digest = { + 0x4d, 0xab, 0xae, 0x99, 0x90, 0x13, 0x3f, 0x4f, + 0x42, 0x0f, 0x19, 0x94, 0xa2, 0x52, 0xfd, 0xaf, + }, + }, + { + .data_len = 64, + .digest = { + 0xf7, 0x49, 0xb9, 0xa7, 0xf9, 0x3e, 0xa0, 0xca, + 0xb2, 0x6c, 0xd7, 0x87, 0x7d, 0x1e, 0xd2, 0xcb, + }, + }, + { + .data_len = 65, + .digest = { + 0x27, 0x2c, 0xb7, 0xc8, 0xdd, 0x26, 0xa9, 0xfe, + 0x37, 0x64, 0x84, 0x38, 0xa5, 0x7e, 0xbc, 0x04, + }, + }, + { + .data_len = 127, + .digest = { + 0xfd, 0x1f, 0x01, 0xa4, 0xea, 0x9b, 0xbd, 0xef, + 0x09, 0x97, 0x57, 0x60, 0x95, 0x23, 0xcc, 0x71, + }, + }, + { + .data_len = 128, + .digest = { + 0x28, 0x21, 0xee, 0x56, 0x9f, 0x38, 0xd6, 0x0e, + 0xe3, 0x22, 0x06, 0x20, 0xad, 0xd8, 0x33, 0x74, + }, + }, + { + .data_len = 129, + .digest = { + 0x07, 0x28, 0x4a, 0x2a, 0xd3, 0x85, 0xa6, 0x87, + 0x5c, 0x01, 0x8c, 0xb9, 0xd3, 0x4b, 0xce, 0x20, + }, + }, + { + .data_len = 256, + .digest = { + 0xe6, 0x12, 0x25, 0x6b, 0xf9, 0x69, 0x4d, 0x5a, + 0x1a, 0xb0, 0xe6, 0x11, 0x46, 0x24, 0x08, 0xdf, + }, + }, + { + .data_len = 511, + .digest = { + 0xce, 0x28, 0x1f, 0x14, 0xb9, 0xcc, 0x7e, 0x1f, + 0xb5, 0x13, 0x2b, 0x45, 0x04, 0x54, 0xe9, 0x5f, + }, + }, + { + .data_len = 513, + .digest = { + 0x63, 0x12, 0xbd, 0x85, 0x60, 0x1b, 0x99, 0x7e, + 0x0a, 0xf7, 0x0f, 0xc1, 0xb5, 0x66, 0xf8, 0x9a, + }, + }, + { + .data_len = 1000, + .digest = { + 0xbd, 0x49, 0x5e, 0x21, 0xc6, 0x58, 0x74, 0x6b, + 0x21, 0xc2, 0x62, 0x6a, 0x15, 0xca, 0x1d, 0x8a, + }, + }, + { + .data_len = 3333, + .digest = { + 0xfe, 0x6b, 0xfa, 0xfc, 0x4c, 0x0b, 0x63, 0x0d, + 0x41, 0x7f, 0xa9, 0xd8, 0xba, 0xe3, 0xce, 0xce, + }, + }, + { + .data_len = 4096, + .digest = { + 0x41, 0x7c, 0xbc, 0x2e, 0x2f, 0xff, 0xdf, 0x09, + 0x31, 0xc5, 0x79, 0x0a, 0x1d, 0x6e, 0x46, 0xec, + }, + }, + { + .data_len = 4128, + .digest = { + 0x6a, 0x9d, 0x86, 0xa8, 0xab, 0xa5, 0xc1, 0xc5, + 0x0d, 0x54, 0xf3, 0x51, 0x10, 0x46, 0x25, 0x5a, + }, + }, + { + .data_len = 4160, + .digest = { + 0x4c, 0x50, 0xff, 0x2a, 0xe9, 0xca, 0x9e, 0x07, + 0x8a, 0x86, 0x67, 0x5e, 0xe5, 0x0a, 0xfd, 0x69, + }, + }, + { + .data_len = 4224, + .digest = { + 0x3a, 0xfa, 0x80, 0x9d, 0x80, 0xe3, 0x1e, 0x95, + 0x53, 0x93, 0x39, 0x17, 0xd3, 0xda, 0x49, 0x15, + }, + }, + { + .data_len = 16384, + .digest = { + 0x48, 0xf4, 0x4f, 0x2d, 0x5d, 0xf2, 0x02, 0xcf, + 0x51, 0x3c, 0x1b, 0x12, 0x80, 0x8f, 0xb0, 0xd6, + }, + }, +}; + +static const u8 hash_testvec_consolidated[AES_BLOCK_SIZE] = { + 0x41, 0xad, 0x25, 0xa1, 0xeb, 0xce, 0x6b, 0x9c, + 0x06, 0xdf, 0x47, 0xc4, 0x3a, 0x59, 0x50, 0x07, +}; diff --git a/lib/crypto/tests/aes_cbc_macs_kunit.c b/lib/crypto/tests/aes_cbc_macs_kunit.c new file mode 100644 index 000000000000..6afdb3a04c17 --- /dev/null +++ b/lib/crypto/tests/aes_cbc_macs_kunit.c @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2026 Google LLC + */ +#include <crypto/aes-cbc-macs.h> +#include "aes-cmac-testvecs.h" + +/* + * A fixed key used when presenting AES-CMAC as an unkeyed hash function in + * order to reuse hash-test-template.h. At the beginning of the test suite, + * this is initialized to a key prepared from bytes generated from a fixed seed. + */ +static struct aes_cmac_key test_key; + +static void aes_cmac_init_withtestkey(struct aes_cmac_ctx *ctx) +{ + aes_cmac_init(ctx, &test_key); +} + +static void aes_cmac_withtestkey(const u8 *data, size_t data_len, + u8 out[AES_BLOCK_SIZE]) +{ + aes_cmac(&test_key, data, data_len, out); +} + +#define HASH aes_cmac_withtestkey +#define HASH_CTX aes_cmac_ctx +#define HASH_SIZE AES_BLOCK_SIZE +#define HASH_INIT aes_cmac_init_withtestkey +#define HASH_UPDATE aes_cmac_update +#define HASH_FINAL aes_cmac_final +#include "hash-test-template.h" + +static int aes_cbc_macs_suite_init(struct kunit_suite *suite) +{ + u8 raw_key[AES_KEYSIZE_256]; + + rand_bytes_seeded_from_len(raw_key, sizeof(raw_key)); + return aes_cmac_preparekey(&test_key, raw_key, sizeof(raw_key)); +} + +/* Verify compatibility of the AES-CMAC implementation with RFC 4493. */ +static void test_aes_cmac_rfc4493(struct kunit *test) +{ + static const u8 raw_key[AES_KEYSIZE_128] = { + 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, + 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c, + }; + static const struct { + size_t data_len; + const u8 data[40]; + const u8 mac[AES_BLOCK_SIZE]; + } testvecs[] = { + { + /* Example 1 from RFC 4493 */ + .data_len = 0, + .mac = { + 0xbb, 0x1d, 0x69, 0x29, 0xe9, 0x59, 0x37, 0x28, + 0x7f, 0xa3, 0x7d, 0x12, 0x9b, 0x75, 0x67, 0x46, + }, + + }, + { + /* Example 2 from RFC 4493 */ + .data = { + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, + 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + }, + .data_len = 16, + .mac = { + 0x07, 0x0a, 0x16, 0xb4, 0x6b, 0x4d, 0x41, 0x44, + 0xf7, 0x9b, 0xdd, 0x9d, 0xd0, 0x4a, 0x28, 0x7c, + }, + }, + { + /* Example 3 from RFC 4493 */ + .data = { + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, + 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, + 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, + }, + .data_len = 40, + .mac = { + 0xdf, 0xa6, 0x67, 0x47, 0xde, 0x9a, 0xe6, 0x30, + 0x30, 0xca, 0x32, 0x61, 0x14, 0x97, 0xc8, 0x27, + }, + }, + }; + struct aes_cmac_key key; + int err; + + err = aes_cmac_preparekey(&key, raw_key, sizeof(raw_key)); + KUNIT_ASSERT_EQ(test, err, 0); + + for (size_t i = 0; i < ARRAY_SIZE(testvecs); i++) { + u8 mac[AES_BLOCK_SIZE]; + + aes_cmac(&key, testvecs[i].data, testvecs[i].data_len, mac); + KUNIT_ASSERT_MEMEQ(test, mac, testvecs[i].mac, AES_BLOCK_SIZE); + } +} + +/* + * Verify compatibility of the AES-XCBC-MAC implementation with RFC 3566. + * + * Additional AES-XCBC-MAC tests are not necessary, since the AES-XCBC-MAC + * implementation is well covered by the AES-CMAC tests already. Only the key + * preparation function differs; the rest of the code is shared. + */ +static void test_aes_xcbcmac_rfc3566(struct kunit *test) +{ + struct aes_cmac_key key; + /* AES-XCBC-MAC Test Case #4 from RFC 3566 */ + static const u8 raw_key[AES_KEYSIZE_128] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + }; + static const u8 message[20] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, + 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, + }; + static const u8 expected_mac[AES_BLOCK_SIZE] = { + 0x47, 0xf5, 0x1b, 0x45, 0x64, 0x96, 0x62, 0x15, + 0xb8, 0x98, 0x5c, 0x63, 0x05, 0x5e, 0xd3, 0x08, + }; + u8 actual_mac[AES_BLOCK_SIZE]; + + aes_xcbcmac_preparekey(&key, raw_key); + aes_cmac(&key, message, sizeof(message), actual_mac); + KUNIT_ASSERT_MEMEQ(test, actual_mac, expected_mac, AES_BLOCK_SIZE); +} + +static void test_aes_cbcmac_rfc3610(struct kunit *test) +{ + /* + * The following AES-CBC-MAC test vector is extracted from RFC 3610 + * Packet Vector #11. It required some rearrangement to get the actual + * input to AES-CBC-MAC from the values given. + */ + static const u8 raw_key[AES_KEYSIZE_128] = { + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, + }; + const size_t unpadded_data_len = 52; + static const u8 data[64] = { + /* clang-format off */ + /* CCM header */ + 0x61, 0x00, 0x00, 0x00, 0x0d, 0x0c, 0x0b, 0x0a, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0x00, 0x14, + /* CCM additional authentication blocks */ + 0x00, 0x0c, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, + 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x00, 0x00, + /* CCM message blocks */ + 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, + 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, + 0x1c, 0x1d, 0x1e, 0x1f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* clang-format on */ + }; + static const u8 expected_mac[AES_BLOCK_SIZE] = { + 0x6b, 0x5e, 0x24, 0x34, 0x12, 0xcc, 0xc2, 0xad, + 0x6f, 0x1b, 0x11, 0xc3, 0xa1, 0xa9, 0xd8, 0xbc, + }; + struct aes_enckey key; + struct aes_cbcmac_ctx ctx; + u8 actual_mac[AES_BLOCK_SIZE]; + int err; + + err = aes_prepareenckey(&key, raw_key, sizeof(raw_key)); + KUNIT_ASSERT_EQ(test, err, 0); + + /* + * Trailing zeroes should not affect the CBC-MAC value, up to the next + * AES block boundary. + */ + for (size_t data_len = unpadded_data_len; data_len <= sizeof(data); + data_len++) { + aes_cbcmac_init(&ctx, &key); + aes_cbcmac_update(&ctx, data, data_len); + aes_cbcmac_final(&ctx, actual_mac); + KUNIT_ASSERT_MEMEQ(test, actual_mac, expected_mac, + AES_BLOCK_SIZE); + + /* Incremental computations should produce the same result. */ + for (size_t part1_len = 0; part1_len <= data_len; part1_len++) { + aes_cbcmac_init(&ctx, &key); + aes_cbcmac_update(&ctx, data, part1_len); + aes_cbcmac_update(&ctx, &data[part1_len], + data_len - part1_len); + aes_cbcmac_final(&ctx, actual_mac); + KUNIT_ASSERT_MEMEQ(test, actual_mac, expected_mac, + AES_BLOCK_SIZE); + } + } +} + +static struct kunit_case aes_cbc_macs_test_cases[] = { + HASH_KUNIT_CASES, + KUNIT_CASE(test_aes_cmac_rfc4493), + KUNIT_CASE(test_aes_xcbcmac_rfc3566), + KUNIT_CASE(test_aes_cbcmac_rfc3610), + KUNIT_CASE(benchmark_hash), + {}, +}; + +static struct kunit_suite aes_cbc_macs_test_suite = { + .name = "aes_cbc_macs", + .test_cases = aes_cbc_macs_test_cases, + .suite_init = aes_cbc_macs_suite_init, +}; +kunit_test_suite(aes_cbc_macs_test_suite); + +MODULE_DESCRIPTION( + "KUnit tests and benchmark for AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC"); +MODULE_IMPORT_NS("CRYPTO_INTERNAL"); +MODULE_LICENSE("GPL"); diff --git a/lib/crypto/tests/aes_ccm_kunit.c b/lib/crypto/tests/aes_ccm_kunit.c new file mode 100644 index 000000000000..e8f2b3446017 --- /dev/null +++ b/lib/crypto/tests/aes_ccm_kunit.c @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * KUnit test suite for AES-CCM + * + * Copyright 2026 Google LLC + */ +#include <crypto/aes-ccm.h> +#include <crypto/blake2s.h> +#include "test-utils.h" + +/* AES-CCM test vectors from external sources */ +static const struct aes_ccm_testvec { + const char *name; + const char *key; + size_t key_len; + const char *nonce; + size_t nonce_len; + const char *ad; + size_t ad_len; + const char *ptext; + const char *ctext; + size_t data_len; + const char *tag; + size_t tag_len; +} aes_ccm_testvecs[] = { + { + .name = "RFC 3610 Packet Vector #1", + .key = "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" + "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf", + .key_len = 16, + .nonce = "\x00\x00\x00\x03\x02\x01\x00\xa0" + "\xa1\xa2\xa3\xa4\xa5", + .nonce_len = 13, + .ad = "\x00\x01\x02\x03\x04\x05\x06\x07", + .ad_len = 8, + .ptext = "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + "\x10\x11\x12\x13\x14\x15\x16\x17" + "\x18\x19\x1a\x1b\x1c\x1d\x1e", + .ctext = "\x58\x8c\x97\x9a\x61\xc6\x63\xd2" + "\xf0\x66\xd0\xc2\xc0\xf9\x89\x80" + "\x6d\x5f\x6b\x61\xda\xc3\x84", + .data_len = 23, + .tag = "\x17\xe8\xd1\x2c\xfd\xf9\x26\xe0", + .tag_len = 8, + }, + { + .name = "RFC 3610 Packet Vector #5", + .key = "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" + "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf", + .key_len = 16, + .nonce = "\x00\x00\x00\x07\x06\x05\x04\xa0" + "\xa1\xa2\xa3\xa4\xa5", + .nonce_len = 13, + .ad = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b", + .ad_len = 12, + .ptext = "\x0c\x0d\x0e\x0f\x10\x11\x12\x13" + "\x14\x15\x16\x17\x18\x19\x1a\x1b" + "\x1c\x1d\x1e\x1f", + .ctext = "\xdc\xf1\xfb\x7b\x5d\x9e\x23\xfb" + "\x9d\x4e\x13\x12\x53\x65\x8a\xd8" + "\x6e\xbd\xca\x3e", + .data_len = 20, + .tag = "\x51\xe8\x3f\x07\x7d\x9c\x2d\x93", + .tag_len = 8, + }, + { + .name = "RFC 3610 Packet Vector #9", + .key = "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" + "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf", + .key_len = 16, + .nonce = "\x00\x00\x00\x0b\x0a\x09\x08\xa0" + "\xa1\xa2\xa3\xa4\xa5", + .nonce_len = 13, + .ad = "\x00\x01\x02\x03\x04\x05\x06\x07", + .ad_len = 8, + .ptext = "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + "\x10\x11\x12\x13\x14\x15\x16\x17" + "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + "\x20", + .ctext = "\x82\x53\x1a\x60\xcc\x24\x94\x5a" + "\x4b\x82\x79\x18\x1a\xb5\xc8\x4d" + "\xf2\x1c\xe7\xf9\xb7\x3f\x42\xe1" + "\x97", + .data_len = 25, + .tag = "\xea\x9c\x07\xe5\x6b\x5e\xb1\x7e" + "\x5f\x4e", + .tag_len = 10, + }, + { + .name = "NIST SP 800-38C Example 1", + .key = "\x40\x41\x42\x43\x44\x45\x46\x47" + "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f", + .key_len = 16, + .nonce = "\x10\x11\x12\x13\x14\x15\x16", + .nonce_len = 7, + .ad = "\x00\x01\x02\x03\x04\x05\x06\x07", + .ad_len = 8, + .ptext = "\x20\x21\x22\x23", + .ctext = "\x71\x62\x01\x5b", + .data_len = 4, + .tag = "\x4d\xac\x25\x5d", + .tag_len = 4, + }, + { + .name = "NIST SP 800-38C Example 2", + .key = "\x40\x41\x42\x43\x44\x45\x46\x47" + "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f", + .key_len = 16, + .nonce = "\x10\x11\x12\x13\x14\x15\x16\x17", + .nonce_len = 8, + .ad = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + .ad_len = 16, + .ptext = "\x20\x21\x22\x23\x24\x25\x26\x27" + "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + .ctext = "\xd2\xa1\xf0\xe0\x51\xea\x5f\x62" + "\x08\x1a\x77\x92\x07\x3d\x59\x3d", + .data_len = 16, + .tag = "\x1f\xc6\x4f\xbf\xac\xcd", + .tag_len = 6, + }, + { + .name = "NIST SP 800-38C Example 3", + .key = "\x40\x41\x42\x43\x44\x45\x46\x47" + "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f", + .key_len = 16, + .nonce = "\x10\x11\x12\x13\x14\x15\x16\x17" + "\x18\x19\x1a\x1b", + .nonce_len = 12, + .ad = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + "\x10\x11\x12\x13", + .ad_len = 20, + .ptext = "\x20\x21\x22\x23\x24\x25\x26\x27" + "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f" + "\x30\x31\x32\x33\x34\x35\x36\x37", + .ctext = "\xe3\xb2\x01\xa9\xf5\xb7\x1a\x7a" + "\x9b\x1c\xea\xec\xcd\x97\xe7\x0b" + "\x61\x76\xaa\xd9\xa4\x42\x8a\xa5", + .data_len = 24, + .tag = "\x48\x43\x92\xfb\xc1\xb0\x99\x51", + .tag_len = 8, + }, +}; + +static void test_aes_ccm_one_test_vector(struct kunit *test, + const struct aes_ccm_testvec *tv) +{ + u8 *ctext = alloc_buf(test, tv->data_len); + u8 *decrypted = alloc_buf(test, tv->data_len); + u8 *tag = alloc_buf(test, tv->tag_len); + struct aes_ccm_key key; + int err; + + err = aes_ccm_preparekey(&key, tv->key, tv->key_len, tv->tag_len); + KUNIT_ASSERT_EQ_MSG(test, 0, err, "Failed to prepare key for %s", + tv->name); + + err = aes_ccm_encrypt(ctext, tv->ptext, tv->data_len, tag, tv->ad, + tv->ad_len, tv->nonce, tv->nonce_len, &key); + KUNIT_ASSERT_EQ_MSG(test, 0, err, "Encryption failed for %s", tv->name); + KUNIT_ASSERT_MEMEQ_MSG(test, tv->ctext, ctext, tv->data_len, + "Wrong ciphertext for %s", tv->name); + KUNIT_ASSERT_MEMEQ_MSG(test, tag, tv->tag, tv->tag_len, + "Wrong tag for %s", tv->name); + + err = aes_ccm_decrypt(decrypted, ctext, tv->data_len, tag, tv->ad, + tv->ad_len, tv->nonce, tv->nonce_len, &key); + KUNIT_ASSERT_EQ_MSG(test, 0, err, "Decryption failed for %s", tv->name); + KUNIT_ASSERT_MEMEQ_MSG(test, tv->ptext, decrypted, tv->data_len, + "Wrong plaintext for %s", tv->name); +} + +static void test_aes_ccm_test_vectors(struct kunit *test) +{ + for (size_t i = 0; i < ARRAY_SIZE(aes_ccm_testvecs); i++) + test_aes_ccm_one_test_vector(test, &aes_ccm_testvecs[i]); +} + +/* + * Test NIST SP 800-38C Example 4, which uses a deterministically-generated + * 65536-byte associated data string. + */ +static void test_aes_ccm_nist_sp800_38c_example4(struct kunit *test) +{ + struct aes_ccm_testvec tv = { + .name = "NIST SP 800-38C Example 4", + .key_len = 16, + .nonce_len = 13, + .ad_len = 65536, + .ctext = "\x69\x91\x5d\xad\x1e\x84\xc6\x37" + "\x6a\x68\xc2\x96\x7e\x4d\xab\x61" + "\x5a\xe0\xfd\x1f\xae\xc4\x4c\xc4" + "\x84\x82\x85\x29\x46\x3c\xcf\x72", + .data_len = 32, + .tag = "\xb4\xac\x6b\xec\x93\xe8\x59\x8e" + "\x7f\x0d\xad\xbc\xea\x5b", + .tag_len = 14, + }; + u8 *key, *nonce, *ad, *ptext; + + key = alloc_buf(test, tv.key_len); + for (size_t i = 0; i < tv.key_len; i++) + key[i] = 0x40 + i; + + nonce = alloc_guarded_buf(test, tv.nonce_len); + for (size_t i = 0; i < tv.nonce_len; i++) + nonce[i] = 0x10 + i; + + ad = alloc_guarded_buf(test, tv.ad_len); + for (size_t i = 0; i < tv.ad_len; i++) + ad[i] = (u8)i; + + ptext = alloc_guarded_buf(test, tv.data_len); + for (size_t i = 0; i < tv.data_len; i++) + ptext[i] = 0x20 + i; + + tv.key = key; + tv.nonce = nonce; + tv.ad = ad; + tv.ptext = ptext; + + test_aes_ccm_one_test_vector(test, &tv); +} + +static const size_t aes_ccm_valid_key_lens[] = { 16, 24, 32 }; +#define AEAD_MAX_KEY_LEN 32 +#define AEAD_VALID_KEY_LENS aes_ccm_valid_key_lens + +static const size_t aes_ccm_valid_nonce_lens[] = { 7, 8, 9, 10, 11, 12, 13 }; +#define AEAD_MAX_NONCE_LEN 13 +#define AEAD_VALID_NONCE_LENS aes_ccm_valid_nonce_lens + +static const size_t aes_ccm_valid_tag_lens[] = { 4, 6, 8, 10, 12, 14, 16 }; +#define AEAD_MAX_TAG_LEN 16 +#define AEAD_VALID_TAG_LENS aes_ccm_valid_tag_lens + +#define AEAD_KEY aes_ccm_key +#define AEAD_CTX aes_ccm_ctx +#define AEAD_PREPAREKEY aes_ccm_preparekey +#define AEAD_ENCRYPT aes_ccm_encrypt +#define AEAD_DECRYPT aes_ccm_decrypt + +#define AEAD_INIT aes_ccm_init +#define AEAD_AUTH_UPDATE aes_ccm_auth_update +#define AEAD_ENCRYPT_UPDATE aes_ccm_encrypt_update +#define AEAD_DECRYPT_UPDATE aes_ccm_decrypt_update +#define AEAD_ENCRYPT_FINAL aes_ccm_encrypt_final +#define AEAD_DECRYPT_FINAL aes_ccm_decrypt_final + +/* This value was generated by gen-aead-testvecs.py. */ +static const u8 aes_ccm_monte_carlo_checksum[BLAKE2S_HASH_SIZE] = { + 0x70, 0x1c, 0xde, 0xa4, 0xe2, 0x03, 0x50, 0xb2, 0xf5, 0x9e, 0x61, + 0x66, 0xe4, 0xe5, 0x13, 0x1a, 0x00, 0x95, 0x34, 0x03, 0xb7, 0x61, + 0x2c, 0xdb, 0xc3, 0x15, 0x36, 0x84, 0x93, 0x7f, 0xb4, 0x5b, +}; +#define AEAD_MONTE_CARLO_CHECKSUM aes_ccm_monte_carlo_checksum + +#include "aead-test-template.h" + +/* + * Test that for each AES-CCM nonce length, the message length is validated + * against the correct corresponding maximum message length. + */ +static void test_aes_ccm_data_len_too_large(struct kunit *test) +{ + static const struct { + size_t nonce_len; + u64 max_data_len; + } lens[] = { + /* clang-format off */ + { 7, 0xffffffffffffffff }, /* U64_MAX */ + { 8, 0xffffffffffffff }, + { 9, 0xffffffffffff }, + { 10, 0xffffffffff }, + { 11, 0xffffffff }, + { 12, 0xffffff }, + { 13, 0xffff }, + /* clang-format on */ + }; + u8 nonce[13] = {}; + u8 raw_key[AES_KEYSIZE_256] = {}; + int err; + struct aes_ccm_key *key = alloc_buf(test, sizeof(*key)); + struct aes_ccm_ctx ctx; + + err = aes_ccm_preparekey(key, raw_key, sizeof(raw_key), 16); + KUNIT_ASSERT_EQ(test, 0, err); + + for (size_t i = 0; i < ARRAY_SIZE(lens); i++) { + size_t nonce_len = lens[i].nonce_len; + u64 max_data_len = lens[i].max_data_len; + + /* data_len <= max_data_len should be accepted. */ + err = aes_ccm_init(&ctx, 0, 0, nonce, nonce_len, key); + KUNIT_ASSERT_EQ_MSG( + test, 0, err, + "data_len=0 wasn't accepted with nonce_len=%zu", + nonce_len); + err = aes_ccm_init(&ctx, max_data_len, 0, nonce, nonce_len, + key); + KUNIT_ASSERT_EQ_MSG( + test, 0, err, + "data_len=%llu wasn't accepted with nonce_len=%zu", + max_data_len, nonce_len); + + /* data_len > max_data_len should be rejected. */ + if (max_data_len == U64_MAX) + continue; + err = aes_ccm_init(&ctx, max_data_len + 1, 0, nonce, nonce_len, + key); + KUNIT_ASSERT_EQ_MSG( + test, -EOVERFLOW, err, + "data_len=%llu wasn't rejected with -EOVERFLOW with nonce_len=%zu (aes_ccm_init)", + max_data_len + 1, nonce_len); + if (max_data_len + 1 <= SIZE_MAX) { + err = aes_ccm_encrypt(NULL, NULL, max_data_len + 1, + NULL, NULL, 0, nonce, nonce_len, + key); + KUNIT_ASSERT_EQ_MSG( + test, -EOVERFLOW, err, + "data_len=%llu wasn't rejected with -EOVERFLOW with nonce_len=%zu (aes_ccm_encrypt)", + max_data_len + 1, nonce_len); + err = aes_ccm_decrypt(NULL, NULL, max_data_len + 1, + NULL, NULL, 0, nonce, nonce_len, + key); + KUNIT_ASSERT_EQ_MSG( + test, -EOVERFLOW, err, + "data_len=%llu wasn't rejected with -EOVERFLOW with nonce_len=%zu (aes_ccm_decrypt)", + max_data_len + 1, nonce_len); + } + err = aes_ccm_init(&ctx, U64_MAX, 0, nonce, nonce_len, key); + KUNIT_ASSERT_EQ_MSG( + test, -EOVERFLOW, err, + "data_len=U64_MAX wasn't rejected with -EOVERFLOW with nonce_len=%zu (aes_ccm_init)", + nonce_len); + } +} + +static struct kunit_case aes_ccm_test_cases[] = { + KUNIT_CASE(test_aes_ccm_test_vectors), + KUNIT_CASE(test_aes_ccm_nist_sp800_38c_example4), + KUNIT_CASE(test_aes_ccm_data_len_too_large), + AEAD_KUNIT_CASES, + {}, +}; + +static struct kunit_suite aes_ccm_test_suite = { + .name = "aes_ccm", + .test_cases = aes_ccm_test_cases, +}; +kunit_test_suite(aes_ccm_test_suite); + +MODULE_DESCRIPTION("KUnit tests and benchmark for AES-CCM"); +MODULE_LICENSE("GPL"); diff --git a/lib/crypto/tests/aes_gcm_kunit.c b/lib/crypto/tests/aes_gcm_kunit.c new file mode 100644 index 000000000000..6959d86b316a --- /dev/null +++ b/lib/crypto/tests/aes_gcm_kunit.c @@ -0,0 +1,472 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * KUnit test suite for AES-GCM + * + * Copyright 2026 Google LLC + */ +#include <crypto/aes-gcm.h> +#include <crypto/blake2s.h> +#include "test-utils.h" + +/* The kernel's AES-GCM implementation supports only 12-byte nonces. */ +#define AES_GCM_NONCE_LEN 12 + +/* + * AES-GCM test vectors from the original paper "The Galois/Counter Mode of + * Operation (GCM)" by McGrew & Viega. Vectors with nonce lengths other than 12 + * bytes are excluded. + */ +static const struct aes_gcm_testvec { + const char *name; + const char *key; + size_t key_len; + const char *nonce; + size_t nonce_len; + const char *ad; + size_t ad_len; + const char *ptext; + const char *ctext; + size_t data_len; + const char *tag; + size_t tag_len; +} aes_gcm_testvecs[] = { + /* AES-128 Test Vectors */ + { + .name = "McGrew & Viega Test Case 1", + .key = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .key_len = 16, + .nonce = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00", + .nonce_len = 12, + .ptext = "", + .ctext = "", + .data_len = 0, + .ad = "", + .ad_len = 0, + .tag = "\x58\xe2\xfc\xce\xfa\x7e\x30\x61" + "\x36\x7f\x1d\x57\xa4\xe7\x45\x5a", + .tag_len = 16, + }, + { + .name = "McGrew & Viega Test Case 2", + .key = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .key_len = 16, + .nonce = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00", + .nonce_len = 12, + .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .ctext = "\x03\x88\xda\xce\x60\xb6\xa3\x92" + "\xf3\x28\xc2\xb9\x71\xb2\xfe\x78", + .data_len = 16, + .ad = "", + .ad_len = 0, + .tag = "\xab\x6e\x47\xd4\x2c\xec\x13\xbd" + "\xf5\x3a\x67\xb2\x12\x57\xbd\xdf", + .tag_len = 16, + }, + { + .name = "McGrew & Viega Test Case 3", + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08", + .key_len = 16, + .nonce = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" + "\xde\xca\xf8\x88", + .nonce_len = 12, + .ptext = "\xd9\x31\x32\x25\xf8\x84\x06\xe5" + "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" + "\x86\xa7\xa9\x53\x15\x34\xf7\xda" + "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" + "\x1c\x3c\x0c\x95\x95\x68\x09\x53" + "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" + "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" + "\xba\x63\x7b\x39\x1a\xaf\xd2\x55", + .ctext = "\x42\x83\x1e\xc2\x21\x77\x74\x24" + "\x4b\x72\x21\xb7\x84\xd0\xd4\x9c" + "\xe3\xaa\x21\x2f\x2c\x02\xa4\xe0" + "\x35\xc1\x7e\x23\x29\xac\xa1\x2e" + "\x21\xd5\x14\xb2\x54\x66\x93\x1c" + "\x7d\x8f\x6a\x5a\xac\x84\xaa\x05" + "\x1b\xa3\x0b\x39\x6a\x0a\xac\x97" + "\x3d\x58\xe0\x91\x47\x3f\x59\x85", + .data_len = 64, + .ad = "", + .ad_len = 0, + .tag = "\x4d\x5c\x2a\xf3\x27\xcd\x64\xa6" + "\x2c\xf3\x5a\xbd\x2b\xa6\xfa\xb4", + .tag_len = 16, + }, + { + .name = "McGrew & Viega Test Case 4", + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08", + .key_len = 16, + .nonce = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" + "\xde\xca\xf8\x88", + .nonce_len = 12, + .ptext = "\xd9\x31\x32\x25\xf8\x84\x06\xe5" + "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" + "\x86\xa7\xa9\x53\x15\x34\xf7\xda" + "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" + "\x1c\x3c\x0c\x95\x95\x68\x09\x53" + "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" + "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" + "\xba\x63\x7b\x39", + .ctext = "\x42\x83\x1e\xc2\x21\x77\x74\x24" + "\x4b\x72\x21\xb7\x84\xd0\xd4\x9c" + "\xe3\xaa\x21\x2f\x2c\x02\xa4\xe0" + "\x35\xc1\x7e\x23\x29\xac\xa1\x2e" + "\x21\xd5\x14\xb2\x54\x66\x93\x1c" + "\x7d\x8f\x6a\x5a\xac\x84\xaa\x05" + "\x1b\xa3\x0b\x39\x6a\x0a\xac\x97" + "\x3d\x58\xe0\x91", + .data_len = 60, + .ad = "\xfe\xed\xfa\xce\xde\xad\xbe\xef" + "\xfe\xed\xfa\xce\xde\xad\xbe\xef" + "\xab\xad\xda\xd2", + .ad_len = 20, + .tag = "\x5b\xc9\x4f\xbc\x32\x21\xa5\xdb" + "\x94\xfa\xe9\x5a\xe7\x12\x1a\x47", + .tag_len = 16, + }, + + /* AES-192 Test Vectors */ + { + .name = "McGrew & Viega Test Case 7", + .key = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .key_len = 24, + .nonce = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00", + .nonce_len = 12, + .ptext = "", + .ctext = "", + .data_len = 0, + .ad = "", + .ad_len = 0, + .tag = "\xcd\x33\xb2\x8a\xc7\x73\xf7\x4b" + "\xa0\x0e\xd1\xf3\x12\x57\x24\x35", + .tag_len = 16, + }, + { + .name = "McGrew & Viega Test Case 8", + .key = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .key_len = 24, + .nonce = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00", + .nonce_len = 12, + .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .ctext = "\x98\xe7\x24\x7c\x07\xf0\xfe\x41" + "\x1c\x26\x7e\x43\x84\xb0\xf6\x00", + .data_len = 16, + .ad = "", + .ad_len = 0, + .tag = "\x2f\xf5\x8d\x80\x03\x39\x27\xab" + "\x8e\xf4\xd4\x58\x75\x14\xf0\xfb", + .tag_len = 16, + }, + { + .name = "McGrew & Viega Test Case 9", + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08" + "\xfe\xff\xe9\x92\x86\x65\x73\x1c", + .key_len = 24, + .nonce = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" + "\xde\xca\xf8\x88", + .nonce_len = 12, + .ptext = "\xd9\x31\x32\x25\xf8\x84\x06\xe5" + "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" + "\x86\xa7\xa9\x53\x15\x34\xf7\xda" + "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" + "\x1c\x3c\x0c\x95\x95\x68\x09\x53" + "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" + "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" + "\xba\x63\x7b\x39\x1a\xaf\xd2\x55", + .ctext = "\x39\x80\xca\x0b\x3c\x00\xe8\x41" + "\xeb\x06\xfa\xc4\x87\x2a\x27\x57" + "\x85\x9e\x1c\xea\xa6\xef\xd9\x84" + "\x62\x85\x93\xb4\x0c\xa1\xe1\x9c" + "\x7d\x77\x3d\x00\xc1\x44\xc5\x25" + "\xac\x61\x9d\x18\xc8\x4a\x3f\x47" + "\x18\xe2\x44\x8b\x2f\xe3\x24\xd9" + "\xcc\xda\x27\x10\xac\xad\xe2\x56", + .data_len = 64, + .ad = "", + .ad_len = 0, + .tag = "\x99\x24\xa7\xc8\x58\x73\x36\xbf" + "\xb1\x18\x02\x4d\xb8\x67\x4a\x14", + .tag_len = 16, + }, + { + .name = "McGrew & Viega Test Case 10", + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08" + "\xfe\xff\xe9\x92\x86\x65\x73\x1c", + .key_len = 24, + .nonce = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" + "\xde\xca\xf8\x88", + .nonce_len = 12, + .ptext = "\xd9\x31\x32\x25\xf8\x84\x06\xe5" + "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" + "\x86\xa7\xa9\x53\x15\x34\xf7\xda" + "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" + "\x1c\x3c\x0c\x95\x95\x68\x09\x53" + "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" + "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" + "\xba\x63\x7b\x39", + .ctext = "\x39\x80\xca\x0b\x3c\x00\xe8\x41" + "\xeb\x06\xfa\xc4\x87\x2a\x27\x57" + "\x85\x9e\x1c\xea\xa6\xef\xd9\x84" + "\x62\x85\x93\xb4\x0c\xa1\xe1\x9c" + "\x7d\x77\x3d\x00\xc1\x44\xc5\x25" + "\xac\x61\x9d\x18\xc8\x4a\x3f\x47" + "\x18\xe2\x44\x8b\x2f\xe3\x24\xd9" + "\xcc\xda\x27\x10", + .data_len = 60, + .ad = "\xfe\xed\xfa\xce\xde\xad\xbe\xef" + "\xfe\xed\xfa\xce\xde\xad\xbe\xef" + "\xab\xad\xda\xd2", + .ad_len = 20, + .tag = "\x25\x19\x49\x8e\x80\xf1\x47\x8f" + "\x37\xba\x55\xbd\x6d\x27\x61\x8c", + .tag_len = 16, + }, + + /* AES-256 Test Vectors */ + { + .name = "McGrew & Viega Test Case 13", + .key = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .key_len = 32, + .nonce = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00", + .nonce_len = 12, + .ptext = "", + .ctext = "", + .data_len = 0, + .ad = "", + .ad_len = 0, + .tag = "\x53\x0f\x8a\xfb\xc7\x45\x36\xb9" + "\xa9\x63\xb4\xf1\xc4\xcb\x73\x8b", + .tag_len = 16, + }, + { + .name = "McGrew & Viega Test Case 14", + .key = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .key_len = 32, + .nonce = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00", + .nonce_len = 12, + .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .ctext = "\xce\xa7\x40\x3d\x4d\x60\x6b\x6e" + "\x07\x4e\xc5\xd3\xba\xf3\x9d\x18", + .data_len = 16, + .ad = "", + .ad_len = 0, + .tag = "\xd0\xd1\xc8\xa7\x99\x99\x6b\xf0" + "\x26\x5b\x98\xb5\xd4\x8a\xb9\x19", + .tag_len = 16, + }, + { + .name = "McGrew & Viega Test Case 15", + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08" + "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08", + .key_len = 32, + .nonce = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" + "\xde\xca\xf8\x88", + .nonce_len = 12, + .ptext = "\xd9\x31\x32\x25\xf8\x84\x06\xe5" + "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" + "\x86\xa7\xa9\x53\x15\x34\xf7\xda" + "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" + "\x1c\x3c\x0c\x95\x95\x68\x09\x53" + "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" + "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" + "\xba\x63\x7b\x39\x1a\xaf\xd2\x55", + .ctext = "\x52\x2d\xc1\xf0\x99\x56\x7d\x07" + "\xf4\x7f\x37\xa3\x2a\x84\x42\x7d" + "\x64\x3a\x8c\xdc\xbf\xe5\xc0\xc9" + "\x75\x98\xa2\xbd\x25\x55\xd1\xaa" + "\x8c\xb0\x8e\x48\x59\x0d\xbb\x3d" + "\xa7\xb0\x8b\x10\x56\x82\x88\x38" + "\xc5\xf6\x1e\x63\x93\xba\x7a\x0a" + "\xbc\xc9\xf6\x62\x89\x80\x15\xad", + .data_len = 64, + .ad = "", + .ad_len = 0, + .tag = "\xb0\x94\xda\xc5\xd9\x34\x71\xbd" + "\xec\x1a\x50\x22\x70\xe3\xcc\x6c", + .tag_len = 16, + }, + { + .name = "McGrew & Viega Test Case 16", + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08" + "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08", + .key_len = 32, + .nonce = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" + "\xde\xca\xf8\x88", + .nonce_len = 12, + .ptext = "\xd9\x31\x32\x25\xf8\x84\x06\xe5" + "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" + "\x86\xa7\xa9\x53\x15\x34\xf7\xda" + "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" + "\x1c\x3c\x0c\x95\x95\x68\x09\x53" + "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" + "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" + "\xba\x63\x7b\x39", + .ctext = "\x52\x2d\xc1\xf0\x99\x56\x7d\x07" + "\xf4\x7f\x37\xa3\x2a\x84\x42\x7d" + "\x64\x3a\x8c\xdc\xbf\xe5\xc0\xc9" + "\x75\x98\xa2\xbd\x25\x55\xd1\xaa" + "\x8c\xb0\x8e\x48\x59\x0d\xbb\x3d" + "\xa7\xb0\x8b\x10\x56\x82\x88\x38" + "\xc5\xf6\x1e\x63\x93\xba\x7a\x0a" + "\xbc\xc9\xf6\x62", + .data_len = 60, + .ad = "\xfe\xed\xfa\xce\xde\xad\xbe\xef" + "\xfe\xed\xfa\xce\xde\xad\xbe\xef" + "\xab\xad\xda\xd2", + .ad_len = 20, + .tag = "\x76\xfc\x6e\xce\x0f\x4e\x17\x68" + "\xcd\xdf\x88\x53\xbb\x2d\x55\x1b", + .tag_len = 16, + }, +}; + +static void test_aes_gcm_one_test_vector(struct kunit *test, + const struct aes_gcm_testvec *tv) +{ + u8 *ctext = alloc_buf(test, tv->data_len); + u8 *decrypted = alloc_buf(test, tv->data_len); + u8 *tag = alloc_buf(test, tv->tag_len); + struct aes_gcm_key key; + int err; + + KUNIT_ASSERT_EQ(test, AES_GCM_NONCE_LEN, tv->nonce_len); + + err = aes_gcm_preparekey(&key, tv->key, tv->key_len, tv->tag_len); + KUNIT_ASSERT_EQ_MSG(test, 0, err, "Failed to prepare key for %s", + tv->name); + + aes_gcm_encrypt(ctext, tv->ptext, tv->data_len, tag, tv->ad, tv->ad_len, + tv->nonce, &key); + KUNIT_ASSERT_MEMEQ_MSG(test, tv->ctext, ctext, tv->data_len, + "Wrong ciphertext for %s", tv->name); + KUNIT_ASSERT_MEMEQ_MSG(test, tag, tv->tag, tv->tag_len, + "Wrong tag for %s", tv->name); + + err = aes_gcm_decrypt(decrypted, ctext, tv->data_len, tag, tv->ad, + tv->ad_len, tv->nonce, &key); + KUNIT_ASSERT_EQ_MSG(test, 0, err, "Decryption failed for %s", tv->name); + KUNIT_ASSERT_MEMEQ_MSG(test, tv->ptext, decrypted, tv->data_len, + "Wrong plaintext for %s", tv->name); +} + +static void test_aes_gcm_test_vectors(struct kunit *test) +{ + for (size_t i = 0; i < ARRAY_SIZE(aes_gcm_testvecs); i++) + test_aes_gcm_one_test_vector(test, &aes_gcm_testvecs[i]); +} + +static int aes_gcm_init_test(struct aes_gcm_ctx *ctx, u64 data_len, u64 ad_len, + const u8 *nonce, size_t nonce_len, + const struct aes_gcm_key *key) +{ + if (nonce_len != AES_GCM_NONCE_LEN) + return -EINVAL; + /* + * Ignore data_len and ad_len. Incremental AES-GCM doesn't need them at + * initialization time. + */ + aes_gcm_init(ctx, nonce, key); + return 0; +} + +static int aes_gcm_encrypt_test(u8 *dst, const u8 *src, size_t data_len, + u8 *authtag, const u8 *ad, size_t ad_len, + const u8 *nonce, size_t nonce_len, + const struct aes_gcm_key *key) +{ + if (nonce_len != AES_GCM_NONCE_LEN) + return -EINVAL; + /* aes_gcm_encrypt() returns void. */ + aes_gcm_encrypt(dst, src, data_len, authtag, ad, ad_len, nonce, key); + return 0; +} + +static int aes_gcm_decrypt_test(u8 *dst, const u8 *src, size_t data_len, + const u8 *authtag, const u8 *ad, size_t ad_len, + const u8 *nonce, size_t nonce_len, + const struct aes_gcm_key *key) +{ + if (nonce_len != AES_GCM_NONCE_LEN) + return -EINVAL; + return aes_gcm_decrypt(dst, src, data_len, authtag, ad, ad_len, nonce, + key); +} + +static const size_t aes_gcm_valid_key_lens[] = { 16, 24, 32 }; +#define AEAD_MAX_KEY_LEN 32 +#define AEAD_VALID_KEY_LENS aes_gcm_valid_key_lens + +static const size_t aes_gcm_valid_nonce_lens[] = { AES_GCM_NONCE_LEN }; +#define AEAD_MAX_NONCE_LEN AES_GCM_NONCE_LEN +#define AEAD_VALID_NONCE_LENS aes_gcm_valid_nonce_lens + +static const size_t aes_gcm_valid_tag_lens[] = { 4, 8, 12, 13, 14, 15, 16 }; +#define AEAD_MAX_TAG_LEN 16 +#define AEAD_VALID_TAG_LENS aes_gcm_valid_tag_lens + +#define AEAD_KEY aes_gcm_key +#define AEAD_CTX aes_gcm_ctx +#define AEAD_PREPAREKEY aes_gcm_preparekey +#define AEAD_ENCRYPT aes_gcm_encrypt_test +#define AEAD_DECRYPT aes_gcm_decrypt_test + +#define AEAD_INIT aes_gcm_init_test +#define AEAD_AUTH_UPDATE aes_gcm_auth_update +#define AEAD_ENCRYPT_UPDATE aes_gcm_encrypt_update +#define AEAD_DECRYPT_UPDATE aes_gcm_decrypt_update +#define AEAD_ENCRYPT_FINAL aes_gcm_encrypt_final +#define AEAD_DECRYPT_FINAL aes_gcm_decrypt_final + +/* This value was generated by gen-aead-testvecs.py. */ +static const u8 aes_gcm_monte_carlo_checksum[BLAKE2S_HASH_SIZE] = { + 0x6d, 0xd0, 0x6e, 0x6b, 0xde, 0x3e, 0x92, 0x9f, 0xae, 0x1f, 0xf1, + 0x84, 0x99, 0x5a, 0x9e, 0x7b, 0xfe, 0x20, 0x9e, 0x22, 0x7c, 0x5f, + 0x15, 0xb3, 0x59, 0x89, 0xd0, 0xb7, 0x74, 0x5d, 0xd2, 0xa5, +}; +#define AEAD_MONTE_CARLO_CHECKSUM aes_gcm_monte_carlo_checksum + +#include "aead-test-template.h" + +static struct kunit_case aes_gcm_test_cases[] = { + KUNIT_CASE(test_aes_gcm_test_vectors), + AEAD_KUNIT_CASES, + {}, +}; + +static struct kunit_suite aes_gcm_test_suite = { + .name = "aes_gcm", + .test_cases = aes_gcm_test_cases, +}; +kunit_test_suite(aes_gcm_test_suite); + +MODULE_DESCRIPTION("KUnit tests and benchmark for AES-GCM"); +MODULE_LICENSE("GPL"); diff --git a/lib/crypto/tests/blake2b_kunit.c b/lib/crypto/tests/blake2b_kunit.c index bc0be7da1e76..e2ce6cf3a99d 100644 --- a/lib/crypto/tests/blake2b_kunit.c +++ b/lib/crypto/tests/blake2b_kunit.c @@ -41,9 +41,9 @@ static void blake2b_init_default(struct blake2b_ctx *ctx) static void test_blake2b_all_key_and_hash_lens(struct kunit *test) { const size_t data_len = 100; - u8 *data = &test_buf[0]; - u8 *key = data + data_len; - u8 *hash = key + BLAKE2B_KEY_SIZE; + u8 *data = alloc_buf(test, data_len); + u8 *key = alloc_buf(test, BLAKE2B_KEY_SIZE); + u8 *hash = alloc_buf(test, BLAKE2B_HASH_SIZE); struct blake2b_ctx main_ctx; u8 main_hash[BLAKE2B_HASH_SIZE]; @@ -68,11 +68,13 @@ static void test_blake2b_all_key_and_hash_lens(struct kunit *test) static void test_blake2b_with_guarded_key_buf(struct kunit *test) { const size_t data_len = 100; + u8 *data = alloc_buf(test, data_len); + u8 *guarded_key_buf = alloc_guarded_buf(test, BLAKE2B_KEY_SIZE); - rand_bytes(test_buf, data_len); + rand_bytes(data, data_len); for (int key_len = 0; key_len <= BLAKE2B_KEY_SIZE; key_len++) { u8 key[BLAKE2B_KEY_SIZE]; - u8 *guarded_key = &test_buf[TEST_BUF_LEN - key_len]; + u8 *guarded_key = &guarded_key_buf[BLAKE2B_KEY_SIZE - key_len]; u8 hash1[BLAKE2B_HASH_SIZE]; u8 hash2[BLAKE2B_HASH_SIZE]; struct blake2b_ctx ctx; @@ -80,14 +82,13 @@ static void test_blake2b_with_guarded_key_buf(struct kunit *test) rand_bytes(key, key_len); memcpy(guarded_key, key, key_len); - blake2b(key, key_len, test_buf, data_len, - hash1, BLAKE2B_HASH_SIZE); - blake2b(guarded_key, key_len, test_buf, data_len, - hash2, BLAKE2B_HASH_SIZE); + blake2b(key, key_len, data, data_len, hash1, BLAKE2B_HASH_SIZE); + blake2b(guarded_key, key_len, data, data_len, hash2, + BLAKE2B_HASH_SIZE); KUNIT_ASSERT_MEMEQ(test, hash1, hash2, BLAKE2B_HASH_SIZE); blake2b_init_key(&ctx, BLAKE2B_HASH_SIZE, guarded_key, key_len); - blake2b_update(&ctx, test_buf, data_len); + blake2b_update(&ctx, data, data_len); blake2b_final(&ctx, hash2); KUNIT_ASSERT_MEMEQ(test, hash1, hash2, BLAKE2B_HASH_SIZE); } @@ -100,14 +101,16 @@ static void test_blake2b_with_guarded_key_buf(struct kunit *test) static void test_blake2b_with_guarded_out_buf(struct kunit *test) { const size_t data_len = 100; + u8 *data = alloc_buf(test, data_len); + u8 *out_buf = alloc_guarded_buf(test, BLAKE2B_HASH_SIZE); - rand_bytes(test_buf, data_len); + rand_bytes(data, data_len); for (int out_len = 1; out_len <= BLAKE2B_HASH_SIZE; out_len++) { u8 hash[BLAKE2B_HASH_SIZE]; - u8 *guarded_hash = &test_buf[TEST_BUF_LEN - out_len]; + u8 *guarded_hash = &out_buf[BLAKE2B_HASH_SIZE - out_len]; - blake2b(NULL, 0, test_buf, data_len, hash, out_len); - blake2b(NULL, 0, test_buf, data_len, guarded_hash, out_len); + blake2b(NULL, 0, data, data_len, hash, out_len); + blake2b(NULL, 0, data, data_len, guarded_hash, out_len); KUNIT_ASSERT_MEMEQ(test, hash, guarded_hash, out_len); } } @@ -124,8 +127,6 @@ static struct kunit_case blake2b_test_cases[] = { static struct kunit_suite blake2b_test_suite = { .name = "blake2b", .test_cases = blake2b_test_cases, - .suite_init = hash_suite_init, - .suite_exit = hash_suite_exit, }; kunit_test_suite(blake2b_test_suite); diff --git a/lib/crypto/tests/blake2s_kunit.c b/lib/crypto/tests/blake2s_kunit.c index 6832d9aa7b82..682f695f8a09 100644 --- a/lib/crypto/tests/blake2s_kunit.c +++ b/lib/crypto/tests/blake2s_kunit.c @@ -41,9 +41,9 @@ static void blake2s_init_default(struct blake2s_ctx *ctx) static void test_blake2s_all_key_and_hash_lens(struct kunit *test) { const size_t data_len = 100; - u8 *data = &test_buf[0]; - u8 *key = data + data_len; - u8 *hash = key + BLAKE2S_KEY_SIZE; + u8 *data = alloc_buf(test, data_len); + u8 *key = alloc_buf(test, BLAKE2S_KEY_SIZE); + u8 *hash = alloc_buf(test, BLAKE2S_HASH_SIZE); struct blake2s_ctx main_ctx; u8 main_hash[BLAKE2S_HASH_SIZE]; @@ -68,11 +68,13 @@ static void test_blake2s_all_key_and_hash_lens(struct kunit *test) static void test_blake2s_with_guarded_key_buf(struct kunit *test) { const size_t data_len = 100; + u8 *data = alloc_buf(test, data_len); + u8 *guarded_key_buf = alloc_guarded_buf(test, BLAKE2S_KEY_SIZE); - rand_bytes(test_buf, data_len); + rand_bytes(data, data_len); for (int key_len = 0; key_len <= BLAKE2S_KEY_SIZE; key_len++) { u8 key[BLAKE2S_KEY_SIZE]; - u8 *guarded_key = &test_buf[TEST_BUF_LEN - key_len]; + u8 *guarded_key = &guarded_key_buf[BLAKE2S_KEY_SIZE - key_len]; u8 hash1[BLAKE2S_HASH_SIZE]; u8 hash2[BLAKE2S_HASH_SIZE]; struct blake2s_ctx ctx; @@ -80,14 +82,13 @@ static void test_blake2s_with_guarded_key_buf(struct kunit *test) rand_bytes(key, key_len); memcpy(guarded_key, key, key_len); - blake2s(key, key_len, test_buf, data_len, - hash1, BLAKE2S_HASH_SIZE); - blake2s(guarded_key, key_len, test_buf, data_len, - hash2, BLAKE2S_HASH_SIZE); + blake2s(key, key_len, data, data_len, hash1, BLAKE2S_HASH_SIZE); + blake2s(guarded_key, key_len, data, data_len, hash2, + BLAKE2S_HASH_SIZE); KUNIT_ASSERT_MEMEQ(test, hash1, hash2, BLAKE2S_HASH_SIZE); blake2s_init_key(&ctx, BLAKE2S_HASH_SIZE, guarded_key, key_len); - blake2s_update(&ctx, test_buf, data_len); + blake2s_update(&ctx, data, data_len); blake2s_final(&ctx, hash2); KUNIT_ASSERT_MEMEQ(test, hash1, hash2, BLAKE2S_HASH_SIZE); } @@ -100,14 +101,16 @@ static void test_blake2s_with_guarded_key_buf(struct kunit *test) static void test_blake2s_with_guarded_out_buf(struct kunit *test) { const size_t data_len = 100; + u8 *data = alloc_buf(test, data_len); + u8 *out_buf = alloc_guarded_buf(test, BLAKE2S_HASH_SIZE); - rand_bytes(test_buf, data_len); + rand_bytes(data, data_len); for (int out_len = 1; out_len <= BLAKE2S_HASH_SIZE; out_len++) { u8 hash[BLAKE2S_HASH_SIZE]; - u8 *guarded_hash = &test_buf[TEST_BUF_LEN - out_len]; + u8 *guarded_hash = &out_buf[BLAKE2S_HASH_SIZE - out_len]; - blake2s(NULL, 0, test_buf, data_len, hash, out_len); - blake2s(NULL, 0, test_buf, data_len, guarded_hash, out_len); + blake2s(NULL, 0, data, data_len, hash, out_len); + blake2s(NULL, 0, data, data_len, guarded_hash, out_len); KUNIT_ASSERT_MEMEQ(test, hash, guarded_hash, out_len); } } @@ -124,8 +127,6 @@ static struct kunit_case blake2s_test_cases[] = { static struct kunit_suite blake2s_test_suite = { .name = "blake2s", .test_cases = blake2s_test_cases, - .suite_init = hash_suite_init, - .suite_exit = hash_suite_exit, }; kunit_test_suite(blake2s_test_suite); diff --git a/lib/crypto/chacha20poly1305-selftest.c b/lib/crypto/tests/chacha20poly1305_kunit.c index e4c85bc5a6d7..d5504d0f4ad7 100644 --- a/lib/crypto/chacha20poly1305-selftest.c +++ b/lib/crypto/tests/chacha20poly1305_kunit.c @@ -6,13 +6,12 @@ #include <crypto/chacha20poly1305.h> #include <crypto/chacha.h> #include <crypto/poly1305.h> - +#include <kunit/test.h> #include <linux/unaligned.h> -#include <linux/bug.h> #include <linux/init.h> #include <linux/mm.h> #include <linux/kernel.h> -#include <linux/slab.h> +#include "test-utils.h" struct chacha20poly1305_testvec { const u8 *input, *output, *assoc, *nonce, *key; @@ -27,7 +26,7 @@ struct chacha20poly1305_testvec { * chapoly construction. */ -static const u8 enc_input001[] __initconst = { +static const u8 enc_input001[] = { 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, @@ -63,7 +62,7 @@ static const u8 enc_input001[] __initconst = { 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, 0x9d }; -static const u8 enc_output001[] __initconst = { +static const u8 enc_output001[] = { 0x64, 0xa0, 0x86, 0x15, 0x75, 0x86, 0x1a, 0xf4, 0x60, 0xf0, 0x62, 0xc7, 0x9b, 0xe6, 0x43, 0xbd, 0x5e, 0x80, 0x5c, 0xfd, 0x34, 0x5c, 0xf3, 0x89, @@ -101,95 +100,95 @@ static const u8 enc_output001[] __initconst = { 0x22, 0x39, 0x23, 0x36, 0xfe, 0xa1, 0x85, 0x1f, 0x38 }; -static const u8 enc_assoc001[] __initconst = { +static const u8 enc_assoc001[] = { 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x91 }; -static const u8 enc_nonce001[] __initconst = { +static const u8 enc_nonce001[] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 }; -static const u8 enc_key001[] __initconst = { +static const u8 enc_key001[] = { 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 }; -static const u8 enc_input002[] __initconst = { }; -static const u8 enc_output002[] __initconst = { +static const u8 enc_input002[] = { }; +static const u8 enc_output002[] = { 0xea, 0xe0, 0x1e, 0x9e, 0x2c, 0x91, 0xaa, 0xe1, 0xdb, 0x5d, 0x99, 0x3f, 0x8a, 0xf7, 0x69, 0x92 }; -static const u8 enc_assoc002[] __initconst = { }; -static const u8 enc_nonce002[] __initconst = { +static const u8 enc_assoc002[] = { }; +static const u8 enc_nonce002[] = { 0xca, 0xbf, 0x33, 0x71, 0x32, 0x45, 0x77, 0x8e }; -static const u8 enc_key002[] __initconst = { +static const u8 enc_key002[] = { 0x4c, 0xf5, 0x96, 0x83, 0x38, 0xe6, 0xae, 0x7f, 0x2d, 0x29, 0x25, 0x76, 0xd5, 0x75, 0x27, 0x86, 0x91, 0x9a, 0x27, 0x7a, 0xfb, 0x46, 0xc5, 0xef, 0x94, 0x81, 0x79, 0x57, 0x14, 0x59, 0x40, 0x68 }; -static const u8 enc_input003[] __initconst = { }; -static const u8 enc_output003[] __initconst = { +static const u8 enc_input003[] = { }; +static const u8 enc_output003[] = { 0xdd, 0x6b, 0x3b, 0x82, 0xce, 0x5a, 0xbd, 0xd6, 0xa9, 0x35, 0x83, 0xd8, 0x8c, 0x3d, 0x85, 0x77 }; -static const u8 enc_assoc003[] __initconst = { +static const u8 enc_assoc003[] = { 0x33, 0x10, 0x41, 0x12, 0x1f, 0xf3, 0xd2, 0x6b }; -static const u8 enc_nonce003[] __initconst = { +static const u8 enc_nonce003[] = { 0x3d, 0x86, 0xb5, 0x6b, 0xc8, 0xa3, 0x1f, 0x1d }; -static const u8 enc_key003[] __initconst = { +static const u8 enc_key003[] = { 0x2d, 0xb0, 0x5d, 0x40, 0xc8, 0xed, 0x44, 0x88, 0x34, 0xd1, 0x13, 0xaf, 0x57, 0xa1, 0xeb, 0x3a, 0x2a, 0x80, 0x51, 0x36, 0xec, 0x5b, 0xbc, 0x08, 0x93, 0x84, 0x21, 0xb5, 0x13, 0x88, 0x3c, 0x0d }; -static const u8 enc_input004[] __initconst = { +static const u8 enc_input004[] = { 0xa4 }; -static const u8 enc_output004[] __initconst = { +static const u8 enc_output004[] = { 0xb7, 0x1b, 0xb0, 0x73, 0x59, 0xb0, 0x84, 0xb2, 0x6d, 0x8e, 0xab, 0x94, 0x31, 0xa1, 0xae, 0xac, 0x89 }; -static const u8 enc_assoc004[] __initconst = { +static const u8 enc_assoc004[] = { 0x6a, 0xe2, 0xad, 0x3f, 0x88, 0x39, 0x5a, 0x40 }; -static const u8 enc_nonce004[] __initconst = { +static const u8 enc_nonce004[] = { 0xd2, 0x32, 0x1f, 0x29, 0x28, 0xc6, 0xc4, 0xc4 }; -static const u8 enc_key004[] __initconst = { +static const u8 enc_key004[] = { 0x4b, 0x28, 0x4b, 0xa3, 0x7b, 0xbe, 0xe9, 0xf8, 0x31, 0x80, 0x82, 0xd7, 0xd8, 0xe8, 0xb5, 0xa1, 0xe2, 0x18, 0x18, 0x8a, 0x9c, 0xfa, 0xa3, 0x3d, 0x25, 0x71, 0x3e, 0x40, 0xbc, 0x54, 0x7a, 0x3e }; -static const u8 enc_input005[] __initconst = { +static const u8 enc_input005[] = { 0x2d }; -static const u8 enc_output005[] __initconst = { +static const u8 enc_output005[] = { 0xbf, 0xe1, 0x5b, 0x0b, 0xdb, 0x6b, 0xf5, 0x5e, 0x6c, 0x5d, 0x84, 0x44, 0x39, 0x81, 0xc1, 0x9c, 0xac }; -static const u8 enc_assoc005[] __initconst = { }; -static const u8 enc_nonce005[] __initconst = { +static const u8 enc_assoc005[] = { }; +static const u8 enc_nonce005[] = { 0x20, 0x1c, 0xaa, 0x5f, 0x9c, 0xbf, 0x92, 0x30 }; -static const u8 enc_key005[] __initconst = { +static const u8 enc_key005[] = { 0x66, 0xca, 0x9c, 0x23, 0x2a, 0x4b, 0x4b, 0x31, 0x0e, 0x92, 0x89, 0x8b, 0xf4, 0x93, 0xc7, 0x87, 0x98, 0xa3, 0xd8, 0x39, 0xf8, 0xf4, 0xa7, 0x01, 0xc0, 0x2e, 0x0a, 0xa6, 0x7e, 0x5a, 0x78, 0x87 }; -static const u8 enc_input006[] __initconst = { +static const u8 enc_input006[] = { 0x33, 0x2f, 0x94, 0xc1, 0xa4, 0xef, 0xcc, 0x2a, 0x5b, 0xa6, 0xe5, 0x8f, 0x1d, 0x40, 0xf0, 0x92, 0x3c, 0xd9, 0x24, 0x11, 0xa9, 0x71, 0xf9, 0x37, @@ -208,7 +207,7 @@ static const u8 enc_input006[] __initconst = { 0x69, 0x21, 0x70, 0xd8, 0xa4, 0x4b, 0xc8, 0xde, 0x8f }; -static const u8 enc_output006[] __initconst = { +static const u8 enc_output006[] = { 0x8b, 0x06, 0xd3, 0x31, 0xb0, 0x93, 0x45, 0xb1, 0x75, 0x6e, 0x26, 0xf9, 0x67, 0xbc, 0x90, 0x15, 0x81, 0x2c, 0xb5, 0xf0, 0xc6, 0x2b, 0xc7, 0x8c, @@ -229,20 +228,20 @@ static const u8 enc_output006[] __initconst = { 0x8d, 0x31, 0xf7, 0x7a, 0x39, 0x4d, 0x8f, 0x9a, 0xeb }; -static const u8 enc_assoc006[] __initconst = { +static const u8 enc_assoc006[] = { 0x70, 0xd3, 0x33, 0xf3, 0x8b, 0x18, 0x0b }; -static const u8 enc_nonce006[] __initconst = { +static const u8 enc_nonce006[] = { 0xdf, 0x51, 0x84, 0x82, 0x42, 0x0c, 0x75, 0x9c }; -static const u8 enc_key006[] __initconst = { +static const u8 enc_key006[] = { 0x68, 0x7b, 0x8d, 0x8e, 0xe3, 0xc4, 0xdd, 0xae, 0xdf, 0x72, 0x7f, 0x53, 0x72, 0x25, 0x1e, 0x78, 0x91, 0xcb, 0x69, 0x76, 0x1f, 0x49, 0x93, 0xf9, 0x6f, 0x21, 0xcc, 0x39, 0x9c, 0xad, 0xb1, 0x01 }; -static const u8 enc_input007[] __initconst = { +static const u8 enc_input007[] = { 0x9b, 0x18, 0xdb, 0xdd, 0x9a, 0x0f, 0x3e, 0xa5, 0x15, 0x17, 0xde, 0xdf, 0x08, 0x9d, 0x65, 0x0a, 0x67, 0x30, 0x12, 0xe2, 0x34, 0x77, 0x4b, 0xc1, @@ -276,7 +275,7 @@ static const u8 enc_input007[] __initconst = { 0x40, 0x5f, 0x99, 0xb7, 0x73, 0xec, 0x9b, 0x2b, 0xf0, 0x65, 0x11, 0xc8, 0xd0, 0x0a, 0x9f, 0xd3 }; -static const u8 enc_output007[] __initconst = { +static const u8 enc_output007[] = { 0x85, 0x04, 0xc2, 0xed, 0x8d, 0xfd, 0x97, 0x5c, 0xd2, 0xb7, 0xe2, 0xc1, 0x6b, 0xa3, 0xba, 0xf8, 0xc9, 0x50, 0xc3, 0xc6, 0xa5, 0xe3, 0xa4, 0x7c, @@ -312,18 +311,18 @@ static const u8 enc_output007[] __initconst = { 0x7a, 0x4b, 0xbc, 0xb0, 0x10, 0x5c, 0x96, 0x42, 0x3a, 0x00, 0x98, 0xcd, 0x15, 0xe8, 0xb7, 0x53 }; -static const u8 enc_assoc007[] __initconst = { }; -static const u8 enc_nonce007[] __initconst = { +static const u8 enc_assoc007[] = { }; +static const u8 enc_nonce007[] = { 0xde, 0x7b, 0xef, 0xc3, 0x65, 0x1b, 0x68, 0xb0 }; -static const u8 enc_key007[] __initconst = { +static const u8 enc_key007[] = { 0x8d, 0xb8, 0x91, 0x48, 0xf0, 0xe7, 0x0a, 0xbd, 0xf9, 0x3f, 0xcd, 0xd9, 0xa0, 0x1e, 0x42, 0x4c, 0xe7, 0xde, 0x25, 0x3d, 0xa3, 0xd7, 0x05, 0x80, 0x8d, 0xf2, 0x82, 0xac, 0x44, 0x16, 0x51, 0x01 }; -static const u8 enc_input008[] __initconst = { +static const u8 enc_input008[] = { 0xc3, 0x09, 0x94, 0x62, 0xe6, 0x46, 0x2e, 0x10, 0xbe, 0x00, 0xe4, 0xfc, 0xf3, 0x40, 0xa3, 0xe2, 0x0f, 0xc2, 0x8b, 0x28, 0xdc, 0xba, 0xb4, 0x3c, @@ -389,7 +388,7 @@ static const u8 enc_input008[] __initconst = { 0x2e, 0x79, 0xb5, 0xe2, 0xb8, 0xe8, 0xb9, 0x7b, 0xd5, 0x10, 0xcb, 0xff, 0x5d, 0x14, 0x73, 0xf3 }; -static const u8 enc_output008[] __initconst = { +static const u8 enc_output008[] = { 0x14, 0xf6, 0x41, 0x37, 0xa6, 0xd4, 0x27, 0xcd, 0xdb, 0x06, 0x3e, 0x9a, 0x4e, 0xab, 0xd5, 0xb1, 0x1e, 0x6b, 0xd2, 0xbc, 0x11, 0xf4, 0x28, 0x93, @@ -457,18 +456,18 @@ static const u8 enc_output008[] __initconst = { 0x54, 0xfd, 0x0d, 0x98, 0x1c, 0x5a, 0x6f, 0x1f, 0x9a, 0x40, 0xcd, 0xa2, 0xff, 0x6a, 0xf1, 0x54 }; -static const u8 enc_assoc008[] __initconst = { }; -static const u8 enc_nonce008[] __initconst = { +static const u8 enc_assoc008[] = { }; +static const u8 enc_nonce008[] = { 0x0e, 0x0d, 0x57, 0xbb, 0x7b, 0x40, 0x54, 0x02 }; -static const u8 enc_key008[] __initconst = { +static const u8 enc_key008[] = { 0xf2, 0xaa, 0x4f, 0x99, 0xfd, 0x3e, 0xa8, 0x53, 0xc1, 0x44, 0xe9, 0x81, 0x18, 0xdc, 0xf5, 0xf0, 0x3e, 0x44, 0x15, 0x59, 0xe0, 0xc5, 0x44, 0x86, 0xc3, 0x91, 0xa8, 0x75, 0xc0, 0x12, 0x46, 0xba }; -static const u8 enc_input009[] __initconst = { +static const u8 enc_input009[] = { 0xe6, 0xc3, 0xdb, 0x63, 0x55, 0x15, 0xe3, 0x5b, 0xb7, 0x4b, 0x27, 0x8b, 0x5a, 0xdd, 0xc2, 0xe8, 0x3a, 0x6b, 0xd7, 0x81, 0x96, 0x35, 0x97, 0xca, @@ -535,7 +534,7 @@ static const u8 enc_input009[] __initconst = { 0x74, 0xb9, 0xe2, 0xd6, 0x1c, 0x80, 0x2c, 0x52, 0x65 }; -static const u8 enc_output009[] __initconst = { +static const u8 enc_output009[] = { 0xfd, 0x81, 0x8d, 0xd0, 0x3d, 0xb4, 0xd5, 0xdf, 0xd3, 0x42, 0x47, 0x5a, 0x6d, 0x19, 0x27, 0x66, 0x4b, 0x2e, 0x0c, 0x27, 0x9c, 0x96, 0x4c, 0x72, @@ -604,21 +603,21 @@ static const u8 enc_output009[] __initconst = { 0x43, 0xb5, 0xd9, 0x90, 0xe1, 0x85, 0xf5, 0xa8, 0xae }; -static const u8 enc_assoc009[] __initconst = { +static const u8 enc_assoc009[] = { 0x5a, 0x27, 0xff, 0xeb, 0xdf, 0x84, 0xb2, 0x9e, 0xef }; -static const u8 enc_nonce009[] __initconst = { +static const u8 enc_nonce009[] = { 0xef, 0x2d, 0x63, 0xee, 0x6b, 0x80, 0x8b, 0x78 }; -static const u8 enc_key009[] __initconst = { +static const u8 enc_key009[] = { 0xea, 0xbc, 0x56, 0x99, 0xe3, 0x50, 0xff, 0xc5, 0xcc, 0x1a, 0xd7, 0xc1, 0x57, 0x72, 0xea, 0x86, 0x5b, 0x89, 0x88, 0x61, 0x3d, 0x2f, 0x9b, 0xb2, 0xe7, 0x9c, 0xec, 0x74, 0x6e, 0x3e, 0xf4, 0x3b }; -static const u8 enc_input010[] __initconst = { +static const u8 enc_input010[] = { 0x42, 0x93, 0xe4, 0xeb, 0x97, 0xb0, 0x57, 0xbf, 0x1a, 0x8b, 0x1f, 0xe4, 0x5f, 0x36, 0x20, 0x3c, 0xef, 0x0a, 0xa9, 0x48, 0x5f, 0x5f, 0x37, 0x22, @@ -748,7 +747,7 @@ static const u8 enc_input010[] __initconst = { 0x5a, 0xc0, 0x33, 0x1f, 0xcb, 0x05, 0x6d, 0x5c, 0x06, 0x87, 0x52, 0xa2, 0x8f, 0x26, 0xd5, 0x4f }; -static const u8 enc_output010[] __initconst = { +static const u8 enc_output010[] = { 0xe5, 0x26, 0xa4, 0x3d, 0xbd, 0x33, 0xd0, 0x4b, 0x6f, 0x05, 0xa7, 0x6e, 0x12, 0x7a, 0xd2, 0x74, 0xa6, 0xdd, 0xbd, 0x95, 0xeb, 0xf9, 0xa4, 0xf1, @@ -880,21 +879,21 @@ static const u8 enc_output010[] __initconst = { 0x80, 0x6b, 0xf6, 0x1f, 0xc3, 0xcc, 0x97, 0xc9, 0x24, 0x9f, 0xf3, 0xaf, 0x43, 0x14, 0xd5, 0xa0 }; -static const u8 enc_assoc010[] __initconst = { +static const u8 enc_assoc010[] = { 0xd2, 0xa1, 0x70, 0xdb, 0x7a, 0xf8, 0xfa, 0x27, 0xba, 0x73, 0x0f, 0xbf, 0x3d, 0x1e, 0x82, 0xb2 }; -static const u8 enc_nonce010[] __initconst = { +static const u8 enc_nonce010[] = { 0xdb, 0x92, 0x0f, 0x7f, 0x17, 0x54, 0x0c, 0x30 }; -static const u8 enc_key010[] __initconst = { +static const u8 enc_key010[] = { 0x47, 0x11, 0xeb, 0x86, 0x2b, 0x2c, 0xab, 0x44, 0x34, 0xda, 0x7f, 0x57, 0x03, 0x39, 0x0c, 0xaf, 0x2c, 0x14, 0xfd, 0x65, 0x23, 0xe9, 0x8e, 0x74, 0xd5, 0x08, 0x68, 0x08, 0xe7, 0xb4, 0x72, 0xd7 }; -static const u8 enc_input011[] __initconst = { +static const u8 enc_input011[] = { 0x7a, 0x57, 0xf2, 0xc7, 0x06, 0x3f, 0x50, 0x7b, 0x36, 0x1a, 0x66, 0x5c, 0xb9, 0x0e, 0x5e, 0x3b, 0x45, 0x60, 0xbe, 0x9a, 0x31, 0x9f, 0xff, 0x5d, @@ -1138,7 +1137,7 @@ static const u8 enc_input011[] __initconst = { 0x5c, 0x42, 0x16, 0xb8, 0x28, 0xeb, 0x1b, 0x61, 0x10, 0x1e, 0xbf, 0xec, 0xa8 }; -static const u8 enc_output011[] __initconst = { +static const u8 enc_output011[] = { 0x6a, 0xfc, 0x4b, 0x25, 0xdf, 0xc0, 0xe4, 0xe8, 0x17, 0x4d, 0x4c, 0xc9, 0x7e, 0xde, 0x3a, 0xcc, 0x3c, 0xba, 0x6a, 0x77, 0x47, 0xdb, 0xe3, 0x74, @@ -1384,20 +1383,20 @@ static const u8 enc_output011[] __initconst = { 0x86, 0xc4, 0xfc, 0x97, 0xae, 0x3f, 0x8f, 0x1e, 0x2b, 0xdf, 0xcd, 0xf9, 0x3c }; -static const u8 enc_assoc011[] __initconst = { +static const u8 enc_assoc011[] = { 0xd6, 0x31, 0xda, 0x5d, 0x42, 0x5e, 0xd7 }; -static const u8 enc_nonce011[] __initconst = { +static const u8 enc_nonce011[] = { 0xfd, 0x87, 0xd4, 0xd8, 0x62, 0xfd, 0xec, 0xaa }; -static const u8 enc_key011[] __initconst = { +static const u8 enc_key011[] = { 0x35, 0x4e, 0xb5, 0x70, 0x50, 0x42, 0x8a, 0x85, 0xf2, 0xfb, 0xed, 0x7b, 0xd0, 0x9e, 0x97, 0xca, 0xfa, 0x98, 0x66, 0x63, 0xee, 0x37, 0xcc, 0x52, 0xfe, 0xd1, 0xdf, 0x95, 0x15, 0x34, 0x29, 0x38 }; -static const u8 enc_input012[] __initconst = { +static const u8 enc_input012[] = { 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, 0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57, @@ -1651,7 +1650,7 @@ static const u8 enc_input012[] __initconst = { 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, 0x78, 0xec, 0x00 }; -static const u8 enc_output012[] __initconst = { +static const u8 enc_output012[] = { 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, 0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1, @@ -1907,7 +1906,7 @@ static const u8 enc_output012[] __initconst = { 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, 0x70, 0xcf, 0xd6 }; -static const u8 enc_assoc012[] __initconst = { +static const u8 enc_assoc012[] = { 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, 0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce, 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, @@ -1917,10 +1916,10 @@ static const u8 enc_assoc012[] __initconst = { 0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b, 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 }; -static const u8 enc_nonce012[] __initconst = { +static const u8 enc_nonce012[] = { 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 }; -static const u8 enc_key012[] __initconst = { +static const u8 enc_key012[] = { 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, 0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d, 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e, @@ -1928,7 +1927,7 @@ static const u8 enc_key012[] __initconst = { }; /* wycheproof - rfc7539 */ -static const u8 enc_input013[] __initconst = { +static const u8 enc_input013[] = { 0x4c, 0x61, 0x64, 0x69, 0x65, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x47, 0x65, 0x6e, 0x74, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x20, 0x6f, 0x66, 0x20, @@ -1945,7 +1944,7 @@ static const u8 enc_input013[] __initconst = { 0x75, 0x6c, 0x64, 0x20, 0x62, 0x65, 0x20, 0x69, 0x74, 0x2e }; -static const u8 enc_output013[] __initconst = { +static const u8 enc_output013[] = { 0xd3, 0x1a, 0x8d, 0x34, 0x64, 0x8e, 0x60, 0xdb, 0x7b, 0x86, 0xaf, 0xbc, 0x53, 0xef, 0x7e, 0xc2, 0xa4, 0xad, 0xed, 0x51, 0x29, 0x6e, 0x08, 0xfe, @@ -1964,15 +1963,15 @@ static const u8 enc_output013[] __initconst = { 0xe2, 0x6a, 0x7e, 0x90, 0x2e, 0xcb, 0xd0, 0x60, 0x06, 0x91 }; -static const u8 enc_assoc013[] __initconst = { +static const u8 enc_assoc013[] = { 0x50, 0x51, 0x52, 0x53, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7 }; -static const u8 enc_nonce013[] __initconst = { +static const u8 enc_nonce013[] = { 0x07, 0x00, 0x00, 0x00, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47 }; -static const u8 enc_key013[] __initconst = { +static const u8 enc_key013[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -1980,17 +1979,17 @@ static const u8 enc_key013[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input014[] __initconst = { }; -static const u8 enc_output014[] __initconst = { +static const u8 enc_input014[] = { }; +static const u8 enc_output014[] = { 0x76, 0xac, 0xb3, 0x42, 0xcf, 0x31, 0x66, 0xa5, 0xb6, 0x3c, 0x0c, 0x0e, 0xa1, 0x38, 0x3c, 0x8d }; -static const u8 enc_assoc014[] __initconst = { }; -static const u8 enc_nonce014[] __initconst = { +static const u8 enc_assoc014[] = { }; +static const u8 enc_nonce014[] = { 0x4d, 0xa5, 0xbf, 0x8d, 0xfd, 0x58, 0x52, 0xc1, 0xea, 0x12, 0x37, 0x9d }; -static const u8 enc_key014[] __initconst = { +static const u8 enc_key014[] = { 0x80, 0xba, 0x31, 0x92, 0xc8, 0x03, 0xce, 0x96, 0x5e, 0xa3, 0x71, 0xd5, 0xff, 0x07, 0x3c, 0xf0, 0xf4, 0x3b, 0x6a, 0x2a, 0xb5, 0x76, 0xb2, 0x08, @@ -1998,19 +1997,19 @@ static const u8 enc_key014[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input015[] __initconst = { }; -static const u8 enc_output015[] __initconst = { +static const u8 enc_input015[] = { }; +static const u8 enc_output015[] = { 0x90, 0x6f, 0xa6, 0x28, 0x4b, 0x52, 0xf8, 0x7b, 0x73, 0x59, 0xcb, 0xaa, 0x75, 0x63, 0xc7, 0x09 }; -static const u8 enc_assoc015[] __initconst = { +static const u8 enc_assoc015[] = { 0xbd, 0x50, 0x67, 0x64, 0xf2, 0xd2, 0xc4, 0x10 }; -static const u8 enc_nonce015[] __initconst = { +static const u8 enc_nonce015[] = { 0xa9, 0x2e, 0xf0, 0xac, 0x99, 0x1d, 0xd5, 0x16, 0xa3, 0xc6, 0xf6, 0x89 }; -static const u8 enc_key015[] __initconst = { +static const u8 enc_key015[] = { 0x7a, 0x4c, 0xd7, 0x59, 0x17, 0x2e, 0x02, 0xeb, 0x20, 0x4d, 0xb2, 0xc3, 0xf5, 0xc7, 0x46, 0x22, 0x7d, 0xf5, 0x84, 0xfc, 0x13, 0x45, 0x19, 0x63, @@ -2018,20 +2017,20 @@ static const u8 enc_key015[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input016[] __initconst = { +static const u8 enc_input016[] = { 0x2a }; -static const u8 enc_output016[] __initconst = { +static const u8 enc_output016[] = { 0x3a, 0xca, 0xc2, 0x7d, 0xec, 0x09, 0x68, 0x80, 0x1e, 0x9f, 0x6e, 0xde, 0xd6, 0x9d, 0x80, 0x75, 0x22 }; -static const u8 enc_assoc016[] __initconst = { }; -static const u8 enc_nonce016[] __initconst = { +static const u8 enc_assoc016[] = { }; +static const u8 enc_nonce016[] = { 0x99, 0xe2, 0x3e, 0xc4, 0x89, 0x85, 0xbc, 0xcd, 0xee, 0xab, 0x60, 0xf1 }; -static const u8 enc_key016[] __initconst = { +static const u8 enc_key016[] = { 0xcc, 0x56, 0xb6, 0x80, 0x55, 0x2e, 0xb7, 0x50, 0x08, 0xf5, 0x48, 0x4b, 0x4c, 0xb8, 0x03, 0xfa, 0x50, 0x63, 0xeb, 0xd6, 0xea, 0xb9, 0x1f, 0x6a, @@ -2039,22 +2038,22 @@ static const u8 enc_key016[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input017[] __initconst = { +static const u8 enc_input017[] = { 0x51 }; -static const u8 enc_output017[] __initconst = { +static const u8 enc_output017[] = { 0xc4, 0x16, 0x83, 0x10, 0xca, 0x45, 0xb1, 0xf7, 0xc6, 0x6c, 0xad, 0x4e, 0x99, 0xe4, 0x3f, 0x72, 0xb9 }; -static const u8 enc_assoc017[] __initconst = { +static const u8 enc_assoc017[] = { 0x91, 0xca, 0x6c, 0x59, 0x2c, 0xbc, 0xca, 0x53 }; -static const u8 enc_nonce017[] __initconst = { +static const u8 enc_nonce017[] = { 0xab, 0x0d, 0xca, 0x71, 0x6e, 0xe0, 0x51, 0xd2, 0x78, 0x2f, 0x44, 0x03 }; -static const u8 enc_key017[] __initconst = { +static const u8 enc_key017[] = { 0x46, 0xf0, 0x25, 0x49, 0x65, 0xf7, 0x69, 0xd5, 0x2b, 0xdb, 0x4a, 0x70, 0xb4, 0x43, 0x19, 0x9f, 0x8e, 0xf2, 0x07, 0x52, 0x0d, 0x12, 0x20, 0xc5, @@ -2062,20 +2061,20 @@ static const u8 enc_key017[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input018[] __initconst = { +static const u8 enc_input018[] = { 0x5c, 0x60 }; -static const u8 enc_output018[] __initconst = { +static const u8 enc_output018[] = { 0x4d, 0x13, 0x91, 0xe8, 0xb6, 0x1e, 0xfb, 0x39, 0xc1, 0x22, 0x19, 0x54, 0x53, 0x07, 0x7b, 0x22, 0xe5, 0xe2 }; -static const u8 enc_assoc018[] __initconst = { }; -static const u8 enc_nonce018[] __initconst = { +static const u8 enc_assoc018[] = { }; +static const u8 enc_nonce018[] = { 0x46, 0x1a, 0xf1, 0x22, 0xe9, 0xf2, 0xe0, 0x34, 0x7e, 0x03, 0xf2, 0xdb }; -static const u8 enc_key018[] __initconst = { +static const u8 enc_key018[] = { 0x2f, 0x7f, 0x7e, 0x4f, 0x59, 0x2b, 0xb3, 0x89, 0x19, 0x49, 0x89, 0x74, 0x35, 0x07, 0xbf, 0x3e, 0xe9, 0xcb, 0xde, 0x17, 0x86, 0xb6, 0x69, 0x5f, @@ -2083,22 +2082,22 @@ static const u8 enc_key018[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input019[] __initconst = { +static const u8 enc_input019[] = { 0xdd, 0xf2 }; -static const u8 enc_output019[] __initconst = { +static const u8 enc_output019[] = { 0xb6, 0x0d, 0xea, 0xd0, 0xfd, 0x46, 0x97, 0xec, 0x2e, 0x55, 0x58, 0x23, 0x77, 0x19, 0xd0, 0x24, 0x37, 0xa2 }; -static const u8 enc_assoc019[] __initconst = { +static const u8 enc_assoc019[] = { 0x88, 0x36, 0x4f, 0xc8, 0x06, 0x05, 0x18, 0xbf }; -static const u8 enc_nonce019[] __initconst = { +static const u8 enc_nonce019[] = { 0x61, 0x54, 0x6b, 0xa5, 0xf1, 0x72, 0x05, 0x90, 0xb6, 0x04, 0x0a, 0xc6 }; -static const u8 enc_key019[] __initconst = { +static const u8 enc_key019[] = { 0xc8, 0x83, 0x3d, 0xce, 0x5e, 0xa9, 0xf2, 0x48, 0xaa, 0x20, 0x30, 0xea, 0xcf, 0xe7, 0x2b, 0xff, 0xe6, 0x9a, 0x62, 0x0c, 0xaf, 0x79, 0x33, 0x44, @@ -2106,20 +2105,20 @@ static const u8 enc_key019[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input020[] __initconst = { +static const u8 enc_input020[] = { 0xab, 0x85, 0xe9, 0xc1, 0x57, 0x17, 0x31 }; -static const u8 enc_output020[] __initconst = { +static const u8 enc_output020[] = { 0x5d, 0xfe, 0x34, 0x40, 0xdb, 0xb3, 0xc3, 0xed, 0x7a, 0x43, 0x4e, 0x26, 0x02, 0xd3, 0x94, 0x28, 0x1e, 0x0a, 0xfa, 0x9f, 0xb7, 0xaa, 0x42 }; -static const u8 enc_assoc020[] __initconst = { }; -static const u8 enc_nonce020[] __initconst = { +static const u8 enc_assoc020[] = { }; +static const u8 enc_nonce020[] = { 0x3c, 0x4e, 0x65, 0x4d, 0x66, 0x3f, 0xa4, 0x59, 0x6d, 0xc5, 0x5b, 0xb7 }; -static const u8 enc_key020[] __initconst = { +static const u8 enc_key020[] = { 0x55, 0x56, 0x81, 0x58, 0xd3, 0xa6, 0x48, 0x3f, 0x1f, 0x70, 0x21, 0xea, 0xb6, 0x9b, 0x70, 0x3f, 0x61, 0x42, 0x51, 0xca, 0xdc, 0x1a, 0xf5, 0xd3, @@ -2127,22 +2126,22 @@ static const u8 enc_key020[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input021[] __initconst = { +static const u8 enc_input021[] = { 0x4e, 0xe5, 0xcd, 0xa2, 0x0d, 0x42, 0x90 }; -static const u8 enc_output021[] __initconst = { +static const u8 enc_output021[] = { 0x4b, 0xd4, 0x72, 0x12, 0x94, 0x1c, 0xe3, 0x18, 0x5f, 0x14, 0x08, 0xee, 0x7f, 0xbf, 0x18, 0xf5, 0xab, 0xad, 0x6e, 0x22, 0x53, 0xa1, 0xba }; -static const u8 enc_assoc021[] __initconst = { +static const u8 enc_assoc021[] = { 0x84, 0xe4, 0x6b, 0xe8, 0xc0, 0x91, 0x90, 0x53 }; -static const u8 enc_nonce021[] __initconst = { +static const u8 enc_nonce021[] = { 0x58, 0x38, 0x93, 0x75, 0xc6, 0x9e, 0xe3, 0x98, 0xde, 0x94, 0x83, 0x96 }; -static const u8 enc_key021[] __initconst = { +static const u8 enc_key021[] = { 0xe3, 0xc0, 0x9e, 0x7f, 0xab, 0x1a, 0xef, 0xb5, 0x16, 0xda, 0x6a, 0x33, 0x02, 0x2a, 0x1d, 0xd4, 0xeb, 0x27, 0x2c, 0x80, 0xd5, 0x40, 0xc5, 0xda, @@ -2150,20 +2149,20 @@ static const u8 enc_key021[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input022[] __initconst = { +static const u8 enc_input022[] = { 0xbe, 0x33, 0x08, 0xf7, 0x2a, 0x2c, 0x6a, 0xed }; -static const u8 enc_output022[] __initconst = { +static const u8 enc_output022[] = { 0x8e, 0x94, 0x39, 0xa5, 0x6e, 0xee, 0xc8, 0x17, 0xfb, 0xe8, 0xa6, 0xed, 0x8f, 0xab, 0xb1, 0x93, 0x75, 0x39, 0xdd, 0x6c, 0x00, 0xe9, 0x00, 0x21 }; -static const u8 enc_assoc022[] __initconst = { }; -static const u8 enc_nonce022[] __initconst = { +static const u8 enc_assoc022[] = { }; +static const u8 enc_nonce022[] = { 0x4f, 0x07, 0xaf, 0xed, 0xfd, 0xc3, 0xb6, 0xc2, 0x36, 0x18, 0x23, 0xd3 }; -static const u8 enc_key022[] __initconst = { +static const u8 enc_key022[] = { 0x51, 0xe4, 0xbf, 0x2b, 0xad, 0x92, 0xb7, 0xaf, 0xf1, 0xa4, 0xbc, 0x05, 0x55, 0x0b, 0xa8, 0x1d, 0xf4, 0xb9, 0x6f, 0xab, 0xf4, 0x1c, 0x12, 0xc7, @@ -2171,22 +2170,22 @@ static const u8 enc_key022[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input023[] __initconst = { +static const u8 enc_input023[] = { 0xa4, 0xc9, 0xc2, 0x80, 0x1b, 0x71, 0xf7, 0xdf }; -static const u8 enc_output023[] __initconst = { +static const u8 enc_output023[] = { 0xb9, 0xb9, 0x10, 0x43, 0x3a, 0xf0, 0x52, 0xb0, 0x45, 0x30, 0xf5, 0x1a, 0xee, 0xe0, 0x24, 0xe0, 0xa4, 0x45, 0xa6, 0x32, 0x8f, 0xa6, 0x7a, 0x18 }; -static const u8 enc_assoc023[] __initconst = { +static const u8 enc_assoc023[] = { 0x66, 0xc0, 0xae, 0x70, 0x07, 0x6c, 0xb1, 0x4d }; -static const u8 enc_nonce023[] __initconst = { +static const u8 enc_nonce023[] = { 0xb4, 0xea, 0x66, 0x6e, 0xe1, 0x19, 0x56, 0x33, 0x66, 0x48, 0x4a, 0x78 }; -static const u8 enc_key023[] __initconst = { +static const u8 enc_key023[] = { 0x11, 0x31, 0xc1, 0x41, 0x85, 0x77, 0xa0, 0x54, 0xde, 0x7a, 0x4a, 0xc5, 0x51, 0x95, 0x0f, 0x1a, 0x05, 0x3f, 0x9a, 0xe4, 0x6e, 0x5b, 0x75, 0xfe, @@ -2194,22 +2193,22 @@ static const u8 enc_key023[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input024[] __initconst = { +static const u8 enc_input024[] = { 0x42, 0xba, 0xae, 0x59, 0x78, 0xfe, 0xaf, 0x5c, 0x36, 0x8d, 0x14, 0xe0 }; -static const u8 enc_output024[] __initconst = { +static const u8 enc_output024[] = { 0xff, 0x7d, 0xc2, 0x03, 0xb2, 0x6c, 0x46, 0x7a, 0x6b, 0x50, 0xdb, 0x33, 0x57, 0x8c, 0x0f, 0x27, 0x58, 0xc2, 0xe1, 0x4e, 0x36, 0xd4, 0xfc, 0x10, 0x6d, 0xcb, 0x29, 0xb4 }; -static const u8 enc_assoc024[] __initconst = { }; -static const u8 enc_nonce024[] __initconst = { +static const u8 enc_assoc024[] = { }; +static const u8 enc_nonce024[] = { 0x9a, 0x59, 0xfc, 0xe2, 0x6d, 0xf0, 0x00, 0x5e, 0x07, 0x53, 0x86, 0x56 }; -static const u8 enc_key024[] __initconst = { +static const u8 enc_key024[] = { 0x99, 0xb6, 0x2b, 0xd5, 0xaf, 0xbe, 0x3f, 0xb0, 0x15, 0xbd, 0xe9, 0x3f, 0x0a, 0xbf, 0x48, 0x39, 0x57, 0xa1, 0xc3, 0xeb, 0x3c, 0xa5, 0x9c, 0xb5, @@ -2217,24 +2216,24 @@ static const u8 enc_key024[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input025[] __initconst = { +static const u8 enc_input025[] = { 0xfd, 0xc8, 0x5b, 0x94, 0xa4, 0xb2, 0xa6, 0xb7, 0x59, 0xb1, 0xa0, 0xda }; -static const u8 enc_output025[] __initconst = { +static const u8 enc_output025[] = { 0x9f, 0x88, 0x16, 0xde, 0x09, 0x94, 0xe9, 0x38, 0xd9, 0xe5, 0x3f, 0x95, 0xd0, 0x86, 0xfc, 0x6c, 0x9d, 0x8f, 0xa9, 0x15, 0xfd, 0x84, 0x23, 0xa7, 0xcf, 0x05, 0x07, 0x2f }; -static const u8 enc_assoc025[] __initconst = { +static const u8 enc_assoc025[] = { 0xa5, 0x06, 0xe1, 0xa5, 0xc6, 0x90, 0x93, 0xf9 }; -static const u8 enc_nonce025[] __initconst = { +static const u8 enc_nonce025[] = { 0x58, 0xdb, 0xd4, 0xad, 0x2c, 0x4a, 0xd3, 0x5d, 0xd9, 0x06, 0xe9, 0xce }; -static const u8 enc_key025[] __initconst = { +static const u8 enc_key025[] = { 0x85, 0xf3, 0x5b, 0x62, 0x82, 0xcf, 0xf4, 0x40, 0xbc, 0x10, 0x20, 0xc8, 0x13, 0x6f, 0xf2, 0x70, 0x31, 0x11, 0x0f, 0xa6, 0x3e, 0xc1, 0x6f, 0x1e, @@ -2242,22 +2241,22 @@ static const u8 enc_key025[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input026[] __initconst = { +static const u8 enc_input026[] = { 0x51, 0xf8, 0xc1, 0xf7, 0x31, 0xea, 0x14, 0xac, 0xdb, 0x21, 0x0a, 0x6d, 0x97, 0x3e, 0x07 }; -static const u8 enc_output026[] __initconst = { +static const u8 enc_output026[] = { 0x0b, 0x29, 0x63, 0x8e, 0x1f, 0xbd, 0xd6, 0xdf, 0x53, 0x97, 0x0b, 0xe2, 0x21, 0x00, 0x42, 0x2a, 0x91, 0x34, 0x08, 0x7d, 0x67, 0xa4, 0x6e, 0x79, 0x17, 0x8d, 0x0a, 0x93, 0xf5, 0xe1, 0xd2 }; -static const u8 enc_assoc026[] __initconst = { }; -static const u8 enc_nonce026[] __initconst = { +static const u8 enc_assoc026[] = { }; +static const u8 enc_nonce026[] = { 0x68, 0xab, 0x7f, 0xdb, 0xf6, 0x19, 0x01, 0xda, 0xd4, 0x61, 0xd2, 0x3c }; -static const u8 enc_key026[] __initconst = { +static const u8 enc_key026[] = { 0x67, 0x11, 0x96, 0x27, 0xbd, 0x98, 0x8e, 0xda, 0x90, 0x62, 0x19, 0xe0, 0x8c, 0x0d, 0x0d, 0x77, 0x9a, 0x07, 0xd2, 0x08, 0xce, 0x8a, 0x4f, 0xe0, @@ -2265,24 +2264,24 @@ static const u8 enc_key026[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input027[] __initconst = { +static const u8 enc_input027[] = { 0x97, 0x46, 0x9d, 0xa6, 0x67, 0xd6, 0x11, 0x0f, 0x9c, 0xbd, 0xa1, 0xd1, 0xa2, 0x06, 0x73 }; -static const u8 enc_output027[] __initconst = { +static const u8 enc_output027[] = { 0x32, 0xdb, 0x66, 0xc4, 0xa3, 0x81, 0x9d, 0x81, 0x55, 0x74, 0x55, 0xe5, 0x98, 0x0f, 0xed, 0xfe, 0xae, 0x30, 0xde, 0xc9, 0x4e, 0x6a, 0xd3, 0xa9, 0xee, 0xa0, 0x6a, 0x0d, 0x70, 0x39, 0x17 }; -static const u8 enc_assoc027[] __initconst = { +static const u8 enc_assoc027[] = { 0x64, 0x53, 0xa5, 0x33, 0x84, 0x63, 0x22, 0x12 }; -static const u8 enc_nonce027[] __initconst = { +static const u8 enc_nonce027[] = { 0xd9, 0x5b, 0x32, 0x43, 0xaf, 0xae, 0xf7, 0x14, 0xc5, 0x03, 0x5b, 0x6a }; -static const u8 enc_key027[] __initconst = { +static const u8 enc_key027[] = { 0xe6, 0xf1, 0x11, 0x8d, 0x41, 0xe4, 0xb4, 0x3f, 0xb5, 0x82, 0x21, 0xb7, 0xed, 0x79, 0x67, 0x38, 0x34, 0xe0, 0xd8, 0xac, 0x5c, 0x4f, 0xa6, 0x0b, @@ -2290,22 +2289,22 @@ static const u8 enc_key027[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input028[] __initconst = { +static const u8 enc_input028[] = { 0x54, 0x9b, 0x36, 0x5a, 0xf9, 0x13, 0xf3, 0xb0, 0x81, 0x13, 0x1c, 0xcb, 0x6b, 0x82, 0x55, 0x88 }; -static const u8 enc_output028[] __initconst = { +static const u8 enc_output028[] = { 0xe9, 0x11, 0x0e, 0x9f, 0x56, 0xab, 0x3c, 0xa4, 0x83, 0x50, 0x0c, 0xea, 0xba, 0xb6, 0x7a, 0x13, 0x83, 0x6c, 0xca, 0xbf, 0x15, 0xa6, 0xa2, 0x2a, 0x51, 0xc1, 0x07, 0x1c, 0xfa, 0x68, 0xfa, 0x0c }; -static const u8 enc_assoc028[] __initconst = { }; -static const u8 enc_nonce028[] __initconst = { +static const u8 enc_assoc028[] = { }; +static const u8 enc_nonce028[] = { 0x2f, 0xcb, 0x1b, 0x38, 0xa9, 0x9e, 0x71, 0xb8, 0x47, 0x40, 0xad, 0x9b }; -static const u8 enc_key028[] __initconst = { +static const u8 enc_key028[] = { 0x59, 0xd4, 0xea, 0xfb, 0x4d, 0xe0, 0xcf, 0xc7, 0xd3, 0xdb, 0x99, 0xa8, 0xf5, 0x4b, 0x15, 0xd7, 0xb3, 0x9f, 0x0a, 0xcc, 0x8d, 0xa6, 0x97, 0x63, @@ -2313,24 +2312,24 @@ static const u8 enc_key028[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input029[] __initconst = { +static const u8 enc_input029[] = { 0x55, 0xa4, 0x65, 0x64, 0x4f, 0x5b, 0x65, 0x09, 0x28, 0xcb, 0xee, 0x7c, 0x06, 0x32, 0x14, 0xd6 }; -static const u8 enc_output029[] __initconst = { +static const u8 enc_output029[] = { 0xe4, 0xb1, 0x13, 0xcb, 0x77, 0x59, 0x45, 0xf3, 0xd3, 0xa8, 0xae, 0x9e, 0xc1, 0x41, 0xc0, 0x0c, 0x7c, 0x43, 0xf1, 0x6c, 0xe0, 0x96, 0xd0, 0xdc, 0x27, 0xc9, 0x58, 0x49, 0xdc, 0x38, 0x3b, 0x7d }; -static const u8 enc_assoc029[] __initconst = { +static const u8 enc_assoc029[] = { 0x03, 0x45, 0x85, 0x62, 0x1a, 0xf8, 0xd7, 0xff }; -static const u8 enc_nonce029[] __initconst = { +static const u8 enc_nonce029[] = { 0x11, 0x8a, 0x69, 0x64, 0xc2, 0xd3, 0xe3, 0x80, 0x07, 0x1f, 0x52, 0x66 }; -static const u8 enc_key029[] __initconst = { +static const u8 enc_key029[] = { 0xb9, 0x07, 0xa4, 0x50, 0x75, 0x51, 0x3f, 0xe8, 0xa8, 0x01, 0x9e, 0xde, 0xe3, 0xf2, 0x59, 0x14, 0x87, 0xb2, 0xa0, 0x30, 0xb0, 0x3c, 0x6e, 0x1d, @@ -2338,24 +2337,24 @@ static const u8 enc_key029[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input030[] __initconst = { +static const u8 enc_input030[] = { 0x3f, 0xf1, 0x51, 0x4b, 0x1c, 0x50, 0x39, 0x15, 0x91, 0x8f, 0x0c, 0x0c, 0x31, 0x09, 0x4a, 0x6e, 0x1f }; -static const u8 enc_output030[] __initconst = { +static const u8 enc_output030[] = { 0x02, 0xcc, 0x3a, 0xcb, 0x5e, 0xe1, 0xfc, 0xdd, 0x12, 0xa0, 0x3b, 0xb8, 0x57, 0x97, 0x64, 0x74, 0xd3, 0xd8, 0x3b, 0x74, 0x63, 0xa2, 0xc3, 0x80, 0x0f, 0xe9, 0x58, 0xc2, 0x8e, 0xaa, 0x29, 0x08, 0x13 }; -static const u8 enc_assoc030[] __initconst = { }; -static const u8 enc_nonce030[] __initconst = { +static const u8 enc_assoc030[] = { }; +static const u8 enc_nonce030[] = { 0x45, 0xaa, 0xa3, 0xe5, 0xd1, 0x6d, 0x2d, 0x42, 0xdc, 0x03, 0x44, 0x5d }; -static const u8 enc_key030[] __initconst = { +static const u8 enc_key030[] = { 0x3b, 0x24, 0x58, 0xd8, 0x17, 0x6e, 0x16, 0x21, 0xc0, 0xcc, 0x24, 0xc0, 0xc0, 0xe2, 0x4c, 0x1e, 0x80, 0xd7, 0x2f, 0x7e, 0xe9, 0x14, 0x9a, 0x4b, @@ -2363,26 +2362,26 @@ static const u8 enc_key030[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input031[] __initconst = { +static const u8 enc_input031[] = { 0x63, 0x85, 0x8c, 0xa3, 0xe2, 0xce, 0x69, 0x88, 0x7b, 0x57, 0x8a, 0x3c, 0x16, 0x7b, 0x42, 0x1c, 0x9c }; -static const u8 enc_output031[] __initconst = { +static const u8 enc_output031[] = { 0x35, 0x76, 0x64, 0x88, 0xd2, 0xbc, 0x7c, 0x2b, 0x8d, 0x17, 0xcb, 0xbb, 0x9a, 0xbf, 0xad, 0x9e, 0x6d, 0x1f, 0x39, 0x1e, 0x65, 0x7b, 0x27, 0x38, 0xdd, 0xa0, 0x84, 0x48, 0xcb, 0xa2, 0x81, 0x1c, 0xeb }; -static const u8 enc_assoc031[] __initconst = { +static const u8 enc_assoc031[] = { 0x9a, 0xaf, 0x29, 0x9e, 0xee, 0xa7, 0x8f, 0x79 }; -static const u8 enc_nonce031[] __initconst = { +static const u8 enc_nonce031[] = { 0xf0, 0x38, 0x4f, 0xb8, 0x76, 0x12, 0x14, 0x10, 0x63, 0x3d, 0x99, 0x3d }; -static const u8 enc_key031[] __initconst = { +static const u8 enc_key031[] = { 0xf6, 0x0c, 0x6a, 0x1b, 0x62, 0x57, 0x25, 0xf7, 0x6c, 0x70, 0x37, 0xb4, 0x8f, 0xe3, 0x57, 0x7f, 0xa7, 0xf7, 0xb8, 0x7b, 0x1b, 0xd5, 0xa9, 0x82, @@ -2390,24 +2389,24 @@ static const u8 enc_key031[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input032[] __initconst = { +static const u8 enc_input032[] = { 0x10, 0xf1, 0xec, 0xf9, 0xc6, 0x05, 0x84, 0x66, 0x5d, 0x9a, 0xe5, 0xef, 0xe2, 0x79, 0xe7, 0xf7, 0x37, 0x7e, 0xea, 0x69, 0x16, 0xd2, 0xb1, 0x11 }; -static const u8 enc_output032[] __initconst = { +static const u8 enc_output032[] = { 0x42, 0xf2, 0x6c, 0x56, 0xcb, 0x4b, 0xe2, 0x1d, 0x9d, 0x8d, 0x0c, 0x80, 0xfc, 0x99, 0xdd, 0xe0, 0x0d, 0x75, 0xf3, 0x80, 0x74, 0xbf, 0xe7, 0x64, 0x54, 0xaa, 0x7e, 0x13, 0xd4, 0x8f, 0xff, 0x7d, 0x75, 0x57, 0x03, 0x94, 0x57, 0x04, 0x0a, 0x3a }; -static const u8 enc_assoc032[] __initconst = { }; -static const u8 enc_nonce032[] __initconst = { +static const u8 enc_assoc032[] = { }; +static const u8 enc_nonce032[] = { 0xe6, 0xb1, 0xad, 0xf2, 0xfd, 0x58, 0xa8, 0x76, 0x2c, 0x65, 0xf3, 0x1b }; -static const u8 enc_key032[] __initconst = { +static const u8 enc_key032[] = { 0x02, 0x12, 0xa8, 0xde, 0x50, 0x07, 0xed, 0x87, 0xb3, 0x3f, 0x1a, 0x70, 0x90, 0xb6, 0x11, 0x4f, 0x9e, 0x08, 0xce, 0xfd, 0x96, 0x07, 0xf2, 0xc2, @@ -2415,26 +2414,26 @@ static const u8 enc_key032[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input033[] __initconst = { +static const u8 enc_input033[] = { 0x92, 0x22, 0xf9, 0x01, 0x8e, 0x54, 0xfd, 0x6d, 0xe1, 0x20, 0x08, 0x06, 0xa9, 0xee, 0x8e, 0x4c, 0xc9, 0x04, 0xd2, 0x9f, 0x25, 0xcb, 0xa1, 0x93 }; -static const u8 enc_output033[] __initconst = { +static const u8 enc_output033[] = { 0x12, 0x30, 0x32, 0x43, 0x7b, 0x4b, 0xfd, 0x69, 0x20, 0xe8, 0xf7, 0xe7, 0xe0, 0x08, 0x7a, 0xe4, 0x88, 0x9e, 0xbe, 0x7a, 0x0a, 0xd0, 0xe9, 0x00, 0x3c, 0xf6, 0x8f, 0x17, 0x95, 0x50, 0xda, 0x63, 0xd3, 0xb9, 0x6c, 0x2d, 0x55, 0x41, 0x18, 0x65 }; -static const u8 enc_assoc033[] __initconst = { +static const u8 enc_assoc033[] = { 0x3e, 0x8b, 0xc5, 0xad, 0xe1, 0x82, 0xff, 0x08 }; -static const u8 enc_nonce033[] __initconst = { +static const u8 enc_nonce033[] = { 0x6b, 0x28, 0x2e, 0xbe, 0xcc, 0x54, 0x1b, 0xcd, 0x78, 0x34, 0xed, 0x55 }; -static const u8 enc_key033[] __initconst = { +static const u8 enc_key033[] = { 0xc5, 0xbc, 0x09, 0x56, 0x56, 0x46, 0xe7, 0xed, 0xda, 0x95, 0x4f, 0x1f, 0x73, 0x92, 0x23, 0xda, 0xda, 0x20, 0xb9, 0x5c, 0x44, 0xab, 0x03, 0x3d, @@ -2442,13 +2441,13 @@ static const u8 enc_key033[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input034[] __initconst = { +static const u8 enc_input034[] = { 0xb0, 0x53, 0x99, 0x92, 0x86, 0xa2, 0x82, 0x4f, 0x42, 0xcc, 0x8c, 0x20, 0x3a, 0xb2, 0x4e, 0x2c, 0x97, 0xa6, 0x85, 0xad, 0xcc, 0x2a, 0xd3, 0x26, 0x62, 0x55, 0x8e, 0x55, 0xa5, 0xc7, 0x29 }; -static const u8 enc_output034[] __initconst = { +static const u8 enc_output034[] = { 0x45, 0xc7, 0xd6, 0xb5, 0x3a, 0xca, 0xd4, 0xab, 0xb6, 0x88, 0x76, 0xa6, 0xe9, 0x6a, 0x48, 0xfb, 0x59, 0x52, 0x4d, 0x2c, 0x92, 0xc9, 0xd8, 0xa1, @@ -2456,12 +2455,12 @@ static const u8 enc_output034[] __initconst = { 0x6d, 0x3c, 0xa1, 0x0e, 0x31, 0x1b, 0x69, 0x5f, 0x3e, 0xae, 0x15, 0x51, 0x65, 0x24, 0x93 }; -static const u8 enc_assoc034[] __initconst = { }; -static const u8 enc_nonce034[] __initconst = { +static const u8 enc_assoc034[] = { }; +static const u8 enc_nonce034[] = { 0x04, 0xa9, 0xbe, 0x03, 0x50, 0x8a, 0x5f, 0x31, 0x37, 0x1a, 0x6f, 0xd2 }; -static const u8 enc_key034[] __initconst = { +static const u8 enc_key034[] = { 0x2e, 0xb5, 0x1c, 0x46, 0x9a, 0xa8, 0xeb, 0x9e, 0x6c, 0x54, 0xa8, 0x34, 0x9b, 0xae, 0x50, 0xa2, 0x0f, 0x0e, 0x38, 0x27, 0x11, 0xbb, 0xa1, 0x15, @@ -2469,13 +2468,13 @@ static const u8 enc_key034[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input035[] __initconst = { +static const u8 enc_input035[] = { 0xf4, 0x52, 0x06, 0xab, 0xc2, 0x55, 0x52, 0xb2, 0xab, 0xc9, 0xab, 0x7f, 0xa2, 0x43, 0x03, 0x5f, 0xed, 0xaa, 0xdd, 0xc3, 0xb2, 0x29, 0x39, 0x56, 0xf1, 0xea, 0x6e, 0x71, 0x56, 0xe7, 0xeb }; -static const u8 enc_output035[] __initconst = { +static const u8 enc_output035[] = { 0x46, 0xa8, 0x0c, 0x41, 0x87, 0x02, 0x47, 0x20, 0x08, 0x46, 0x27, 0x58, 0x00, 0x80, 0xdd, 0xe5, 0xa3, 0xf4, 0xa1, 0x10, 0x93, 0xa7, 0x07, 0x6e, @@ -2483,14 +2482,14 @@ static const u8 enc_output035[] __initconst = { 0x4d, 0x4a, 0xa2, 0x83, 0x5a, 0x52, 0xe7, 0x2d, 0x14, 0xdf, 0x0e, 0x4f, 0x47, 0xf2, 0x5f }; -static const u8 enc_assoc035[] __initconst = { +static const u8 enc_assoc035[] = { 0x37, 0x46, 0x18, 0xa0, 0x6e, 0xa9, 0x8a, 0x48 }; -static const u8 enc_nonce035[] __initconst = { +static const u8 enc_nonce035[] = { 0x47, 0x0a, 0x33, 0x9e, 0xcb, 0x32, 0x19, 0xb8, 0xb8, 0x1a, 0x1f, 0x8b }; -static const u8 enc_key035[] __initconst = { +static const u8 enc_key035[] = { 0x7f, 0x5b, 0x74, 0xc0, 0x7e, 0xd1, 0xb4, 0x0f, 0xd1, 0x43, 0x58, 0xfe, 0x2f, 0xf2, 0xa7, 0x40, 0xc1, 0x16, 0xc7, 0x70, 0x65, 0x10, 0xe6, 0xa4, @@ -2498,13 +2497,13 @@ static const u8 enc_key035[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input036[] __initconst = { +static const u8 enc_input036[] = { 0xb9, 0xc5, 0x54, 0xcb, 0xc3, 0x6a, 0xc1, 0x8a, 0xe8, 0x97, 0xdf, 0x7b, 0xee, 0xca, 0xc1, 0xdb, 0xeb, 0x4e, 0xaf, 0xa1, 0x56, 0xbb, 0x60, 0xce, 0x2e, 0x5d, 0x48, 0xf0, 0x57, 0x15, 0xe6, 0x78 }; -static const u8 enc_output036[] __initconst = { +static const u8 enc_output036[] = { 0xea, 0x29, 0xaf, 0xa4, 0x9d, 0x36, 0xe8, 0x76, 0x0f, 0x5f, 0xe1, 0x97, 0x23, 0xb9, 0x81, 0x1e, 0xd5, 0xd5, 0x19, 0x93, 0x4a, 0x44, 0x0f, 0x50, @@ -2512,12 +2511,12 @@ static const u8 enc_output036[] __initconst = { 0x22, 0x25, 0x41, 0xaf, 0x46, 0xb8, 0x65, 0x33, 0xc6, 0xb6, 0x8d, 0x2f, 0xf1, 0x08, 0xa7, 0xea }; -static const u8 enc_assoc036[] __initconst = { }; -static const u8 enc_nonce036[] __initconst = { +static const u8 enc_assoc036[] = { }; +static const u8 enc_nonce036[] = { 0x72, 0xcf, 0xd9, 0x0e, 0xf3, 0x02, 0x6c, 0xa2, 0x2b, 0x7e, 0x6e, 0x6a }; -static const u8 enc_key036[] __initconst = { +static const u8 enc_key036[] = { 0xe1, 0x73, 0x1d, 0x58, 0x54, 0xe1, 0xb7, 0x0c, 0xb3, 0xff, 0xe8, 0xb7, 0x86, 0xa2, 0xb3, 0xeb, 0xf0, 0x99, 0x43, 0x70, 0x95, 0x47, 0x57, 0xb9, @@ -2525,13 +2524,13 @@ static const u8 enc_key036[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input037[] __initconst = { +static const u8 enc_input037[] = { 0x6b, 0x26, 0x04, 0x99, 0x6c, 0xd3, 0x0c, 0x14, 0xa1, 0x3a, 0x52, 0x57, 0xed, 0x6c, 0xff, 0xd3, 0xbc, 0x5e, 0x29, 0xd6, 0xb9, 0x7e, 0xb1, 0x79, 0x9e, 0xb3, 0x35, 0xe2, 0x81, 0xea, 0x45, 0x1e }; -static const u8 enc_output037[] __initconst = { +static const u8 enc_output037[] = { 0x6d, 0xad, 0x63, 0x78, 0x97, 0x54, 0x4d, 0x8b, 0xf6, 0xbe, 0x95, 0x07, 0xed, 0x4d, 0x1b, 0xb2, 0xe9, 0x54, 0xbc, 0x42, 0x7e, 0x5d, 0xe7, 0x29, @@ -2539,14 +2538,14 @@ static const u8 enc_output037[] __initconst = { 0x7b, 0x99, 0x7d, 0x93, 0xc9, 0x82, 0x18, 0x9d, 0x70, 0x95, 0xdc, 0x79, 0x4c, 0x74, 0x62, 0x32 }; -static const u8 enc_assoc037[] __initconst = { +static const u8 enc_assoc037[] = { 0x23, 0x33, 0xe5, 0xce, 0x0f, 0x93, 0xb0, 0x59 }; -static const u8 enc_nonce037[] __initconst = { +static const u8 enc_nonce037[] = { 0x26, 0x28, 0x80, 0xd4, 0x75, 0xf3, 0xda, 0xc5, 0x34, 0x0d, 0xd1, 0xb8 }; -static const u8 enc_key037[] __initconst = { +static const u8 enc_key037[] = { 0x27, 0xd8, 0x60, 0x63, 0x1b, 0x04, 0x85, 0xa4, 0x10, 0x70, 0x2f, 0xea, 0x61, 0xbc, 0x87, 0x3f, 0x34, 0x42, 0x26, 0x0c, 0xad, 0xed, 0x4a, 0xbd, @@ -2554,7 +2553,7 @@ static const u8 enc_key037[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input038[] __initconst = { +static const u8 enc_input038[] = { 0x97, 0x3d, 0x0c, 0x75, 0x38, 0x26, 0xba, 0xe4, 0x66, 0xcf, 0x9a, 0xbb, 0x34, 0x93, 0x15, 0x2e, 0x9d, 0xe7, 0x81, 0x9e, 0x2b, 0xd0, 0xc7, 0x11, @@ -2562,7 +2561,7 @@ static const u8 enc_input038[] __initconst = { 0x1a, 0xa3, 0xce, 0xdc, 0x0d, 0xfd, 0x7b, 0x46, 0x7e, 0x26, 0x22, 0x8b, 0xc8, 0x6c, 0x9a }; -static const u8 enc_output038[] __initconst = { +static const u8 enc_output038[] = { 0xfb, 0xa7, 0x8a, 0xe4, 0xf9, 0xd8, 0x08, 0xa6, 0x2e, 0x3d, 0xa4, 0x0b, 0xe2, 0xcb, 0x77, 0x00, 0xc3, 0x61, 0x3d, 0x9e, 0xb2, 0xc5, 0x29, 0xc6, @@ -2572,12 +2571,12 @@ static const u8 enc_output038[] __initconst = { 0x04, 0x69, 0x56, 0xdb, 0x3a, 0x51, 0x29, 0x08, 0xbd, 0x7a, 0xfc, 0x8f, 0x2a, 0xb0, 0xa9 }; -static const u8 enc_assoc038[] __initconst = { }; -static const u8 enc_nonce038[] __initconst = { +static const u8 enc_assoc038[] = { }; +static const u8 enc_nonce038[] = { 0xe7, 0x4a, 0x51, 0x5e, 0x7e, 0x21, 0x02, 0xb9, 0x0b, 0xef, 0x55, 0xd2 }; -static const u8 enc_key038[] __initconst = { +static const u8 enc_key038[] = { 0xcf, 0x0d, 0x40, 0xa4, 0x64, 0x4e, 0x5f, 0x51, 0x81, 0x51, 0x65, 0xd5, 0x30, 0x1b, 0x22, 0x63, 0x1f, 0x45, 0x44, 0xc4, 0x9a, 0x18, 0x78, 0xe3, @@ -2585,7 +2584,7 @@ static const u8 enc_key038[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input039[] __initconst = { +static const u8 enc_input039[] = { 0xa9, 0x89, 0x95, 0x50, 0x4d, 0xf1, 0x6f, 0x74, 0x8b, 0xfb, 0x77, 0x85, 0xff, 0x91, 0xee, 0xb3, 0xb6, 0x60, 0xea, 0x9e, 0xd3, 0x45, 0x0c, 0x3d, @@ -2593,7 +2592,7 @@ static const u8 enc_input039[] __initconst = { 0xa9, 0x97, 0x8d, 0x75, 0x54, 0x2e, 0xf9, 0x1c, 0x45, 0x67, 0x62, 0x21, 0x56, 0x40, 0xb9 }; -static const u8 enc_output039[] __initconst = { +static const u8 enc_output039[] = { 0xa1, 0xff, 0xed, 0x80, 0x76, 0x18, 0x29, 0xec, 0xce, 0x24, 0x2e, 0x0e, 0x88, 0xb1, 0x38, 0x04, 0x90, 0x16, 0xbc, 0xa0, 0x18, 0xda, 0x2b, 0x6e, @@ -2603,14 +2602,14 @@ static const u8 enc_output039[] __initconst = { 0xcb, 0xf0, 0xbe, 0xfd, 0xa0, 0xb7, 0x02, 0x42, 0xc6, 0x40, 0xd7, 0xcd, 0x02, 0xd7, 0xa3 }; -static const u8 enc_assoc039[] __initconst = { +static const u8 enc_assoc039[] = { 0xb3, 0xe4, 0x06, 0x46, 0x83, 0xb0, 0x2d, 0x84 }; -static const u8 enc_nonce039[] __initconst = { +static const u8 enc_nonce039[] = { 0xd4, 0xd8, 0x07, 0x34, 0x16, 0x83, 0x82, 0x5b, 0x31, 0xcd, 0x4d, 0x95 }; -static const u8 enc_key039[] __initconst = { +static const u8 enc_key039[] = { 0x6c, 0xbf, 0xd7, 0x1c, 0x64, 0x5d, 0x18, 0x4c, 0xf5, 0xd2, 0x3c, 0x40, 0x2b, 0xdb, 0x0d, 0x25, 0xec, 0x54, 0x89, 0x8c, 0x8a, 0x02, 0x73, 0xd4, @@ -2618,7 +2617,7 @@ static const u8 enc_key039[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input040[] __initconst = { +static const u8 enc_input040[] = { 0xd0, 0x96, 0x80, 0x31, 0x81, 0xbe, 0xef, 0x9e, 0x00, 0x8f, 0xf8, 0x5d, 0x5d, 0xdc, 0x38, 0xdd, 0xac, 0xf0, 0xf0, 0x9e, 0xe5, 0xf7, 0xe0, 0x7f, @@ -2628,7 +2627,7 @@ static const u8 enc_input040[] __initconst = { 0x18, 0xf1, 0x18, 0x55, 0x86, 0xbf, 0xea, 0x9d, 0x4c, 0x68, 0x5d, 0x50, 0xe4, 0xbb, 0x9a, 0x82 }; -static const u8 enc_output040[] __initconst = { +static const u8 enc_output040[] = { 0x9a, 0x4e, 0xf2, 0x2b, 0x18, 0x16, 0x77, 0xb5, 0x75, 0x5c, 0x08, 0xf7, 0x47, 0xc0, 0xf8, 0xd8, 0xe8, 0xd4, 0xc1, 0x8a, 0x9c, 0xc2, 0x40, 0x5c, @@ -2640,12 +2639,12 @@ static const u8 enc_output040[] __initconst = { 0x9f, 0xf3, 0x42, 0x7a, 0x0f, 0x32, 0xfa, 0x56, 0x6d, 0x9c, 0xa0, 0xa7, 0x8a, 0xef, 0xc0, 0x13 }; -static const u8 enc_assoc040[] __initconst = { }; -static const u8 enc_nonce040[] __initconst = { +static const u8 enc_assoc040[] = { }; +static const u8 enc_nonce040[] = { 0xd6, 0x10, 0x40, 0xa3, 0x13, 0xed, 0x49, 0x28, 0x23, 0xcc, 0x06, 0x5b }; -static const u8 enc_key040[] __initconst = { +static const u8 enc_key040[] = { 0x5b, 0x1d, 0x10, 0x35, 0xc0, 0xb1, 0x7e, 0xe0, 0xb0, 0x44, 0x47, 0x67, 0xf8, 0x0a, 0x25, 0xb8, 0xc1, 0xb7, 0x41, 0xf4, 0xb5, 0x0a, 0x4d, 0x30, @@ -2653,7 +2652,7 @@ static const u8 enc_key040[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input041[] __initconst = { +static const u8 enc_input041[] = { 0x94, 0xee, 0x16, 0x6d, 0x6d, 0x6e, 0xcf, 0x88, 0x32, 0x43, 0x71, 0x36, 0xb4, 0xae, 0x80, 0x5d, 0x42, 0x88, 0x64, 0x35, 0x95, 0x86, 0xd9, 0x19, @@ -2663,7 +2662,7 @@ static const u8 enc_input041[] __initconst = { 0x4a, 0x10, 0x8c, 0x7d, 0x7c, 0xe3, 0x4e, 0x6c, 0x6f, 0x8e, 0xa1, 0xbe, 0xc0, 0x56, 0x73, 0x17 }; -static const u8 enc_output041[] __initconst = { +static const u8 enc_output041[] = { 0x5f, 0xbb, 0xde, 0xcc, 0x34, 0xbe, 0x20, 0x16, 0x14, 0xf6, 0x36, 0x03, 0x1e, 0xeb, 0x42, 0xf1, 0xca, 0xce, 0x3c, 0x79, 0xa1, 0x2c, 0xff, 0xd8, @@ -2675,14 +2674,14 @@ static const u8 enc_output041[] __initconst = { 0x09, 0x26, 0x3d, 0xa7, 0xb4, 0xcb, 0x92, 0x14, 0x52, 0xf9, 0x7d, 0xca, 0x40, 0xf5, 0x80, 0xec }; -static const u8 enc_assoc041[] __initconst = { +static const u8 enc_assoc041[] = { 0x71, 0x93, 0xf6, 0x23, 0x66, 0x33, 0x21, 0xa2 }; -static const u8 enc_nonce041[] __initconst = { +static const u8 enc_nonce041[] = { 0xd3, 0x1c, 0x21, 0xab, 0xa1, 0x75, 0xb7, 0x0d, 0xe4, 0xeb, 0xb1, 0x9c }; -static const u8 enc_key041[] __initconst = { +static const u8 enc_key041[] = { 0x97, 0xd6, 0x35, 0xc4, 0xf4, 0x75, 0x74, 0xd9, 0x99, 0x8a, 0x90, 0x87, 0x5d, 0xa1, 0xd3, 0xa2, 0x84, 0xb7, 0x55, 0xb2, 0xd3, 0x92, 0x97, 0xa5, @@ -2690,7 +2689,7 @@ static const u8 enc_key041[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input042[] __initconst = { +static const u8 enc_input042[] = { 0xb4, 0x29, 0xeb, 0x80, 0xfb, 0x8f, 0xe8, 0xba, 0xed, 0xa0, 0xc8, 0x5b, 0x9c, 0x33, 0x34, 0x58, 0xe7, 0xc2, 0x99, 0x2e, 0x55, 0x84, 0x75, 0x06, @@ -2705,7 +2704,7 @@ static const u8 enc_input042[] __initconst = { 0xc3, 0xea, 0x46, 0xc8, 0x76, 0x1d, 0x50, 0xf5, 0x92 }; -static const u8 enc_output042[] __initconst = { +static const u8 enc_output042[] = { 0xd0, 0x10, 0x2f, 0x6c, 0x25, 0x8b, 0xf4, 0x97, 0x42, 0xce, 0xc3, 0x4c, 0xf2, 0xd0, 0xfe, 0xdf, 0x23, 0xd1, 0x05, 0xfb, 0x4c, 0x84, 0xcf, 0x98, @@ -2722,12 +2721,12 @@ static const u8 enc_output042[] __initconst = { 0x19, 0x61, 0x21, 0x27, 0xce, 0x49, 0x99, 0x3b, 0xb0 }; -static const u8 enc_assoc042[] __initconst = { }; -static const u8 enc_nonce042[] __initconst = { +static const u8 enc_assoc042[] = { }; +static const u8 enc_nonce042[] = { 0x17, 0xc8, 0x6a, 0x8a, 0xbb, 0xb7, 0xe0, 0x03, 0xac, 0xde, 0x27, 0x99 }; -static const u8 enc_key042[] __initconst = { +static const u8 enc_key042[] = { 0xfe, 0x6e, 0x55, 0xbd, 0xae, 0xd1, 0xf7, 0x28, 0x4c, 0xa5, 0xfc, 0x0f, 0x8c, 0x5f, 0x2b, 0x8d, 0xf5, 0x6d, 0xc0, 0xf4, 0x9e, 0x8c, 0xa6, 0x6a, @@ -2735,7 +2734,7 @@ static const u8 enc_key042[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input043[] __initconst = { +static const u8 enc_input043[] = { 0xce, 0xb5, 0x34, 0xce, 0x50, 0xdc, 0x23, 0xff, 0x63, 0x8a, 0xce, 0x3e, 0xf6, 0x3a, 0xb2, 0xcc, 0x29, 0x73, 0xee, 0xad, 0xa8, 0x07, 0x85, 0xfc, @@ -2750,7 +2749,7 @@ static const u8 enc_input043[] __initconst = { 0xdc, 0x47, 0x97, 0x0d, 0x84, 0xa6, 0xad, 0x31, 0x76 }; -static const u8 enc_output043[] __initconst = { +static const u8 enc_output043[] = { 0x75, 0x45, 0x39, 0x1b, 0x51, 0xde, 0x01, 0xd5, 0xc5, 0x3d, 0xfa, 0xca, 0x77, 0x79, 0x09, 0x06, 0x3e, 0x58, 0xed, 0xee, 0x4b, 0xb1, 0x22, 0x7e, @@ -2767,14 +2766,14 @@ static const u8 enc_output043[] __initconst = { 0xc7, 0xbb, 0xf1, 0x0d, 0xca, 0xdd, 0x7f, 0xac, 0xf6 }; -static const u8 enc_assoc043[] __initconst = { +static const u8 enc_assoc043[] = { 0xa1, 0x1c, 0x40, 0xb6, 0x03, 0x76, 0x73, 0x30 }; -static const u8 enc_nonce043[] __initconst = { +static const u8 enc_nonce043[] = { 0x46, 0x36, 0x2f, 0x45, 0xd6, 0x37, 0x9e, 0x63, 0xe5, 0x22, 0x94, 0x60 }; -static const u8 enc_key043[] __initconst = { +static const u8 enc_key043[] = { 0xaa, 0xbc, 0x06, 0x34, 0x74, 0xe6, 0x5c, 0x4c, 0x3e, 0x9b, 0xdc, 0x48, 0x0d, 0xea, 0x97, 0xb4, 0x51, 0x10, 0xc8, 0x61, 0x88, 0x46, 0xff, 0x6b, @@ -2782,24 +2781,24 @@ static const u8 enc_key043[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input044[] __initconst = { +static const u8 enc_input044[] = { 0xe5, 0xcc, 0xaa, 0x44, 0x1b, 0xc8, 0x14, 0x68, 0x8f, 0x8f, 0x6e, 0x8f, 0x28, 0xb5, 0x00, 0xb2 }; -static const u8 enc_output044[] __initconst = { +static const u8 enc_output044[] = { 0x7e, 0x72, 0xf5, 0xa1, 0x85, 0xaf, 0x16, 0xa6, 0x11, 0x92, 0x1b, 0x43, 0x8f, 0x74, 0x9f, 0x0b, 0x12, 0x42, 0xc6, 0x70, 0x73, 0x23, 0x34, 0x02, 0x9a, 0xdf, 0xe1, 0xc5, 0x00, 0x16, 0x51, 0xe4 }; -static const u8 enc_assoc044[] __initconst = { +static const u8 enc_assoc044[] = { 0x02 }; -static const u8 enc_nonce044[] __initconst = { +static const u8 enc_nonce044[] = { 0x87, 0x34, 0x5f, 0x10, 0x55, 0xfd, 0x9e, 0x21, 0x02, 0xd5, 0x06, 0x56 }; -static const u8 enc_key044[] __initconst = { +static const u8 enc_key044[] = { 0x7d, 0x00, 0xb4, 0x80, 0x95, 0xad, 0xfa, 0x32, 0x72, 0x05, 0x06, 0x07, 0xb2, 0x64, 0x18, 0x50, 0x02, 0xba, 0x99, 0x95, 0x7c, 0x49, 0x8b, 0xe0, @@ -2807,24 +2806,24 @@ static const u8 enc_key044[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input045[] __initconst = { +static const u8 enc_input045[] = { 0x02, 0xcd, 0xe1, 0x68, 0xfb, 0xa3, 0xf5, 0x44, 0xbb, 0xd0, 0x33, 0x2f, 0x7a, 0xde, 0xad, 0xa8 }; -static const u8 enc_output045[] __initconst = { +static const u8 enc_output045[] = { 0x85, 0xf2, 0x9a, 0x71, 0x95, 0x57, 0xcd, 0xd1, 0x4d, 0x1f, 0x8f, 0xff, 0xab, 0x6d, 0x9e, 0x60, 0x73, 0x2c, 0xa3, 0x2b, 0xec, 0xd5, 0x15, 0xa1, 0xed, 0x35, 0x3f, 0x54, 0x2e, 0x99, 0x98, 0x58 }; -static const u8 enc_assoc045[] __initconst = { +static const u8 enc_assoc045[] = { 0xb6, 0x48 }; -static const u8 enc_nonce045[] __initconst = { +static const u8 enc_nonce045[] = { 0x87, 0xa3, 0x16, 0x3e, 0xc0, 0x59, 0x8a, 0xd9, 0x5b, 0x3a, 0xa7, 0x13 }; -static const u8 enc_key045[] __initconst = { +static const u8 enc_key045[] = { 0x64, 0x32, 0x71, 0x7f, 0x1d, 0xb8, 0x5e, 0x41, 0xac, 0x78, 0x36, 0xbc, 0xe2, 0x51, 0x85, 0xa0, 0x80, 0xd5, 0x76, 0x2b, 0x9e, 0x2b, 0x18, 0x44, @@ -2832,25 +2831,25 @@ static const u8 enc_key045[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input046[] __initconst = { +static const u8 enc_input046[] = { 0x16, 0xdd, 0xd2, 0x3f, 0xf5, 0x3f, 0x3d, 0x23, 0xc0, 0x63, 0x34, 0x48, 0x70, 0x40, 0xeb, 0x47 }; -static const u8 enc_output046[] __initconst = { +static const u8 enc_output046[] = { 0xc1, 0xb2, 0x95, 0x93, 0x6d, 0x56, 0xfa, 0xda, 0xc0, 0x3e, 0x5f, 0x74, 0x2b, 0xff, 0x73, 0xa1, 0x39, 0xc4, 0x57, 0xdb, 0xab, 0x66, 0x38, 0x2b, 0xab, 0xb3, 0xb5, 0x58, 0x00, 0xcd, 0xa5, 0xb8 }; -static const u8 enc_assoc046[] __initconst = { +static const u8 enc_assoc046[] = { 0xbd, 0x4c, 0xd0, 0x2f, 0xc7, 0x50, 0x2b, 0xbd, 0xbd, 0xf6, 0xc9, 0xa3, 0xcb, 0xe8, 0xf0 }; -static const u8 enc_nonce046[] __initconst = { +static const u8 enc_nonce046[] = { 0x6f, 0x57, 0x3a, 0xa8, 0x6b, 0xaa, 0x49, 0x2b, 0xa4, 0x65, 0x96, 0xdf }; -static const u8 enc_key046[] __initconst = { +static const u8 enc_key046[] = { 0x8e, 0x34, 0xcf, 0x73, 0xd2, 0x45, 0xa1, 0x08, 0x2a, 0x92, 0x0b, 0x86, 0x36, 0x4e, 0xb8, 0x96, 0xc4, 0x94, 0x64, 0x67, 0xbc, 0xb3, 0xd5, 0x89, @@ -2858,25 +2857,25 @@ static const u8 enc_key046[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input047[] __initconst = { +static const u8 enc_input047[] = { 0x62, 0x3b, 0x78, 0x50, 0xc3, 0x21, 0xe2, 0xcf, 0x0c, 0x6f, 0xbc, 0xc8, 0xdf, 0xd1, 0xaf, 0xf2 }; -static const u8 enc_output047[] __initconst = { +static const u8 enc_output047[] = { 0xc8, 0x4c, 0x9b, 0xb7, 0xc6, 0x1c, 0x1b, 0xcb, 0x17, 0x77, 0x2a, 0x1c, 0x50, 0x0c, 0x50, 0x95, 0xdb, 0xad, 0xf7, 0xa5, 0x13, 0x8c, 0xa0, 0x34, 0x59, 0xa2, 0xcd, 0x65, 0x83, 0x1e, 0x09, 0x2f }; -static const u8 enc_assoc047[] __initconst = { +static const u8 enc_assoc047[] = { 0x89, 0xcc, 0xe9, 0xfb, 0x47, 0x44, 0x1d, 0x07, 0xe0, 0x24, 0x5a, 0x66, 0xfe, 0x8b, 0x77, 0x8b }; -static const u8 enc_nonce047[] __initconst = { +static const u8 enc_nonce047[] = { 0x1a, 0x65, 0x18, 0xf0, 0x2e, 0xde, 0x1d, 0xa6, 0x80, 0x92, 0x66, 0xd9 }; -static const u8 enc_key047[] __initconst = { +static const u8 enc_key047[] = { 0xcb, 0x55, 0x75, 0xf5, 0xc7, 0xc4, 0x5c, 0x91, 0xcf, 0x32, 0x0b, 0x13, 0x9f, 0xb5, 0x94, 0x23, 0x75, 0x60, 0xd0, 0xa3, 0xe6, 0xf8, 0x65, 0xa6, @@ -2884,26 +2883,26 @@ static const u8 enc_key047[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input048[] __initconst = { +static const u8 enc_input048[] = { 0x87, 0xb3, 0xa4, 0xd7, 0xb2, 0x6d, 0x8d, 0x32, 0x03, 0xa0, 0xde, 0x1d, 0x64, 0xef, 0x82, 0xe3 }; -static const u8 enc_output048[] __initconst = { +static const u8 enc_output048[] = { 0x94, 0xbc, 0x80, 0x62, 0x1e, 0xd1, 0xe7, 0x1b, 0x1f, 0xd2, 0xb5, 0xc3, 0xa1, 0x5e, 0x35, 0x68, 0x33, 0x35, 0x11, 0x86, 0x17, 0x96, 0x97, 0x84, 0x01, 0x59, 0x8b, 0x96, 0x37, 0x22, 0xf5, 0xb3 }; -static const u8 enc_assoc048[] __initconst = { +static const u8 enc_assoc048[] = { 0xd1, 0x9f, 0x2d, 0x98, 0x90, 0x95, 0xf7, 0xab, 0x03, 0xa5, 0xfd, 0xe8, 0x44, 0x16, 0xe0, 0x0c, 0x0e }; -static const u8 enc_nonce048[] __initconst = { +static const u8 enc_nonce048[] = { 0x56, 0x4d, 0xee, 0x49, 0xab, 0x00, 0xd2, 0x40, 0xfc, 0x10, 0x68, 0xc3 }; -static const u8 enc_key048[] __initconst = { +static const u8 enc_key048[] = { 0xa5, 0x56, 0x9e, 0x72, 0x9a, 0x69, 0xb2, 0x4b, 0xa6, 0xe0, 0xff, 0x15, 0xc4, 0x62, 0x78, 0x97, 0x43, 0x68, 0x24, 0xc9, 0x41, 0xe9, 0xd0, 0x0b, @@ -2911,27 +2910,27 @@ static const u8 enc_key048[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input049[] __initconst = { +static const u8 enc_input049[] = { 0xe6, 0x01, 0xb3, 0x85, 0x57, 0x79, 0x7d, 0xa2, 0xf8, 0xa4, 0x10, 0x6a, 0x08, 0x9d, 0x1d, 0xa6 }; -static const u8 enc_output049[] __initconst = { +static const u8 enc_output049[] = { 0x29, 0x9b, 0x5d, 0x3f, 0x3d, 0x03, 0xc0, 0x87, 0x20, 0x9a, 0x16, 0xe2, 0x85, 0x14, 0x31, 0x11, 0x4b, 0x45, 0x4e, 0xd1, 0x98, 0xde, 0x11, 0x7e, 0x83, 0xec, 0x49, 0xfa, 0x8d, 0x85, 0x08, 0xd6 }; -static const u8 enc_assoc049[] __initconst = { +static const u8 enc_assoc049[] = { 0x5e, 0x64, 0x70, 0xfa, 0xcd, 0x99, 0xc1, 0xd8, 0x1e, 0x37, 0xcd, 0x44, 0x01, 0x5f, 0xe1, 0x94, 0x80, 0xa2, 0xa4, 0xd3, 0x35, 0x2a, 0x4f, 0xf5, 0x60, 0xc0, 0x64, 0x0f, 0xdb, 0xda }; -static const u8 enc_nonce049[] __initconst = { +static const u8 enc_nonce049[] = { 0xdf, 0x87, 0x13, 0xe8, 0x7e, 0xc3, 0xdb, 0xcf, 0xad, 0x14, 0xd5, 0x3e }; -static const u8 enc_key049[] __initconst = { +static const u8 enc_key049[] = { 0x56, 0x20, 0x74, 0x65, 0xb4, 0xe4, 0x8e, 0x6d, 0x04, 0x63, 0x0f, 0x4a, 0x42, 0xf3, 0x5c, 0xfc, 0x16, 0x3a, 0xb2, 0x89, 0xc2, 0x2a, 0x2b, 0x47, @@ -2939,27 +2938,27 @@ static const u8 enc_key049[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input050[] __initconst = { +static const u8 enc_input050[] = { 0xdc, 0x9e, 0x9e, 0xaf, 0x11, 0xe3, 0x14, 0x18, 0x2d, 0xf6, 0xa4, 0xeb, 0xa1, 0x7a, 0xec, 0x9c }; -static const u8 enc_output050[] __initconst = { +static const u8 enc_output050[] = { 0x60, 0x5b, 0xbf, 0x90, 0xae, 0xb9, 0x74, 0xf6, 0x60, 0x2b, 0xc7, 0x78, 0x05, 0x6f, 0x0d, 0xca, 0x38, 0xea, 0x23, 0xd9, 0x90, 0x54, 0xb4, 0x6b, 0x42, 0xff, 0xe0, 0x04, 0x12, 0x9d, 0x22, 0x04 }; -static const u8 enc_assoc050[] __initconst = { +static const u8 enc_assoc050[] = { 0xba, 0x44, 0x6f, 0x6f, 0x9a, 0x0c, 0xed, 0x22, 0x45, 0x0f, 0xeb, 0x10, 0x73, 0x7d, 0x90, 0x07, 0xfd, 0x69, 0xab, 0xc1, 0x9b, 0x1d, 0x4d, 0x90, 0x49, 0xa5, 0x55, 0x1e, 0x86, 0xec, 0x2b, 0x37 }; -static const u8 enc_nonce050[] __initconst = { +static const u8 enc_nonce050[] = { 0x8d, 0xf4, 0xb1, 0x5a, 0x88, 0x8c, 0x33, 0x28, 0x6a, 0x7b, 0x76, 0x51 }; -static const u8 enc_key050[] __initconst = { +static const u8 enc_key050[] = { 0x39, 0x37, 0x98, 0x6a, 0xf8, 0x6d, 0xaf, 0xc1, 0xba, 0x0c, 0x46, 0x72, 0xd8, 0xab, 0xc4, 0x6c, 0x20, 0x70, 0x62, 0x68, 0x2d, 0x9c, 0x26, 0x4a, @@ -2967,28 +2966,28 @@ static const u8 enc_key050[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input051[] __initconst = { +static const u8 enc_input051[] = { 0x81, 0xce, 0x84, 0xed, 0xe9, 0xb3, 0x58, 0x59, 0xcc, 0x8c, 0x49, 0xa8, 0xf6, 0xbe, 0x7d, 0xc6 }; -static const u8 enc_output051[] __initconst = { +static const u8 enc_output051[] = { 0x7b, 0x7c, 0xe0, 0xd8, 0x24, 0x80, 0x9a, 0x70, 0xde, 0x32, 0x56, 0x2c, 0xcf, 0x2c, 0x2b, 0xbd, 0x15, 0xd4, 0x4a, 0x00, 0xce, 0x0d, 0x19, 0xb4, 0x23, 0x1f, 0x92, 0x1e, 0x22, 0xbc, 0x0a, 0x43 }; -static const u8 enc_assoc051[] __initconst = { +static const u8 enc_assoc051[] = { 0xd4, 0x1a, 0x82, 0x8d, 0x5e, 0x71, 0x82, 0x92, 0x47, 0x02, 0x19, 0x05, 0x40, 0x2e, 0xa2, 0x57, 0xdc, 0xcb, 0xc3, 0xb8, 0x0f, 0xcd, 0x56, 0x75, 0x05, 0x6b, 0x68, 0xbb, 0x59, 0xe6, 0x2e, 0x88, 0x73 }; -static const u8 enc_nonce051[] __initconst = { +static const u8 enc_nonce051[] = { 0xbe, 0x40, 0xe5, 0xf1, 0xa1, 0x18, 0x17, 0xa0, 0xa8, 0xfa, 0x89, 0x49 }; -static const u8 enc_key051[] __initconst = { +static const u8 enc_key051[] = { 0x36, 0x37, 0x2a, 0xbc, 0xdb, 0x78, 0xe0, 0x27, 0x96, 0x46, 0xac, 0x3d, 0x17, 0x6b, 0x96, 0x74, 0xe9, 0x15, 0x4e, 0xec, 0xf0, 0xd5, 0x46, 0x9c, @@ -2996,17 +2995,17 @@ static const u8 enc_key051[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input052[] __initconst = { +static const u8 enc_input052[] = { 0xa6, 0x67, 0x47, 0xc8, 0x9e, 0x85, 0x7a, 0xf3, 0xa1, 0x8e, 0x2c, 0x79, 0x50, 0x00, 0x87, 0xed }; -static const u8 enc_output052[] __initconst = { +static const u8 enc_output052[] = { 0xca, 0x82, 0xbf, 0xf3, 0xe2, 0xf3, 0x10, 0xcc, 0xc9, 0x76, 0x67, 0x2c, 0x44, 0x15, 0xe6, 0x9b, 0x57, 0x63, 0x8c, 0x62, 0xa5, 0xd8, 0x5d, 0xed, 0x77, 0x4f, 0x91, 0x3c, 0x81, 0x3e, 0xa0, 0x32 }; -static const u8 enc_assoc052[] __initconst = { +static const u8 enc_assoc052[] = { 0x3f, 0x2d, 0xd4, 0x9b, 0xbf, 0x09, 0xd6, 0x9a, 0x78, 0xa3, 0xd8, 0x0e, 0xa2, 0x56, 0x66, 0x14, 0xfc, 0x37, 0x94, 0x74, 0x19, 0x6c, 0x1a, 0xae, @@ -3014,11 +3013,11 @@ static const u8 enc_assoc052[] __initconst = { 0x6f, 0x42, 0xca, 0x42, 0x05, 0x6a, 0x97, 0x92, 0xcc, 0x1b, 0x9f, 0xb3, 0xc7, 0xd2, 0x61 }; -static const u8 enc_nonce052[] __initconst = { +static const u8 enc_nonce052[] = { 0x84, 0xc8, 0x7d, 0xae, 0x4e, 0xee, 0x27, 0x73, 0x0e, 0xc3, 0x5d, 0x12 }; -static const u8 enc_key052[] __initconst = { +static const u8 enc_key052[] = { 0x9f, 0x14, 0x79, 0xed, 0x09, 0x7d, 0x7f, 0xe5, 0x29, 0xc1, 0x1f, 0x2f, 0x5a, 0xdd, 0x9a, 0xaf, 0xf4, 0xa1, 0xca, 0x0b, 0x68, 0x99, 0x7a, 0x2c, @@ -3026,13 +3025,13 @@ static const u8 enc_key052[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input053[] __initconst = { +static const u8 enc_input053[] = { 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, 0x80, 0xe0, 0x30, 0xbe, 0xeb, 0xd3, 0x29, 0xbe }; -static const u8 enc_output053[] __initconst = { +static const u8 enc_output053[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -3040,14 +3039,14 @@ static const u8 enc_output053[] __initconst = { 0xe6, 0xd3, 0xd7, 0x32, 0x4a, 0x1c, 0xbb, 0xa7, 0x77, 0xbb, 0xb0, 0xec, 0xdd, 0xa3, 0x78, 0x07 }; -static const u8 enc_assoc053[] __initconst = { +static const u8 enc_assoc053[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce053[] __initconst = { +static const u8 enc_nonce053[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key053[] __initconst = { +static const u8 enc_key053[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3055,7 +3054,7 @@ static const u8 enc_key053[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input054[] __initconst = { +static const u8 enc_input054[] = { 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, @@ -3065,7 +3064,7 @@ static const u8 enc_input054[] __initconst = { 0x08, 0x69, 0xff, 0xd2, 0xec, 0x5e, 0x26, 0xe5, 0x53, 0xb7, 0xb2, 0x27, 0xfe, 0x87, 0xfd, 0xbd }; -static const u8 enc_output054[] __initconst = { +static const u8 enc_output054[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -3077,14 +3076,14 @@ static const u8 enc_output054[] __initconst = { 0x06, 0x2d, 0xe6, 0x79, 0x5f, 0x27, 0x4f, 0xd2, 0xa3, 0x05, 0xd7, 0x69, 0x80, 0xbc, 0x9c, 0xce }; -static const u8 enc_assoc054[] __initconst = { +static const u8 enc_assoc054[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce054[] __initconst = { +static const u8 enc_nonce054[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key054[] __initconst = { +static const u8 enc_key054[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3092,7 +3091,7 @@ static const u8 enc_key054[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input055[] __initconst = { +static const u8 enc_input055[] = { 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, @@ -3110,7 +3109,7 @@ static const u8 enc_input055[] __initconst = { 0x77, 0xd3, 0x0b, 0xc5, 0x76, 0x92, 0xed, 0x38, 0xfb, 0xac, 0x01, 0x88, 0x38, 0x04, 0x88, 0xc7 }; -static const u8 enc_output055[] __initconst = { +static const u8 enc_output055[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -3130,14 +3129,14 @@ static const u8 enc_output055[] __initconst = { 0xd8, 0xb4, 0x79, 0x02, 0xba, 0xae, 0xaf, 0xb3, 0x42, 0x03, 0x05, 0x15, 0x29, 0xaf, 0x28, 0x2e }; -static const u8 enc_assoc055[] __initconst = { +static const u8 enc_assoc055[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce055[] __initconst = { +static const u8 enc_nonce055[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key055[] __initconst = { +static const u8 enc_key055[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3145,13 +3144,13 @@ static const u8 enc_key055[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input056[] __initconst = { +static const u8 enc_input056[] = { 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, 0x7f, 0x1f, 0xcf, 0x41, 0x14, 0x2c, 0xd6, 0x41 }; -static const u8 enc_output056[] __initconst = { +static const u8 enc_output056[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3159,14 +3158,14 @@ static const u8 enc_output056[] __initconst = { 0xb3, 0x89, 0x1c, 0x84, 0x9c, 0xb5, 0x2c, 0x27, 0x74, 0x7e, 0xdf, 0xcf, 0x31, 0x21, 0x3b, 0xb6 }; -static const u8 enc_assoc056[] __initconst = { +static const u8 enc_assoc056[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce056[] __initconst = { +static const u8 enc_nonce056[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key056[] __initconst = { +static const u8 enc_key056[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3174,7 +3173,7 @@ static const u8 enc_key056[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input057[] __initconst = { +static const u8 enc_input057[] = { 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, @@ -3184,7 +3183,7 @@ static const u8 enc_input057[] __initconst = { 0xf7, 0x96, 0x00, 0x2d, 0x13, 0xa1, 0xd9, 0x1a, 0xac, 0x48, 0x4d, 0xd8, 0x01, 0x78, 0x02, 0x42 }; -static const u8 enc_output057[] __initconst = { +static const u8 enc_output057[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3196,14 +3195,14 @@ static const u8 enc_output057[] __initconst = { 0xf0, 0xc1, 0x2d, 0x26, 0xef, 0x03, 0x02, 0x9b, 0x62, 0xc0, 0x08, 0xda, 0x27, 0xc5, 0xdc, 0x68 }; -static const u8 enc_assoc057[] __initconst = { +static const u8 enc_assoc057[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce057[] __initconst = { +static const u8 enc_nonce057[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key057[] __initconst = { +static const u8 enc_key057[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3211,7 +3210,7 @@ static const u8 enc_key057[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input058[] __initconst = { +static const u8 enc_input058[] = { 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, @@ -3229,7 +3228,7 @@ static const u8 enc_input058[] __initconst = { 0x88, 0x2c, 0xf4, 0x3a, 0x89, 0x6d, 0x12, 0xc7, 0x04, 0x53, 0xfe, 0x77, 0xc7, 0xfb, 0x77, 0x38 }; -static const u8 enc_output058[] __initconst = { +static const u8 enc_output058[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3249,14 +3248,14 @@ static const u8 enc_output058[] __initconst = { 0xee, 0x65, 0x78, 0x30, 0x01, 0xc2, 0x56, 0x91, 0xfa, 0x28, 0xd0, 0xf5, 0xf1, 0xc1, 0xd7, 0x62 }; -static const u8 enc_assoc058[] __initconst = { +static const u8 enc_assoc058[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce058[] __initconst = { +static const u8 enc_nonce058[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key058[] __initconst = { +static const u8 enc_key058[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3264,13 +3263,13 @@ static const u8 enc_key058[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input059[] __initconst = { +static const u8 enc_input059[] = { 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, 0x80, 0xe0, 0x30, 0x3e, 0xeb, 0xd3, 0x29, 0x3e }; -static const u8 enc_output059[] __initconst = { +static const u8 enc_output059[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, @@ -3278,14 +3277,14 @@ static const u8 enc_output059[] __initconst = { 0x79, 0xba, 0x7a, 0x29, 0xf5, 0xa7, 0xbb, 0x75, 0x79, 0x7a, 0xf8, 0x7a, 0x61, 0x01, 0x29, 0xa4 }; -static const u8 enc_assoc059[] __initconst = { +static const u8 enc_assoc059[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 }; -static const u8 enc_nonce059[] __initconst = { +static const u8 enc_nonce059[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key059[] __initconst = { +static const u8 enc_key059[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3293,7 +3292,7 @@ static const u8 enc_key059[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input060[] __initconst = { +static const u8 enc_input060[] = { 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, @@ -3303,7 +3302,7 @@ static const u8 enc_input060[] __initconst = { 0x08, 0x69, 0xff, 0x52, 0xec, 0x5e, 0x26, 0x65, 0x53, 0xb7, 0xb2, 0xa7, 0xfe, 0x87, 0xfd, 0x3d }; -static const u8 enc_output060[] __initconst = { +static const u8 enc_output060[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, @@ -3315,14 +3314,14 @@ static const u8 enc_output060[] __initconst = { 0x36, 0xb1, 0x74, 0x38, 0x19, 0xe1, 0xb9, 0xba, 0x15, 0x51, 0xe8, 0xed, 0x92, 0x2a, 0x95, 0x9a }; -static const u8 enc_assoc060[] __initconst = { +static const u8 enc_assoc060[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 }; -static const u8 enc_nonce060[] __initconst = { +static const u8 enc_nonce060[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key060[] __initconst = { +static const u8 enc_key060[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3330,7 +3329,7 @@ static const u8 enc_key060[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input061[] __initconst = { +static const u8 enc_input061[] = { 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, @@ -3348,7 +3347,7 @@ static const u8 enc_input061[] __initconst = { 0x77, 0xd3, 0x0b, 0x45, 0x76, 0x92, 0xed, 0xb8, 0xfb, 0xac, 0x01, 0x08, 0x38, 0x04, 0x88, 0x47 }; -static const u8 enc_output061[] __initconst = { +static const u8 enc_output061[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, @@ -3368,14 +3367,14 @@ static const u8 enc_output061[] __initconst = { 0xfe, 0xac, 0x49, 0x55, 0x55, 0x4e, 0x80, 0x6f, 0x3a, 0x19, 0x02, 0xe2, 0x44, 0x32, 0xc0, 0x8a }; -static const u8 enc_assoc061[] __initconst = { +static const u8 enc_assoc061[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 }; -static const u8 enc_nonce061[] __initconst = { +static const u8 enc_nonce061[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key061[] __initconst = { +static const u8 enc_key061[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3383,13 +3382,13 @@ static const u8 enc_key061[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input062[] __initconst = { +static const u8 enc_input062[] = { 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, 0x7f, 0x1f, 0xcf, 0xc1, 0x14, 0x2c, 0xd6, 0xc1 }; -static const u8 enc_output062[] __initconst = { +static const u8 enc_output062[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, @@ -3397,14 +3396,14 @@ static const u8 enc_output062[] __initconst = { 0x20, 0xa3, 0x79, 0x8d, 0xf1, 0x29, 0x2c, 0x59, 0x72, 0xbf, 0x97, 0x41, 0xae, 0xc3, 0x8a, 0x19 }; -static const u8 enc_assoc062[] __initconst = { +static const u8 enc_assoc062[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f }; -static const u8 enc_nonce062[] __initconst = { +static const u8 enc_nonce062[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key062[] __initconst = { +static const u8 enc_key062[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3412,7 +3411,7 @@ static const u8 enc_key062[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input063[] __initconst = { +static const u8 enc_input063[] = { 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, @@ -3422,7 +3421,7 @@ static const u8 enc_input063[] __initconst = { 0xf7, 0x96, 0x00, 0xad, 0x13, 0xa1, 0xd9, 0x9a, 0xac, 0x48, 0x4d, 0x58, 0x01, 0x78, 0x02, 0xc2 }; -static const u8 enc_output063[] __initconst = { +static const u8 enc_output063[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, @@ -3434,14 +3433,14 @@ static const u8 enc_output063[] __initconst = { 0xc0, 0x3d, 0x9f, 0x67, 0x35, 0x4a, 0x97, 0xb2, 0xf0, 0x74, 0xf7, 0x55, 0x15, 0x57, 0xe4, 0x9c }; -static const u8 enc_assoc063[] __initconst = { +static const u8 enc_assoc063[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f }; -static const u8 enc_nonce063[] __initconst = { +static const u8 enc_nonce063[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key063[] __initconst = { +static const u8 enc_key063[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3449,7 +3448,7 @@ static const u8 enc_key063[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input064[] __initconst = { +static const u8 enc_input064[] = { 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, @@ -3467,7 +3466,7 @@ static const u8 enc_input064[] __initconst = { 0x88, 0x2c, 0xf4, 0xba, 0x89, 0x6d, 0x12, 0x47, 0x04, 0x53, 0xfe, 0xf7, 0xc7, 0xfb, 0x77, 0xb8 }; -static const u8 enc_output064[] __initconst = { +static const u8 enc_output064[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, @@ -3487,14 +3486,14 @@ static const u8 enc_output064[] __initconst = { 0xc8, 0x6d, 0xa8, 0xdd, 0x65, 0x22, 0x86, 0xd5, 0x02, 0x13, 0xd3, 0x28, 0xd6, 0x3e, 0x40, 0x06 }; -static const u8 enc_assoc064[] __initconst = { +static const u8 enc_assoc064[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f }; -static const u8 enc_nonce064[] __initconst = { +static const u8 enc_nonce064[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key064[] __initconst = { +static const u8 enc_key064[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3502,13 +3501,13 @@ static const u8 enc_key064[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input065[] __initconst = { +static const u8 enc_input065[] = { 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, 0xff, 0x1f, 0xcf, 0x41, 0x94, 0x2c, 0xd6, 0x41 }; -static const u8 enc_output065[] __initconst = { +static const u8 enc_output065[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, @@ -3516,14 +3515,14 @@ static const u8 enc_output065[] __initconst = { 0xbe, 0xde, 0x90, 0x83, 0xce, 0xb3, 0x6d, 0xdf, 0xe5, 0xfa, 0x81, 0x1f, 0x95, 0x47, 0x1c, 0x67 }; -static const u8 enc_assoc065[] __initconst = { +static const u8 enc_assoc065[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff }; -static const u8 enc_nonce065[] __initconst = { +static const u8 enc_nonce065[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key065[] __initconst = { +static const u8 enc_key065[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3531,7 +3530,7 @@ static const u8 enc_key065[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input066[] __initconst = { +static const u8 enc_input066[] = { 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, @@ -3541,7 +3540,7 @@ static const u8 enc_input066[] __initconst = { 0x77, 0x96, 0x00, 0x2d, 0x93, 0xa1, 0xd9, 0x1a, 0x2c, 0x48, 0x4d, 0xd8, 0x81, 0x78, 0x02, 0x42 }; -static const u8 enc_output066[] __initconst = { +static const u8 enc_output066[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, @@ -3553,14 +3552,14 @@ static const u8 enc_output066[] __initconst = { 0x30, 0x08, 0x74, 0xbb, 0x06, 0x92, 0xb6, 0x89, 0xde, 0xad, 0x9a, 0xe1, 0x5b, 0x06, 0x73, 0x90 }; -static const u8 enc_assoc066[] __initconst = { +static const u8 enc_assoc066[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff }; -static const u8 enc_nonce066[] __initconst = { +static const u8 enc_nonce066[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key066[] __initconst = { +static const u8 enc_key066[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3568,7 +3567,7 @@ static const u8 enc_key066[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input067[] __initconst = { +static const u8 enc_input067[] = { 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, @@ -3586,7 +3585,7 @@ static const u8 enc_input067[] __initconst = { 0x08, 0x2c, 0xf4, 0x3a, 0x09, 0x6d, 0x12, 0xc7, 0x84, 0x53, 0xfe, 0x77, 0x47, 0xfb, 0x77, 0x38 }; -static const u8 enc_output067[] __initconst = { +static const u8 enc_output067[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, @@ -3606,14 +3605,14 @@ static const u8 enc_output067[] __initconst = { 0x99, 0xca, 0xd8, 0x5f, 0x45, 0xca, 0x40, 0x94, 0x2d, 0x0d, 0x4d, 0x5e, 0x95, 0x0a, 0xde, 0x22 }; -static const u8 enc_assoc067[] __initconst = { +static const u8 enc_assoc067[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff }; -static const u8 enc_nonce067[] __initconst = { +static const u8 enc_nonce067[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key067[] __initconst = { +static const u8 enc_key067[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3621,13 +3620,13 @@ static const u8 enc_key067[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input068[] __initconst = { +static const u8 enc_input068[] = { 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, 0x80, 0xe0, 0x30, 0xbe, 0x14, 0x2c, 0xd6, 0x41 }; -static const u8 enc_output068[] __initconst = { +static const u8 enc_output068[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, @@ -3635,14 +3634,14 @@ static const u8 enc_output068[] __initconst = { 0x8b, 0xbe, 0x14, 0x52, 0x72, 0xe7, 0xc2, 0xd9, 0xa1, 0x89, 0x1a, 0x3a, 0xb0, 0x98, 0x3d, 0x9d }; -static const u8 enc_assoc068[] __initconst = { +static const u8 enc_assoc068[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce068[] __initconst = { +static const u8 enc_nonce068[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key068[] __initconst = { +static const u8 enc_key068[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3650,7 +3649,7 @@ static const u8 enc_key068[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input069[] __initconst = { +static const u8 enc_input069[] = { 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, @@ -3660,7 +3659,7 @@ static const u8 enc_input069[] __initconst = { 0x08, 0x69, 0xff, 0xd2, 0x13, 0xa1, 0xd9, 0x1a, 0x53, 0xb7, 0xb2, 0x27, 0x01, 0x78, 0x02, 0x42 }; -static const u8 enc_output069[] __initconst = { +static const u8 enc_output069[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, @@ -3672,14 +3671,14 @@ static const u8 enc_output069[] __initconst = { 0x3b, 0x41, 0x86, 0x19, 0x13, 0xa8, 0xf6, 0xde, 0x7f, 0x61, 0xe2, 0x25, 0x63, 0x1b, 0xc3, 0x82 }; -static const u8 enc_assoc069[] __initconst = { +static const u8 enc_assoc069[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce069[] __initconst = { +static const u8 enc_nonce069[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key069[] __initconst = { +static const u8 enc_key069[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3687,7 +3686,7 @@ static const u8 enc_key069[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input070[] __initconst = { +static const u8 enc_input070[] = { 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, @@ -3705,7 +3704,7 @@ static const u8 enc_input070[] __initconst = { 0x77, 0xd3, 0x0b, 0xc5, 0x89, 0x6d, 0x12, 0xc7, 0xfb, 0xac, 0x01, 0x88, 0xc7, 0xfb, 0x77, 0x38 }; -static const u8 enc_output070[] __initconst = { +static const u8 enc_output070[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, @@ -3725,14 +3724,14 @@ static const u8 enc_output070[] __initconst = { 0x84, 0x28, 0xbc, 0xf0, 0x23, 0xec, 0x6b, 0xf3, 0x1f, 0xd9, 0xef, 0xb2, 0x03, 0xff, 0x08, 0x71 }; -static const u8 enc_assoc070[] __initconst = { +static const u8 enc_assoc070[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce070[] __initconst = { +static const u8 enc_nonce070[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key070[] __initconst = { +static const u8 enc_key070[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3740,13 +3739,13 @@ static const u8 enc_key070[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input071[] __initconst = { +static const u8 enc_input071[] = { 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, 0x7f, 0x1f, 0xcf, 0x41, 0xeb, 0xd3, 0x29, 0xbe }; -static const u8 enc_output071[] __initconst = { +static const u8 enc_output071[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, @@ -3754,14 +3753,14 @@ static const u8 enc_output071[] __initconst = { 0x13, 0x9f, 0xdf, 0x64, 0x74, 0xea, 0x24, 0xf5, 0x49, 0xb0, 0x75, 0x82, 0x5f, 0x2c, 0x76, 0x20 }; -static const u8 enc_assoc071[] __initconst = { +static const u8 enc_assoc071[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce071[] __initconst = { +static const u8 enc_nonce071[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key071[] __initconst = { +static const u8 enc_key071[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3769,7 +3768,7 @@ static const u8 enc_key071[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input072[] __initconst = { +static const u8 enc_input072[] = { 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, @@ -3779,7 +3778,7 @@ static const u8 enc_input072[] __initconst = { 0xf7, 0x96, 0x00, 0x2d, 0xec, 0x5e, 0x26, 0xe5, 0xac, 0x48, 0x4d, 0xd8, 0xfe, 0x87, 0xfd, 0xbd }; -static const u8 enc_output072[] __initconst = { +static const u8 enc_output072[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, @@ -3791,14 +3790,14 @@ static const u8 enc_output072[] __initconst = { 0xbb, 0xad, 0x8d, 0x86, 0x3b, 0x83, 0x5a, 0x8e, 0x86, 0x64, 0xfd, 0x1d, 0x45, 0x66, 0xb6, 0xb4 }; -static const u8 enc_assoc072[] __initconst = { +static const u8 enc_assoc072[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce072[] __initconst = { +static const u8 enc_nonce072[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key072[] __initconst = { +static const u8 enc_key072[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3806,7 +3805,7 @@ static const u8 enc_key072[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input073[] __initconst = { +static const u8 enc_input073[] = { 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, @@ -3824,7 +3823,7 @@ static const u8 enc_input073[] __initconst = { 0x88, 0x2c, 0xf4, 0x3a, 0x76, 0x92, 0xed, 0x38, 0x04, 0x53, 0xfe, 0x77, 0x38, 0x04, 0x88, 0xc7 }; -static const u8 enc_output073[] __initconst = { +static const u8 enc_output073[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, @@ -3844,14 +3843,14 @@ static const u8 enc_output073[] __initconst = { 0x42, 0xf2, 0x35, 0x42, 0x97, 0x84, 0x9a, 0x51, 0x1d, 0x53, 0xe5, 0x57, 0x17, 0x72, 0xf7, 0x1f }; -static const u8 enc_assoc073[] __initconst = { +static const u8 enc_assoc073[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce073[] __initconst = { +static const u8 enc_nonce073[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key073[] __initconst = { +static const u8 enc_key073[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3859,7 +3858,7 @@ static const u8 enc_key073[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input074[] __initconst = { +static const u8 enc_input074[] = { 0xd4, 0x50, 0x0b, 0xf0, 0x09, 0x49, 0x35, 0x51, 0xc3, 0x80, 0xad, 0xf5, 0x2c, 0x57, 0x3a, 0x69, 0xdf, 0x7e, 0x8b, 0x76, 0x24, 0x63, 0x33, 0x0f, @@ -3877,7 +3876,7 @@ static const u8 enc_input074[] __initconst = { 0x93, 0x76, 0x71, 0xa0, 0x62, 0x9b, 0xd9, 0x5c, 0x99, 0x15, 0xc7, 0x85, 0x55, 0x77, 0x1e, 0x7a }; -static const u8 enc_output074[] __initconst = { +static const u8 enc_output074[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3897,7 +3896,7 @@ static const u8 enc_output074[] __initconst = { 0x0b, 0x30, 0x0d, 0x8d, 0xa5, 0x6c, 0x21, 0x85, 0x75, 0x52, 0x79, 0x55, 0x3c, 0x4c, 0x82, 0xca }; -static const u8 enc_assoc074[] __initconst = { +static const u8 enc_assoc074[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3907,11 +3906,11 @@ static const u8 enc_assoc074[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce074[] __initconst = { +static const u8 enc_nonce074[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x00, 0x02, 0x50, 0x6e }; -static const u8 enc_key074[] __initconst = { +static const u8 enc_key074[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -3919,7 +3918,7 @@ static const u8 enc_key074[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input075[] __initconst = { +static const u8 enc_input075[] = { 0x7d, 0xe8, 0x7f, 0x67, 0x29, 0x94, 0x52, 0x75, 0xd0, 0x65, 0x5d, 0xa4, 0xc7, 0xfd, 0xe4, 0x56, 0x9e, 0x16, 0xf1, 0x11, 0xb5, 0xeb, 0x26, 0xc2, @@ -3937,7 +3936,7 @@ static const u8 enc_input075[] __initconst = { 0x3a, 0xc1, 0xee, 0x54, 0xc2, 0x9e, 0xe4, 0xc1, 0x70, 0xde, 0x40, 0x8f, 0x66, 0x69, 0x21, 0x94 }; -static const u8 enc_output075[] __initconst = { +static const u8 enc_output075[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3957,7 +3956,7 @@ static const u8 enc_output075[] __initconst = { 0xc5, 0x78, 0xe2, 0xaa, 0x44, 0xd3, 0x09, 0xb7, 0xb6, 0xa5, 0x19, 0x3b, 0xdc, 0x61, 0x18, 0xf5 }; -static const u8 enc_assoc075[] __initconst = { +static const u8 enc_assoc075[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3967,11 +3966,11 @@ static const u8 enc_assoc075[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce075[] __initconst = { +static const u8 enc_nonce075[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x00, 0x03, 0x18, 0xa5 }; -static const u8 enc_key075[] __initconst = { +static const u8 enc_key075[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -3979,7 +3978,7 @@ static const u8 enc_key075[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input076[] __initconst = { +static const u8 enc_input076[] = { 0x1b, 0x99, 0x6f, 0x9a, 0x3c, 0xcc, 0x67, 0x85, 0xde, 0x22, 0xff, 0x5b, 0x8a, 0xdd, 0x95, 0x02, 0xce, 0x03, 0xa0, 0xfa, 0xf5, 0x99, 0x2a, 0x09, @@ -3997,7 +3996,7 @@ static const u8 enc_input076[] __initconst = { 0xc3, 0x53, 0xd0, 0xcf, 0x93, 0x8d, 0xcc, 0xb9, 0xef, 0xad, 0x8f, 0xed, 0xbe, 0x46, 0xda, 0xa5 }; -static const u8 enc_output076[] __initconst = { +static const u8 enc_output076[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4017,7 +4016,7 @@ static const u8 enc_output076[] __initconst = { 0x4b, 0x0b, 0xda, 0x8a, 0xd0, 0x43, 0x83, 0x0d, 0x83, 0x19, 0xab, 0x82, 0xc5, 0x0c, 0x76, 0x63 }; -static const u8 enc_assoc076[] __initconst = { +static const u8 enc_assoc076[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4027,10 +4026,10 @@ static const u8 enc_assoc076[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce076[] __initconst = { +static const u8 enc_nonce076[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xb4, 0xf0 }; -static const u8 enc_key076[] __initconst = { +static const u8 enc_key076[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4038,7 +4037,7 @@ static const u8 enc_key076[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input077[] __initconst = { +static const u8 enc_input077[] = { 0x86, 0xcb, 0xac, 0xae, 0x4d, 0x3f, 0x74, 0xae, 0x01, 0x21, 0x3e, 0x05, 0x51, 0xcc, 0x15, 0x16, 0x0e, 0xa1, 0xbe, 0x84, 0x08, 0xe3, 0xd5, 0xd7, @@ -4056,7 +4055,7 @@ static const u8 enc_input077[] __initconst = { 0x0e, 0x76, 0x2b, 0x43, 0x0c, 0x4d, 0x51, 0x7c, 0x97, 0x10, 0x70, 0x68, 0xf4, 0x98, 0xef, 0x7f }; -static const u8 enc_output077[] __initconst = { +static const u8 enc_output077[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4076,7 +4075,7 @@ static const u8 enc_output077[] __initconst = { 0x4b, 0xc9, 0x8f, 0x72, 0xc4, 0x94, 0xc2, 0xa4, 0x3c, 0x2b, 0x15, 0xa1, 0x04, 0x3f, 0x1c, 0xfa }; -static const u8 enc_assoc077[] __initconst = { +static const u8 enc_assoc077[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4086,10 +4085,10 @@ static const u8 enc_assoc077[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce077[] __initconst = { +static const u8 enc_nonce077[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0xfb, 0x66 }; -static const u8 enc_key077[] __initconst = { +static const u8 enc_key077[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4097,7 +4096,7 @@ static const u8 enc_key077[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input078[] __initconst = { +static const u8 enc_input078[] = { 0xfa, 0xb1, 0xcd, 0xdf, 0x4f, 0xe1, 0x98, 0xef, 0x63, 0xad, 0xd8, 0x81, 0xd6, 0xea, 0xd6, 0xc5, 0x76, 0x37, 0xbb, 0xe9, 0x20, 0x18, 0xca, 0x7c, @@ -4115,7 +4114,7 @@ static const u8 enc_input078[] __initconst = { 0xc3, 0x24, 0x60, 0x41, 0x43, 0x21, 0x43, 0xe9, 0xab, 0x3a, 0x6d, 0x2c, 0xcc, 0x2f, 0x4d, 0x62 }; -static const u8 enc_output078[] __initconst = { +static const u8 enc_output078[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4135,7 +4134,7 @@ static const u8 enc_output078[] __initconst = { 0xf7, 0xe9, 0xe1, 0x51, 0xb0, 0x25, 0x33, 0xc7, 0x46, 0x58, 0xbf, 0xc7, 0x73, 0x7c, 0x68, 0x0d }; -static const u8 enc_assoc078[] __initconst = { +static const u8 enc_assoc078[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4145,10 +4144,10 @@ static const u8 enc_assoc078[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce078[] __initconst = { +static const u8 enc_nonce078[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0xbb, 0x90 }; -static const u8 enc_key078[] __initconst = { +static const u8 enc_key078[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4156,7 +4155,7 @@ static const u8 enc_key078[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input079[] __initconst = { +static const u8 enc_input079[] = { 0x22, 0x72, 0x02, 0xbe, 0x7f, 0x35, 0x15, 0xe9, 0xd1, 0xc0, 0x2e, 0xea, 0x2f, 0x19, 0x50, 0xb6, 0x48, 0x1b, 0x04, 0x8a, 0x4c, 0x91, 0x50, 0x6c, @@ -4174,7 +4173,7 @@ static const u8 enc_input079[] __initconst = { 0xfb, 0xe1, 0xee, 0x45, 0xb1, 0xb2, 0x1f, 0x71, 0x62, 0xe2, 0xfc, 0xaa, 0x74, 0x2a, 0xbe, 0xfd }; -static const u8 enc_output079[] __initconst = { +static const u8 enc_output079[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4194,7 +4193,7 @@ static const u8 enc_output079[] __initconst = { 0x79, 0x5b, 0xcf, 0xf6, 0x47, 0xc5, 0x53, 0xc2, 0xe4, 0xeb, 0x6e, 0x0e, 0xaf, 0xd9, 0xe0, 0x4e }; -static const u8 enc_assoc079[] __initconst = { +static const u8 enc_assoc079[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4204,10 +4203,10 @@ static const u8 enc_assoc079[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce079[] __initconst = { +static const u8 enc_nonce079[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x48, 0x4a }; -static const u8 enc_key079[] __initconst = { +static const u8 enc_key079[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4215,7 +4214,7 @@ static const u8 enc_key079[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input080[] __initconst = { +static const u8 enc_input080[] = { 0xfa, 0xe5, 0x83, 0x45, 0xc1, 0x6c, 0xb0, 0xf5, 0xcc, 0x53, 0x7f, 0x2b, 0x1b, 0x34, 0x69, 0xc9, 0x69, 0x46, 0x3b, 0x3e, 0xa7, 0x1b, 0xcf, 0x6b, @@ -4233,7 +4232,7 @@ static const u8 enc_input080[] __initconst = { 0x18, 0x01, 0xce, 0x33, 0xc4, 0xe4, 0xa7, 0x7d, 0x83, 0x1d, 0x3c, 0xe3, 0x4e, 0x84, 0x10, 0xe1 }; -static const u8 enc_output080[] __initconst = { +static const u8 enc_output080[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4253,7 +4252,7 @@ static const u8 enc_output080[] __initconst = { 0x19, 0x46, 0xd6, 0x53, 0x96, 0x0f, 0x94, 0x7a, 0x74, 0xd3, 0xe8, 0x09, 0x3c, 0xf4, 0x85, 0x02 }; -static const u8 enc_assoc080[] __initconst = { +static const u8 enc_assoc080[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4263,10 +4262,10 @@ static const u8 enc_assoc080[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce080[] __initconst = { +static const u8 enc_nonce080[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0x2f, 0x40 }; -static const u8 enc_key080[] __initconst = { +static const u8 enc_key080[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4274,7 +4273,7 @@ static const u8 enc_key080[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input081[] __initconst = { +static const u8 enc_input081[] = { 0xeb, 0xb2, 0x16, 0xdd, 0xd7, 0xca, 0x70, 0x92, 0x15, 0xf5, 0x03, 0xdf, 0x9c, 0xe6, 0x3c, 0x5c, 0xd2, 0x19, 0x4e, 0x7d, 0x90, 0x99, 0xe8, 0xa9, @@ -4292,7 +4291,7 @@ static const u8 enc_input081[] __initconst = { 0x83, 0xfd, 0xca, 0x39, 0xd2, 0xe1, 0x4f, 0x23, 0xd0, 0x0a, 0x58, 0x26, 0x64, 0xf4, 0xec, 0xb1 }; -static const u8 enc_output081[] __initconst = { +static const u8 enc_output081[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4312,7 +4311,7 @@ static const u8 enc_output081[] __initconst = { 0x36, 0xc3, 0x00, 0x29, 0x85, 0xdd, 0x21, 0xba, 0xf8, 0x95, 0xd6, 0x33, 0x57, 0x3f, 0x12, 0xc0 }; -static const u8 enc_assoc081[] __initconst = { +static const u8 enc_assoc081[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4322,10 +4321,10 @@ static const u8 enc_assoc081[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce081[] __initconst = { +static const u8 enc_nonce081[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0xe2, 0x93, 0x35 }; -static const u8 enc_key081[] __initconst = { +static const u8 enc_key081[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4333,7 +4332,7 @@ static const u8 enc_key081[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input082[] __initconst = { +static const u8 enc_input082[] = { 0x40, 0x8a, 0xe6, 0xef, 0x1c, 0x7e, 0xf0, 0xfb, 0x2c, 0x2d, 0x61, 0x08, 0x16, 0xfc, 0x78, 0x49, 0xef, 0xa5, 0x8f, 0x78, 0x27, 0x3f, 0x5f, 0x16, @@ -4351,7 +4350,7 @@ static const u8 enc_input082[] __initconst = { 0x2b, 0x68, 0x9f, 0x93, 0x55, 0xd9, 0xc1, 0x83, 0x80, 0x1f, 0x6a, 0xcc, 0x31, 0x3f, 0x89, 0x07 }; -static const u8 enc_output082[] __initconst = { +static const u8 enc_output082[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4371,7 +4370,7 @@ static const u8 enc_output082[] __initconst = { 0x65, 0x14, 0x51, 0x8e, 0x0a, 0x26, 0x41, 0x42, 0xe0, 0xb7, 0x35, 0x1f, 0x96, 0x7f, 0xc2, 0xae }; -static const u8 enc_assoc082[] __initconst = { +static const u8 enc_assoc082[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4381,10 +4380,10 @@ static const u8 enc_assoc082[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce082[] __initconst = { +static const u8 enc_nonce082[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0xf7, 0xd5 }; -static const u8 enc_key082[] __initconst = { +static const u8 enc_key082[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4392,7 +4391,7 @@ static const u8 enc_key082[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input083[] __initconst = { +static const u8 enc_input083[] = { 0x0a, 0x0a, 0x24, 0x49, 0x9b, 0xca, 0xde, 0x58, 0xcf, 0x15, 0x76, 0xc3, 0x12, 0xac, 0xa9, 0x84, 0x71, 0x8c, 0xb4, 0xcc, 0x7e, 0x01, 0x53, 0xf5, @@ -4410,7 +4409,7 @@ static const u8 enc_input083[] __initconst = { 0xb3, 0x06, 0xa2, 0x1b, 0x42, 0xd4, 0xc3, 0xba, 0x6e, 0x6f, 0x0c, 0xbc, 0xc8, 0x1e, 0x87, 0x7a }; -static const u8 enc_output083[] __initconst = { +static const u8 enc_output083[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4430,7 +4429,7 @@ static const u8 enc_output083[] __initconst = { 0x4c, 0x19, 0x4d, 0xa6, 0xa9, 0x9f, 0xd6, 0x5b, 0x40, 0xe9, 0xca, 0xd7, 0x98, 0xf4, 0x4b, 0x19 }; -static const u8 enc_assoc083[] __initconst = { +static const u8 enc_assoc083[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4440,10 +4439,10 @@ static const u8 enc_assoc083[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce083[] __initconst = { +static const u8 enc_nonce083[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0xfc, 0xe4 }; -static const u8 enc_key083[] __initconst = { +static const u8 enc_key083[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4451,7 +4450,7 @@ static const u8 enc_key083[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input084[] __initconst = { +static const u8 enc_input084[] = { 0x4a, 0x0a, 0xaf, 0xf8, 0x49, 0x47, 0x29, 0x18, 0x86, 0x91, 0x70, 0x13, 0x40, 0xf3, 0xce, 0x2b, 0x8a, 0x78, 0xee, 0xd3, 0xa0, 0xf0, 0x65, 0x99, @@ -4469,7 +4468,7 @@ static const u8 enc_input084[] __initconst = { 0x5d, 0x7e, 0x51, 0x3b, 0xe5, 0xb8, 0xea, 0x97, 0x13, 0x10, 0xd5, 0xbf, 0x16, 0xba, 0x7a, 0xee }; -static const u8 enc_output084[] __initconst = { +static const u8 enc_output084[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4489,7 +4488,7 @@ static const u8 enc_output084[] __initconst = { 0xc8, 0xae, 0x77, 0x88, 0xcd, 0x28, 0x74, 0xab, 0xc1, 0x38, 0x54, 0x1e, 0x11, 0xfd, 0x05, 0x87 }; -static const u8 enc_assoc084[] __initconst = { +static const u8 enc_assoc084[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4499,10 +4498,10 @@ static const u8 enc_assoc084[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce084[] __initconst = { +static const u8 enc_nonce084[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0x84, 0x86, 0xa8 }; -static const u8 enc_key084[] __initconst = { +static const u8 enc_key084[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4510,7 +4509,7 @@ static const u8 enc_key084[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input085[] __initconst = { +static const u8 enc_input085[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x78, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -4528,7 +4527,7 @@ static const u8 enc_input085[] __initconst = { 0xf7, 0x4d, 0x5b, 0xf8, 0x67, 0x1c, 0x5a, 0x8a, 0x50, 0x92, 0xf6, 0x1d, 0x54, 0xc9, 0xaa, 0x5b }; -static const u8 enc_output085[] __initconst = { +static const u8 enc_output085[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4548,7 +4547,7 @@ static const u8 enc_output085[] __initconst = { 0x93, 0x3a, 0x51, 0x63, 0xc7, 0xf6, 0x23, 0x68, 0x32, 0x7b, 0x3f, 0xbc, 0x10, 0x36, 0xc9, 0x43 }; -static const u8 enc_assoc085[] __initconst = { +static const u8 enc_assoc085[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4558,10 +4557,10 @@ static const u8 enc_assoc085[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce085[] __initconst = { +static const u8 enc_nonce085[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key085[] __initconst = { +static const u8 enc_key085[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4569,7 +4568,7 @@ static const u8 enc_key085[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input086[] __initconst = { +static const u8 enc_input086[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4579,7 +4578,7 @@ static const u8 enc_input086[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output086[] __initconst = { +static const u8 enc_output086[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4591,17 +4590,17 @@ static const u8 enc_output086[] __initconst = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }; -static const u8 enc_assoc086[] __initconst = { +static const u8 enc_assoc086[] = { 0x85, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa6, 0x90, 0x2f, 0xcb, 0xc8, 0x83, 0xbb, 0xc1, 0x80, 0xb2, 0x56, 0xae, 0x34, 0xad, 0x7f, 0x00 }; -static const u8 enc_nonce086[] __initconst = { +static const u8 enc_nonce086[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key086[] __initconst = { +static const u8 enc_key086[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4609,7 +4608,7 @@ static const u8 enc_key086[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input087[] __initconst = { +static const u8 enc_input087[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4619,7 +4618,7 @@ static const u8 enc_input087[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output087[] __initconst = { +static const u8 enc_output087[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4631,17 +4630,17 @@ static const u8 enc_output087[] __initconst = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_assoc087[] __initconst = { +static const u8 enc_assoc087[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x24, 0x7e, 0x50, 0x64, 0x2a, 0x1c, 0x0a, 0x2f, 0x8f, 0x77, 0x21, 0x96, 0x09, 0xdb, 0xa9, 0x58 }; -static const u8 enc_nonce087[] __initconst = { +static const u8 enc_nonce087[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key087[] __initconst = { +static const u8 enc_key087[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4649,7 +4648,7 @@ static const u8 enc_key087[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input088[] __initconst = { +static const u8 enc_input088[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4659,7 +4658,7 @@ static const u8 enc_input088[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output088[] __initconst = { +static const u8 enc_output088[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4671,17 +4670,17 @@ static const u8 enc_output088[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_assoc088[] __initconst = { +static const u8 enc_assoc088[] = { 0x7c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xd9, 0xe7, 0x2c, 0x06, 0x4a, 0xc8, 0x96, 0x1f, 0x3f, 0xa5, 0x85, 0xe0, 0xe2, 0xab, 0xd6, 0x00 }; -static const u8 enc_nonce088[] __initconst = { +static const u8 enc_nonce088[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key088[] __initconst = { +static const u8 enc_key088[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4689,7 +4688,7 @@ static const u8 enc_key088[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input089[] __initconst = { +static const u8 enc_input089[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4699,7 +4698,7 @@ static const u8 enc_input089[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output089[] __initconst = { +static const u8 enc_output089[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4711,17 +4710,17 @@ static const u8 enc_output089[] __initconst = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 }; -static const u8 enc_assoc089[] __initconst = { +static const u8 enc_assoc089[] = { 0x65, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x95, 0xaf, 0x0f, 0x4d, 0x0b, 0x68, 0x6e, 0xae, 0xcc, 0xca, 0x43, 0x07, 0xd5, 0x96, 0xf5, 0x02 }; -static const u8 enc_nonce089[] __initconst = { +static const u8 enc_nonce089[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key089[] __initconst = { +static const u8 enc_key089[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4729,7 +4728,7 @@ static const u8 enc_key089[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input090[] __initconst = { +static const u8 enc_input090[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4739,7 +4738,7 @@ static const u8 enc_input090[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output090[] __initconst = { +static const u8 enc_output090[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4751,17 +4750,17 @@ static const u8 enc_output090[] __initconst = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f }; -static const u8 enc_assoc090[] __initconst = { +static const u8 enc_assoc090[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x85, 0x40, 0xb4, 0x64, 0x35, 0x77, 0x07, 0xbe, 0x3a, 0x39, 0xd5, 0x5c, 0x34, 0xf8, 0xbc, 0xb3 }; -static const u8 enc_nonce090[] __initconst = { +static const u8 enc_nonce090[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key090[] __initconst = { +static const u8 enc_key090[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4769,7 +4768,7 @@ static const u8 enc_key090[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input091[] __initconst = { +static const u8 enc_input091[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4779,7 +4778,7 @@ static const u8 enc_input091[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output091[] __initconst = { +static const u8 enc_output091[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4791,17 +4790,17 @@ static const u8 enc_output091[] __initconst = { 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00 }; -static const u8 enc_assoc091[] __initconst = { +static const u8 enc_assoc091[] = { 0x4f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x66, 0x23, 0xd9, 0x90, 0xb8, 0x98, 0xd8, 0x30, 0xd2, 0x12, 0xaf, 0x23, 0x83, 0x33, 0x07, 0x01 }; -static const u8 enc_nonce091[] __initconst = { +static const u8 enc_nonce091[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key091[] __initconst = { +static const u8 enc_key091[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4809,7 +4808,7 @@ static const u8 enc_key091[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input092[] __initconst = { +static const u8 enc_input092[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4819,7 +4818,7 @@ static const u8 enc_input092[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output092[] __initconst = { +static const u8 enc_output092[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4831,17 +4830,17 @@ static const u8 enc_output092[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_assoc092[] __initconst = { +static const u8 enc_assoc092[] = { 0x83, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x5f, 0x16, 0xd0, 0x9f, 0x17, 0x78, 0x72, 0x11, 0xb7, 0xd4, 0x84, 0xe0, 0x24, 0xf8, 0x97, 0x01 }; -static const u8 enc_nonce092[] __initconst = { +static const u8 enc_nonce092[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key092[] __initconst = { +static const u8 enc_key092[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4849,7 +4848,7 @@ static const u8 enc_key092[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input093[] __initconst = { +static const u8 enc_input093[] = { 0x00, 0x52, 0x35, 0xd2, 0xa9, 0x19, 0xf2, 0x8d, 0x3d, 0xb7, 0x66, 0x4a, 0x34, 0xae, 0x6b, 0x44, 0x4d, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -4861,7 +4860,7 @@ static const u8 enc_input093[] __initconst = { 0x83, 0xdc, 0xe9, 0xf3, 0x07, 0x3e, 0xfa, 0xdb, 0x7d, 0x23, 0xb8, 0x7a, 0xce, 0x35, 0x16, 0x8c }; -static const u8 enc_output093[] __initconst = { +static const u8 enc_output093[] = { 0x00, 0x39, 0xe2, 0xfd, 0x2f, 0xd3, 0x12, 0x14, 0x9e, 0x98, 0x98, 0x80, 0x88, 0x48, 0x13, 0xe7, 0xca, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4875,13 +4874,13 @@ static const u8 enc_output093[] __initconst = { 0xa5, 0x19, 0xac, 0x1a, 0x35, 0xb4, 0xa5, 0x77, 0x87, 0x51, 0x0a, 0xf7, 0x8d, 0x8d, 0x20, 0x0a }; -static const u8 enc_assoc093[] __initconst = { +static const u8 enc_assoc093[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce093[] __initconst = { +static const u8 enc_nonce093[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key093[] __initconst = { +static const u8 enc_key093[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4889,7 +4888,7 @@ static const u8 enc_key093[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input094[] __initconst = { +static const u8 enc_input094[] = { 0xd3, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xe5, 0xda, 0x78, 0x76, 0x6f, 0xa1, 0x92, 0x90, @@ -4903,7 +4902,7 @@ static const u8 enc_input094[] __initconst = { 0x01, 0x49, 0xef, 0x50, 0x4b, 0x71, 0xb1, 0x20, 0xca, 0x4f, 0xf3, 0x95, 0x19, 0xc2, 0xc2, 0x10 }; -static const u8 enc_output094[] __initconst = { +static const u8 enc_output094[] = { 0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x62, 0x18, 0xb2, 0x7f, 0x83, 0xb8, 0xb4, 0x66, @@ -4919,13 +4918,13 @@ static const u8 enc_output094[] __initconst = { 0x30, 0x2f, 0xe8, 0x2a, 0xb0, 0xa0, 0x9a, 0xf6, 0x44, 0x00, 0xd0, 0x15, 0xae, 0x83, 0xd9, 0xcc }; -static const u8 enc_assoc094[] __initconst = { +static const u8 enc_assoc094[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce094[] __initconst = { +static const u8 enc_nonce094[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key094[] __initconst = { +static const u8 enc_key094[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4933,7 +4932,7 @@ static const u8 enc_key094[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input095[] __initconst = { +static const u8 enc_input095[] = { 0xe9, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x6d, 0xf1, 0x39, 0x4e, 0xdc, 0x53, 0x9b, 0x5b, @@ -4947,7 +4946,7 @@ static const u8 enc_input095[] __initconst = { 0x99, 0x52, 0xae, 0x08, 0x18, 0xc3, 0x89, 0x79, 0xc0, 0x74, 0x13, 0x71, 0x1a, 0x9a, 0xf7, 0x13 }; -static const u8 enc_output095[] __initconst = { +static const u8 enc_output095[] = { 0xe9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xea, 0x33, 0xf3, 0x47, 0x30, 0x4a, 0xbd, 0xad, @@ -4963,13 +4962,13 @@ static const u8 enc_output095[] __initconst = { 0x98, 0xa7, 0xe8, 0x36, 0xe0, 0xee, 0x4d, 0x02, 0x35, 0x00, 0xd0, 0x55, 0x7e, 0xc2, 0xcb, 0xe0 }; -static const u8 enc_assoc095[] __initconst = { +static const u8 enc_assoc095[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce095[] __initconst = { +static const u8 enc_nonce095[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key095[] __initconst = { +static const u8 enc_key095[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4977,7 +4976,7 @@ static const u8 enc_key095[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input096[] __initconst = { +static const u8 enc_input096[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x64, 0xf9, 0x0f, 0x5b, 0x26, 0x92, 0xb8, 0x60, @@ -4987,7 +4986,7 @@ static const u8 enc_input096[] __initconst = { 0x03, 0x35, 0x61, 0xe7, 0xca, 0xca, 0x6d, 0x94, 0x1d, 0xc3, 0xcd, 0x69, 0x14, 0xad, 0x69, 0x04 }; -static const u8 enc_output096[] __initconst = { +static const u8 enc_output096[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe3, 0x3b, 0xc5, 0x52, 0xca, 0x8b, 0x9e, 0x96, @@ -4999,13 +4998,13 @@ static const u8 enc_output096[] __initconst = { 0x6a, 0xb8, 0xdc, 0xe2, 0xc5, 0x9d, 0xa4, 0x73, 0x71, 0x30, 0xb0, 0x25, 0x2f, 0x68, 0xa8, 0xd8 }; -static const u8 enc_assoc096[] __initconst = { +static const u8 enc_assoc096[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce096[] __initconst = { +static const u8 enc_nonce096[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key096[] __initconst = { +static const u8 enc_key096[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5013,7 +5012,7 @@ static const u8 enc_key096[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input097[] __initconst = { +static const u8 enc_input097[] = { 0x68, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xb0, 0x8f, 0x25, 0x67, 0x5b, 0x9b, 0xcb, 0xf6, @@ -5027,7 +5026,7 @@ static const u8 enc_input097[] __initconst = { 0x65, 0x0e, 0xc6, 0x2d, 0x75, 0x70, 0x72, 0xce, 0xe6, 0xff, 0x23, 0x31, 0x86, 0xdd, 0x1c, 0x8f }; -static const u8 enc_output097[] __initconst = { +static const u8 enc_output097[] = { 0x68, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x37, 0x4d, 0xef, 0x6e, 0xb7, 0x82, 0xed, 0x00, @@ -5043,13 +5042,13 @@ static const u8 enc_output097[] __initconst = { 0x04, 0x4d, 0xea, 0x60, 0x88, 0x80, 0x41, 0x2b, 0xfd, 0xff, 0xcf, 0x35, 0x57, 0x9e, 0x9b, 0x26 }; -static const u8 enc_assoc097[] __initconst = { +static const u8 enc_assoc097[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce097[] __initconst = { +static const u8 enc_nonce097[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key097[] __initconst = { +static const u8 enc_key097[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5057,7 +5056,7 @@ static const u8 enc_key097[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input098[] __initconst = { +static const u8 enc_input098[] = { 0x6d, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xa1, 0x61, 0xb5, 0xab, 0x04, 0x09, 0x00, 0x62, @@ -5071,7 +5070,7 @@ static const u8 enc_input098[] __initconst = { 0x8e, 0xdc, 0x36, 0x6c, 0xd6, 0x97, 0x65, 0x6f, 0xca, 0x81, 0xfb, 0x13, 0x3c, 0xed, 0x79, 0xa1 }; -static const u8 enc_output098[] __initconst = { +static const u8 enc_output098[] = { 0x6d, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x26, 0xa3, 0x7f, 0xa2, 0xe8, 0x10, 0x26, 0x94, @@ -5087,13 +5086,13 @@ static const u8 enc_output098[] __initconst = { 0x1e, 0x6b, 0xea, 0x63, 0x14, 0x54, 0x2e, 0x2e, 0xf9, 0xff, 0xcf, 0x45, 0x0b, 0x2e, 0x98, 0x2b }; -static const u8 enc_assoc098[] __initconst = { +static const u8 enc_assoc098[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce098[] __initconst = { +static const u8 enc_nonce098[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key098[] __initconst = { +static const u8 enc_key098[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5101,7 +5100,7 @@ static const u8 enc_key098[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input099[] __initconst = { +static const u8 enc_input099[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xfc, 0x01, 0xb8, 0x91, 0xe5, 0xf0, 0xf9, 0x12, @@ -5111,7 +5110,7 @@ static const u8 enc_input099[] __initconst = { 0x60, 0xcd, 0x9e, 0xa1, 0x0c, 0x29, 0xa3, 0x66, 0x54, 0xe7, 0xa2, 0x8e, 0x76, 0x1b, 0xec, 0xd8 }; -static const u8 enc_output099[] __initconst = { +static const u8 enc_output099[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7b, 0xc3, 0x72, 0x98, 0x09, 0xe9, 0xdf, 0xe4, @@ -5123,13 +5122,13 @@ static const u8 enc_output099[] __initconst = { 0xed, 0x20, 0x17, 0xc8, 0xdb, 0xa4, 0x77, 0x56, 0x29, 0x04, 0x9d, 0x78, 0x6e, 0x3b, 0xce, 0xb1 }; -static const u8 enc_assoc099[] __initconst = { +static const u8 enc_assoc099[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce099[] __initconst = { +static const u8 enc_nonce099[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key099[] __initconst = { +static const u8 enc_key099[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5137,7 +5136,7 @@ static const u8 enc_key099[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input100[] __initconst = { +static const u8 enc_input100[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x6b, 0x6d, 0xc9, 0xd2, 0x1a, 0x81, 0x9e, 0x70, @@ -5147,7 +5146,7 @@ static const u8 enc_input100[] __initconst = { 0x10, 0xb9, 0x2f, 0x5f, 0xfe, 0xf9, 0x8b, 0x84, 0x7c, 0xf1, 0x7a, 0x9c, 0x98, 0xd8, 0x83, 0xe5 }; -static const u8 enc_output100[] __initconst = { +static const u8 enc_output100[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xec, 0xaf, 0x03, 0xdb, 0xf6, 0x98, 0xb8, 0x86, @@ -5159,13 +5158,13 @@ static const u8 enc_output100[] __initconst = { 0x07, 0x3f, 0x17, 0xcb, 0x67, 0x78, 0x64, 0x59, 0x25, 0x04, 0x9d, 0x88, 0x22, 0xcb, 0xca, 0xb6 }; -static const u8 enc_assoc100[] __initconst = { +static const u8 enc_assoc100[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce100[] __initconst = { +static const u8 enc_nonce100[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key100[] __initconst = { +static const u8 enc_key100[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5173,7 +5172,7 @@ static const u8 enc_key100[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input101[] __initconst = { +static const u8 enc_input101[] = { 0xff, 0xcb, 0x2b, 0x11, 0x06, 0xf8, 0x23, 0x4c, 0x5e, 0x99, 0xd4, 0xdb, 0x4c, 0x70, 0x48, 0xde, 0x32, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5185,7 +5184,7 @@ static const u8 enc_input101[] __initconst = { 0xce, 0xbe, 0xf5, 0xe9, 0x88, 0x5a, 0x80, 0xea, 0x76, 0xd9, 0x75, 0xc1, 0x44, 0xa4, 0x18, 0x88 }; -static const u8 enc_output101[] __initconst = { +static const u8 enc_output101[] = { 0xff, 0xa0, 0xfc, 0x3e, 0x80, 0x32, 0xc3, 0xd5, 0xfd, 0xb6, 0x2a, 0x11, 0xf0, 0x96, 0x30, 0x7d, 0xb5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5199,13 +5198,13 @@ static const u8 enc_output101[] __initconst = { 0x8b, 0x9b, 0xb4, 0xb4, 0x86, 0x12, 0x89, 0x65, 0x8c, 0x69, 0x6a, 0x83, 0x40, 0x15, 0x04, 0x05 }; -static const u8 enc_assoc101[] __initconst = { +static const u8 enc_assoc101[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce101[] __initconst = { +static const u8 enc_nonce101[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key101[] __initconst = { +static const u8 enc_key101[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5213,7 +5212,7 @@ static const u8 enc_key101[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input102[] __initconst = { +static const u8 enc_input102[] = { 0x6f, 0x9e, 0x70, 0xed, 0x3b, 0x8b, 0xac, 0xa0, 0x26, 0xe4, 0x6a, 0x5a, 0x09, 0x43, 0x15, 0x8d, 0x21, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5225,7 +5224,7 @@ static const u8 enc_input102[] __initconst = { 0xd4, 0x36, 0x51, 0xfd, 0x14, 0x9c, 0x26, 0x0b, 0xcb, 0xdd, 0x7b, 0x12, 0x68, 0x01, 0x31, 0x8c }; -static const u8 enc_output102[] __initconst = { +static const u8 enc_output102[] = { 0x6f, 0xf5, 0xa7, 0xc2, 0xbd, 0x41, 0x4c, 0x39, 0x85, 0xcb, 0x94, 0x90, 0xb5, 0xa5, 0x6d, 0x2e, 0xa6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5239,13 +5238,13 @@ static const u8 enc_output102[] __initconst = { 0x8b, 0x3b, 0xbd, 0x51, 0x64, 0x44, 0x59, 0x56, 0x8d, 0x81, 0xca, 0x1f, 0xa7, 0x2c, 0xe4, 0x04 }; -static const u8 enc_assoc102[] __initconst = { +static const u8 enc_assoc102[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce102[] __initconst = { +static const u8 enc_nonce102[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key102[] __initconst = { +static const u8 enc_key102[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5253,7 +5252,7 @@ static const u8 enc_key102[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input103[] __initconst = { +static const u8 enc_input103[] = { 0x41, 0x2b, 0x08, 0x0a, 0x3e, 0x19, 0xc1, 0x0d, 0x44, 0xa1, 0xaf, 0x1e, 0xab, 0xde, 0xb4, 0xce, 0x35, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5265,7 +5264,7 @@ static const u8 enc_input103[] __initconst = { 0xb3, 0xd4, 0xe9, 0x90, 0x90, 0x34, 0xc6, 0x14, 0xb1, 0x0a, 0xff, 0x55, 0x25, 0xd0, 0x9d, 0x8d }; -static const u8 enc_output103[] __initconst = { +static const u8 enc_output103[] = { 0x41, 0x40, 0xdf, 0x25, 0xb8, 0xd3, 0x21, 0x94, 0xe7, 0x8e, 0x51, 0xd4, 0x17, 0x38, 0xcc, 0x6d, 0xb2, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5279,13 +5278,13 @@ static const u8 enc_output103[] __initconst = { 0x86, 0xfb, 0xab, 0x2b, 0x4a, 0x94, 0xf4, 0x7a, 0xa5, 0x6f, 0x0a, 0xea, 0x65, 0xd1, 0x10, 0x08 }; -static const u8 enc_assoc103[] __initconst = { +static const u8 enc_assoc103[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce103[] __initconst = { +static const u8 enc_nonce103[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key103[] __initconst = { +static const u8 enc_key103[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5293,7 +5292,7 @@ static const u8 enc_key103[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input104[] __initconst = { +static const u8 enc_input104[] = { 0xb2, 0x47, 0xa7, 0x47, 0x23, 0x49, 0x1a, 0xac, 0xac, 0xaa, 0xd7, 0x09, 0xc9, 0x1e, 0x93, 0x2b, 0x31, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5305,7 +5304,7 @@ static const u8 enc_input104[] __initconst = { 0x42, 0x89, 0x79, 0x44, 0xc2, 0xa2, 0x8f, 0xa1, 0x76, 0x11, 0xd7, 0xfa, 0x5c, 0x22, 0xad, 0x8f }; -static const u8 enc_output104[] __initconst = { +static const u8 enc_output104[] = { 0xb2, 0x2c, 0x70, 0x68, 0xa5, 0x83, 0xfa, 0x35, 0x0f, 0x85, 0x29, 0xc3, 0x75, 0xf8, 0xeb, 0x88, 0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5319,13 +5318,13 @@ static const u8 enc_output104[] __initconst = { 0xa0, 0x19, 0xac, 0x2e, 0xd6, 0x67, 0xe1, 0x7d, 0xa1, 0x6f, 0x0a, 0xfa, 0x19, 0x61, 0x0d, 0x0d }; -static const u8 enc_assoc104[] __initconst = { +static const u8 enc_assoc104[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce104[] __initconst = { +static const u8 enc_nonce104[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key104[] __initconst = { +static const u8 enc_key104[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5333,7 +5332,7 @@ static const u8 enc_key104[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input105[] __initconst = { +static const u8 enc_input105[] = { 0x74, 0x0f, 0x9e, 0x49, 0xf6, 0x10, 0xef, 0xa5, 0x85, 0xb6, 0x59, 0xca, 0x6e, 0xd8, 0xb4, 0x99, 0x2d, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5345,7 +5344,7 @@ static const u8 enc_input105[] __initconst = { 0x99, 0x7a, 0xeb, 0x0c, 0x27, 0x95, 0x62, 0x46, 0x69, 0xc3, 0x87, 0xf9, 0x11, 0x6a, 0xc1, 0x8d }; -static const u8 enc_output105[] __initconst = { +static const u8 enc_output105[] = { 0x74, 0x64, 0x49, 0x66, 0x70, 0xda, 0x0f, 0x3c, 0x26, 0x99, 0xa7, 0x00, 0xd2, 0x3e, 0xcc, 0x3a, 0xaa, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5359,13 +5358,13 @@ static const u8 enc_output105[] __initconst = { 0x73, 0x6e, 0x18, 0x18, 0x16, 0x96, 0xa5, 0x88, 0x9c, 0x31, 0x59, 0xfa, 0xab, 0xab, 0x20, 0xfd }; -static const u8 enc_assoc105[] __initconst = { +static const u8 enc_assoc105[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce105[] __initconst = { +static const u8 enc_nonce105[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key105[] __initconst = { +static const u8 enc_key105[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5373,7 +5372,7 @@ static const u8 enc_key105[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input106[] __initconst = { +static const u8 enc_input106[] = { 0xad, 0xba, 0x5d, 0x10, 0x5b, 0xc8, 0xaa, 0x06, 0x2c, 0x23, 0x36, 0xcb, 0x88, 0x9d, 0xdb, 0xd5, 0x37, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5385,7 +5384,7 @@ static const u8 enc_input106[] __initconst = { 0xcf, 0x2b, 0x22, 0x5d, 0xb1, 0x60, 0x7a, 0x10, 0xe6, 0xd5, 0x40, 0x1e, 0x53, 0xb4, 0x2a, 0x8d }; -static const u8 enc_output106[] __initconst = { +static const u8 enc_output106[] = { 0xad, 0xd1, 0x8a, 0x3f, 0xdd, 0x02, 0x4a, 0x9f, 0x8f, 0x0c, 0xc8, 0x01, 0x34, 0x7b, 0xa3, 0x76, 0xb0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5399,13 +5398,13 @@ static const u8 enc_output106[] __initconst = { 0xba, 0xd5, 0x8f, 0x10, 0xa9, 0x1e, 0x6a, 0x88, 0x9a, 0xba, 0x32, 0xfd, 0x17, 0xd8, 0x33, 0x1a }; -static const u8 enc_assoc106[] __initconst = { +static const u8 enc_assoc106[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce106[] __initconst = { +static const u8 enc_nonce106[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key106[] __initconst = { +static const u8 enc_key106[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5413,7 +5412,7 @@ static const u8 enc_key106[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input107[] __initconst = { +static const u8 enc_input107[] = { 0xfe, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xc0, 0x01, 0xed, 0xc5, 0xda, 0x44, 0x2e, 0x71, @@ -5427,7 +5426,7 @@ static const u8 enc_input107[] __initconst = { 0x00, 0x26, 0x6e, 0xa1, 0xe4, 0x36, 0x44, 0xa3, 0x4d, 0x8d, 0xd1, 0xdc, 0x93, 0xf2, 0xfa, 0x13 }; -static const u8 enc_output107[] __initconst = { +static const u8 enc_output107[] = { 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x47, 0xc3, 0x27, 0xcc, 0x36, 0x5d, 0x08, 0x87, @@ -5443,13 +5442,13 @@ static const u8 enc_output107[] __initconst = { 0xd6, 0x8c, 0xe1, 0x74, 0x07, 0x9a, 0xdd, 0x02, 0x8d, 0xd0, 0x5c, 0xf8, 0x14, 0x63, 0x04, 0x88 }; -static const u8 enc_assoc107[] __initconst = { +static const u8 enc_assoc107[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce107[] __initconst = { +static const u8 enc_nonce107[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key107[] __initconst = { +static const u8 enc_key107[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5457,7 +5456,7 @@ static const u8 enc_key107[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input108[] __initconst = { +static const u8 enc_input108[] = { 0xb5, 0x13, 0xb0, 0x6a, 0xb9, 0xac, 0x14, 0x43, 0x5a, 0xcb, 0x8a, 0xa3, 0xa3, 0x7a, 0xfd, 0xb6, 0x54, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5469,7 +5468,7 @@ static const u8 enc_input108[] __initconst = { 0xb9, 0xc2, 0x7c, 0x30, 0x28, 0xaa, 0x8d, 0x69, 0xef, 0x06, 0xaf, 0xc0, 0xb5, 0x9e, 0xda, 0x8e }; -static const u8 enc_output108[] __initconst = { +static const u8 enc_output108[] = { 0xb5, 0x78, 0x67, 0x45, 0x3f, 0x66, 0xf4, 0xda, 0xf9, 0xe4, 0x74, 0x69, 0x1f, 0x9c, 0x85, 0x15, 0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5483,13 +5482,13 @@ static const u8 enc_output108[] __initconst = { 0xaa, 0x48, 0xa3, 0x88, 0x7d, 0x4b, 0x05, 0x96, 0x99, 0xc2, 0xfd, 0xf9, 0xc6, 0x78, 0x7e, 0x0a }; -static const u8 enc_assoc108[] __initconst = { +static const u8 enc_assoc108[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce108[] __initconst = { +static const u8 enc_nonce108[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key108[] __initconst = { +static const u8 enc_key108[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5497,7 +5496,7 @@ static const u8 enc_key108[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input109[] __initconst = { +static const u8 enc_input109[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xd4, 0xf1, 0x09, 0xe8, 0x14, 0xce, 0xa8, 0x5a, @@ -5511,7 +5510,7 @@ static const u8 enc_input109[] __initconst = { 0x1b, 0x64, 0x89, 0xba, 0x84, 0xd8, 0xf5, 0x59, 0x82, 0x9e, 0xd9, 0xbd, 0xa2, 0x29, 0x0f, 0x16 }; -static const u8 enc_output109[] __initconst = { +static const u8 enc_output109[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x53, 0x33, 0xc3, 0xe1, 0xf8, 0xd7, 0x8e, 0xac, @@ -5527,13 +5526,13 @@ static const u8 enc_output109[] __initconst = { 0xb9, 0x36, 0xa8, 0x17, 0xf2, 0x21, 0x1a, 0xf1, 0x29, 0xe2, 0xcf, 0x16, 0x0f, 0xd4, 0x2b, 0xcb }; -static const u8 enc_assoc109[] __initconst = { +static const u8 enc_assoc109[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce109[] __initconst = { +static const u8 enc_nonce109[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key109[] __initconst = { +static const u8 enc_key109[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5541,7 +5540,7 @@ static const u8 enc_key109[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input110[] __initconst = { +static const u8 enc_input110[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xdf, 0x4c, 0x62, 0x03, 0x2d, 0x41, 0x19, 0xb5, @@ -5555,7 +5554,7 @@ static const u8 enc_input110[] __initconst = { 0xb2, 0xa0, 0xc1, 0x84, 0x4b, 0x4e, 0x35, 0xd4, 0x1e, 0x5d, 0xa2, 0x10, 0xf6, 0x2f, 0x84, 0x12 }; -static const u8 enc_output110[] __initconst = { +static const u8 enc_output110[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x58, 0x8e, 0xa8, 0x0a, 0xc1, 0x58, 0x3f, 0x43, @@ -5571,13 +5570,13 @@ static const u8 enc_output110[] __initconst = { 0x9f, 0x7a, 0xc4, 0x35, 0x1f, 0x6b, 0x91, 0xe6, 0x30, 0x97, 0xa7, 0x13, 0x11, 0x5d, 0x05, 0xbe }; -static const u8 enc_assoc110[] __initconst = { +static const u8 enc_assoc110[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce110[] __initconst = { +static const u8 enc_nonce110[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key110[] __initconst = { +static const u8 enc_key110[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5585,7 +5584,7 @@ static const u8 enc_key110[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input111[] __initconst = { +static const u8 enc_input111[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x13, 0xf8, 0x0a, 0x00, 0x6d, 0xc1, 0xbb, 0xda, @@ -5599,7 +5598,7 @@ static const u8 enc_input111[] __initconst = { 0x2b, 0x6c, 0x89, 0x1d, 0x37, 0xc7, 0xe1, 0x1a, 0x56, 0x41, 0x91, 0x9c, 0x49, 0x4d, 0x95, 0x16 }; -static const u8 enc_output111[] __initconst = { +static const u8 enc_output111[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x94, 0x3a, 0xc0, 0x09, 0x81, 0xd8, 0x9d, 0x2c, @@ -5615,13 +5614,13 @@ static const u8 enc_output111[] __initconst = { 0x9a, 0x18, 0xa8, 0x28, 0x07, 0x02, 0x69, 0xf4, 0x47, 0x00, 0xd0, 0x09, 0xe7, 0x17, 0x1c, 0xc9 }; -static const u8 enc_assoc111[] __initconst = { +static const u8 enc_assoc111[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce111[] __initconst = { +static const u8 enc_nonce111[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key111[] __initconst = { +static const u8 enc_key111[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5629,7 +5628,7 @@ static const u8 enc_key111[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input112[] __initconst = { +static const u8 enc_input112[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x82, 0xe5, 0x9b, 0x45, 0x82, 0x91, 0x50, 0x38, @@ -5643,7 +5642,7 @@ static const u8 enc_input112[] __initconst = { 0xfe, 0x55, 0xf9, 0x2a, 0xdc, 0x08, 0xb5, 0xaa, 0x95, 0x48, 0xa9, 0x2d, 0x63, 0xaf, 0xe1, 0x13 }; -static const u8 enc_output112[] __initconst = { +static const u8 enc_output112[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x05, 0x27, 0x51, 0x4c, 0x6e, 0x88, 0x76, 0xce, @@ -5659,13 +5658,13 @@ static const u8 enc_output112[] __initconst = { 0xb4, 0x36, 0xa8, 0x2b, 0x93, 0xd5, 0x55, 0xf7, 0x43, 0x00, 0xd0, 0x19, 0x9b, 0xa7, 0x18, 0xce }; -static const u8 enc_assoc112[] __initconst = { +static const u8 enc_assoc112[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce112[] __initconst = { +static const u8 enc_nonce112[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key112[] __initconst = { +static const u8 enc_key112[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5673,7 +5672,7 @@ static const u8 enc_key112[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input113[] __initconst = { +static const u8 enc_input113[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xf1, 0xd1, 0x28, 0x87, 0xb7, 0x21, 0x69, 0x86, @@ -5687,7 +5686,7 @@ static const u8 enc_input113[] __initconst = { 0xef, 0xe3, 0x69, 0x79, 0xed, 0x9e, 0x7d, 0x3e, 0xc9, 0x52, 0x41, 0x4e, 0x49, 0xb1, 0x30, 0x16 }; -static const u8 enc_output113[] __initconst = { +static const u8 enc_output113[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x76, 0x13, 0xe2, 0x8e, 0x5b, 0x38, 0x4f, 0x70, @@ -5703,13 +5702,13 @@ static const u8 enc_output113[] __initconst = { 0xce, 0x54, 0xa8, 0x2e, 0x1f, 0xa9, 0x42, 0xfa, 0x3f, 0x00, 0xd0, 0x29, 0x4f, 0x37, 0x15, 0xd3 }; -static const u8 enc_assoc113[] __initconst = { +static const u8 enc_assoc113[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce113[] __initconst = { +static const u8 enc_nonce113[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key113[] __initconst = { +static const u8 enc_key113[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5717,7 +5716,7 @@ static const u8 enc_key113[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input114[] __initconst = { +static const u8 enc_input114[] = { 0xcb, 0xf1, 0xda, 0x9e, 0x0b, 0xa9, 0x37, 0x73, 0x74, 0xe6, 0x9e, 0x1c, 0x0e, 0x60, 0x0c, 0xfc, 0x34, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5729,7 +5728,7 @@ static const u8 enc_input114[] __initconst = { 0x66, 0x68, 0xdb, 0xc8, 0xf5, 0xf2, 0x0e, 0xf2, 0xb3, 0xf3, 0x8f, 0x00, 0xe2, 0x03, 0x17, 0x88 }; -static const u8 enc_output114[] __initconst = { +static const u8 enc_output114[] = { 0xcb, 0x9a, 0x0d, 0xb1, 0x8d, 0x63, 0xd7, 0xea, 0xd7, 0xc9, 0x60, 0xd6, 0xb2, 0x86, 0x74, 0x5f, 0xb3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5743,13 +5742,13 @@ static const u8 enc_output114[] __initconst = { 0x23, 0x83, 0xab, 0x0b, 0x79, 0x92, 0x05, 0x69, 0x9b, 0x51, 0x0a, 0xa7, 0x09, 0xbf, 0x31, 0xf1 }; -static const u8 enc_assoc114[] __initconst = { +static const u8 enc_assoc114[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce114[] __initconst = { +static const u8 enc_nonce114[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key114[] __initconst = { +static const u8 enc_key114[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5757,7 +5756,7 @@ static const u8 enc_key114[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input115[] __initconst = { +static const u8 enc_input115[] = { 0x8f, 0x27, 0x86, 0x94, 0xc4, 0xe9, 0xda, 0xeb, 0xd5, 0x8d, 0x3e, 0x5b, 0x96, 0x6e, 0x8b, 0x68, 0x42, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5769,7 +5768,7 @@ static const u8 enc_input115[] __initconst = { 0xde, 0x04, 0x9a, 0x00, 0xa8, 0x64, 0x06, 0x4b, 0xbc, 0xd4, 0x6f, 0xe4, 0xe4, 0x5b, 0x42, 0x8f }; -static const u8 enc_output115[] __initconst = { +static const u8 enc_output115[] = { 0x8f, 0x4c, 0x51, 0xbb, 0x42, 0x23, 0x3a, 0x72, 0x76, 0xa2, 0xc0, 0x91, 0x2a, 0x88, 0xf3, 0xcb, 0xc5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5783,13 +5782,13 @@ static const u8 enc_output115[] __initconst = { 0x8b, 0xfb, 0xab, 0x17, 0xa9, 0xe0, 0xb8, 0x74, 0x8b, 0x51, 0x0a, 0xe7, 0xd9, 0xfd, 0x23, 0x05 }; -static const u8 enc_assoc115[] __initconst = { +static const u8 enc_assoc115[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce115[] __initconst = { +static const u8 enc_nonce115[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key115[] __initconst = { +static const u8 enc_key115[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5797,7 +5796,7 @@ static const u8 enc_key115[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input116[] __initconst = { +static const u8 enc_input116[] = { 0xd5, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x9a, 0x22, 0xd7, 0x0a, 0x48, 0xe2, 0x4f, 0xdd, @@ -5811,7 +5810,7 @@ static const u8 enc_input116[] __initconst = { 0x3f, 0x91, 0xf8, 0xe7, 0xc7, 0xb1, 0x96, 0x25, 0x64, 0x61, 0x9c, 0x5e, 0x7e, 0x9b, 0xf6, 0x13 }; -static const u8 enc_output116[] __initconst = { +static const u8 enc_output116[] = { 0xd5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x1d, 0xe0, 0x1d, 0x03, 0xa4, 0xfb, 0x69, 0x2b, @@ -5827,13 +5826,13 @@ static const u8 enc_output116[] __initconst = { 0x49, 0xbc, 0x6e, 0x9f, 0xc5, 0x1c, 0x4d, 0x50, 0x30, 0x36, 0x64, 0x4d, 0x84, 0x27, 0x73, 0xd2 }; -static const u8 enc_assoc116[] __initconst = { +static const u8 enc_assoc116[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce116[] __initconst = { +static const u8 enc_nonce116[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key116[] __initconst = { +static const u8 enc_key116[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5841,7 +5840,7 @@ static const u8 enc_key116[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input117[] __initconst = { +static const u8 enc_input117[] = { 0xdb, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x75, 0xd5, 0x64, 0x3a, 0xa5, 0xaf, 0x93, 0x4d, @@ -5855,7 +5854,7 @@ static const u8 enc_input117[] __initconst = { 0x28, 0x3f, 0x6b, 0x32, 0x18, 0x07, 0x5f, 0xc9, 0x5f, 0x6b, 0xb4, 0xff, 0x45, 0x6d, 0xc1, 0x11 }; -static const u8 enc_output117[] __initconst = { +static const u8 enc_output117[] = { 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf2, 0x17, 0xae, 0x33, 0x49, 0xb6, 0xb5, 0xbb, @@ -5871,13 +5870,13 @@ static const u8 enc_output117[] __initconst = { 0x63, 0xda, 0x6e, 0xa2, 0x51, 0xf0, 0x39, 0x53, 0x2c, 0x36, 0x64, 0x5d, 0x38, 0xb7, 0x6f, 0xd7 }; -static const u8 enc_assoc117[] __initconst = { +static const u8 enc_assoc117[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce117[] __initconst = { +static const u8 enc_nonce117[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key117[] __initconst = { +static const u8 enc_key117[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5885,7 +5884,7 @@ static const u8 enc_key117[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input118[] __initconst = { +static const u8 enc_input118[] = { 0x93, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x62, 0x48, 0x39, 0x60, 0x42, 0x16, 0xe4, 0x03, @@ -5899,7 +5898,7 @@ static const u8 enc_input118[] __initconst = { 0x90, 0xec, 0xf2, 0x1a, 0x04, 0xe6, 0x30, 0x85, 0x8b, 0xb6, 0x56, 0x52, 0xb5, 0xb1, 0x80, 0x16 }; -static const u8 enc_output118[] __initconst = { +static const u8 enc_output118[] = { 0x93, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe5, 0x8a, 0xf3, 0x69, 0xae, 0x0f, 0xc2, 0xf5, @@ -5915,13 +5914,13 @@ static const u8 enc_output118[] __initconst = { 0x73, 0xeb, 0x27, 0x24, 0xb5, 0xc4, 0x05, 0xf0, 0x4d, 0x00, 0xd0, 0xf1, 0x58, 0x40, 0xa1, 0xc1 }; -static const u8 enc_assoc118[] __initconst = { +static const u8 enc_assoc118[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce118[] __initconst = { +static const u8 enc_nonce118[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key118[] __initconst = { +static const u8 enc_key118[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5929,7 +5928,7 @@ static const u8 enc_key118[] __initconst = { }; static const struct chacha20poly1305_testvec -chacha20poly1305_enc_vectors[] __initconst = { +chacha20poly1305_enc_vectors[] = { { enc_input001, enc_output001, enc_assoc001, enc_nonce001, enc_key001, sizeof(enc_input001), sizeof(enc_assoc001), sizeof(enc_nonce001) }, { enc_input002, enc_output002, enc_assoc002, enc_nonce002, enc_key002, @@ -6168,7 +6167,7 @@ chacha20poly1305_enc_vectors[] __initconst = { sizeof(enc_input118), sizeof(enc_assoc118), sizeof(enc_nonce118) } }; -static const u8 dec_input001[] __initconst = { +static const u8 dec_input001[] = { 0x64, 0xa0, 0x86, 0x15, 0x75, 0x86, 0x1a, 0xf4, 0x60, 0xf0, 0x62, 0xc7, 0x9b, 0xe6, 0x43, 0xbd, 0x5e, 0x80, 0x5c, 0xfd, 0x34, 0x5c, 0xf3, 0x89, @@ -6206,7 +6205,7 @@ static const u8 dec_input001[] __initconst = { 0x22, 0x39, 0x23, 0x36, 0xfe, 0xa1, 0x85, 0x1f, 0x38 }; -static const u8 dec_output001[] __initconst = { +static const u8 dec_output001[] = { 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, @@ -6242,95 +6241,95 @@ static const u8 dec_output001[] __initconst = { 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, 0x9d }; -static const u8 dec_assoc001[] __initconst = { +static const u8 dec_assoc001[] = { 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x91 }; -static const u8 dec_nonce001[] __initconst = { +static const u8 dec_nonce001[] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 }; -static const u8 dec_key001[] __initconst = { +static const u8 dec_key001[] = { 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 }; -static const u8 dec_input002[] __initconst = { +static const u8 dec_input002[] = { 0xea, 0xe0, 0x1e, 0x9e, 0x2c, 0x91, 0xaa, 0xe1, 0xdb, 0x5d, 0x99, 0x3f, 0x8a, 0xf7, 0x69, 0x92 }; -static const u8 dec_output002[] __initconst = { }; -static const u8 dec_assoc002[] __initconst = { }; -static const u8 dec_nonce002[] __initconst = { +static const u8 dec_output002[] = { }; +static const u8 dec_assoc002[] = { }; +static const u8 dec_nonce002[] = { 0xca, 0xbf, 0x33, 0x71, 0x32, 0x45, 0x77, 0x8e }; -static const u8 dec_key002[] __initconst = { +static const u8 dec_key002[] = { 0x4c, 0xf5, 0x96, 0x83, 0x38, 0xe6, 0xae, 0x7f, 0x2d, 0x29, 0x25, 0x76, 0xd5, 0x75, 0x27, 0x86, 0x91, 0x9a, 0x27, 0x7a, 0xfb, 0x46, 0xc5, 0xef, 0x94, 0x81, 0x79, 0x57, 0x14, 0x59, 0x40, 0x68 }; -static const u8 dec_input003[] __initconst = { +static const u8 dec_input003[] = { 0xdd, 0x6b, 0x3b, 0x82, 0xce, 0x5a, 0xbd, 0xd6, 0xa9, 0x35, 0x83, 0xd8, 0x8c, 0x3d, 0x85, 0x77 }; -static const u8 dec_output003[] __initconst = { }; -static const u8 dec_assoc003[] __initconst = { +static const u8 dec_output003[] = { }; +static const u8 dec_assoc003[] = { 0x33, 0x10, 0x41, 0x12, 0x1f, 0xf3, 0xd2, 0x6b }; -static const u8 dec_nonce003[] __initconst = { +static const u8 dec_nonce003[] = { 0x3d, 0x86, 0xb5, 0x6b, 0xc8, 0xa3, 0x1f, 0x1d }; -static const u8 dec_key003[] __initconst = { +static const u8 dec_key003[] = { 0x2d, 0xb0, 0x5d, 0x40, 0xc8, 0xed, 0x44, 0x88, 0x34, 0xd1, 0x13, 0xaf, 0x57, 0xa1, 0xeb, 0x3a, 0x2a, 0x80, 0x51, 0x36, 0xec, 0x5b, 0xbc, 0x08, 0x93, 0x84, 0x21, 0xb5, 0x13, 0x88, 0x3c, 0x0d }; -static const u8 dec_input004[] __initconst = { +static const u8 dec_input004[] = { 0xb7, 0x1b, 0xb0, 0x73, 0x59, 0xb0, 0x84, 0xb2, 0x6d, 0x8e, 0xab, 0x94, 0x31, 0xa1, 0xae, 0xac, 0x89 }; -static const u8 dec_output004[] __initconst = { +static const u8 dec_output004[] = { 0xa4 }; -static const u8 dec_assoc004[] __initconst = { +static const u8 dec_assoc004[] = { 0x6a, 0xe2, 0xad, 0x3f, 0x88, 0x39, 0x5a, 0x40 }; -static const u8 dec_nonce004[] __initconst = { +static const u8 dec_nonce004[] = { 0xd2, 0x32, 0x1f, 0x29, 0x28, 0xc6, 0xc4, 0xc4 }; -static const u8 dec_key004[] __initconst = { +static const u8 dec_key004[] = { 0x4b, 0x28, 0x4b, 0xa3, 0x7b, 0xbe, 0xe9, 0xf8, 0x31, 0x80, 0x82, 0xd7, 0xd8, 0xe8, 0xb5, 0xa1, 0xe2, 0x18, 0x18, 0x8a, 0x9c, 0xfa, 0xa3, 0x3d, 0x25, 0x71, 0x3e, 0x40, 0xbc, 0x54, 0x7a, 0x3e }; -static const u8 dec_input005[] __initconst = { +static const u8 dec_input005[] = { 0xbf, 0xe1, 0x5b, 0x0b, 0xdb, 0x6b, 0xf5, 0x5e, 0x6c, 0x5d, 0x84, 0x44, 0x39, 0x81, 0xc1, 0x9c, 0xac }; -static const u8 dec_output005[] __initconst = { +static const u8 dec_output005[] = { 0x2d }; -static const u8 dec_assoc005[] __initconst = { }; -static const u8 dec_nonce005[] __initconst = { +static const u8 dec_assoc005[] = { }; +static const u8 dec_nonce005[] = { 0x20, 0x1c, 0xaa, 0x5f, 0x9c, 0xbf, 0x92, 0x30 }; -static const u8 dec_key005[] __initconst = { +static const u8 dec_key005[] = { 0x66, 0xca, 0x9c, 0x23, 0x2a, 0x4b, 0x4b, 0x31, 0x0e, 0x92, 0x89, 0x8b, 0xf4, 0x93, 0xc7, 0x87, 0x98, 0xa3, 0xd8, 0x39, 0xf8, 0xf4, 0xa7, 0x01, 0xc0, 0x2e, 0x0a, 0xa6, 0x7e, 0x5a, 0x78, 0x87 }; -static const u8 dec_input006[] __initconst = { +static const u8 dec_input006[] = { 0x8b, 0x06, 0xd3, 0x31, 0xb0, 0x93, 0x45, 0xb1, 0x75, 0x6e, 0x26, 0xf9, 0x67, 0xbc, 0x90, 0x15, 0x81, 0x2c, 0xb5, 0xf0, 0xc6, 0x2b, 0xc7, 0x8c, @@ -6351,7 +6350,7 @@ static const u8 dec_input006[] __initconst = { 0x8d, 0x31, 0xf7, 0x7a, 0x39, 0x4d, 0x8f, 0x9a, 0xeb }; -static const u8 dec_output006[] __initconst = { +static const u8 dec_output006[] = { 0x33, 0x2f, 0x94, 0xc1, 0xa4, 0xef, 0xcc, 0x2a, 0x5b, 0xa6, 0xe5, 0x8f, 0x1d, 0x40, 0xf0, 0x92, 0x3c, 0xd9, 0x24, 0x11, 0xa9, 0x71, 0xf9, 0x37, @@ -6370,20 +6369,20 @@ static const u8 dec_output006[] __initconst = { 0x69, 0x21, 0x70, 0xd8, 0xa4, 0x4b, 0xc8, 0xde, 0x8f }; -static const u8 dec_assoc006[] __initconst = { +static const u8 dec_assoc006[] = { 0x70, 0xd3, 0x33, 0xf3, 0x8b, 0x18, 0x0b }; -static const u8 dec_nonce006[] __initconst = { +static const u8 dec_nonce006[] = { 0xdf, 0x51, 0x84, 0x82, 0x42, 0x0c, 0x75, 0x9c }; -static const u8 dec_key006[] __initconst = { +static const u8 dec_key006[] = { 0x68, 0x7b, 0x8d, 0x8e, 0xe3, 0xc4, 0xdd, 0xae, 0xdf, 0x72, 0x7f, 0x53, 0x72, 0x25, 0x1e, 0x78, 0x91, 0xcb, 0x69, 0x76, 0x1f, 0x49, 0x93, 0xf9, 0x6f, 0x21, 0xcc, 0x39, 0x9c, 0xad, 0xb1, 0x01 }; -static const u8 dec_input007[] __initconst = { +static const u8 dec_input007[] = { 0x85, 0x04, 0xc2, 0xed, 0x8d, 0xfd, 0x97, 0x5c, 0xd2, 0xb7, 0xe2, 0xc1, 0x6b, 0xa3, 0xba, 0xf8, 0xc9, 0x50, 0xc3, 0xc6, 0xa5, 0xe3, 0xa4, 0x7c, @@ -6419,7 +6418,7 @@ static const u8 dec_input007[] __initconst = { 0x7a, 0x4b, 0xbc, 0xb0, 0x10, 0x5c, 0x96, 0x42, 0x3a, 0x00, 0x98, 0xcd, 0x15, 0xe8, 0xb7, 0x53 }; -static const u8 dec_output007[] __initconst = { +static const u8 dec_output007[] = { 0x9b, 0x18, 0xdb, 0xdd, 0x9a, 0x0f, 0x3e, 0xa5, 0x15, 0x17, 0xde, 0xdf, 0x08, 0x9d, 0x65, 0x0a, 0x67, 0x30, 0x12, 0xe2, 0x34, 0x77, 0x4b, 0xc1, @@ -6453,18 +6452,18 @@ static const u8 dec_output007[] __initconst = { 0x40, 0x5f, 0x99, 0xb7, 0x73, 0xec, 0x9b, 0x2b, 0xf0, 0x65, 0x11, 0xc8, 0xd0, 0x0a, 0x9f, 0xd3 }; -static const u8 dec_assoc007[] __initconst = { }; -static const u8 dec_nonce007[] __initconst = { +static const u8 dec_assoc007[] = { }; +static const u8 dec_nonce007[] = { 0xde, 0x7b, 0xef, 0xc3, 0x65, 0x1b, 0x68, 0xb0 }; -static const u8 dec_key007[] __initconst = { +static const u8 dec_key007[] = { 0x8d, 0xb8, 0x91, 0x48, 0xf0, 0xe7, 0x0a, 0xbd, 0xf9, 0x3f, 0xcd, 0xd9, 0xa0, 0x1e, 0x42, 0x4c, 0xe7, 0xde, 0x25, 0x3d, 0xa3, 0xd7, 0x05, 0x80, 0x8d, 0xf2, 0x82, 0xac, 0x44, 0x16, 0x51, 0x01 }; -static const u8 dec_input008[] __initconst = { +static const u8 dec_input008[] = { 0x14, 0xf6, 0x41, 0x37, 0xa6, 0xd4, 0x27, 0xcd, 0xdb, 0x06, 0x3e, 0x9a, 0x4e, 0xab, 0xd5, 0xb1, 0x1e, 0x6b, 0xd2, 0xbc, 0x11, 0xf4, 0x28, 0x93, @@ -6532,7 +6531,7 @@ static const u8 dec_input008[] __initconst = { 0x54, 0xfd, 0x0d, 0x98, 0x1c, 0x5a, 0x6f, 0x1f, 0x9a, 0x40, 0xcd, 0xa2, 0xff, 0x6a, 0xf1, 0x54 }; -static const u8 dec_output008[] __initconst = { +static const u8 dec_output008[] = { 0xc3, 0x09, 0x94, 0x62, 0xe6, 0x46, 0x2e, 0x10, 0xbe, 0x00, 0xe4, 0xfc, 0xf3, 0x40, 0xa3, 0xe2, 0x0f, 0xc2, 0x8b, 0x28, 0xdc, 0xba, 0xb4, 0x3c, @@ -6598,18 +6597,18 @@ static const u8 dec_output008[] __initconst = { 0x2e, 0x79, 0xb5, 0xe2, 0xb8, 0xe8, 0xb9, 0x7b, 0xd5, 0x10, 0xcb, 0xff, 0x5d, 0x14, 0x73, 0xf3 }; -static const u8 dec_assoc008[] __initconst = { }; -static const u8 dec_nonce008[] __initconst = { +static const u8 dec_assoc008[] = { }; +static const u8 dec_nonce008[] = { 0x0e, 0x0d, 0x57, 0xbb, 0x7b, 0x40, 0x54, 0x02 }; -static const u8 dec_key008[] __initconst = { +static const u8 dec_key008[] = { 0xf2, 0xaa, 0x4f, 0x99, 0xfd, 0x3e, 0xa8, 0x53, 0xc1, 0x44, 0xe9, 0x81, 0x18, 0xdc, 0xf5, 0xf0, 0x3e, 0x44, 0x15, 0x59, 0xe0, 0xc5, 0x44, 0x86, 0xc3, 0x91, 0xa8, 0x75, 0xc0, 0x12, 0x46, 0xba }; -static const u8 dec_input009[] __initconst = { +static const u8 dec_input009[] = { 0xfd, 0x81, 0x8d, 0xd0, 0x3d, 0xb4, 0xd5, 0xdf, 0xd3, 0x42, 0x47, 0x5a, 0x6d, 0x19, 0x27, 0x66, 0x4b, 0x2e, 0x0c, 0x27, 0x9c, 0x96, 0x4c, 0x72, @@ -6678,7 +6677,7 @@ static const u8 dec_input009[] __initconst = { 0x43, 0xb5, 0xd9, 0x90, 0xe1, 0x85, 0xf5, 0xa8, 0xae }; -static const u8 dec_output009[] __initconst = { +static const u8 dec_output009[] = { 0xe6, 0xc3, 0xdb, 0x63, 0x55, 0x15, 0xe3, 0x5b, 0xb7, 0x4b, 0x27, 0x8b, 0x5a, 0xdd, 0xc2, 0xe8, 0x3a, 0x6b, 0xd7, 0x81, 0x96, 0x35, 0x97, 0xca, @@ -6745,21 +6744,21 @@ static const u8 dec_output009[] __initconst = { 0x74, 0xb9, 0xe2, 0xd6, 0x1c, 0x80, 0x2c, 0x52, 0x65 }; -static const u8 dec_assoc009[] __initconst = { +static const u8 dec_assoc009[] = { 0x5a, 0x27, 0xff, 0xeb, 0xdf, 0x84, 0xb2, 0x9e, 0xef }; -static const u8 dec_nonce009[] __initconst = { +static const u8 dec_nonce009[] = { 0xef, 0x2d, 0x63, 0xee, 0x6b, 0x80, 0x8b, 0x78 }; -static const u8 dec_key009[] __initconst = { +static const u8 dec_key009[] = { 0xea, 0xbc, 0x56, 0x99, 0xe3, 0x50, 0xff, 0xc5, 0xcc, 0x1a, 0xd7, 0xc1, 0x57, 0x72, 0xea, 0x86, 0x5b, 0x89, 0x88, 0x61, 0x3d, 0x2f, 0x9b, 0xb2, 0xe7, 0x9c, 0xec, 0x74, 0x6e, 0x3e, 0xf4, 0x3b }; -static const u8 dec_input010[] __initconst = { +static const u8 dec_input010[] = { 0xe5, 0x26, 0xa4, 0x3d, 0xbd, 0x33, 0xd0, 0x4b, 0x6f, 0x05, 0xa7, 0x6e, 0x12, 0x7a, 0xd2, 0x74, 0xa6, 0xdd, 0xbd, 0x95, 0xeb, 0xf9, 0xa4, 0xf1, @@ -6891,7 +6890,7 @@ static const u8 dec_input010[] __initconst = { 0x80, 0x6b, 0xf6, 0x1f, 0xc3, 0xcc, 0x97, 0xc9, 0x24, 0x9f, 0xf3, 0xaf, 0x43, 0x14, 0xd5, 0xa0 }; -static const u8 dec_output010[] __initconst = { +static const u8 dec_output010[] = { 0x42, 0x93, 0xe4, 0xeb, 0x97, 0xb0, 0x57, 0xbf, 0x1a, 0x8b, 0x1f, 0xe4, 0x5f, 0x36, 0x20, 0x3c, 0xef, 0x0a, 0xa9, 0x48, 0x5f, 0x5f, 0x37, 0x22, @@ -7021,21 +7020,21 @@ static const u8 dec_output010[] __initconst = { 0x5a, 0xc0, 0x33, 0x1f, 0xcb, 0x05, 0x6d, 0x5c, 0x06, 0x87, 0x52, 0xa2, 0x8f, 0x26, 0xd5, 0x4f }; -static const u8 dec_assoc010[] __initconst = { +static const u8 dec_assoc010[] = { 0xd2, 0xa1, 0x70, 0xdb, 0x7a, 0xf8, 0xfa, 0x27, 0xba, 0x73, 0x0f, 0xbf, 0x3d, 0x1e, 0x82, 0xb2 }; -static const u8 dec_nonce010[] __initconst = { +static const u8 dec_nonce010[] = { 0xdb, 0x92, 0x0f, 0x7f, 0x17, 0x54, 0x0c, 0x30 }; -static const u8 dec_key010[] __initconst = { +static const u8 dec_key010[] = { 0x47, 0x11, 0xeb, 0x86, 0x2b, 0x2c, 0xab, 0x44, 0x34, 0xda, 0x7f, 0x57, 0x03, 0x39, 0x0c, 0xaf, 0x2c, 0x14, 0xfd, 0x65, 0x23, 0xe9, 0x8e, 0x74, 0xd5, 0x08, 0x68, 0x08, 0xe7, 0xb4, 0x72, 0xd7 }; -static const u8 dec_input011[] __initconst = { +static const u8 dec_input011[] = { 0x6a, 0xfc, 0x4b, 0x25, 0xdf, 0xc0, 0xe4, 0xe8, 0x17, 0x4d, 0x4c, 0xc9, 0x7e, 0xde, 0x3a, 0xcc, 0x3c, 0xba, 0x6a, 0x77, 0x47, 0xdb, 0xe3, 0x74, @@ -7281,7 +7280,7 @@ static const u8 dec_input011[] __initconst = { 0x86, 0xc4, 0xfc, 0x97, 0xae, 0x3f, 0x8f, 0x1e, 0x2b, 0xdf, 0xcd, 0xf9, 0x3c }; -static const u8 dec_output011[] __initconst = { +static const u8 dec_output011[] = { 0x7a, 0x57, 0xf2, 0xc7, 0x06, 0x3f, 0x50, 0x7b, 0x36, 0x1a, 0x66, 0x5c, 0xb9, 0x0e, 0x5e, 0x3b, 0x45, 0x60, 0xbe, 0x9a, 0x31, 0x9f, 0xff, 0x5d, @@ -7525,20 +7524,20 @@ static const u8 dec_output011[] __initconst = { 0x5c, 0x42, 0x16, 0xb8, 0x28, 0xeb, 0x1b, 0x61, 0x10, 0x1e, 0xbf, 0xec, 0xa8 }; -static const u8 dec_assoc011[] __initconst = { +static const u8 dec_assoc011[] = { 0xd6, 0x31, 0xda, 0x5d, 0x42, 0x5e, 0xd7 }; -static const u8 dec_nonce011[] __initconst = { +static const u8 dec_nonce011[] = { 0xfd, 0x87, 0xd4, 0xd8, 0x62, 0xfd, 0xec, 0xaa }; -static const u8 dec_key011[] __initconst = { +static const u8 dec_key011[] = { 0x35, 0x4e, 0xb5, 0x70, 0x50, 0x42, 0x8a, 0x85, 0xf2, 0xfb, 0xed, 0x7b, 0xd0, 0x9e, 0x97, 0xca, 0xfa, 0x98, 0x66, 0x63, 0xee, 0x37, 0xcc, 0x52, 0xfe, 0xd1, 0xdf, 0x95, 0x15, 0x34, 0x29, 0x38 }; -static const u8 dec_input012[] __initconst = { +static const u8 dec_input012[] = { 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, 0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1, @@ -7794,7 +7793,7 @@ static const u8 dec_input012[] __initconst = { 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, 0x70, 0xcf, 0xd6 }; -static const u8 dec_output012[] __initconst = { +static const u8 dec_output012[] = { 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, 0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57, @@ -8048,7 +8047,7 @@ static const u8 dec_output012[] __initconst = { 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, 0x78, 0xec, 0x00 }; -static const u8 dec_assoc012[] __initconst = { +static const u8 dec_assoc012[] = { 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, 0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce, 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, @@ -8058,17 +8057,17 @@ static const u8 dec_assoc012[] __initconst = { 0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b, 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 }; -static const u8 dec_nonce012[] __initconst = { +static const u8 dec_nonce012[] = { 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 }; -static const u8 dec_key012[] __initconst = { +static const u8 dec_key012[] = { 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, 0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d, 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e, 0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64 }; -static const u8 dec_input013[] __initconst = { +static const u8 dec_input013[] = { 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, 0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1, @@ -8324,7 +8323,7 @@ static const u8 dec_input013[] __initconst = { 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, 0x70, 0xcf, 0xd7 }; -static const u8 dec_output013[] __initconst = { +static const u8 dec_output013[] = { 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, 0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57, @@ -8578,7 +8577,7 @@ static const u8 dec_output013[] __initconst = { 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, 0x78, 0xec, 0x00 }; -static const u8 dec_assoc013[] __initconst = { +static const u8 dec_assoc013[] = { 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, 0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce, 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, @@ -8588,10 +8587,10 @@ static const u8 dec_assoc013[] __initconst = { 0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b, 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 }; -static const u8 dec_nonce013[] __initconst = { +static const u8 dec_nonce013[] = { 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 }; -static const u8 dec_key013[] __initconst = { +static const u8 dec_key013[] = { 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, 0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d, 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e, @@ -8599,7 +8598,7 @@ static const u8 dec_key013[] __initconst = { }; static const struct chacha20poly1305_testvec -chacha20poly1305_dec_vectors[] __initconst = { +chacha20poly1305_dec_vectors[] = { { dec_input001, dec_output001, dec_assoc001, dec_nonce001, dec_key001, sizeof(dec_input001), sizeof(dec_assoc001), sizeof(dec_nonce001) }, { dec_input002, dec_output002, dec_assoc002, dec_nonce002, dec_key002, @@ -8629,7 +8628,7 @@ chacha20poly1305_dec_vectors[] __initconst = { true } }; -static const u8 xenc_input001[] __initconst = { +static const u8 xenc_input001[] = { 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, @@ -8665,7 +8664,7 @@ static const u8 xenc_input001[] __initconst = { 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, 0x9d }; -static const u8 xenc_output001[] __initconst = { +static const u8 xenc_output001[] = { 0x1a, 0x6e, 0x3a, 0xd9, 0xfd, 0x41, 0x3f, 0x77, 0x54, 0x72, 0x0a, 0x70, 0x9a, 0xa0, 0x29, 0x92, 0x2e, 0xed, 0x93, 0xcf, 0x0f, 0x71, 0x88, 0x18, @@ -8703,16 +8702,16 @@ static const u8 xenc_output001[] __initconst = { 0x93, 0xa6, 0x85, 0x5b, 0x40, 0x39, 0x5c, 0xc5, 0x9c }; -static const u8 xenc_assoc001[] __initconst = { +static const u8 xenc_assoc001[] = { 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x91 }; -static const u8 xenc_nonce001[] __initconst = { +static const u8 xenc_nonce001[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 }; -static const u8 xenc_key001[] __initconst = { +static const u8 xenc_key001[] = { 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, @@ -8720,12 +8719,12 @@ static const u8 xenc_key001[] __initconst = { }; static const struct chacha20poly1305_testvec -xchacha20poly1305_enc_vectors[] __initconst = { +xchacha20poly1305_enc_vectors[] = { { xenc_input001, xenc_output001, xenc_assoc001, xenc_nonce001, xenc_key001, sizeof(xenc_input001), sizeof(xenc_assoc001), sizeof(xenc_nonce001) } }; -static const u8 xdec_input001[] __initconst = { +static const u8 xdec_input001[] = { 0x1a, 0x6e, 0x3a, 0xd9, 0xfd, 0x41, 0x3f, 0x77, 0x54, 0x72, 0x0a, 0x70, 0x9a, 0xa0, 0x29, 0x92, 0x2e, 0xed, 0x93, 0xcf, 0x0f, 0x71, 0x88, 0x18, @@ -8763,7 +8762,7 @@ static const u8 xdec_input001[] __initconst = { 0x93, 0xa6, 0x85, 0x5b, 0x40, 0x39, 0x5c, 0xc5, 0x9c }; -static const u8 xdec_output001[] __initconst = { +static const u8 xdec_output001[] = { 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, @@ -8799,16 +8798,16 @@ static const u8 xdec_output001[] __initconst = { 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, 0x9d }; -static const u8 xdec_assoc001[] __initconst = { +static const u8 xdec_assoc001[] = { 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x91 }; -static const u8 xdec_nonce001[] __initconst = { +static const u8 xdec_nonce001[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 }; -static const u8 xdec_key001[] __initconst = { +static const u8 xdec_key001[] = { 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, @@ -8816,15 +8815,15 @@ static const u8 xdec_key001[] __initconst = { }; static const struct chacha20poly1305_testvec -xchacha20poly1305_dec_vectors[] __initconst = { +xchacha20poly1305_dec_vectors[] = { { xdec_input001, xdec_output001, xdec_assoc001, xdec_nonce001, xdec_key001, sizeof(xdec_input001), sizeof(xdec_assoc001), sizeof(xdec_nonce001) } }; -/* This is for the selftests-only, since it is only useful for the purpose of +/* This is for the tests only, since it is only useful for the purpose of * testing the underlying primitives and interactions. */ -static void __init +static void chacha20poly1305_encrypt_bignonce(u8 *dst, const u8 *src, const size_t src_len, const u8 *ad, const size_t ad_len, const u8 nonce[12], @@ -8858,11 +8857,12 @@ chacha20poly1305_encrypt_bignonce(u8 *dst, const u8 *src, const size_t src_len, poly1305_final(&poly1305_state, dst + src_len); } -static void __init -chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len, - const u8 *ad, const size_t ad_len, - const u8 *nonce, const size_t nonce_len, - const u8 key[CHACHA20POLY1305_KEY_SIZE]) +static void +chacha20poly1305_test_encrypt(struct kunit *test, u8 *dst, + const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u8 *nonce, const size_t nonce_len, + const u8 key[CHACHA20POLY1305_KEY_SIZE]) { if (nonce_len == 8) chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, @@ -8871,10 +8871,10 @@ chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len, chacha20poly1305_encrypt_bignonce(dst, src, src_len, ad, ad_len, nonce, key); else - BUG(); + KUNIT_FAIL(test, "bad nonce_len: %zu", nonce_len); } -static bool __init +static bool decryption_success(bool func_ret, bool expect_failure, int memcmp_result) { if (expect_failure) @@ -8882,25 +8882,20 @@ decryption_success(bool func_ret, bool expect_failure, int memcmp_result) return func_ret && !memcmp_result; } -bool __init chacha20poly1305_selftest(void) +static void test_chacha20poly1305(struct kunit *test) { enum { MAXIMUM_TEST_BUFFER_LEN = 1UL << 12 }; size_t i, j, k, total_len; u8 *computed_output = NULL, *input = NULL; - bool success = true, ret; + bool ret; struct scatterlist sg_src[3]; - computed_output = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); - input = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); - if (!computed_output || !input) { - pr_err("chacha20poly1305 self-test malloc: FAIL\n"); - success = false; - goto out; - } + computed_output = alloc_buf(test, MAXIMUM_TEST_BUFFER_LEN); + input = alloc_buf(test, MAXIMUM_TEST_BUFFER_LEN); for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) { memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); - chacha20poly1305_selftest_encrypt(computed_output, + chacha20poly1305_test_encrypt(test, computed_output, chacha20poly1305_enc_vectors[i].input, chacha20poly1305_enc_vectors[i].ilen, chacha20poly1305_enc_vectors[i].assoc, @@ -8908,14 +8903,13 @@ bool __init chacha20poly1305_selftest(void) chacha20poly1305_enc_vectors[i].nonce, chacha20poly1305_enc_vectors[i].nlen, chacha20poly1305_enc_vectors[i].key); - if (memcmp(computed_output, - chacha20poly1305_enc_vectors[i].output, - chacha20poly1305_enc_vectors[i].ilen + - POLY1305_DIGEST_SIZE)) { - pr_err("chacha20poly1305 encryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + KUNIT_EXPECT_TRUE_MSG( + test, + memcmp(computed_output, + chacha20poly1305_enc_vectors[i].output, + chacha20poly1305_enc_vectors[i].ilen + + POLY1305_DIGEST_SIZE) == 0, + "chacha20poly1305 encryption test %zu: FAIL", i + 1); } for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) { @@ -8931,14 +8925,13 @@ bool __init chacha20poly1305_selftest(void) chacha20poly1305_enc_vectors[i].alen, get_unaligned_le64(chacha20poly1305_enc_vectors[i].nonce), chacha20poly1305_enc_vectors[i].key); - if (!ret || memcmp(computed_output, - chacha20poly1305_enc_vectors[i].output, - chacha20poly1305_enc_vectors[i].ilen + - POLY1305_DIGEST_SIZE)) { - pr_err("chacha20poly1305 sg encryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + KUNIT_EXPECT_TRUE_MSG( + test, + ret && memcmp(computed_output, + chacha20poly1305_enc_vectors[i].output, + chacha20poly1305_enc_vectors[i].ilen + + POLY1305_DIGEST_SIZE) == 0, + "chacha20poly1305 sg encryption test %zu: FAIL", i + 1); } for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) { @@ -8950,16 +8943,15 @@ bool __init chacha20poly1305_selftest(void) chacha20poly1305_dec_vectors[i].alen, get_unaligned_le64(chacha20poly1305_dec_vectors[i].nonce), chacha20poly1305_dec_vectors[i].key); - if (!decryption_success(ret, - chacha20poly1305_dec_vectors[i].failure, + KUNIT_EXPECT_TRUE_MSG( + test, + decryption_success( + ret, chacha20poly1305_dec_vectors[i].failure, memcmp(computed_output, chacha20poly1305_dec_vectors[i].output, chacha20poly1305_dec_vectors[i].ilen - - POLY1305_DIGEST_SIZE))) { - pr_err("chacha20poly1305 decryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + POLY1305_DIGEST_SIZE)), + "chacha20poly1305 decryption test %zu: FAIL", i + 1); } for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) { @@ -8973,15 +8965,15 @@ bool __init chacha20poly1305_selftest(void) chacha20poly1305_dec_vectors[i].alen, get_unaligned_le64(chacha20poly1305_dec_vectors[i].nonce), chacha20poly1305_dec_vectors[i].key); - if (!decryption_success(ret, - chacha20poly1305_dec_vectors[i].failure, - memcmp(computed_output, chacha20poly1305_dec_vectors[i].output, - chacha20poly1305_dec_vectors[i].ilen - - POLY1305_DIGEST_SIZE))) { - pr_err("chacha20poly1305 sg decryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + KUNIT_EXPECT_TRUE_MSG( + test, + decryption_success( + ret, chacha20poly1305_dec_vectors[i].failure, + memcmp(computed_output, + chacha20poly1305_dec_vectors[i].output, + chacha20poly1305_dec_vectors[i].ilen - + POLY1305_DIGEST_SIZE)), + "chacha20poly1305 sg decryption test %zu: FAIL", i + 1); } for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_enc_vectors); ++i) { @@ -8993,14 +8985,13 @@ bool __init chacha20poly1305_selftest(void) xchacha20poly1305_enc_vectors[i].alen, xchacha20poly1305_enc_vectors[i].nonce, xchacha20poly1305_enc_vectors[i].key); - if (memcmp(computed_output, - xchacha20poly1305_enc_vectors[i].output, - xchacha20poly1305_enc_vectors[i].ilen + - POLY1305_DIGEST_SIZE)) { - pr_err("xchacha20poly1305 encryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + KUNIT_EXPECT_TRUE_MSG( + test, + memcmp(computed_output, + xchacha20poly1305_enc_vectors[i].output, + xchacha20poly1305_enc_vectors[i].ilen + + POLY1305_DIGEST_SIZE) == 0, + "xchacha20poly1305 encryption test %zu: FAIL", i + 1); } for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_dec_vectors); ++i) { @@ -9012,16 +9003,15 @@ bool __init chacha20poly1305_selftest(void) xchacha20poly1305_dec_vectors[i].alen, xchacha20poly1305_dec_vectors[i].nonce, xchacha20poly1305_dec_vectors[i].key); - if (!decryption_success(ret, - xchacha20poly1305_dec_vectors[i].failure, + KUNIT_EXPECT_TRUE_MSG( + test, + decryption_success( + ret, xchacha20poly1305_dec_vectors[i].failure, memcmp(computed_output, xchacha20poly1305_dec_vectors[i].output, xchacha20poly1305_dec_vectors[i].ilen - - POLY1305_DIGEST_SIZE))) { - pr_err("xchacha20poly1305 decryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + POLY1305_DIGEST_SIZE)), + "xchacha20poly1305 decryption test %zu: FAIL", i + 1); } for (total_len = POLY1305_DIGEST_SIZE; IS_ENABLED(DEBUG_CHACHA20POLY1305_SLOW_CHUNK_TEST) @@ -9067,16 +9057,26 @@ bool __init chacha20poly1305_selftest(void) continue; chunkfail: - pr_err("chacha20poly1305 chunked self-test %zu/%zu/%zu: FAIL\n", - total_len, i, j); - success = false; + KUNIT_FAIL( + test, + "chacha20poly1305 chunked test %zu/%zu/%zu: FAIL\n", + total_len, i, j); } } } - -out: - kfree(computed_output); - kfree(input); - return success; } + +static struct kunit_case chacha20poly1305_test_cases[] = { + KUNIT_CASE(test_chacha20poly1305), + {}, +}; + +static struct kunit_suite chacha20poly1305_test_suite = { + .name = "chacha20poly1305", + .test_cases = chacha20poly1305_test_cases, +}; +kunit_test_suite(chacha20poly1305_test_suite); + +MODULE_DESCRIPTION("KUnit tests for ChaCha20Poly1305"); +MODULE_LICENSE("GPL"); diff --git a/lib/crypto/tests/ghash-testvecs.h b/lib/crypto/tests/ghash-testvecs.h new file mode 100644 index 000000000000..759eb4072336 --- /dev/null +++ b/lib/crypto/tests/ghash-testvecs.h @@ -0,0 +1,186 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* This file was generated by: ./scripts/crypto/gen-hash-testvecs.py ghash */ + +static const struct { + size_t data_len; + u8 digest[GHASH_DIGEST_SIZE]; +} hash_testvecs[] = { + { + .data_len = 0, + .digest = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, + }, + { + .data_len = 1, + .digest = { + 0x13, 0x91, 0xa1, 0x11, 0x08, 0xc3, 0x7e, 0xeb, + 0x21, 0x42, 0x4a, 0xd6, 0x45, 0x0f, 0x41, 0xa7, + }, + }, + { + .data_len = 2, + .digest = { + 0xde, 0x00, 0x63, 0x3f, 0x71, 0x0f, 0xc6, 0x29, + 0x53, 0x2e, 0x49, 0xd9, 0xc2, 0xb7, 0x73, 0xce, + }, + }, + { + .data_len = 3, + .digest = { + 0xcf, 0xc7, 0xa8, 0x20, 0x24, 0xe9, 0x7a, 0x6c, + 0x2c, 0x2a, 0x34, 0x70, 0x26, 0xba, 0xd5, 0x9a, + }, + }, + { + .data_len = 16, + .digest = { + 0xaa, 0xe0, 0xdc, 0x7f, 0xcf, 0x8b, 0xe6, 0x0c, + 0x2e, 0x93, 0x89, 0x7d, 0x68, 0x4e, 0xc2, 0x63, + }, + }, + { + .data_len = 32, + .digest = { + 0x4b, 0x8b, 0x93, 0x5c, 0x79, 0xad, 0x85, 0x08, + 0xd3, 0x8a, 0xcd, 0xdd, 0x4c, 0x6e, 0x0e, 0x6f, + }, + }, + { + .data_len = 48, + .digest = { + 0xfa, 0xa0, 0x25, 0xdd, 0x61, 0x9a, 0x52, 0x9a, + 0xea, 0xee, 0xc6, 0x62, 0xb2, 0xba, 0x11, 0x49, + }, + }, + { + .data_len = 49, + .digest = { + 0x23, 0xf1, 0x05, 0xeb, 0x30, 0x40, 0xb9, 0x1d, + 0xe6, 0x35, 0x51, 0x4e, 0x0f, 0xc0, 0x1b, 0x9e, + }, + }, + { + .data_len = 63, + .digest = { + 0x8d, 0xcf, 0xa0, 0xc8, 0x83, 0x21, 0x06, 0x81, + 0xc6, 0x36, 0xd5, 0x62, 0xbf, 0xa0, 0xcd, 0x9c, + }, + }, + { + .data_len = 64, + .digest = { + 0xe7, 0xca, 0xbe, 0xe7, 0x66, 0xc8, 0x85, 0xad, + 0xbc, 0xaf, 0x58, 0x21, 0xd7, 0x67, 0x82, 0x15, + }, + }, + { + .data_len = 65, + .digest = { + 0x9f, 0x48, 0x10, 0xd9, 0xa2, 0x6b, 0x9d, 0xe0, + 0xb1, 0x87, 0xe1, 0x39, 0xc3, 0xd7, 0xee, 0x09, + }, + }, + { + .data_len = 127, + .digest = { + 0xa4, 0x36, 0xb7, 0x82, 0xd2, 0x67, 0x7e, 0xaf, + 0x5d, 0xfd, 0x67, 0x9c, 0x1d, 0x9f, 0xe4, 0xf7, + }, + }, + { + .data_len = 128, + .digest = { + 0x57, 0xe7, 0x1d, 0x78, 0xf0, 0x8e, 0xc7, 0x0c, + 0x15, 0xee, 0x18, 0xc4, 0xd1, 0x75, 0x90, 0xaa, + }, + }, + { + .data_len = 129, + .digest = { + 0x9b, 0xad, 0x81, 0xa9, 0x22, 0xb2, 0x19, 0x53, + 0x60, 0x30, 0xe7, 0xa0, 0x4f, 0xd6, 0x72, 0x42, + }, + }, + { + .data_len = 256, + .digest = { + 0xf7, 0x33, 0x42, 0xbf, 0x58, 0xde, 0x88, 0x0f, + 0x8d, 0x3d, 0xa6, 0x11, 0x14, 0xc3, 0xf1, 0xdc, + }, + }, + { + .data_len = 511, + .digest = { + 0x59, 0xdc, 0xa9, 0xc0, 0x4e, 0xd6, 0x97, 0xb3, + 0x60, 0xaf, 0xa8, 0xa0, 0xea, 0x54, 0x8e, 0xc3, + }, + }, + { + .data_len = 513, + .digest = { + 0xa2, 0x23, 0x37, 0xcc, 0x97, 0xec, 0xea, 0xbe, + 0xd6, 0xc7, 0x13, 0xf7, 0x93, 0x73, 0xc0, 0x64, + }, + }, + { + .data_len = 1000, + .digest = { + 0x46, 0x8b, 0x43, 0x77, 0x9b, 0xc2, 0xfc, 0xa4, + 0x68, 0x6a, 0x6c, 0x07, 0xa4, 0x6f, 0x47, 0x65, + }, + }, + { + .data_len = 3333, + .digest = { + 0x69, 0x7f, 0x19, 0xc3, 0xb9, 0xa4, 0xff, 0x40, + 0xe3, 0x03, 0x71, 0xa3, 0x88, 0x8a, 0xf1, 0xbd, + }, + }, + { + .data_len = 4096, + .digest = { + 0x4d, 0x65, 0xe6, 0x9c, 0xeb, 0x6a, 0x46, 0x8d, + 0xe9, 0x32, 0x96, 0x72, 0xb3, 0x0d, 0x08, 0xa9, + }, + }, + { + .data_len = 4128, + .digest = { + 0xfc, 0xa1, 0x74, 0x46, 0x21, 0x64, 0xa7, 0x64, + 0xbe, 0x47, 0x03, 0x1e, 0x05, 0xf7, 0xd8, 0x37, + }, + }, + { + .data_len = 4160, + .digest = { + 0x70, 0x5b, 0xe9, 0x17, 0xab, 0xd5, 0xa2, 0xee, + 0xcb, 0x39, 0xa4, 0x81, 0x2f, 0x41, 0x70, 0xae, + }, + }, + { + .data_len = 4224, + .digest = { + 0x07, 0xbd, 0xb6, 0x52, 0xe2, 0x75, 0x2c, 0x33, + 0x6d, 0x1b, 0x63, 0x56, 0x58, 0xda, 0x98, 0x55, + }, + }, + { + .data_len = 16384, + .digest = { + 0x9c, 0xb5, 0xf4, 0x14, 0xe8, 0xa8, 0x4a, 0xde, + 0xee, 0x7b, 0xbb, 0xd6, 0x21, 0x6d, 0x6a, 0x69, + }, + }, +}; + +static const u8 hash_testvec_consolidated[GHASH_DIGEST_SIZE] = { + 0x08, 0xef, 0xf5, 0x27, 0xb1, 0xca, 0xd4, 0x1d, + 0xad, 0x38, 0x69, 0x88, 0x6b, 0x16, 0xdf, 0xa8, +}; + +static const u8 ghash_allones_hashofhashes[GHASH_DIGEST_SIZE] = { + 0xef, 0x85, 0x58, 0xf8, 0x54, 0x9c, 0x5e, 0x54, + 0xd9, 0xbe, 0x04, 0x1f, 0xff, 0xff, 0xff, 0xff, +}; diff --git a/lib/crypto/tests/ghash_kunit.c b/lib/crypto/tests/ghash_kunit.c new file mode 100644 index 000000000000..4d7aa79b3cb2 --- /dev/null +++ b/lib/crypto/tests/ghash_kunit.c @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2026 Google LLC + */ +#include <crypto/gf128hash.h> +#include "ghash-testvecs.h" + +/* + * A fixed key used when presenting GHASH as an unkeyed hash function in order + * to reuse hash-test-template.h. At the beginning of the test suite, this is + * initialized to a key prepared from bytes generated from a fixed seed. + */ +static struct ghash_key test_key; + +static void ghash_init_withtestkey(struct ghash_ctx *ctx) +{ + ghash_init(ctx, &test_key); +} + +static void ghash_withtestkey(const u8 *data, size_t len, + u8 out[GHASH_BLOCK_SIZE]) +{ + ghash(&test_key, data, len, out); +} + +/* Generate the HASH_KUNIT_CASES using hash-test-template.h. */ +#define HASH ghash_withtestkey +#define HASH_CTX ghash_ctx +#define HASH_SIZE GHASH_BLOCK_SIZE +#define HASH_INIT ghash_init_withtestkey +#define HASH_UPDATE ghash_update +#define HASH_FINAL ghash_final +#include "hash-test-template.h" + +/* + * Test a key and messages containing all one bits. This is useful to detect + * overflow bugs in implementations that emulate carryless multiplication using + * a series of standard multiplications with the bits spread out. + */ +static void test_ghash_allones_key_and_message(struct kunit *test) +{ + const size_t max_len = 4096; + u8 *data = alloc_buf(test, max_len); + struct ghash_key key; + struct ghash_ctx hashofhashes_ctx; + u8 hash[GHASH_BLOCK_SIZE]; + + memset(data, 0xff, max_len); + + ghash_preparekey(&key, data); + ghash_init(&hashofhashes_ctx, &key); + for (size_t len = 0; len <= max_len; len += 16) { + ghash(&key, data, len, hash); + ghash_update(&hashofhashes_ctx, hash, sizeof(hash)); + } + ghash_final(&hashofhashes_ctx, hash); + KUNIT_ASSERT_MEMEQ(test, hash, ghash_allones_hashofhashes, + sizeof(hash)); +} + +#define MAX_LEN_FOR_KEY_CHECK 1024 + +/* + * Given two prepared keys which should be identical (but may differ in + * alignment and/or whether they are followed by a guard page or not), verify + * that they produce consistent results on various data lengths. + */ +static void check_key_consistency(struct kunit *test, + const struct ghash_key *key1, + const struct ghash_key *key2) +{ + u8 *data = alloc_buf(test, MAX_LEN_FOR_KEY_CHECK); + u8 hash1[GHASH_BLOCK_SIZE]; + u8 hash2[GHASH_BLOCK_SIZE]; + + rand_bytes(data, MAX_LEN_FOR_KEY_CHECK); + KUNIT_ASSERT_MEMEQ(test, key1, key2, sizeof(*key1)); + + for (int i = 0; i < 100; i++) { + size_t len = rand_length(MAX_LEN_FOR_KEY_CHECK); + + ghash(key1, data, len, hash1); + ghash(key2, data, len, hash2); + KUNIT_ASSERT_MEMEQ(test, hash1, hash2, sizeof(hash1)); + } +} + +/* Test that no buffer overreads occur on either raw_key or ghash_key. */ +static void test_ghash_with_guarded_key(struct kunit *test) +{ + u8 raw_key[GHASH_BLOCK_SIZE]; + u8 *guarded_raw_key = alloc_guarded_buf(test, sizeof(raw_key)); + struct ghash_key key1, key2; + struct ghash_key *guarded_key = + alloc_guarded_buf(test, sizeof(*guarded_key)); + + /* Prepare with regular buffers. */ + rand_bytes(raw_key, sizeof(raw_key)); + ghash_preparekey(&key1, raw_key); + + /* Prepare with guarded raw_key, then check that it works. */ + memcpy(guarded_raw_key, raw_key, sizeof(raw_key)); + ghash_preparekey(&key2, guarded_raw_key); + check_key_consistency(test, &key1, &key2); + + /* Prepare guarded ghash_key, then check that it works. */ + ghash_preparekey(guarded_key, raw_key); + check_key_consistency(test, &key1, guarded_key); +} + +/* + * Test that ghash_key only needs to be aligned to + * __alignof__(struct ghash_key), i.e. 8 bytes. The assembly code may prefer + * 16-byte or higher alignment, but it mustn't require it. + */ +static void test_ghash_with_minimally_aligned_key(struct kunit *test) +{ + u8 raw_key[GHASH_BLOCK_SIZE]; + struct ghash_key key; + const size_t align = __alignof__(struct ghash_key); + u8 *key_buf = alloc_buf(test, sizeof(struct ghash_key) + 3 * align); + struct ghash_key *minaligned_key = + (struct ghash_key *)(PTR_ALIGN(key_buf, 2 * align) + align); + + KUNIT_ASSERT_TRUE(test, IS_ALIGNED((uintptr_t)minaligned_key, align)); + KUNIT_ASSERT_TRUE(test, + !IS_ALIGNED((uintptr_t)minaligned_key, 2 * align)); + + rand_bytes(raw_key, sizeof(raw_key)); + ghash_preparekey(&key, raw_key); + ghash_preparekey(minaligned_key, raw_key); + check_key_consistency(test, &key, minaligned_key); +} + +struct ghash_irq_test_state { + struct ghash_key expected_key; + u8 raw_key[GHASH_BLOCK_SIZE]; +}; + +static bool ghash_irq_test_func(void *state_) +{ + struct ghash_irq_test_state *state = state_; + struct ghash_key key; + + ghash_preparekey(&key, state->raw_key); + return memcmp(&key, &state->expected_key, sizeof(key)) == 0; +} + +/* + * Test that ghash_preparekey() produces the same output regardless of whether + * FPU or vector registers are usable when it is called. + */ +static void test_ghash_preparekey_in_irqs(struct kunit *test) +{ + struct ghash_irq_test_state state; + + rand_bytes(state.raw_key, sizeof(state.raw_key)); + ghash_preparekey(&state.expected_key, state.raw_key); + kunit_run_irq_test(test, ghash_irq_test_func, 200000, &state); +} + +static int ghash_suite_init(struct kunit_suite *suite) +{ + u8 raw_key[GHASH_BLOCK_SIZE]; + + rand_bytes_seeded_from_len(raw_key, sizeof(raw_key)); + ghash_preparekey(&test_key, raw_key); + return 0; +} + +static struct kunit_case ghash_test_cases[] = { + HASH_KUNIT_CASES, + KUNIT_CASE(test_ghash_allones_key_and_message), + KUNIT_CASE(test_ghash_with_guarded_key), + KUNIT_CASE(test_ghash_with_minimally_aligned_key), + KUNIT_CASE(test_ghash_preparekey_in_irqs), + KUNIT_CASE(benchmark_hash), + {}, +}; + +static struct kunit_suite ghash_test_suite = { + .name = "ghash", + .test_cases = ghash_test_cases, + .suite_init = ghash_suite_init, +}; +kunit_test_suite(ghash_test_suite); + +MODULE_DESCRIPTION("KUnit tests and benchmark for GHASH"); +MODULE_LICENSE("GPL"); diff --git a/lib/crypto/tests/hash-test-template.h b/lib/crypto/tests/hash-test-template.h index 61b43e62779f..bb6eaa509d0f 100644 --- a/lib/crypto/tests/hash-test-template.h +++ b/lib/crypto/tests/hash-test-template.h @@ -7,93 +7,7 @@ */ #include <kunit/run-in-irq-context.h> #include <kunit/test.h> -#include <linux/vmalloc.h> - -/* test_buf is a guarded buffer, i.e. &test_buf[TEST_BUF_LEN] is not mapped. */ -#define TEST_BUF_LEN 16384 -static u8 *test_buf; - -static u8 *orig_test_buf; - -static u64 random_seed; - -/* - * This is a simple linear congruential generator. It is used only for testing, - * which does not require cryptographically secure random numbers. A hard-coded - * algorithm is used instead of <linux/prandom.h> so that it matches the - * algorithm used by the test vector generation script. This allows the input - * data in random test vectors to be concisely stored as just the seed. - */ -static u32 rand32(void) -{ - random_seed = (random_seed * 25214903917 + 11) & ((1ULL << 48) - 1); - return random_seed >> 16; -} - -static void rand_bytes(u8 *out, size_t len) -{ - for (size_t i = 0; i < len; i++) - out[i] = rand32(); -} - -static void rand_bytes_seeded_from_len(u8 *out, size_t len) -{ - random_seed = len; - rand_bytes(out, len); -} - -static bool rand_bool(void) -{ - return rand32() % 2; -} - -/* Generate a random length, preferring small lengths. */ -static size_t rand_length(size_t max_len) -{ - size_t len; - - switch (rand32() % 3) { - case 0: - len = rand32() % 128; - break; - case 1: - len = rand32() % 3072; - break; - default: - len = rand32(); - break; - } - return len % (max_len + 1); -} - -static size_t rand_offset(size_t max_offset) -{ - return min(rand32() % 128, max_offset); -} - -static int hash_suite_init(struct kunit_suite *suite) -{ - /* - * Allocate the test buffer using vmalloc() with a page-aligned length - * so that it is immediately followed by a guard page. This allows - * buffer overreads to be detected, even in assembly code. - */ - size_t alloc_len = round_up(TEST_BUF_LEN, PAGE_SIZE); - - orig_test_buf = vmalloc(alloc_len); - if (!orig_test_buf) - return -ENOMEM; - - test_buf = orig_test_buf + alloc_len - TEST_BUF_LEN; - return 0; -} - -static void hash_suite_exit(struct kunit_suite *suite) -{ - vfree(orig_test_buf); - orig_test_buf = NULL; - test_buf = NULL; -} +#include "test-utils.h" /* * Test the hash function against a list of test vectors. @@ -104,14 +18,16 @@ static void hash_suite_exit(struct kunit_suite *suite) */ static void test_hash_test_vectors(struct kunit *test) { + const size_t max_len = 16384; + u8 *data = alloc_buf(test, max_len); + for (size_t i = 0; i < ARRAY_SIZE(hash_testvecs); i++) { size_t data_len = hash_testvecs[i].data_len; u8 actual_hash[HASH_SIZE]; - KUNIT_ASSERT_LE(test, data_len, TEST_BUF_LEN); - rand_bytes_seeded_from_len(test_buf, data_len); - - HASH(test_buf, data_len, actual_hash); + KUNIT_ASSERT_LE(test, data_len, max_len); + rand_bytes_seeded_from_len(data, data_len); + HASH(data, data_len, actual_hash); KUNIT_ASSERT_MEMEQ_MSG( test, actual_hash, hash_testvecs[i].digest, HASH_SIZE, "Wrong result with test vector %zu; data_len=%zu", i, @@ -127,14 +43,15 @@ static void test_hash_test_vectors(struct kunit *test) */ static void test_hash_all_lens_up_to_4096(struct kunit *test) { + const size_t max_len = 4096; + u8 *data = alloc_buf(test, max_len); struct HASH_CTX ctx; u8 hash[HASH_SIZE]; - static_assert(TEST_BUF_LEN >= 4096); - rand_bytes_seeded_from_len(test_buf, 4096); + rand_bytes_seeded_from_len(data, max_len); HASH_INIT(&ctx); - for (size_t len = 0; len <= 4096; len++) { - HASH(test_buf, len, hash); + for (size_t len = 0; len <= max_len; len++) { + HASH(data, len, hash); HASH_UPDATE(&ctx, hash, HASH_SIZE); } HASH_FINAL(&ctx, hash); @@ -147,6 +64,9 @@ static void test_hash_all_lens_up_to_4096(struct kunit *test) */ static void test_hash_incremental_updates(struct kunit *test) { + const size_t max_len = 16384; + u8 *data = alloc_guarded_buf(test, max_len); + for (int i = 0; i < 1000; i++) { size_t total_len, offset; struct HASH_CTX ctx; @@ -155,12 +75,12 @@ static void test_hash_incremental_updates(struct kunit *test) size_t num_parts = 0; size_t remaining_len, cur_offset; - total_len = rand_length(TEST_BUF_LEN); - offset = rand_offset(TEST_BUF_LEN - total_len); - rand_bytes(&test_buf[offset], total_len); + total_len = rand_length(max_len); + offset = rand_offset(max_len - total_len); + rand_bytes(&data[offset], total_len); /* Compute the hash value in one shot. */ - HASH(&test_buf[offset], total_len, hash1); + HASH(&data[offset], total_len, hash1); /* * Compute the hash value incrementally, using a randomly @@ -172,13 +92,13 @@ static void test_hash_incremental_updates(struct kunit *test) while (rand_bool()) { size_t part_len = rand_length(remaining_len); - HASH_UPDATE(&ctx, &test_buf[cur_offset], part_len); + HASH_UPDATE(&ctx, &data[cur_offset], part_len); num_parts++; cur_offset += part_len; remaining_len -= part_len; } if (remaining_len != 0 || rand_bool()) { - HASH_UPDATE(&ctx, &test_buf[cur_offset], remaining_len); + HASH_UPDATE(&ctx, &data[cur_offset], remaining_len); num_parts++; } HASH_FINAL(&ctx, hash2); @@ -197,11 +117,13 @@ static void test_hash_incremental_updates(struct kunit *test) */ static void test_hash_buffer_overruns(struct kunit *test) { - const size_t max_tested_len = TEST_BUF_LEN - sizeof(struct HASH_CTX); - void *const buf_end = &test_buf[TEST_BUF_LEN]; + const size_t buf_len = 16384; + u8 *buf = alloc_guarded_buf(test, buf_len); + void *const buf_end = &buf[buf_len]; + const size_t max_tested_len = buf_len - sizeof(struct HASH_CTX); struct HASH_CTX *guarded_ctx = buf_end - sizeof(*guarded_ctx); - rand_bytes(test_buf, TEST_BUF_LEN); + rand_bytes(buf, buf_len); for (int i = 0; i < 100; i++) { size_t len = rand_length(max_tested_len); @@ -215,14 +137,14 @@ static void test_hash_buffer_overruns(struct kunit *test) HASH_FINAL(&ctx, hash); /* Check for overruns of the hash value buffer. */ - HASH(test_buf, len, buf_end - HASH_SIZE); + HASH(buf, len, buf_end - HASH_SIZE); HASH_INIT(&ctx); - HASH_UPDATE(&ctx, test_buf, len); + HASH_UPDATE(&ctx, buf, len); HASH_FINAL(&ctx, buf_end - HASH_SIZE); - /* Check for overuns of the hash context. */ + /* Check for overruns of the hash context. */ HASH_INIT(guarded_ctx); - HASH_UPDATE(guarded_ctx, test_buf, len); + HASH_UPDATE(guarded_ctx, buf, len); HASH_FINAL(guarded_ctx, hash); } } @@ -233,30 +155,32 @@ static void test_hash_buffer_overruns(struct kunit *test) */ static void test_hash_overlaps(struct kunit *test) { - const size_t max_tested_len = TEST_BUF_LEN - HASH_SIZE; + const size_t buf_len = 16384; + u8 *buf = alloc_guarded_buf(test, buf_len); + const size_t max_tested_len = buf_len - HASH_SIZE; struct HASH_CTX ctx; u8 hash[HASH_SIZE]; - rand_bytes(test_buf, TEST_BUF_LEN); + rand_bytes(buf, buf_len); for (int i = 0; i < 100; i++) { size_t len = rand_length(max_tested_len); size_t offset = HASH_SIZE + rand_offset(max_tested_len - len); bool left_end = rand_bool(); - u8 *ovl_hash = left_end ? &test_buf[offset] : - &test_buf[offset + len - HASH_SIZE]; + u8 *ovl_hash = left_end ? &buf[offset] : + &buf[offset + len - HASH_SIZE]; - HASH(&test_buf[offset], len, hash); - HASH(&test_buf[offset], len, ovl_hash); + HASH(&buf[offset], len, hash); + HASH(&buf[offset], len, ovl_hash); KUNIT_ASSERT_MEMEQ_MSG( test, hash, ovl_hash, HASH_SIZE, "Overlap test 1 failed with len=%zu offset=%zu left_end=%d", len, offset, left_end); /* Repeat the above test, but this time use init+update+final */ - HASH(&test_buf[offset], len, hash); + HASH(&buf[offset], len, hash); HASH_INIT(&ctx); - HASH_UPDATE(&ctx, &test_buf[offset], len); + HASH_UPDATE(&ctx, &buf[offset], len); HASH_FINAL(&ctx, ovl_hash); KUNIT_ASSERT_MEMEQ_MSG( test, hash, ovl_hash, HASH_SIZE, @@ -264,10 +188,10 @@ static void test_hash_overlaps(struct kunit *test) len, offset, left_end); /* Test modifying the source data after it was used. */ - HASH(&test_buf[offset], len, hash); + HASH(&buf[offset], len, hash); HASH_INIT(&ctx); - HASH_UPDATE(&ctx, &test_buf[offset], len); - rand_bytes(&test_buf[offset], len); + HASH_UPDATE(&ctx, &buf[offset], len); + rand_bytes(&buf[offset], len); HASH_FINAL(&ctx, ovl_hash); KUNIT_ASSERT_MEMEQ_MSG( test, hash, ovl_hash, HASH_SIZE, @@ -282,20 +206,22 @@ static void test_hash_overlaps(struct kunit *test) */ static void test_hash_alignment_consistency(struct kunit *test) { + const size_t max_len = 16384; + u8 *data = alloc_guarded_buf(test, max_len); u8 hash1[128 + HASH_SIZE]; u8 hash2[128 + HASH_SIZE]; for (int i = 0; i < 100; i++) { - size_t len = rand_length(TEST_BUF_LEN); - size_t data_offs1 = rand_offset(TEST_BUF_LEN - len); - size_t data_offs2 = rand_offset(TEST_BUF_LEN - len); + size_t len = rand_length(max_len); + size_t data_offs1 = rand_offset(max_len - len); + size_t data_offs2 = rand_offset(max_len - len); size_t hash_offs1 = rand_offset(128); size_t hash_offs2 = rand_offset(128); - rand_bytes(&test_buf[data_offs1], len); - HASH(&test_buf[data_offs1], len, &hash1[hash_offs1]); - memmove(&test_buf[data_offs2], &test_buf[data_offs1], len); - HASH(&test_buf[data_offs2], len, &hash2[hash_offs2]); + rand_bytes(&data[data_offs1], len); + HASH(&data[data_offs1], len, &hash1[hash_offs1]); + memmove(&data[data_offs2], &data[data_offs1], len); + HASH(&data[data_offs2], len, &hash2[hash_offs2]); KUNIT_ASSERT_MEMEQ_MSG( test, &hash1[hash_offs1], &hash2[hash_offs2], HASH_SIZE, "Alignment consistency test failed with len=%zu data_offs=(%zu,%zu) hash_offs=(%zu,%zu)", @@ -308,11 +234,14 @@ static void test_hash_ctx_zeroization(struct kunit *test) { static const u8 zeroes[sizeof(struct HASH_CTX)]; struct HASH_CTX ctx; + const size_t data_len = 128; + u8 *data = alloc_buf(test, data_len); + u8 hash[HASH_SIZE]; - rand_bytes(test_buf, 128); + rand_bytes(data, data_len); HASH_INIT(&ctx); - HASH_UPDATE(&ctx, test_buf, 128); - HASH_FINAL(&ctx, test_buf); + HASH_UPDATE(&ctx, data, data_len); + HASH_FINAL(&ctx, hash); KUNIT_ASSERT_MEMEQ_MSG(test, &ctx, zeroes, sizeof(ctx), "Hash context was not zeroized by finalization"); } @@ -321,6 +250,7 @@ static void test_hash_ctx_zeroization(struct kunit *test) #define IRQ_TEST_NUM_BUFFERS 3 /* matches max concurrency level */ struct hash_irq_test1_state { + u8 *data; u8 expected_hashes[IRQ_TEST_NUM_BUFFERS][HASH_SIZE]; atomic_t seqno; }; @@ -336,7 +266,8 @@ static bool hash_irq_test1_func(void *state_) u32 i = (u32)atomic_inc_return(&state->seqno) % IRQ_TEST_NUM_BUFFERS; u8 actual_hash[HASH_SIZE]; - HASH(&test_buf[i * IRQ_TEST_DATA_LEN], IRQ_TEST_DATA_LEN, actual_hash); + HASH(&state->data[i * IRQ_TEST_DATA_LEN], IRQ_TEST_DATA_LEN, + actual_hash); return memcmp(actual_hash, state->expected_hashes[i], HASH_SIZE) == 0; } @@ -346,12 +277,14 @@ static bool hash_irq_test1_func(void *state_) */ static void test_hash_interrupt_context_1(struct kunit *test) { + const size_t total_data_len = IRQ_TEST_NUM_BUFFERS * IRQ_TEST_DATA_LEN; struct hash_irq_test1_state state = {}; /* Prepare some test messages and compute the expected hash of each. */ - rand_bytes(test_buf, IRQ_TEST_NUM_BUFFERS * IRQ_TEST_DATA_LEN); + state.data = alloc_buf(test, total_data_len); + rand_bytes(state.data, total_data_len); for (int i = 0; i < IRQ_TEST_NUM_BUFFERS; i++) - HASH(&test_buf[i * IRQ_TEST_DATA_LEN], IRQ_TEST_DATA_LEN, + HASH(&state.data[i * IRQ_TEST_DATA_LEN], IRQ_TEST_DATA_LEN, state.expected_hashes[i]); kunit_run_irq_test(test, hash_irq_test1_func, 100000, &state); @@ -365,6 +298,8 @@ struct hash_irq_test2_hash_ctx { }; struct hash_irq_test2_state { + u8 *data; + size_t data_len; struct hash_irq_test2_hash_ctx ctxs[IRQ_TEST_NUM_BUFFERS]; u8 expected_hash[HASH_SIZE]; u16 update_lens[32]; @@ -397,7 +332,7 @@ static bool hash_irq_test2_func(void *state_) ctx->step++; } else if (ctx->step < state->num_steps - 1) { /* Update step */ - HASH_UPDATE(&ctx->hash_ctx, &test_buf[ctx->offset], + HASH_UPDATE(&ctx->hash_ctx, &state->data[ctx->offset], state->update_lens[ctx->step - 1]); ctx->offset += state->update_lens[ctx->step - 1]; ctx->step++; @@ -405,7 +340,7 @@ static bool hash_irq_test2_func(void *state_) /* Final step */ u8 actual_hash[HASH_SIZE]; - if (WARN_ON_ONCE(ctx->offset != TEST_BUF_LEN)) + if (WARN_ON_ONCE(ctx->offset != state->data_len)) ret = false; HASH_FINAL(&ctx->hash_ctx, actual_hash); if (memcmp(actual_hash, state->expected_hash, HASH_SIZE) != 0) @@ -426,20 +361,23 @@ static bool hash_irq_test2_func(void *state_) */ static void test_hash_interrupt_context_2(struct kunit *test) { + const size_t data_len = 16384; struct hash_irq_test2_state *state; - int remaining = TEST_BUF_LEN; + size_t remaining = data_len; state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, state); + state->data_len = data_len; + state->data = alloc_buf(test, data_len); - rand_bytes(test_buf, TEST_BUF_LEN); - HASH(test_buf, TEST_BUF_LEN, state->expected_hash); + rand_bytes(state->data, data_len); + HASH(state->data, data_len, state->expected_hash); /* * Generate a list of update lengths to use. Ensure that it contains * multiple entries but is limited to a maximum length. */ - static_assert(TEST_BUF_LEN / 4096 > 1); + KUNIT_ASSERT_GT(test, data_len / 4096, 1); for (state->num_steps = 0; state->num_steps < ARRAY_SIZE(state->update_lens) - 1 && remaining; state->num_steps++) { @@ -485,21 +423,23 @@ static void test_hash_interrupt_context_2(struct kunit *test) */ static void test_hmac(struct kunit *test) { + const size_t max_data_len = 4096; + const size_t max_key_len = 293; + const size_t outer_key_len = 32; + u8 *data = alloc_guarded_buf(test, max_data_len); + u8 *raw_key = alloc_guarded_buf(test, max_key_len); static const u8 zeroes[sizeof(struct HMAC_CTX)]; - u8 *raw_key; struct HMAC_KEY key; struct HMAC_CTX ctx; u8 mac[HASH_SIZE]; u8 mac2[HASH_SIZE]; - static_assert(TEST_BUF_LEN >= 4096 + 293); - rand_bytes_seeded_from_len(test_buf, 4096); - raw_key = &test_buf[4096]; + rand_bytes_seeded_from_len(data, max_data_len); + rand_bytes_seeded_from_len(raw_key, outer_key_len); - rand_bytes_seeded_from_len(raw_key, 32); - HMAC_PREPAREKEY(&key, raw_key, 32); + HMAC_PREPAREKEY(&key, raw_key, outer_key_len); HMAC_INIT(&ctx, &key); - for (size_t data_len = 0; data_len <= 4096; data_len++) { + for (size_t data_len = 0; data_len <= max_data_len; data_len++) { /* * Cycle through key lengths as well. Somewhat arbitrarily go * up to 293, which is somewhat larger than the largest hash @@ -507,17 +447,17 @@ static void test_hmac(struct kunit *test) * hashed down to one block); going higher would not be useful. * To reduce correlation with data_len, use a prime number here. */ - size_t key_len = data_len % 293; + size_t key_len = data_len % max_key_len; - HMAC_UPDATE(&ctx, test_buf, data_len); + HMAC_UPDATE(&ctx, data, data_len); rand_bytes_seeded_from_len(raw_key, key_len); - HMAC_USINGRAWKEY(raw_key, key_len, test_buf, data_len, mac); + HMAC_USINGRAWKEY(raw_key, key_len, data, data_len, mac); HMAC_UPDATE(&ctx, mac, HASH_SIZE); /* Verify that HMAC() is consistent with HMAC_USINGRAWKEY(). */ HMAC_PREPAREKEY(&key, raw_key, key_len); - HMAC(&key, test_buf, data_len, mac2); + HMAC(&key, data, data_len, mac2); KUNIT_ASSERT_MEMEQ_MSG( test, mac, mac2, HASH_SIZE, "HMAC gave different results with raw and prepared keys"); @@ -540,14 +480,17 @@ static void benchmark_hash(struct kunit *test) 1, 16, 64, 127, 128, 200, 256, 511, 512, 1024, 3173, 4096, 16384, }; + const size_t max_len = 16384; + u8 *data = alloc_buf(test, max_len); u8 hash[HASH_SIZE]; if (!IS_ENABLED(CONFIG_CRYPTO_LIB_BENCHMARK)) kunit_skip(test, "not enabled"); /* Warm-up */ - for (size_t i = 0; i < 10000000; i += TEST_BUF_LEN) - HASH(test_buf, TEST_BUF_LEN, hash); + memset(data, 0, max_len); + for (size_t i = 0; i < 10000000; i += max_len) + HASH(data, max_len, hash); for (size_t i = 0; i < ARRAY_SIZE(lens_to_test); i++) { size_t len = lens_to_test[i]; @@ -555,11 +498,11 @@ static void benchmark_hash(struct kunit *test) size_t num_iters = 10000000 / (len + 128); u64 t; - KUNIT_ASSERT_LE(test, len, TEST_BUF_LEN); + KUNIT_ASSERT_LE(test, len, max_len); preempt_disable(); t = ktime_get_ns(); for (size_t j = 0; j < num_iters; j++) - HASH(test_buf, len, hash); + HASH(data, len, hash); t = ktime_get_ns() - t; preempt_enable(); kunit_info(test, "len=%zu: %llu MB/s", len, diff --git a/lib/crypto/tests/md5_kunit.c b/lib/crypto/tests/md5_kunit.c index 38bd52c25ae3..1598f8585b45 100644 --- a/lib/crypto/tests/md5_kunit.c +++ b/lib/crypto/tests/md5_kunit.c @@ -30,8 +30,6 @@ static struct kunit_case hash_test_cases[] = { static struct kunit_suite hash_test_suite = { .name = "md5", .test_cases = hash_test_cases, - .suite_init = hash_suite_init, - .suite_exit = hash_suite_exit, }; kunit_test_suite(hash_test_suite); diff --git a/lib/crypto/tests/mldsa_kunit.c b/lib/crypto/tests/mldsa_kunit.c index 67f8f93e3dc6..ec64e10c97f2 100644 --- a/lib/crypto/tests/mldsa_kunit.c +++ b/lib/crypto/tests/mldsa_kunit.c @@ -8,6 +8,7 @@ #include <kunit/test.h> #include <linux/random.h> #include <linux/unaligned.h> +#include "test-utils.h" #define Q 8380417 /* The prime q = 2^23 - 2^13 + 1 */ @@ -60,14 +61,6 @@ static void do_mldsa_and_assert_success(struct kunit *test, KUNIT_ASSERT_EQ(test, err, 0); } -static u8 *kunit_kmemdup_or_fail(struct kunit *test, const u8 *src, size_t len) -{ - u8 *dst = kunit_kmalloc(test, len, GFP_KERNEL); - - KUNIT_ASSERT_NOT_NULL(test, dst); - return memcpy(dst, src, len); -} - /* * Test that changing coefficients in a valid signature's z vector results in * the following behavior from mldsa_verify(): @@ -83,7 +76,7 @@ static u8 *kunit_kmemdup_or_fail(struct kunit *test, const u8 *src, size_t len) static void test_mldsa_z_range(struct kunit *test, const struct mldsa_testvector *tv) { - u8 *sig = kunit_kmemdup_or_fail(test, tv->sig, tv->sig_len); + u8 *sig = memdup_buf(test, tv->sig, tv->sig_len); const int lambda = params[tv->alg].lambda; const s32 gamma1 = params[tv->alg].gamma1; const int beta = params[tv->alg].beta; @@ -146,7 +139,7 @@ static void test_mldsa_bad_hints(struct kunit *test, { const int omega = params[tv->alg].omega; const int k = params[tv->alg].k; - u8 *sig = kunit_kmemdup_or_fail(test, tv->sig, tv->sig_len); + u8 *sig = memdup_buf(test, tv->sig, tv->sig_len); /* Pointer to the encoded hint vector in the signature */ u8 *hintvec = &sig[tv->sig_len - omega - k]; u8 h; @@ -202,9 +195,9 @@ static void test_mldsa_mutation(struct kunit *test, const int msg_len = tv->msg_len; const int pk_len = tv->pk_len; const int num_iter = 200; - u8 *sig = kunit_kmemdup_or_fail(test, tv->sig, sig_len); - u8 *msg = kunit_kmemdup_or_fail(test, tv->msg, msg_len); - u8 *pk = kunit_kmemdup_or_fail(test, tv->pk, pk_len); + u8 *sig = memdup_buf(test, tv->sig, sig_len); + u8 *msg = memdup_buf(test, tv->msg, msg_len); + u8 *pk = memdup_buf(test, tv->pk, pk_len); /* Initially the signature is valid. */ do_mldsa_and_assert_success(test, tv); diff --git a/lib/crypto/tests/nh_kunit.c b/lib/crypto/tests/nh_kunit.c index a8a3c3f345cb..49e0fb3dd294 100644 --- a/lib/crypto/tests/nh_kunit.c +++ b/lib/crypto/tests/nh_kunit.c @@ -5,14 +5,13 @@ #include <crypto/nh.h> #include <kunit/test.h> #include "nh-testvecs.h" +#include "test-utils.h" static void test_nh(struct kunit *test) { - u32 *key = kunit_kmalloc(test, NH_KEY_BYTES, GFP_KERNEL); + u32 *key = memdup_buf(test, nh_test_key, NH_KEY_BYTES); __le64 hash[NH_NUM_PASSES]; - KUNIT_ASSERT_NOT_NULL(test, key); - memcpy(key, nh_test_key, NH_KEY_BYTES); le32_to_cpu_array(key, NH_KEY_WORDS); nh(key, nh_test_msg, 16, hash); diff --git a/lib/crypto/tests/poly1305_kunit.c b/lib/crypto/tests/poly1305_kunit.c index 7ac191bd96b6..f3cb6245bc29 100644 --- a/lib/crypto/tests/poly1305_kunit.c +++ b/lib/crypto/tests/poly1305_kunit.c @@ -46,12 +46,7 @@ static void poly1305_withtestkey(const u8 *data, size_t len, static int poly1305_suite_init(struct kunit_suite *suite) { rand_bytes_seeded_from_len(test_key, POLY1305_KEY_SIZE); - return hash_suite_init(suite); -} - -static void poly1305_suite_exit(struct kunit_suite *suite) -{ - hash_suite_exit(suite); + return 0; } /* @@ -81,19 +76,20 @@ static void poly1305_suite_exit(struct kunit_suite *suite) */ static void test_poly1305_allones_keys_and_message(struct kunit *test) { + const size_t max_len = 4096; + u8 *data = alloc_buf(test, max_len); struct poly1305_desc_ctx mac_ctx, macofmacs_ctx; u8 mac[POLY1305_DIGEST_SIZE]; - static_assert(TEST_BUF_LEN >= 4096); - memset(test_buf, 0xff, 4096); + memset(data, 0xff, max_len); - poly1305_init(&mac_ctx, test_buf); - poly1305_init(&macofmacs_ctx, test_buf); + poly1305_init(&mac_ctx, data); + poly1305_init(&macofmacs_ctx, data); for (int i = 0; i < 32; i++) { - for (size_t len = 0; len <= 4096; len += 16) { + for (size_t len = 0; len <= max_len; len += 16) { struct poly1305_desc_ctx tmp_ctx; - poly1305_update(&mac_ctx, test_buf, len); + poly1305_update(&mac_ctx, data, len); tmp_ctx = mac_ctx; poly1305_final(&tmp_ctx, mac); poly1305_update(&macofmacs_ctx, mac, @@ -157,7 +153,6 @@ static struct kunit_suite poly1305_test_suite = { .name = "poly1305", .test_cases = poly1305_test_cases, .suite_init = poly1305_suite_init, - .suite_exit = poly1305_suite_exit, }; kunit_test_suite(poly1305_test_suite); diff --git a/lib/crypto/tests/polyval_kunit.c b/lib/crypto/tests/polyval_kunit.c index f47f41a39a41..2835929e890c 100644 --- a/lib/crypto/tests/polyval_kunit.c +++ b/lib/crypto/tests/polyval_kunit.c @@ -2,7 +2,7 @@ /* * Copyright 2025 Google LLC */ -#include <crypto/polyval.h> +#include <crypto/gf128hash.h> #include "polyval-testvecs.h" /* @@ -66,17 +66,18 @@ static void test_polyval_rfc8452_testvec(struct kunit *test) */ static void test_polyval_allones_key_and_message(struct kunit *test) { + const size_t max_len = 4096; + u8 *data = alloc_buf(test, max_len); struct polyval_key key; struct polyval_ctx hashofhashes_ctx; u8 hash[POLYVAL_BLOCK_SIZE]; - static_assert(TEST_BUF_LEN >= 4096); - memset(test_buf, 0xff, 4096); + memset(data, 0xff, max_len); - polyval_preparekey(&key, test_buf); + polyval_preparekey(&key, data); polyval_init(&hashofhashes_ctx, &key); - for (size_t len = 0; len <= 4096; len += 16) { - polyval(&key, test_buf, len, hash); + for (size_t len = 0; len <= max_len; len += 16) { + polyval(&key, data, len, hash); polyval_update(&hashofhashes_ctx, hash, sizeof(hash)); } polyval_final(&hashofhashes_ctx, hash); @@ -95,7 +96,7 @@ static void check_key_consistency(struct kunit *test, const struct polyval_key *key1, const struct polyval_key *key2) { - u8 *data = test_buf; + u8 *data = alloc_buf(test, MAX_LEN_FOR_KEY_CHECK); u8 hash1[POLYVAL_BLOCK_SIZE]; u8 hash2[POLYVAL_BLOCK_SIZE]; @@ -115,10 +116,10 @@ static void check_key_consistency(struct kunit *test, static void test_polyval_with_guarded_key(struct kunit *test) { u8 raw_key[POLYVAL_BLOCK_SIZE]; - u8 *guarded_raw_key = &test_buf[TEST_BUF_LEN - sizeof(raw_key)]; + u8 *guarded_raw_key = alloc_guarded_buf(test, sizeof(raw_key)); struct polyval_key key1, key2; struct polyval_key *guarded_key = - (struct polyval_key *)&test_buf[TEST_BUF_LEN - sizeof(key1)]; + alloc_guarded_buf(test, sizeof(*guarded_key)); /* Prepare with regular buffers. */ rand_bytes(raw_key, sizeof(raw_key)); @@ -137,21 +138,20 @@ static void test_polyval_with_guarded_key(struct kunit *test) /* * Test that polyval_key only needs to be aligned to * __alignof__(struct polyval_key), i.e. 8 bytes. The assembly code may prefer - * 16-byte or higher alignment, but it musn't require it. + * 16-byte or higher alignment, but it mustn't require it. */ static void test_polyval_with_minimally_aligned_key(struct kunit *test) { u8 raw_key[POLYVAL_BLOCK_SIZE]; struct polyval_key key; + const size_t align = __alignof__(struct polyval_key); + u8 *key_buf = alloc_buf(test, sizeof(struct polyval_key) + 3 * align); struct polyval_key *minaligned_key = - (struct polyval_key *)&test_buf[MAX_LEN_FOR_KEY_CHECK + - __alignof__(struct polyval_key)]; + (struct polyval_key *)(PTR_ALIGN(key_buf, 2 * align) + align); - KUNIT_ASSERT_TRUE(test, IS_ALIGNED((uintptr_t)minaligned_key, - __alignof__(struct polyval_key))); + KUNIT_ASSERT_TRUE(test, IS_ALIGNED((uintptr_t)minaligned_key, align)); KUNIT_ASSERT_TRUE(test, - !IS_ALIGNED((uintptr_t)minaligned_key, - 2 * __alignof__(struct polyval_key))); + !IS_ALIGNED((uintptr_t)minaligned_key, 2 * align)); rand_bytes(raw_key, sizeof(raw_key)); polyval_preparekey(&key, raw_key); @@ -192,12 +192,7 @@ static int polyval_suite_init(struct kunit_suite *suite) rand_bytes_seeded_from_len(raw_key, sizeof(raw_key)); polyval_preparekey(&test_key, raw_key); - return hash_suite_init(suite); -} - -static void polyval_suite_exit(struct kunit_suite *suite) -{ - hash_suite_exit(suite); + return 0; } static struct kunit_case polyval_test_cases[] = { @@ -215,7 +210,6 @@ static struct kunit_suite polyval_test_suite = { .name = "polyval", .test_cases = polyval_test_cases, .suite_init = polyval_suite_init, - .suite_exit = polyval_suite_exit, }; kunit_test_suite(polyval_test_suite); diff --git a/lib/crypto/tests/sha1_kunit.c b/lib/crypto/tests/sha1_kunit.c index 24ba8d5669c8..27286afaa407 100644 --- a/lib/crypto/tests/sha1_kunit.c +++ b/lib/crypto/tests/sha1_kunit.c @@ -30,8 +30,6 @@ static struct kunit_case hash_test_cases[] = { static struct kunit_suite hash_test_suite = { .name = "sha1", .test_cases = hash_test_cases, - .suite_init = hash_suite_init, - .suite_exit = hash_suite_exit, }; kunit_test_suite(hash_test_suite); diff --git a/lib/crypto/tests/sha224_kunit.c b/lib/crypto/tests/sha224_kunit.c index 962ad46b9c99..bcf8b90f9ae2 100644 --- a/lib/crypto/tests/sha224_kunit.c +++ b/lib/crypto/tests/sha224_kunit.c @@ -30,8 +30,6 @@ static struct kunit_case hash_test_cases[] = { static struct kunit_suite hash_test_suite = { .name = "sha224", .test_cases = hash_test_cases, - .suite_init = hash_suite_init, - .suite_exit = hash_suite_exit, }; kunit_test_suite(hash_test_suite); diff --git a/lib/crypto/tests/sha256_kunit.c b/lib/crypto/tests/sha256_kunit.c index 5dccdee79693..17daa09adfe7 100644 --- a/lib/crypto/tests/sha256_kunit.c +++ b/lib/crypto/tests/sha256_kunit.c @@ -4,6 +4,7 @@ */ #include <crypto/sha2.h> #include "sha256-testvecs.h" +#include "test-utils.h" /* Generate the HASH_KUNIT_CASES using hash-test-template.h. */ #define HASH sha256 @@ -22,26 +23,6 @@ #define HMAC_USINGRAWKEY hmac_sha256_usingrawkey #include "hash-test-template.h" -static void free_guarded_buf(void *buf) -{ - vfree(buf); -} - -/* - * Allocate a KUnit-managed buffer that has length @len bytes immediately - * followed by an unmapped page, and assert that the allocation succeeds. - */ -static void *alloc_guarded_buf(struct kunit *test, size_t len) -{ - size_t full_len = round_up(len, PAGE_SIZE); - void *buf = vmalloc(full_len); - - KUNIT_ASSERT_NOT_NULL(test, buf); - KUNIT_ASSERT_EQ(test, 0, - kunit_add_action_or_reset(test, free_guarded_buf, buf)); - return buf + full_len - len; -} - /* * Test for sha256_finup_2x(). Specifically, choose various data lengths and * salt lengths, and for each one, verify that sha256_finup_2x() produces the @@ -105,19 +86,20 @@ static void test_sha256_finup_2x(struct kunit *test) static void test_sha256_finup_2x_defaultctx(struct kunit *test) { const size_t data_len = 128; + u8 *data = alloc_buf(test, 2 * data_len); struct sha256_ctx ctx; u8 hash1_a[SHA256_DIGEST_SIZE]; u8 hash2_a[SHA256_DIGEST_SIZE]; u8 hash1_b[SHA256_DIGEST_SIZE]; u8 hash2_b[SHA256_DIGEST_SIZE]; - rand_bytes(test_buf, 2 * data_len); + rand_bytes(data, 2 * data_len); sha256_init(&ctx); - sha256_finup_2x(&ctx, test_buf, &test_buf[data_len], data_len, hash1_a, + sha256_finup_2x(&ctx, data, &data[data_len], data_len, hash1_a, hash2_a); - sha256_finup_2x(NULL, test_buf, &test_buf[data_len], data_len, hash1_b, + sha256_finup_2x(NULL, data, &data[data_len], data_len, hash1_b, hash2_b); KUNIT_ASSERT_MEMEQ(test, hash1_a, hash1_b, SHA256_DIGEST_SIZE); @@ -131,18 +113,19 @@ static void test_sha256_finup_2x_defaultctx(struct kunit *test) static void test_sha256_finup_2x_hugelen(struct kunit *test) { const size_t data_len = 4 * SHA256_BLOCK_SIZE; + u8 *data = alloc_buf(test, data_len); struct sha256_ctx ctx = {}; u8 expected_hash[SHA256_DIGEST_SIZE]; u8 hash[SHA256_DIGEST_SIZE]; - rand_bytes(test_buf, data_len); + rand_bytes(data, data_len); for (size_t align = 0; align < SHA256_BLOCK_SIZE; align++) { sha256_init(&ctx); ctx.ctx.bytecount = 0x123456789abcd00 + align; - sha256_finup_2x(&ctx, test_buf, test_buf, data_len, hash, hash); + sha256_finup_2x(&ctx, data, data, data_len, hash, hash); - sha256_update(&ctx, test_buf, data_len); + sha256_update(&ctx, data, data_len); sha256_final(&ctx, expected_hash); KUNIT_ASSERT_MEMEQ(test, hash, expected_hash, @@ -161,6 +144,7 @@ static void benchmark_sha256_finup_2x(struct kunit *test) static const size_t salt_lens_to_test[] = { 0, 32, 64 }; const size_t data_len = 4096; const size_t num_iters = 4096; + u8 *data = alloc_buf(test, data_len * 2); struct sha256_ctx ctx; u8 hash1[SHA256_DIGEST_SIZE]; u8 hash2[SHA256_DIGEST_SIZE]; @@ -170,12 +154,12 @@ static void benchmark_sha256_finup_2x(struct kunit *test) if (!sha256_finup_2x_is_optimized()) kunit_skip(test, "not relevant"); - rand_bytes(test_buf, data_len * 2); + rand_bytes(data, data_len * 2); /* Warm-up */ for (size_t i = 0; i < num_iters; i++) - sha256_finup_2x(NULL, &test_buf[0], &test_buf[data_len], - data_len, hash1, hash2); + sha256_finup_2x(NULL, &data[0], &data[data_len], data_len, + hash1, hash2); for (size_t i = 0; i < ARRAY_SIZE(salt_lens_to_test); i++) { size_t salt_len = salt_lens_to_test[i]; @@ -186,12 +170,12 @@ static void benchmark_sha256_finup_2x(struct kunit *test) * not measured; we're just interested in sha256_finup_2x(). */ sha256_init(&ctx); - sha256_update(&ctx, test_buf, salt_len); + sha256_update(&ctx, data, salt_len); preempt_disable(); t0 = ktime_get_ns(); for (size_t j = 0; j < num_iters; j++) - sha256_finup_2x(&ctx, &test_buf[0], &test_buf[data_len], + sha256_finup_2x(&ctx, &data[0], &data[data_len], data_len, hash1, hash2); t1 = ktime_get_ns(); preempt_enable(); @@ -215,8 +199,6 @@ static struct kunit_case hash_test_cases[] = { static struct kunit_suite hash_test_suite = { .name = "sha256", .test_cases = hash_test_cases, - .suite_init = hash_suite_init, - .suite_exit = hash_suite_exit, }; kunit_test_suite(hash_test_suite); diff --git a/lib/crypto/tests/sha384_kunit.c b/lib/crypto/tests/sha384_kunit.c index e1ef5c995bb6..76409822d965 100644 --- a/lib/crypto/tests/sha384_kunit.c +++ b/lib/crypto/tests/sha384_kunit.c @@ -30,8 +30,6 @@ static struct kunit_case hash_test_cases[] = { static struct kunit_suite hash_test_suite = { .name = "sha384", .test_cases = hash_test_cases, - .suite_init = hash_suite_init, - .suite_exit = hash_suite_exit, }; kunit_test_suite(hash_test_suite); diff --git a/lib/crypto/tests/sha3_kunit.c b/lib/crypto/tests/sha3_kunit.c index ed5fbe80337f..81b113079755 100644 --- a/lib/crypto/tests/sha3_kunit.c +++ b/lib/crypto/tests/sha3_kunit.c @@ -273,12 +273,10 @@ static void test_shake_all_lens_up_to_4096(struct kunit *test) { struct sha3_ctx main_ctx; const size_t max_len = 4096; - u8 *const in = test_buf; - u8 *const out = &test_buf[TEST_BUF_LEN - max_len]; + u8 *const in = alloc_buf(test, max_len); + u8 *const out = alloc_buf(test, max_len); u8 main_hash[SHA3_256_DIGEST_SIZE]; - KUNIT_ASSERT_LE(test, 2 * max_len, TEST_BUF_LEN); - rand_bytes_seeded_from_len(in, max_len); for (int alg = 0; alg < 2; alg++) { sha3_256_init(&main_ctx); @@ -309,12 +307,8 @@ static void test_shake_all_lens_up_to_4096(struct kunit *test) static void test_shake_multiple_squeezes(struct kunit *test) { const size_t max_len = 512; - u8 *ref_out; - - KUNIT_ASSERT_GE(test, TEST_BUF_LEN, 2 * max_len); - - ref_out = kunit_kzalloc(test, max_len, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, ref_out); + u8 *buf = alloc_buf(test, max_len); + u8 *ref_out = alloc_buf(test, max_len); for (int i = 0; i < 2000; i++) { const int alg = rand32() % 2; @@ -322,8 +316,8 @@ static void test_shake_multiple_squeezes(struct kunit *test) const size_t out_len = rand_length(max_len); const size_t in_offs = rand_offset(max_len - in_len); const size_t out_offs = rand_offset(max_len - out_len); - u8 *const in = &test_buf[in_offs]; - u8 *const out = &test_buf[out_offs]; + u8 *const in = &buf[in_offs]; + u8 *const out = &buf[out_offs]; struct shake_ctx ctx; size_t remaining_len, j, num_parts; @@ -368,16 +362,12 @@ static void test_shake_multiple_squeezes(struct kunit *test) static void test_shake_with_guarded_bufs(struct kunit *test) { const size_t max_len = 512; - u8 *reg_buf; - - KUNIT_ASSERT_GE(test, TEST_BUF_LEN, max_len); - - reg_buf = kunit_kzalloc(test, max_len, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, reg_buf); + u8 *buf = alloc_guarded_buf(test, max_len); + u8 *reg_buf = alloc_buf(test, max_len); for (int alg = 0; alg < 2; alg++) { for (size_t len = 0; len <= max_len; len++) { - u8 *guarded_buf = &test_buf[TEST_BUF_LEN - len]; + u8 *guarded_buf = &buf[max_len - len]; rand_bytes(reg_buf, len); memcpy(guarded_buf, reg_buf, len); @@ -413,8 +403,6 @@ static struct kunit_case sha3_test_cases[] = { static struct kunit_suite sha3_test_suite = { .name = "sha3", .test_cases = sha3_test_cases, - .suite_init = hash_suite_init, - .suite_exit = hash_suite_exit, }; kunit_test_suite(sha3_test_suite); diff --git a/lib/crypto/tests/sha512_kunit.c b/lib/crypto/tests/sha512_kunit.c index 8923e2d7d3d4..e03926c24403 100644 --- a/lib/crypto/tests/sha512_kunit.c +++ b/lib/crypto/tests/sha512_kunit.c @@ -30,8 +30,6 @@ static struct kunit_case hash_test_cases[] = { static struct kunit_suite hash_test_suite = { .name = "sha512", .test_cases = hash_test_cases, - .suite_init = hash_suite_init, - .suite_exit = hash_suite_exit, }; kunit_test_suite(hash_test_suite); diff --git a/lib/crypto/tests/sm3-testvecs.h b/lib/crypto/tests/sm3-testvecs.h new file mode 100644 index 000000000000..5e788c29f487 --- /dev/null +++ b/lib/crypto/tests/sm3-testvecs.h @@ -0,0 +1,231 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* This file was generated by: ./scripts/crypto/gen-hash-testvecs.py sm3 */ + +static const struct { + size_t data_len; + u8 digest[SM3_DIGEST_SIZE]; +} hash_testvecs[] = { + { + .data_len = 0, + .digest = { + 0x1a, 0xb2, 0x1d, 0x83, 0x55, 0xcf, 0xa1, 0x7f, + 0x8e, 0x61, 0x19, 0x48, 0x31, 0xe8, 0x1a, 0x8f, + 0x22, 0xbe, 0xc8, 0xc7, 0x28, 0xfe, 0xfb, 0x74, + 0x7e, 0xd0, 0x35, 0xeb, 0x50, 0x82, 0xaa, 0x2b, + }, + }, + { + .data_len = 1, + .digest = { + 0xb6, 0x22, 0x2c, 0x39, 0xdc, 0x14, 0x9a, 0xee, + 0x01, 0x9a, 0xcb, 0x0d, 0xe6, 0xc6, 0x75, 0x6e, + 0x8f, 0x18, 0x7b, 0x0e, 0xe8, 0x98, 0x61, 0x71, + 0x2b, 0xd8, 0x38, 0xa9, 0xee, 0x2c, 0x1e, 0x93, + }, + }, + { + .data_len = 2, + .digest = { + 0x62, 0x0c, 0x66, 0x77, 0x67, 0x28, 0x74, 0x8a, + 0xe3, 0x64, 0xea, 0x44, 0x6a, 0x3f, 0x34, 0x61, + 0x55, 0xc5, 0xaa, 0xb2, 0x6c, 0x67, 0x97, 0x68, + 0x68, 0xae, 0x4d, 0x64, 0xa8, 0xb6, 0x72, 0x3e, + }, + }, + { + .data_len = 3, + .digest = { + 0x71, 0xd4, 0x63, 0xb1, 0xfa, 0x27, 0xc7, 0xae, + 0x65, 0xed, 0x5c, 0x93, 0x70, 0xe0, 0x09, 0x34, + 0x2f, 0x42, 0xe6, 0x71, 0x16, 0x8e, 0x90, 0x90, + 0x9a, 0xdd, 0xa6, 0x44, 0x66, 0x71, 0x18, 0xf9, + }, + }, + { + .data_len = 16, + .digest = { + 0x79, 0x0b, 0x68, 0xb5, 0x41, 0xc1, 0x97, 0xa0, + 0x50, 0xe6, 0x93, 0x70, 0xf6, 0x98, 0x49, 0xea, + 0x92, 0xc9, 0xd0, 0xb1, 0x46, 0xbd, 0x4a, 0x0c, + 0x8e, 0xe8, 0xf3, 0xe4, 0x8f, 0x90, 0x33, 0x3c, + }, + }, + { + .data_len = 32, + .digest = { + 0x32, 0x9f, 0xa3, 0x18, 0x18, 0x45, 0xe0, 0x28, + 0xd3, 0xa4, 0x41, 0x3a, 0x25, 0x62, 0x9c, 0x95, + 0xab, 0xfe, 0x02, 0xe0, 0x37, 0x7d, 0x3c, 0xc4, + 0xce, 0x69, 0x57, 0x5a, 0x07, 0x0e, 0xb9, 0xf5, + }, + }, + { + .data_len = 48, + .digest = { + 0x0c, 0xcf, 0x7c, 0x48, 0x44, 0xa0, 0xb0, 0x8d, + 0xdf, 0xbe, 0x22, 0x14, 0x7e, 0xd4, 0xc3, 0x8d, + 0x6a, 0x23, 0xfc, 0x44, 0x0e, 0x0f, 0xde, 0xa5, + 0x7c, 0x8b, 0xc4, 0x8b, 0xab, 0x8c, 0x87, 0x41, + }, + }, + { + .data_len = 49, + .digest = { + 0xb3, 0x76, 0x8b, 0x19, 0xf9, 0x10, 0xa9, 0x56, + 0x4f, 0xce, 0x27, 0xaa, 0x65, 0x96, 0xe5, 0xdb, + 0x90, 0x9b, 0x92, 0xcd, 0x32, 0x0d, 0x16, 0xac, + 0xf8, 0xd0, 0x66, 0x62, 0x10, 0xf0, 0x44, 0xdf, + }, + }, + { + .data_len = 63, + .digest = { + 0x07, 0xc9, 0x45, 0x65, 0x9f, 0x68, 0x75, 0xc3, + 0x74, 0xb2, 0x3b, 0x0c, 0x97, 0x05, 0xd3, 0x13, + 0xc0, 0xb6, 0x21, 0xed, 0xf6, 0x10, 0x7a, 0xed, + 0xec, 0xd8, 0x10, 0x29, 0xbf, 0x7a, 0x78, 0x37, + }, + }, + { + .data_len = 64, + .digest = { + 0x3e, 0x69, 0x18, 0x45, 0xd8, 0x25, 0x6f, 0x44, + 0xc0, 0x02, 0xe5, 0xcf, 0xcd, 0x94, 0x42, 0xa9, + 0xd5, 0x12, 0x62, 0x10, 0x15, 0xa0, 0xf9, 0x16, + 0x19, 0x2d, 0x8d, 0x63, 0x31, 0xf2, 0x2f, 0x36, + }, + }, + { + .data_len = 65, + .digest = { + 0x6b, 0x3e, 0xc0, 0x20, 0xb7, 0x74, 0x30, 0xa0, + 0xc6, 0x5c, 0xee, 0xbe, 0xdc, 0xe6, 0xe5, 0x4f, + 0x3c, 0x61, 0x8d, 0x91, 0xac, 0x31, 0x4b, 0x2a, + 0xdf, 0x1c, 0xef, 0x24, 0xdc, 0x0a, 0x10, 0xe8, + }, + }, + { + .data_len = 127, + .digest = { + 0xab, 0xd6, 0xa1, 0xbf, 0x39, 0x43, 0x75, 0xda, + 0xbf, 0xc7, 0x22, 0xcc, 0x4e, 0xfc, 0xe4, 0x42, + 0x6d, 0x1b, 0x87, 0x25, 0x64, 0x7f, 0x88, 0xf7, + 0xc3, 0x0a, 0x0a, 0x4c, 0xd6, 0xa7, 0x68, 0xae, + }, + }, + { + .data_len = 128, + .digest = { + 0x1b, 0x70, 0xd4, 0x5f, 0x6c, 0xe4, 0x2d, 0x58, + 0x2d, 0x0f, 0x9a, 0x12, 0x34, 0xbb, 0x5e, 0x38, + 0xd8, 0x1f, 0x6a, 0x46, 0x8a, 0xef, 0xdb, 0x68, + 0x18, 0x62, 0xbb, 0x85, 0xfc, 0xc4, 0x6e, 0x2e, + }, + }, + { + .data_len = 129, + .digest = { + 0x33, 0x62, 0xba, 0xa7, 0x4a, 0xbc, 0xd7, 0x7b, + 0xd4, 0x67, 0x6d, 0x3e, 0xea, 0xe8, 0xb0, 0x64, + 0x0d, 0xf3, 0xae, 0x1d, 0x52, 0x24, 0x11, 0x9f, + 0xda, 0xa9, 0x7f, 0xd5, 0x22, 0x1a, 0xde, 0x8a, + }, + }, + { + .data_len = 256, + .digest = { + 0x70, 0xa8, 0xa6, 0x2b, 0xfb, 0x1f, 0x3b, 0x5a, + 0xcc, 0x71, 0x76, 0x9e, 0x25, 0x4c, 0xfa, 0x8f, + 0x39, 0x4a, 0x21, 0x8a, 0x9d, 0x74, 0x8d, 0x2c, + 0x31, 0xa5, 0xb5, 0xff, 0x30, 0xc1, 0x14, 0xc4, + }, + }, + { + .data_len = 511, + .digest = { + 0x39, 0xd0, 0x8c, 0x5f, 0xfc, 0x36, 0xc2, 0x7c, + 0xdb, 0x8b, 0x2e, 0xdc, 0x9d, 0x1b, 0xd1, 0xba, + 0x9b, 0x52, 0x6b, 0x35, 0x46, 0x46, 0x75, 0x73, + 0xe5, 0x62, 0x96, 0x6e, 0xf3, 0xba, 0xd9, 0x19, + }, + }, + { + .data_len = 513, + .digest = { + 0x76, 0xa0, 0x3d, 0xa2, 0x5f, 0xd4, 0xa6, 0xbe, + 0x6b, 0xdb, 0xed, 0x14, 0x9e, 0xa8, 0x15, 0x77, + 0xa9, 0x38, 0x30, 0x6b, 0x68, 0xfa, 0xb6, 0xe2, + 0x93, 0x19, 0x24, 0x72, 0x67, 0x20, 0x72, 0xc3, + }, + }, + { + .data_len = 1000, + .digest = { + 0x16, 0xbc, 0x33, 0x77, 0x0b, 0xcf, 0x93, 0x5e, + 0xec, 0x7d, 0x8d, 0x3c, 0xae, 0xd9, 0x75, 0xdf, + 0x46, 0x24, 0x17, 0x7e, 0x03, 0x88, 0xf2, 0x75, + 0xa9, 0x18, 0xa6, 0x1c, 0x7a, 0x74, 0x0d, 0xf3, + }, + }, + { + .data_len = 3333, + .digest = { + 0xdb, 0x54, 0x89, 0xe7, 0x1c, 0x50, 0xf2, 0xbf, + 0xde, 0x3a, 0xbf, 0x5b, 0xee, 0x5a, 0x46, 0x62, + 0x20, 0xb1, 0x80, 0x48, 0xac, 0x56, 0x33, 0xb3, + 0xbb, 0x3f, 0xfa, 0x02, 0xc6, 0x43, 0xb5, 0x8c, + }, + }, + { + .data_len = 4096, + .digest = { + 0xdf, 0x0d, 0xed, 0x3b, 0x8f, 0xea, 0x81, 0xfd, + 0xd6, 0x34, 0xae, 0x74, 0x24, 0x3a, 0x15, 0x38, + 0xe7, 0xcf, 0x45, 0x7e, 0x8f, 0xf5, 0x50, 0x6c, + 0xaa, 0x27, 0x23, 0x92, 0x6d, 0xab, 0x3b, 0xde, + }, + }, + { + .data_len = 4128, + .digest = { + 0x6a, 0xbd, 0x56, 0x5a, 0xf1, 0xc6, 0x40, 0x4d, + 0xf3, 0x50, 0x77, 0x87, 0x86, 0x63, 0x1b, 0x4d, + 0x21, 0x99, 0x96, 0xad, 0x24, 0x62, 0xce, 0xc0, + 0x3e, 0xb7, 0x35, 0x52, 0x56, 0x0e, 0x55, 0x85, + }, + }, + { + .data_len = 4160, + .digest = { + 0x5b, 0xc1, 0x1f, 0x25, 0x43, 0xa3, 0x1c, 0xa0, + 0x8c, 0xfc, 0x41, 0xf1, 0xcc, 0xb3, 0x95, 0x95, + 0xe0, 0xb9, 0xd3, 0x29, 0xf4, 0x08, 0x31, 0x47, + 0x6d, 0x09, 0xa8, 0x2e, 0xa5, 0xf4, 0xf1, 0x8d, + }, + }, + { + .data_len = 4224, + .digest = { + 0xec, 0x56, 0x1e, 0xa6, 0x1f, 0xb2, 0x87, 0xb2, + 0x7e, 0x15, 0xd6, 0x30, 0x08, 0x74, 0xb0, 0x48, + 0x90, 0x2a, 0xbe, 0x2f, 0x80, 0x3a, 0x88, 0xcc, + 0xd7, 0xc5, 0x87, 0x8c, 0x04, 0xef, 0x78, 0x71, + }, + }, + { + .data_len = 16384, + .digest = { + 0xe7, 0xb8, 0x84, 0x20, 0xff, 0xd5, 0x53, 0xe6, + 0x14, 0x31, 0x12, 0x75, 0xb7, 0x9a, 0x4f, 0x63, + 0x63, 0x00, 0xfe, 0x2c, 0x54, 0xee, 0x06, 0xfc, + 0x12, 0x16, 0xe5, 0xdc, 0xa4, 0x40, 0x62, 0x12, + }, + }, +}; + +static const u8 hash_testvec_consolidated[SM3_DIGEST_SIZE] = { + 0xe6, 0x58, 0xd4, 0x8e, 0x74, 0x92, 0xdf, 0xfe, + 0x58, 0x05, 0xe5, 0x29, 0x83, 0xfb, 0xb7, 0x51, + 0x7e, 0x66, 0x0c, 0x49, 0x3e, 0x11, 0x7e, 0x9b, + 0xb1, 0x83, 0x3a, 0xa6, 0xb0, 0x3c, 0xf5, 0xe0, +}; diff --git a/lib/crypto/tests/sm3_kunit.c b/lib/crypto/tests/sm3_kunit.c new file mode 100644 index 000000000000..1cba384946e9 --- /dev/null +++ b/lib/crypto/tests/sm3_kunit.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2026 Google LLC + */ +#include <crypto/sm3.h> +#include "sm3-testvecs.h" + +#define HASH sm3 +#define HASH_CTX sm3_ctx +#define HASH_SIZE SM3_DIGEST_SIZE +#define HASH_INIT sm3_init +#define HASH_UPDATE sm3_update +#define HASH_FINAL sm3_final +#include "hash-test-template.h" + +static struct kunit_case sm3_test_cases[] = { + HASH_KUNIT_CASES, + KUNIT_CASE(benchmark_hash), + {}, +}; + +static struct kunit_suite sm3_test_suite = { + .name = "sm3", + .test_cases = sm3_test_cases, +}; +kunit_test_suite(sm3_test_suite); + +MODULE_DESCRIPTION("KUnit tests and benchmark for SM3"); +MODULE_LICENSE("GPL"); diff --git a/lib/crypto/tests/test-utils.h b/lib/crypto/tests/test-utils.h new file mode 100644 index 000000000000..73ca21f9176e --- /dev/null +++ b/lib/crypto/tests/test-utils.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Test utility functions shared by the crypto library tests. + * + * For now this is simply a header that's included into the KUnit test suites + * that need it. If this gets too large it could be made its own translation + * unit and libcrypto_test_utils module, but that seems overkill for now. + */ +#ifndef LIB_CRYPTO_TEST_UTILS_H +#define LIB_CRYPTO_TEST_UTILS_H + +#include <kunit/test.h> +#include <linux/math.h> +#include <linux/minmax.h> +#include <linux/string.h> +#include <linux/vmalloc.h> + +static u64 random_seed; + +static __maybe_unused void action_free_guarded_buf(void *buf) +{ + vfree(buf); +} + +/* + * Allocate a KUnit-managed buffer that has length @size bytes (> 0) immediately + * followed by an unmapped page, and assert that the allocation succeeds. + */ +static __maybe_unused void *alloc_guarded_buf(struct kunit *test, size_t size) +{ + size_t full_size = round_up(size, PAGE_SIZE); + void *buf = vmalloc(full_size); + + KUNIT_ASSERT_NOT_NULL(test, buf); + KUNIT_ASSERT_EQ(test, 0, + kunit_add_action_or_reset(test, action_free_guarded_buf, + buf)); + return buf + full_size - size; +} + +static __maybe_unused void *alloc_buf(struct kunit *test, size_t size) +{ + void *buf = kunit_kmalloc(test, size, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, buf); + return buf; +} + +static __maybe_unused void *memdup_buf(struct kunit *test, const void *src, + size_t size) +{ + void *dst = alloc_buf(test, size); + + return memcpy(dst, src, size); +} + +/* + * This is a simple linear congruential generator. It is used only for testing, + * which does not require cryptographically secure random numbers. A hard-coded + * algorithm is used instead of <linux/prandom.h> so that it matches the + * algorithm used by the test vector generation script. This allows the input + * data in random test vectors to be concisely stored as just the seed. + */ +static __maybe_unused u32 rand32(void) +{ + random_seed = (random_seed * 25214903917 + 11) & ((1ULL << 48) - 1); + return random_seed >> 16; +} + +static __maybe_unused void rand_bytes(u8 *out, size_t len) +{ + for (size_t i = 0; i < len; i++) + out[i] = rand32(); +} + +static __maybe_unused void rand_bytes_seeded_from_len(u8 *out, size_t len) +{ + random_seed = len; + rand_bytes(out, len); +} + +static __maybe_unused bool rand_bool(void) +{ + return rand32() % 2; +} + +/* Generate a random length, preferring small lengths. */ +static __maybe_unused size_t rand_length(size_t max_len) +{ + size_t len; + + switch (rand32() % 3) { + case 0: + len = rand32() % 128; + break; + case 1: + len = rand32() % 3072; + break; + default: + len = rand32(); + break; + } + return len % (max_len + 1); +} + +static __maybe_unused size_t rand_offset(size_t max_offset) +{ + return min(rand32() % 128, max_offset); +} + +#endif /* LIB_CRYPTO_TEST_UTILS_H */ diff --git a/lib/crypto/x86/polyval.h b/lib/crypto/x86/gf128hash.h index ef8797521420..6b79b06caab0 100644 --- a/lib/crypto/x86/polyval.h +++ b/lib/crypto/x86/gf128hash.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * POLYVAL library functions, x86_64 optimized + * GHASH and POLYVAL, x86_64 optimized * * Copyright 2025 Google LLC */ @@ -9,14 +9,22 @@ #define NUM_H_POWERS 8 +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul_avx); +asmlinkage void polyval_mul_pclmul(struct polyval_elem *a, + const struct polyval_elem *b); asmlinkage void polyval_mul_pclmul_avx(struct polyval_elem *a, const struct polyval_elem *b); + +asmlinkage void ghash_blocks_pclmul(struct polyval_elem *acc, + const struct polyval_elem *key, + const u8 *data, size_t nblocks); asmlinkage void polyval_blocks_pclmul_avx(struct polyval_elem *acc, const struct polyval_key *key, const u8 *data, size_t nblocks); +#define polyval_preparekey_arch polyval_preparekey_arch static void polyval_preparekey_arch(struct polyval_key *key, const u8 raw_key[POLYVAL_BLOCK_SIZE]) { @@ -40,18 +48,58 @@ static void polyval_preparekey_arch(struct polyval_key *key, } } -static void polyval_mul_arch(struct polyval_elem *acc, - const struct polyval_key *key) +static void polyval_mul_x86(struct polyval_elem *a, + const struct polyval_elem *b) { - if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) { + if (static_branch_likely(&have_pclmul) && irq_fpu_usable()) { kernel_fpu_begin(); - polyval_mul_pclmul_avx(acc, &key->h_powers[NUM_H_POWERS - 1]); + if (static_branch_likely(&have_pclmul_avx)) + polyval_mul_pclmul_avx(a, b); + else + polyval_mul_pclmul(a, b); kernel_fpu_end(); } else { - polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]); + polyval_mul_generic(a, b); + } +} + +#define ghash_mul_arch ghash_mul_arch +static void ghash_mul_arch(struct polyval_elem *acc, + const struct ghash_key *key) +{ + polyval_mul_x86(acc, &key->h); +} + +#define polyval_mul_arch polyval_mul_arch +static void polyval_mul_arch(struct polyval_elem *acc, + const struct polyval_key *key) +{ + polyval_mul_x86(acc, &key->h_powers[NUM_H_POWERS - 1]); +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_pclmul) && irq_fpu_usable()) { + do { + /* Allow rescheduling every 4 KiB. */ + size_t n = min_t(size_t, nblocks, + 4096 / GHASH_BLOCK_SIZE); + + kernel_fpu_begin(); + ghash_blocks_pclmul(acc, &key->h, data, n); + kernel_fpu_end(); + data += n * GHASH_BLOCK_SIZE; + nblocks -= n; + } while (nblocks); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); } } +#define polyval_blocks_arch polyval_blocks_arch static void polyval_blocks_arch(struct polyval_elem *acc, const struct polyval_key *key, const u8 *data, size_t nblocks) @@ -74,10 +122,12 @@ static void polyval_blocks_arch(struct polyval_elem *acc, } } -#define polyval_mod_init_arch polyval_mod_init_arch -static void polyval_mod_init_arch(void) +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) { - if (boot_cpu_has(X86_FEATURE_PCLMULQDQ) && - boot_cpu_has(X86_FEATURE_AVX)) - static_branch_enable(&have_pclmul_avx); + if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) { + static_branch_enable(&have_pclmul); + if (boot_cpu_has(X86_FEATURE_AVX)) + static_branch_enable(&have_pclmul_avx); + } } diff --git a/lib/crypto/x86/ghash-pclmul.S b/lib/crypto/x86/ghash-pclmul.S new file mode 100644 index 000000000000..6ffb5aea6063 --- /dev/null +++ b/lib/crypto/x86/ghash-pclmul.S @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Accelerated GHASH implementation with Intel PCLMULQDQ-NI + * instructions. This file contains accelerated part of ghash + * implementation. More information about PCLMULQDQ can be found at: + * + * https://www.intel.com/content/dam/develop/external/us/en/documents/clmul-wp-rev-2-02-2014-04-20.pdf + * + * Copyright (c) 2009 Intel Corp. + * Author: Huang Ying <ying.huang@intel.com> + * Vinodh Gopal + * Erdinc Ozturk + * Deniz Karakoyunlu + */ + +#include <linux/linkage.h> +#include <asm/frame.h> + +.section .rodata.cst16.bswap_mask, "aM", @progbits, 16 +.align 16 +.Lbswap_mask: + .octa 0x000102030405060708090a0b0c0d0e0f + +#define ACC %xmm0 +#define KEY %xmm1 +#define T1 %xmm2 +#define T2 %xmm3 +#define T3 %xmm4 +#define BSWAP %xmm5 +#define IN1 %xmm6 + +.text + +/* + * __clmul_gf128mul_ble: internal ABI + * input: + * ACC: operand1 + * KEY: operand2, hash_key << 1 mod poly + * output: + * ACC: operand1 * operand2 mod poly + * changed: + * T1 + * T2 + * T3 + */ +SYM_FUNC_START_LOCAL(__clmul_gf128mul_ble) + movaps ACC, T1 + pshufd $0b01001110, ACC, T2 + pshufd $0b01001110, KEY, T3 + pxor ACC, T2 + pxor KEY, T3 + + pclmulqdq $0x00, KEY, ACC # ACC = a0 * b0 + pclmulqdq $0x11, KEY, T1 # T1 = a1 * b1 + pclmulqdq $0x00, T3, T2 # T2 = (a1 + a0) * (b1 + b0) + pxor ACC, T2 + pxor T1, T2 # T2 = a0 * b1 + a1 * b0 + + movaps T2, T3 + pslldq $8, T3 + psrldq $8, T2 + pxor T3, ACC + pxor T2, T1 # <T1:ACC> is result of + # carry-less multiplication + + # first phase of the reduction + movaps ACC, T3 + psllq $1, T3 + pxor ACC, T3 + psllq $5, T3 + pxor ACC, T3 + psllq $57, T3 + movaps T3, T2 + pslldq $8, T2 + psrldq $8, T3 + pxor T2, ACC + pxor T3, T1 + + # second phase of the reduction + movaps ACC, T2 + psrlq $5, T2 + pxor ACC, T2 + psrlq $1, T2 + pxor ACC, T2 + psrlq $1, T2 + pxor T2, T1 + pxor T1, ACC + RET +SYM_FUNC_END(__clmul_gf128mul_ble) + +/* + * void polyval_mul_pclmul(struct polyval_elem *a, + * const struct polyval_elem *b) + */ +SYM_FUNC_START(polyval_mul_pclmul) + FRAME_BEGIN + movups (%rdi), ACC + movups (%rsi), KEY + call __clmul_gf128mul_ble + movups ACC, (%rdi) + FRAME_END + RET +SYM_FUNC_END(polyval_mul_pclmul) + +/* + * void ghash_blocks_pclmul(struct polyval_elem *acc, + * const struct polyval_elem *key, + * const u8 *data, size_t nblocks) + */ +SYM_FUNC_START(ghash_blocks_pclmul) + FRAME_BEGIN + movaps .Lbswap_mask(%rip), BSWAP + movups (%rdi), ACC + movups (%rsi), KEY +.align 4 +.Lnext_block: + movups (%rdx), IN1 + pshufb BSWAP, IN1 + pxor IN1, ACC + call __clmul_gf128mul_ble + add $16, %rdx + dec %rcx + jnz .Lnext_block + movups ACC, (%rdi) + FRAME_END + RET +SYM_FUNC_END(ghash_blocks_pclmul) diff --git a/lib/crypto/x86/sha256.h b/lib/crypto/x86/sha256.h index 38e33b22a092..0ee69d8e39fe 100644 --- a/lib/crypto/x86/sha256.h +++ b/lib/crypto/x86/sha256.h @@ -31,6 +31,27 @@ DEFINE_X86_SHA256_FN(sha256_blocks_avx, sha256_transform_avx); DEFINE_X86_SHA256_FN(sha256_blocks_avx2, sha256_transform_rorx); DEFINE_X86_SHA256_FN(sha256_blocks_ni, sha256_ni_transform); +#define PHE_ALIGNMENT 16 +static void sha256_blocks_phe(struct sha256_block_state *state, + const u8 *data, size_t nblocks) +{ + /* + * On Zhaoxin processors, XSHA256 requires the %rdi register + * in 64-bit mode (or %edi in 32-bit mode) to point to + * a 32-byte, 16-byte-aligned buffer. + */ + u8 buf[32 + PHE_ALIGNMENT - 1]; + u8 *dst = PTR_ALIGN(&buf[0], PHE_ALIGNMENT); + size_t padding = -1; + + memcpy(dst, state, SHA256_DIGEST_SIZE); + asm volatile(".byte 0xf3,0x0f,0xa6,0xd0" /* REP XSHA256 */ + : "+a"(padding), "+c"(nblocks), "+S"(data) + : "D"(dst) + : "memory"); + memcpy(state, dst, SHA256_DIGEST_SIZE); +} + static void sha256_blocks(struct sha256_block_state *state, const u8 *data, size_t nblocks) { @@ -79,6 +100,10 @@ static void sha256_mod_init_arch(void) if (boot_cpu_has(X86_FEATURE_SHA_NI)) { static_call_update(sha256_blocks_x86, sha256_blocks_ni); static_branch_enable(&have_sha_ni); + } else if (IS_ENABLED(CONFIG_CPU_SUP_ZHAOXIN) && + boot_cpu_has(X86_FEATURE_PHE_EN) && + boot_cpu_data.x86 >= 0x07) { + static_call_update(sha256_blocks_x86, sha256_blocks_phe); } else if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL) && boot_cpu_has(X86_FEATURE_AVX)) { diff --git a/lib/crypto/x86/sm3-avx-asm_64.S b/lib/crypto/x86/sm3-avx-asm_64.S new file mode 100644 index 000000000000..a1925b136010 --- /dev/null +++ b/lib/crypto/x86/sm3-avx-asm_64.S @@ -0,0 +1,516 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SM3 AVX accelerated transform. + * specified in: https://datatracker.ietf.org/doc/html/draft-sca-cfrg-sm3-02 + * + * Copyright (C) 2021 Jussi Kivilinna <jussi.kivilinna@iki.fi> + * Copyright (C) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> + */ + +/* Based on SM3 AES/BMI2 accelerated work by libgcrypt at: + * https://gnupg.org/software/libgcrypt/index.html + */ + +#include <linux/linkage.h> +#include <asm/frame.h> + +/* State structure */ + +#define state_h0 0 +#define state_h1 4 +#define state_h2 8 +#define state_h3 12 +#define state_h4 16 +#define state_h5 20 +#define state_h6 24 +#define state_h7 28 + +/* Constants */ + +/* Round constant macros */ + +#define K0 2043430169 /* 0x79cc4519 */ +#define K1 -208106958 /* 0xf3988a32 */ +#define K2 -416213915 /* 0xe7311465 */ +#define K3 -832427829 /* 0xce6228cb */ +#define K4 -1664855657 /* 0x9cc45197 */ +#define K5 965255983 /* 0x3988a32f */ +#define K6 1930511966 /* 0x7311465e */ +#define K7 -433943364 /* 0xe6228cbc */ +#define K8 -867886727 /* 0xcc451979 */ +#define K9 -1735773453 /* 0x988a32f3 */ +#define K10 823420391 /* 0x311465e7 */ +#define K11 1646840782 /* 0x6228cbce */ +#define K12 -1001285732 /* 0xc451979c */ +#define K13 -2002571463 /* 0x88a32f39 */ +#define K14 289824371 /* 0x11465e73 */ +#define K15 579648742 /* 0x228cbce6 */ +#define K16 -1651869049 /* 0x9d8a7a87 */ +#define K17 991229199 /* 0x3b14f50f */ +#define K18 1982458398 /* 0x7629ea1e */ +#define K19 -330050500 /* 0xec53d43c */ +#define K20 -660100999 /* 0xd8a7a879 */ +#define K21 -1320201997 /* 0xb14f50f3 */ +#define K22 1654563303 /* 0x629ea1e7 */ +#define K23 -985840690 /* 0xc53d43ce */ +#define K24 -1971681379 /* 0x8a7a879d */ +#define K25 351604539 /* 0x14f50f3b */ +#define K26 703209078 /* 0x29ea1e76 */ +#define K27 1406418156 /* 0x53d43cec */ +#define K28 -1482130984 /* 0xa7a879d8 */ +#define K29 1330705329 /* 0x4f50f3b1 */ +#define K30 -1633556638 /* 0x9ea1e762 */ +#define K31 1027854021 /* 0x3d43cec5 */ +#define K32 2055708042 /* 0x7a879d8a */ +#define K33 -183551212 /* 0xf50f3b14 */ +#define K34 -367102423 /* 0xea1e7629 */ +#define K35 -734204845 /* 0xd43cec53 */ +#define K36 -1468409689 /* 0xa879d8a7 */ +#define K37 1358147919 /* 0x50f3b14f */ +#define K38 -1578671458 /* 0xa1e7629e */ +#define K39 1137624381 /* 0x43cec53d */ +#define K40 -2019718534 /* 0x879d8a7a */ +#define K41 255530229 /* 0x0f3b14f5 */ +#define K42 511060458 /* 0x1e7629ea */ +#define K43 1022120916 /* 0x3cec53d4 */ +#define K44 2044241832 /* 0x79d8a7a8 */ +#define K45 -206483632 /* 0xf3b14f50 */ +#define K46 -412967263 /* 0xe7629ea1 */ +#define K47 -825934525 /* 0xcec53d43 */ +#define K48 -1651869049 /* 0x9d8a7a87 */ +#define K49 991229199 /* 0x3b14f50f */ +#define K50 1982458398 /* 0x7629ea1e */ +#define K51 -330050500 /* 0xec53d43c */ +#define K52 -660100999 /* 0xd8a7a879 */ +#define K53 -1320201997 /* 0xb14f50f3 */ +#define K54 1654563303 /* 0x629ea1e7 */ +#define K55 -985840690 /* 0xc53d43ce */ +#define K56 -1971681379 /* 0x8a7a879d */ +#define K57 351604539 /* 0x14f50f3b */ +#define K58 703209078 /* 0x29ea1e76 */ +#define K59 1406418156 /* 0x53d43cec */ +#define K60 -1482130984 /* 0xa7a879d8 */ +#define K61 1330705329 /* 0x4f50f3b1 */ +#define K62 -1633556638 /* 0x9ea1e762 */ +#define K63 1027854021 /* 0x3d43cec5 */ + +/* Register macros */ + +#define RSTATE %rdi +#define RDATA %rsi +#define RNBLKS %rdx + +#define t0 %eax +#define t1 %ebx +#define t2 %ecx + +#define a %r8d +#define b %r9d +#define c %r10d +#define d %r11d +#define e %r12d +#define f %r13d +#define g %r14d +#define h %r15d + +#define W0 %xmm0 +#define W1 %xmm1 +#define W2 %xmm2 +#define W3 %xmm3 +#define W4 %xmm4 +#define W5 %xmm5 + +#define XTMP0 %xmm6 +#define XTMP1 %xmm7 +#define XTMP2 %xmm8 +#define XTMP3 %xmm9 +#define XTMP4 %xmm10 +#define XTMP5 %xmm11 +#define XTMP6 %xmm12 + +#define BSWAP_REG %xmm15 + +/* Stack structure */ + +#define STACK_W_SIZE (32 * 2 * 3) +#define STACK_REG_SAVE_SIZE (64) + +#define STACK_W (0) +#define STACK_REG_SAVE (STACK_W + STACK_W_SIZE) +#define STACK_SIZE (STACK_REG_SAVE + STACK_REG_SAVE_SIZE) + +/* Instruction helpers. */ + +#define roll2(v, reg) \ + roll $(v), reg; + +#define roll3mov(v, src, dst) \ + movl src, dst; \ + roll $(v), dst; + +#define roll3(v, src, dst) \ + rorxl $(32-(v)), src, dst; + +#define addl2(a, out) \ + leal (a, out), out; + +/* Round function macros. */ + +#define GG1(x, y, z, o, t) \ + movl x, o; \ + xorl y, o; \ + xorl z, o; + +#define FF1(x, y, z, o, t) GG1(x, y, z, o, t) + +#define GG2(x, y, z, o, t) \ + andnl z, x, o; \ + movl y, t; \ + andl x, t; \ + addl2(t, o); + +#define FF2(x, y, z, o, t) \ + movl y, o; \ + xorl x, o; \ + movl y, t; \ + andl x, t; \ + andl z, o; \ + xorl t, o; + +#define R(i, a, b, c, d, e, f, g, h, round, widx, wtype) \ + /* rol(a, 12) => t0 */ \ + roll3mov(12, a, t0); /* rorxl here would reduce perf by 6% on zen3 */ \ + /* rol (t0 + e + t), 7) => t1 */ \ + leal K##round(t0, e, 1), t1; \ + roll2(7, t1); \ + /* h + w1 => h */ \ + addl wtype##_W1_ADDR(round, widx), h; \ + /* h + t1 => h */ \ + addl2(t1, h); \ + /* t1 ^ t0 => t0 */ \ + xorl t1, t0; \ + /* w1w2 + d => d */ \ + addl wtype##_W1W2_ADDR(round, widx), d; \ + /* FF##i(a,b,c) => t1 */ \ + FF##i(a, b, c, t1, t2); \ + /* d + t1 => d */ \ + addl2(t1, d); \ + /* GG#i(e,f,g) => t2 */ \ + GG##i(e, f, g, t2, t1); \ + /* h + t2 => h */ \ + addl2(t2, h); \ + /* rol (f, 19) => f */ \ + roll2(19, f); \ + /* d + t0 => d */ \ + addl2(t0, d); \ + /* rol (b, 9) => b */ \ + roll2(9, b); \ + /* P0(h) => h */ \ + roll3(9, h, t2); \ + roll3(17, h, t1); \ + xorl t2, h; \ + xorl t1, h; + +#define R1(a, b, c, d, e, f, g, h, round, widx, wtype) \ + R(1, a, b, c, d, e, f, g, h, round, widx, wtype) + +#define R2(a, b, c, d, e, f, g, h, round, widx, wtype) \ + R(2, a, b, c, d, e, f, g, h, round, widx, wtype) + +/* Input expansion macros. */ + +/* Byte-swapped input address. */ +#define IW_W_ADDR(round, widx, offs) \ + (STACK_W + ((round) / 4) * 64 + (offs) + ((widx) * 4))(%rsp) + +/* Expanded input address. */ +#define XW_W_ADDR(round, widx, offs) \ + (STACK_W + ((((round) / 3) - 4) % 2) * 64 + (offs) + ((widx) * 4))(%rsp) + +/* Rounds 1-12, byte-swapped input block addresses. */ +#define IW_W1_ADDR(round, widx) IW_W_ADDR(round, widx, 0) +#define IW_W1W2_ADDR(round, widx) IW_W_ADDR(round, widx, 32) + +/* Rounds 1-12, expanded input block addresses. */ +#define XW_W1_ADDR(round, widx) XW_W_ADDR(round, widx, 0) +#define XW_W1W2_ADDR(round, widx) XW_W_ADDR(round, widx, 32) + +/* Input block loading. */ +#define LOAD_W_XMM_1() \ + vmovdqu 0*16(RDATA), XTMP0; /* XTMP0: w3, w2, w1, w0 */ \ + vmovdqu 1*16(RDATA), XTMP1; /* XTMP1: w7, w6, w5, w4 */ \ + vmovdqu 2*16(RDATA), XTMP2; /* XTMP2: w11, w10, w9, w8 */ \ + vmovdqu 3*16(RDATA), XTMP3; /* XTMP3: w15, w14, w13, w12 */ \ + vpshufb BSWAP_REG, XTMP0, XTMP0; \ + vpshufb BSWAP_REG, XTMP1, XTMP1; \ + vpshufb BSWAP_REG, XTMP2, XTMP2; \ + vpshufb BSWAP_REG, XTMP3, XTMP3; \ + vpxor XTMP0, XTMP1, XTMP4; \ + vpxor XTMP1, XTMP2, XTMP5; \ + vpxor XTMP2, XTMP3, XTMP6; \ + leaq 64(RDATA), RDATA; \ + vmovdqa XTMP0, IW_W1_ADDR(0, 0); \ + vmovdqa XTMP4, IW_W1W2_ADDR(0, 0); \ + vmovdqa XTMP1, IW_W1_ADDR(4, 0); \ + vmovdqa XTMP5, IW_W1W2_ADDR(4, 0); + +#define LOAD_W_XMM_2() \ + vmovdqa XTMP2, IW_W1_ADDR(8, 0); \ + vmovdqa XTMP6, IW_W1W2_ADDR(8, 0); + +#define LOAD_W_XMM_3() \ + vpshufd $0b00000000, XTMP0, W0; /* W0: xx, w0, xx, xx */ \ + vpshufd $0b11111001, XTMP0, W1; /* W1: xx, w3, w2, w1 */ \ + vmovdqa XTMP1, W2; /* W2: xx, w6, w5, w4 */ \ + vpalignr $12, XTMP1, XTMP2, W3; /* W3: xx, w9, w8, w7 */ \ + vpalignr $8, XTMP2, XTMP3, W4; /* W4: xx, w12, w11, w10 */ \ + vpshufd $0b11111001, XTMP3, W5; /* W5: xx, w15, w14, w13 */ + +/* Message scheduling. Note: 3 words per XMM register. */ +#define SCHED_W_0(round, w0, w1, w2, w3, w4, w5) \ + /* Load (w[i - 16]) => XTMP0 */ \ + vpshufd $0b10111111, w0, XTMP0; \ + vpalignr $12, XTMP0, w1, XTMP0; /* XTMP0: xx, w2, w1, w0 */ \ + /* Load (w[i - 13]) => XTMP1 */ \ + vpshufd $0b10111111, w1, XTMP1; \ + vpalignr $12, XTMP1, w2, XTMP1; \ + /* w[i - 9] == w3 */ \ + /* XMM3 ^ XTMP0 => XTMP0 */ \ + vpxor w3, XTMP0, XTMP0; + +#define SCHED_W_1(round, w0, w1, w2, w3, w4, w5) \ + /* w[i - 3] == w5 */ \ + /* rol(XMM5, 15) ^ XTMP0 => XTMP0 */ \ + vpslld $15, w5, XTMP2; \ + vpsrld $(32-15), w5, XTMP3; \ + vpxor XTMP2, XTMP3, XTMP3; \ + vpxor XTMP3, XTMP0, XTMP0; \ + /* rol(XTMP1, 7) => XTMP1 */ \ + vpslld $7, XTMP1, XTMP5; \ + vpsrld $(32-7), XTMP1, XTMP1; \ + vpxor XTMP5, XTMP1, XTMP1; \ + /* XMM4 ^ XTMP1 => XTMP1 */ \ + vpxor w4, XTMP1, XTMP1; \ + /* w[i - 6] == XMM4 */ \ + /* P1(XTMP0) ^ XTMP1 => XMM0 */ \ + vpslld $15, XTMP0, XTMP5; \ + vpsrld $(32-15), XTMP0, XTMP6; \ + vpslld $23, XTMP0, XTMP2; \ + vpsrld $(32-23), XTMP0, XTMP3; \ + vpxor XTMP0, XTMP1, XTMP1; \ + vpxor XTMP6, XTMP5, XTMP5; \ + vpxor XTMP3, XTMP2, XTMP2; \ + vpxor XTMP2, XTMP5, XTMP5; \ + vpxor XTMP5, XTMP1, w0; + +#define SCHED_W_2(round, w0, w1, w2, w3, w4, w5) \ + /* W1 in XMM12 */ \ + vpshufd $0b10111111, w4, XTMP4; \ + vpalignr $12, XTMP4, w5, XTMP4; \ + vmovdqa XTMP4, XW_W1_ADDR((round), 0); \ + /* W1 ^ W2 => XTMP1 */ \ + vpxor w0, XTMP4, XTMP1; \ + vmovdqa XTMP1, XW_W1W2_ADDR((round), 0); + + +.section .rodata.cst16, "aM", @progbits, 16 +.align 16 + +.Lbe32mask: + .long 0x00010203, 0x04050607, 0x08090a0b, 0x0c0d0e0f + +.text + +/* + * Transform nblocks*64 bytes (nblocks*16 32-bit words) at DATA. + * + * void sm3_transform_avx(struct sm3_block_state *state, + * const u8 *data, size_t nblocks); + */ +SYM_FUNC_START(sm3_transform_avx) + /* input: + * %rdi: state + * %rsi: data + * %rdx: nblocks + */ + vzeroupper; + + pushq %rbp; + movq %rsp, %rbp; + + movq %rdx, RNBLKS; + + subq $STACK_SIZE, %rsp; + andq $(~63), %rsp; + + movq %rbx, (STACK_REG_SAVE + 0 * 8)(%rsp); + movq %r15, (STACK_REG_SAVE + 1 * 8)(%rsp); + movq %r14, (STACK_REG_SAVE + 2 * 8)(%rsp); + movq %r13, (STACK_REG_SAVE + 3 * 8)(%rsp); + movq %r12, (STACK_REG_SAVE + 4 * 8)(%rsp); + + vmovdqa .Lbe32mask (%rip), BSWAP_REG; + + /* Get the values of the chaining variables. */ + movl state_h0(RSTATE), a; + movl state_h1(RSTATE), b; + movl state_h2(RSTATE), c; + movl state_h3(RSTATE), d; + movl state_h4(RSTATE), e; + movl state_h5(RSTATE), f; + movl state_h6(RSTATE), g; + movl state_h7(RSTATE), h; + +.align 16 +.Loop: + /* Load data part1. */ + LOAD_W_XMM_1(); + + leaq -1(RNBLKS), RNBLKS; + + /* Transform 0-3 + Load data part2. */ + R1(a, b, c, d, e, f, g, h, 0, 0, IW); LOAD_W_XMM_2(); + R1(d, a, b, c, h, e, f, g, 1, 1, IW); + R1(c, d, a, b, g, h, e, f, 2, 2, IW); + R1(b, c, d, a, f, g, h, e, 3, 3, IW); LOAD_W_XMM_3(); + + /* Transform 4-7 + Precalc 12-14. */ + R1(a, b, c, d, e, f, g, h, 4, 0, IW); + R1(d, a, b, c, h, e, f, g, 5, 1, IW); + R1(c, d, a, b, g, h, e, f, 6, 2, IW); SCHED_W_0(12, W0, W1, W2, W3, W4, W5); + R1(b, c, d, a, f, g, h, e, 7, 3, IW); SCHED_W_1(12, W0, W1, W2, W3, W4, W5); + + /* Transform 8-11 + Precalc 12-17. */ + R1(a, b, c, d, e, f, g, h, 8, 0, IW); SCHED_W_2(12, W0, W1, W2, W3, W4, W5); + R1(d, a, b, c, h, e, f, g, 9, 1, IW); SCHED_W_0(15, W1, W2, W3, W4, W5, W0); + R1(c, d, a, b, g, h, e, f, 10, 2, IW); SCHED_W_1(15, W1, W2, W3, W4, W5, W0); + R1(b, c, d, a, f, g, h, e, 11, 3, IW); SCHED_W_2(15, W1, W2, W3, W4, W5, W0); + + /* Transform 12-14 + Precalc 18-20 */ + R1(a, b, c, d, e, f, g, h, 12, 0, XW); SCHED_W_0(18, W2, W3, W4, W5, W0, W1); + R1(d, a, b, c, h, e, f, g, 13, 1, XW); SCHED_W_1(18, W2, W3, W4, W5, W0, W1); + R1(c, d, a, b, g, h, e, f, 14, 2, XW); SCHED_W_2(18, W2, W3, W4, W5, W0, W1); + + /* Transform 15-17 + Precalc 21-23 */ + R1(b, c, d, a, f, g, h, e, 15, 0, XW); SCHED_W_0(21, W3, W4, W5, W0, W1, W2); + R2(a, b, c, d, e, f, g, h, 16, 1, XW); SCHED_W_1(21, W3, W4, W5, W0, W1, W2); + R2(d, a, b, c, h, e, f, g, 17, 2, XW); SCHED_W_2(21, W3, W4, W5, W0, W1, W2); + + /* Transform 18-20 + Precalc 24-26 */ + R2(c, d, a, b, g, h, e, f, 18, 0, XW); SCHED_W_0(24, W4, W5, W0, W1, W2, W3); + R2(b, c, d, a, f, g, h, e, 19, 1, XW); SCHED_W_1(24, W4, W5, W0, W1, W2, W3); + R2(a, b, c, d, e, f, g, h, 20, 2, XW); SCHED_W_2(24, W4, W5, W0, W1, W2, W3); + + /* Transform 21-23 + Precalc 27-29 */ + R2(d, a, b, c, h, e, f, g, 21, 0, XW); SCHED_W_0(27, W5, W0, W1, W2, W3, W4); + R2(c, d, a, b, g, h, e, f, 22, 1, XW); SCHED_W_1(27, W5, W0, W1, W2, W3, W4); + R2(b, c, d, a, f, g, h, e, 23, 2, XW); SCHED_W_2(27, W5, W0, W1, W2, W3, W4); + + /* Transform 24-26 + Precalc 30-32 */ + R2(a, b, c, d, e, f, g, h, 24, 0, XW); SCHED_W_0(30, W0, W1, W2, W3, W4, W5); + R2(d, a, b, c, h, e, f, g, 25, 1, XW); SCHED_W_1(30, W0, W1, W2, W3, W4, W5); + R2(c, d, a, b, g, h, e, f, 26, 2, XW); SCHED_W_2(30, W0, W1, W2, W3, W4, W5); + + /* Transform 27-29 + Precalc 33-35 */ + R2(b, c, d, a, f, g, h, e, 27, 0, XW); SCHED_W_0(33, W1, W2, W3, W4, W5, W0); + R2(a, b, c, d, e, f, g, h, 28, 1, XW); SCHED_W_1(33, W1, W2, W3, W4, W5, W0); + R2(d, a, b, c, h, e, f, g, 29, 2, XW); SCHED_W_2(33, W1, W2, W3, W4, W5, W0); + + /* Transform 30-32 + Precalc 36-38 */ + R2(c, d, a, b, g, h, e, f, 30, 0, XW); SCHED_W_0(36, W2, W3, W4, W5, W0, W1); + R2(b, c, d, a, f, g, h, e, 31, 1, XW); SCHED_W_1(36, W2, W3, W4, W5, W0, W1); + R2(a, b, c, d, e, f, g, h, 32, 2, XW); SCHED_W_2(36, W2, W3, W4, W5, W0, W1); + + /* Transform 33-35 + Precalc 39-41 */ + R2(d, a, b, c, h, e, f, g, 33, 0, XW); SCHED_W_0(39, W3, W4, W5, W0, W1, W2); + R2(c, d, a, b, g, h, e, f, 34, 1, XW); SCHED_W_1(39, W3, W4, W5, W0, W1, W2); + R2(b, c, d, a, f, g, h, e, 35, 2, XW); SCHED_W_2(39, W3, W4, W5, W0, W1, W2); + + /* Transform 36-38 + Precalc 42-44 */ + R2(a, b, c, d, e, f, g, h, 36, 0, XW); SCHED_W_0(42, W4, W5, W0, W1, W2, W3); + R2(d, a, b, c, h, e, f, g, 37, 1, XW); SCHED_W_1(42, W4, W5, W0, W1, W2, W3); + R2(c, d, a, b, g, h, e, f, 38, 2, XW); SCHED_W_2(42, W4, W5, W0, W1, W2, W3); + + /* Transform 39-41 + Precalc 45-47 */ + R2(b, c, d, a, f, g, h, e, 39, 0, XW); SCHED_W_0(45, W5, W0, W1, W2, W3, W4); + R2(a, b, c, d, e, f, g, h, 40, 1, XW); SCHED_W_1(45, W5, W0, W1, W2, W3, W4); + R2(d, a, b, c, h, e, f, g, 41, 2, XW); SCHED_W_2(45, W5, W0, W1, W2, W3, W4); + + /* Transform 42-44 + Precalc 48-50 */ + R2(c, d, a, b, g, h, e, f, 42, 0, XW); SCHED_W_0(48, W0, W1, W2, W3, W4, W5); + R2(b, c, d, a, f, g, h, e, 43, 1, XW); SCHED_W_1(48, W0, W1, W2, W3, W4, W5); + R2(a, b, c, d, e, f, g, h, 44, 2, XW); SCHED_W_2(48, W0, W1, W2, W3, W4, W5); + + /* Transform 45-47 + Precalc 51-53 */ + R2(d, a, b, c, h, e, f, g, 45, 0, XW); SCHED_W_0(51, W1, W2, W3, W4, W5, W0); + R2(c, d, a, b, g, h, e, f, 46, 1, XW); SCHED_W_1(51, W1, W2, W3, W4, W5, W0); + R2(b, c, d, a, f, g, h, e, 47, 2, XW); SCHED_W_2(51, W1, W2, W3, W4, W5, W0); + + /* Transform 48-50 + Precalc 54-56 */ + R2(a, b, c, d, e, f, g, h, 48, 0, XW); SCHED_W_0(54, W2, W3, W4, W5, W0, W1); + R2(d, a, b, c, h, e, f, g, 49, 1, XW); SCHED_W_1(54, W2, W3, W4, W5, W0, W1); + R2(c, d, a, b, g, h, e, f, 50, 2, XW); SCHED_W_2(54, W2, W3, W4, W5, W0, W1); + + /* Transform 51-53 + Precalc 57-59 */ + R2(b, c, d, a, f, g, h, e, 51, 0, XW); SCHED_W_0(57, W3, W4, W5, W0, W1, W2); + R2(a, b, c, d, e, f, g, h, 52, 1, XW); SCHED_W_1(57, W3, W4, W5, W0, W1, W2); + R2(d, a, b, c, h, e, f, g, 53, 2, XW); SCHED_W_2(57, W3, W4, W5, W0, W1, W2); + + /* Transform 54-56 + Precalc 60-62 */ + R2(c, d, a, b, g, h, e, f, 54, 0, XW); SCHED_W_0(60, W4, W5, W0, W1, W2, W3); + R2(b, c, d, a, f, g, h, e, 55, 1, XW); SCHED_W_1(60, W4, W5, W0, W1, W2, W3); + R2(a, b, c, d, e, f, g, h, 56, 2, XW); SCHED_W_2(60, W4, W5, W0, W1, W2, W3); + + /* Transform 57-59 + Precalc 63 */ + R2(d, a, b, c, h, e, f, g, 57, 0, XW); SCHED_W_0(63, W5, W0, W1, W2, W3, W4); + R2(c, d, a, b, g, h, e, f, 58, 1, XW); + R2(b, c, d, a, f, g, h, e, 59, 2, XW); SCHED_W_1(63, W5, W0, W1, W2, W3, W4); + + /* Transform 60-62 + Precalc 63 */ + R2(a, b, c, d, e, f, g, h, 60, 0, XW); + R2(d, a, b, c, h, e, f, g, 61, 1, XW); SCHED_W_2(63, W5, W0, W1, W2, W3, W4); + R2(c, d, a, b, g, h, e, f, 62, 2, XW); + + /* Transform 63 */ + R2(b, c, d, a, f, g, h, e, 63, 0, XW); + + /* Update the chaining variables. */ + xorl state_h0(RSTATE), a; + xorl state_h1(RSTATE), b; + xorl state_h2(RSTATE), c; + xorl state_h3(RSTATE), d; + movl a, state_h0(RSTATE); + movl b, state_h1(RSTATE); + movl c, state_h2(RSTATE); + movl d, state_h3(RSTATE); + xorl state_h4(RSTATE), e; + xorl state_h5(RSTATE), f; + xorl state_h6(RSTATE), g; + xorl state_h7(RSTATE), h; + movl e, state_h4(RSTATE); + movl f, state_h5(RSTATE); + movl g, state_h6(RSTATE); + movl h, state_h7(RSTATE); + + cmpq $0, RNBLKS; + jne .Loop; + + vzeroall; + + movq (STACK_REG_SAVE + 0 * 8)(%rsp), %rbx; + movq (STACK_REG_SAVE + 1 * 8)(%rsp), %r15; + movq (STACK_REG_SAVE + 2 * 8)(%rsp), %r14; + movq (STACK_REG_SAVE + 3 * 8)(%rsp), %r13; + movq (STACK_REG_SAVE + 4 * 8)(%rsp), %r12; + + vmovdqa %xmm0, IW_W1_ADDR(0, 0); + vmovdqa %xmm0, IW_W1W2_ADDR(0, 0); + vmovdqa %xmm0, IW_W1_ADDR(4, 0); + vmovdqa %xmm0, IW_W1W2_ADDR(4, 0); + vmovdqa %xmm0, IW_W1_ADDR(8, 0); + vmovdqa %xmm0, IW_W1W2_ADDR(8, 0); + + movq %rbp, %rsp; + popq %rbp; + RET; +SYM_FUNC_END(sm3_transform_avx) diff --git a/lib/crypto/x86/sm3.h b/lib/crypto/x86/sm3.h new file mode 100644 index 000000000000..3834780f2f6a --- /dev/null +++ b/lib/crypto/x86/sm3.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SM3 optimized for x86_64 + * + * Copyright 2026 Google LLC + */ +#include <asm/fpu/api.h> +#include <linux/static_call.h> + +asmlinkage void sm3_transform_avx(struct sm3_block_state *state, + const u8 *data, size_t nblocks); + +static void sm3_blocks_avx(struct sm3_block_state *state, + const u8 *data, size_t nblocks) +{ + if (likely(irq_fpu_usable())) { + kernel_fpu_begin(); + sm3_transform_avx(state, data, nblocks); + kernel_fpu_end(); + } else { + sm3_blocks_generic(state, data, nblocks); + } +} + +DEFINE_STATIC_CALL(sm3_blocks_x86, sm3_blocks_generic); + +static void sm3_blocks(struct sm3_block_state *state, + const u8 *data, size_t nblocks) +{ + static_call(sm3_blocks_x86)(state, data, nblocks); +} + +#define sm3_mod_init_arch sm3_mod_init_arch +static void sm3_mod_init_arch(void) +{ + if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_BMI2) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) + static_call_update(sm3_blocks_x86, sm3_blocks_avx); +} diff --git a/lib/debugobjects.c b/lib/debugobjects.c index 12f50de85b62..877f7675ba26 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -711,6 +711,50 @@ static struct debug_obj *lookup_object_or_alloc(void *addr, struct debug_bucket return NULL; } +static inline bool debug_objects_is_pi_blocked_on(void) +{ +#ifdef CONFIG_RT_MUTEXES + return current->pi_blocked_on != NULL; +#else + return false; +#endif +} + +static inline bool can_fill_pool(void) +{ + /* + * On !RT enabled kernels there are no restrictions and spinlock_t and + * raw_spinlock_t are the same types. + */ + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + return true; + + /* + * On RT enabled kernels, the task must not be blocked on a lock as + * that could corrupt the PI state when blocking on a lock in the + * allocation path. + */ + if (debug_objects_is_pi_blocked_on()) + return false; + + /* + * On RT enabled kernels the pool refill should happen in preemptible + * context. + */ + if (preemptible()) + return true; + + /* + * Though during system boot before scheduling is set up, preemption is + * disabled and the pool can get exhausted. Before scheduling is active + * a task cannot be blocked on a sleeping lock, but it might hold a lock + * and if interrupted then hard interrupt context might run into a lock + * inversion. So exclude hard interrupt context from allocations before + * scheduling is active. + */ + return system_state < SYSTEM_SCHEDULING && !in_hardirq(); +} + static void debug_objects_fill_pool(void) { if (!static_branch_likely(&obj_cache_enabled)) @@ -725,17 +769,11 @@ static void debug_objects_fill_pool(void) if (likely(!pool_should_refill(&pool_global))) return; - /* - * On RT enabled kernels the pool refill must happen in preemptible - * context -- for !RT kernels we rely on the fact that spinlock_t and - * raw_spinlock_t are basically the same type and this lock-type - * inversion works just fine. - */ - if (!IS_ENABLED(CONFIG_PREEMPT_RT) || preemptible() || system_state < SYSTEM_SCHEDULING) { + if (can_fill_pool()) { /* * Annotate away the spinlock_t inside raw_spinlock_t warning * by temporarily raising the wait-type to LD_WAIT_CONFIG, matching - * the preemptible() condition above. + * the preemptible() condition in can_fill_pool(). */ static DEFINE_WAIT_OVERRIDE_MAP(fill_pool_map, LD_WAIT_CONFIG); lock_map_acquire_try(&fill_pool_map); @@ -856,6 +894,14 @@ int debug_object_activate(void *addr, const struct debug_obj_descr *descr) } raw_spin_unlock_irqrestore(&db->lock, flags); + + /* + * lookup_object_or_alloc() might have raced with a concurrent + * allocation failure which disabled debug objects. + */ + if (!debug_objects_enabled) + return 0; + debug_print_object(&o, "activate"); switch (o.state) { @@ -1024,7 +1070,7 @@ void debug_object_assert_init(void *addr, const struct debug_obj_descr *descr) raw_spin_lock_irqsave(&db->lock, flags); obj = lookup_object_or_alloc(addr, db, descr, false, true); raw_spin_unlock_irqrestore(&db->lock, flags); - if (likely(!IS_ERR_OR_NULL(obj))) + if (!IS_ERR_OR_NULL(obj)) return; /* If NULL the allocation has hit OOM */ @@ -1033,6 +1079,15 @@ void debug_object_assert_init(void *addr, const struct debug_obj_descr *descr) return; } + /* + * lookup_object_or_alloc() might have raced with a concurrent + * allocation failure which disabled debug objects. Don't run the fixup + * as it might turn a valid object useless. See for example + * hrtimer_fixup_assert_init(). + */ + if (!debug_objects_enabled) + return; + /* Object is neither tracked nor static. It's not initialized. */ debug_print_object(&o, "assert_init"); debug_object_fixup(descr->fixup_assert_init, addr, ODEBUG_STATE_NOTAVAILABLE); @@ -1212,7 +1267,7 @@ struct self_test { static __initconst const struct debug_obj_descr descr_type_test; -static bool __init is_static_object(void *addr) +static __noipa bool __init is_static_object(void *addr) { struct self_test *obj = addr; diff --git a/lib/decompress_bunzip2.c b/lib/decompress_bunzip2.c index ca736166f100..1288f146661f 100644 --- a/lib/decompress_bunzip2.c +++ b/lib/decompress_bunzip2.c @@ -135,7 +135,7 @@ static unsigned int INIT get_bits(struct bunzip_data *bd, char bits_wanted) } /* Avoid 32-bit overflow (dump bit buffer to top of output) */ if (bd->inbufBitCount >= 24) { - bits = bd->inbufBits&((1 << bd->inbufBitCount)-1); + bits = bd->inbufBits & ((1ULL << bd->inbufBitCount) - 1); bits_wanted -= bd->inbufBitCount; bits <<= bits_wanted; bd->inbufBitCount = 0; @@ -146,7 +146,7 @@ static unsigned int INIT get_bits(struct bunzip_data *bd, char bits_wanted) } /* Calculate result */ bd->inbufBitCount -= bits_wanted; - bits |= (bd->inbufBits >> bd->inbufBitCount)&((1 << bits_wanted)-1); + bits |= (bd->inbufBits >> bd->inbufBitCount) & ((1ULL << bits_wanted) - 1); return bits; } diff --git a/lib/embedded-cmdline.S b/lib/embedded-cmdline.S new file mode 100644 index 000000000000..bda81b4a42be --- /dev/null +++ b/lib/embedded-cmdline.S @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Embed the build-time-rendered bootconfig "kernel" subtree as a flat + * cmdline string. setup_arch() prepends this to boot_command_line on + * architectures that select ARCH_SUPPORTS_CMDLINE_FROM_BOOTCONFIG. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates + * Copyright (c) 2026 Breno Leitao <leitao@debian.org> + */ + .section .init.rodata.embed_cmdline, "a", %progbits + .global embedded_kernel_cmdline +embedded_kernel_cmdline: + .incbin "lib/embedded_cmdline.bin" + .byte 0 + .global embedded_kernel_cmdline_end +embedded_kernel_cmdline_end: diff --git a/lib/error-inject.c b/lib/error-inject.c index f3d1b70be605..32f3d1ca9ea2 100644 --- a/lib/error-inject.c +++ b/lib/error-inject.c @@ -219,9 +219,9 @@ static int __init ei_debugfs_init(void) dir = debugfs_create_dir("error_injection", NULL); file = debugfs_create_file("list", 0444, dir, NULL, &ei_fops); - if (!file) { + if (IS_ERR(file)) { debugfs_remove(dir); - return -ENOMEM; + return PTR_ERR(file); } return 0; diff --git a/lib/find_bit.c b/lib/find_bit.c index 5a0066c26d9a..5ac52dfce730 100644 --- a/lib/find_bit.c +++ b/lib/find_bit.c @@ -172,13 +172,6 @@ unsigned long __find_nth_and_bit(const unsigned long *addr1, const unsigned long } EXPORT_SYMBOL(__find_nth_and_bit); -unsigned long __find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, - unsigned long size, unsigned long n) -{ - return FIND_NTH_BIT(addr1[idx] & ~addr2[idx], size, n); -} -EXPORT_SYMBOL(__find_nth_andnot_bit); - unsigned long __find_nth_and_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, const unsigned long *addr3, diff --git a/lib/find_bit_benchmark.c b/lib/find_bit_benchmark.c index 402e160e7186..05305e655f99 100644 --- a/lib/find_bit_benchmark.c +++ b/lib/find_bit_benchmark.c @@ -30,18 +30,20 @@ static DECLARE_BITMAP(bitmap, BITMAP_LEN) __initdata; static DECLARE_BITMAP(bitmap2, BITMAP_LEN) __initdata; /* - * This is Schlemiel the Painter's algorithm. It should be called after - * all other tests for the same bitmap because it sets all bits of bitmap to 1. + * This is Schlemiel the Painter's algorithm. */ -static int __init test_find_first_bit(void *bitmap, unsigned long len) +static int __init test_find_first_bit(const void *bitmap, unsigned long len) { + static DECLARE_BITMAP(cp, BITMAP_LEN) __initdata; unsigned long i, cnt; ktime_t time; + bitmap_copy(cp, bitmap, BITMAP_LEN); + time = ktime_get(); for (cnt = i = 0; i < len; cnt++) { - i = find_first_bit(bitmap, len); - __clear_bit(i, bitmap); + i = find_first_bit(cp, len); + __clear_bit(i, cp); } time = ktime_get() - time; pr_err("find_first_bit: %18llu ns, %6ld iterations\n", time, cnt); @@ -49,7 +51,8 @@ static int __init test_find_first_bit(void *bitmap, unsigned long len) return 0; } -static int __init test_find_first_and_bit(void *bitmap, const void *bitmap2, unsigned long len) +static int __init test_find_first_and_bit(const void *bitmap, const void *bitmap2, + unsigned long len) { static DECLARE_BITMAP(cp, BITMAP_LEN) __initdata; unsigned long i, cnt; @@ -146,6 +149,21 @@ static int __init test_find_next_and_bit(const void *bitmap, return 0; } +static int __init +test_bitmap_find_next_zero_area_off(unsigned long *bitmap, unsigned long len) +{ + unsigned long i, cnt; + ktime_t time; + + time = ktime_get(); + for (cnt = i = 0; i < BITMAP_LEN; cnt++) + i = bitmap_find_next_zero_area_off(bitmap, BITMAP_LEN, i, 8, 0, 0) + 1; + time = ktime_get() - time; + pr_err("bitmap_find_next_zero_area_off:%7llu ns, %6ld iterations\n", time, cnt); + + return 0; +} + static int __init find_bit_test(void) { unsigned long nbits = BITMAP_LEN / SPARSE; @@ -155,6 +173,7 @@ static int __init find_bit_test(void) get_random_bytes(bitmap, sizeof(bitmap)); get_random_bytes(bitmap2, sizeof(bitmap2)); + test_bitmap_find_next_zero_area_off(bitmap, BITMAP_LEN); test_find_next_bit(bitmap, BITMAP_LEN); test_find_next_zero_bit(bitmap, BITMAP_LEN); test_find_last_bit(bitmap, BITMAP_LEN); @@ -178,6 +197,7 @@ static int __init find_bit_test(void) __set_bit(get_random_u32_below(BITMAP_LEN), bitmap2); } + test_bitmap_find_next_zero_area_off(bitmap, BITMAP_LEN); test_find_next_bit(bitmap, BITMAP_LEN); test_find_next_zero_bit(bitmap, BITMAP_LEN); test_find_last_bit(bitmap, BITMAP_LEN); diff --git a/lib/fonts/Makefile b/lib/fonts/Makefile index 30a85a4292fa..7202a70a56ef 100644 --- a/lib/fonts/Makefile +++ b/lib/fonts/Makefile @@ -1,23 +1,23 @@ # SPDX-License-Identifier: GPL-2.0 # Font handling -font-objs := fonts.o +font-y := fonts.o +font-$(CONFIG_FRAMEBUFFER_CONSOLE_ROTATION) += font_rotate.o -font-objs-$(CONFIG_FONT_SUN8x16) += font_sun8x16.o -font-objs-$(CONFIG_FONT_SUN12x22) += font_sun12x22.o -font-objs-$(CONFIG_FONT_8x8) += font_8x8.o -font-objs-$(CONFIG_FONT_8x16) += font_8x16.o -font-objs-$(CONFIG_FONT_6x11) += font_6x11.o -font-objs-$(CONFIG_FONT_7x14) += font_7x14.o -font-objs-$(CONFIG_FONT_10x18) += font_10x18.o -font-objs-$(CONFIG_FONT_PEARL_8x8) += font_pearl_8x8.o -font-objs-$(CONFIG_FONT_ACORN_8x8) += font_acorn_8x8.o -font-objs-$(CONFIG_FONT_MINI_4x6) += font_mini_4x6.o -font-objs-$(CONFIG_FONT_6x10) += font_6x10.o -font-objs-$(CONFIG_FONT_TER10x18) += font_ter10x18.o -font-objs-$(CONFIG_FONT_TER16x32) += font_ter16x32.o -font-objs-$(CONFIG_FONT_6x8) += font_6x8.o +# Built-in fonts; sorted by Family-Size in ascending order +font-$(CONFIG_FONT_6x8) += font_6x8.o +font-$(CONFIG_FONT_6x10) += font_6x10.o +font-$(CONFIG_FONT_6x11) += font_6x11.o +font-$(CONFIG_FONT_7x14) += font_7x14.o +font-$(CONFIG_FONT_8x8) += font_8x8.o +font-$(CONFIG_FONT_8x16) += font_8x16.o +font-$(CONFIG_FONT_10x18) += font_10x18.o +font-$(CONFIG_FONT_ACORN_8x8) += font_acorn_8x8.o +font-$(CONFIG_FONT_MINI_4x6) += font_mini_4x6.o +font-$(CONFIG_FONT_PEARL_8x8) += font_pearl_8x8.o +font-$(CONFIG_FONT_SUN8x16) += font_sun8x16.o +font-$(CONFIG_FONT_SUN12x22) += font_sun12x22.o +font-$(CONFIG_FONT_TER10x18) += font_ter10x18.o +font-$(CONFIG_FONT_TER16x32) += font_ter16x32.o -font-objs += $(font-objs-y) - -obj-$(CONFIG_FONT_SUPPORT) += font.o +obj-$(CONFIG_FONT_SUPPORT) += font.o diff --git a/lib/fonts/font.h b/lib/fonts/font.h new file mode 100644 index 000000000000..4f1adf0b6b54 --- /dev/null +++ b/lib/fonts/font.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _LIB_FONTS_FONT_H +#define _LIB_FONTS_FONT_H + +#include <linux/font.h> + +/* + * Font data + */ + +#define FONT_EXTRA_WORDS 4 + +struct font_data { + unsigned int extra[FONT_EXTRA_WORDS]; + unsigned char data[]; +} __packed; + +/* + * Built-in fonts + */ + +#define VGA8x8_IDX 0 +#define VGA8x16_IDX 1 +#define PEARL8x8_IDX 2 +#define VGA6x11_IDX 3 +#define FONT7x14_IDX 4 +#define FONT10x18_IDX 5 +#define SUN8x16_IDX 6 +#define SUN12x22_IDX 7 +#define ACORN8x8_IDX 8 +#define MINI4x6_IDX 9 +#define FONT6x10_IDX 10 +#define TER16x32_IDX 11 +#define FONT6x8_IDX 12 +#define TER10x18_IDX 13 + +#endif diff --git a/lib/fonts/font_10x18.c b/lib/fonts/font_10x18.c index 5d940db626e7..10edebc4bb74 100644 --- a/lib/fonts/font_10x18.c +++ b/lib/fonts/font_10x18.c @@ -4,7 +4,7 @@ * by Jurriaan Kalkman 06-2005 * ********************************/ -#include <linux/font.h> +#include "font.h" #define FONTDATAMAX 9216 diff --git a/lib/fonts/font_6x10.c b/lib/fonts/font_6x10.c index e65df019e0d2..660d3a371b30 100644 --- a/lib/fonts/font_6x10.c +++ b/lib/fonts/font_6x10.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 -#include <linux/font.h> + +#include "font.h" #define FONTDATAMAX 2560 diff --git a/lib/fonts/font_6x11.c b/lib/fonts/font_6x11.c index bd76b3f6b635..671487ccc172 100644 --- a/lib/fonts/font_6x11.c +++ b/lib/fonts/font_6x11.c @@ -5,7 +5,7 @@ /* */ /**********************************************/ -#include <linux/font.h> +#include "font.h" #define FONTDATAMAX (11*256) diff --git a/lib/fonts/font_6x8.c b/lib/fonts/font_6x8.c index 06ace7792521..5811ee07f4d8 100644 --- a/lib/fonts/font_6x8.c +++ b/lib/fonts/font_6x8.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 -#include <linux/font.h> + +#include "font.h" #define FONTDATAMAX 2048 diff --git a/lib/fonts/font_7x14.c b/lib/fonts/font_7x14.c index a2f561c9fa04..0c7475d643c8 100644 --- a/lib/fonts/font_7x14.c +++ b/lib/fonts/font_7x14.c @@ -4,7 +4,7 @@ /* by Jurriaan Kalkman 05-2005 */ /**************************************/ -#include <linux/font.h> +#include "font.h" #define FONTDATAMAX 3584 diff --git a/lib/fonts/font_8x16.c b/lib/fonts/font_8x16.c index 06ae14088514..523e95c75569 100644 --- a/lib/fonts/font_8x16.c +++ b/lib/fonts/font_8x16.c @@ -5,9 +5,10 @@ /* */ /**********************************************/ -#include <linux/font.h> #include <linux/module.h> +#include "font.h" + #define FONTDATAMAX 4096 static const struct font_data fontdata_8x16 = { diff --git a/lib/fonts/font_8x8.c b/lib/fonts/font_8x8.c index 69570b8c31af..e5b697fc9675 100644 --- a/lib/fonts/font_8x8.c +++ b/lib/fonts/font_8x8.c @@ -5,7 +5,7 @@ /* */ /**********************************************/ -#include <linux/font.h> +#include "font.h" #define FONTDATAMAX 2048 diff --git a/lib/fonts/font_acorn_8x8.c b/lib/fonts/font_acorn_8x8.c index 18755c33d249..4ff52c79f8c4 100644 --- a/lib/fonts/font_acorn_8x8.c +++ b/lib/fonts/font_acorn_8x8.c @@ -1,11 +1,16 @@ // SPDX-License-Identifier: GPL-2.0 /* Acorn-like font definition, with PC graphics characters */ -#include <linux/font.h> +#include "font.h" #define FONTDATAMAX 2048 +#ifdef BOOTLOADER +/* The acorndata_8x8 symbol is needed by the ARM bootloader too. */ +const struct font_data acorndata_8x8 = { +#else static const struct font_data acorndata_8x8 = { +#endif { 0, 0, FONTDATAMAX, 0 }, { /* 00 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* ^@ */ /* 01 */ 0x7e, 0x81, 0xa5, 0x81, 0xbd, 0x99, 0x81, 0x7e, /* ^A */ @@ -68,7 +73,7 @@ static const struct font_data acorndata_8x8 = { /* 3A */ 0x00, 0x00, 0x18, 0x18, 0x00, 0x18, 0x18, 0x00, /* : */ /* 3B */ 0x00, 0x00, 0x18, 0x18, 0x00, 0x18, 0x18, 0x30, /* ; */ /* 3C */ 0x0C, 0x18, 0x30, 0x60, 0x30, 0x18, 0x0C, 0x00, /* < */ -/* 3D */ 0x00, 0x00, 0x7E, 0x00, 0x7E, 0x00, 0x00, 0x00, /* = */ +/* 3D */ 0x00, 0x00, 0x7E, 0x00, 0x7E, 0x00, 0x00, 0x00, /* = */ /* 3E */ 0x30, 0x18, 0x0C, 0x06, 0x0C, 0x18, 0x30, 0x00, /* > */ /* 3F */ 0x3C, 0x66, 0x0C, 0x18, 0x18, 0x00, 0x18, 0x00, /* ? */ /* 40 */ 0x3C, 0x66, 0x6E, 0x6A, 0x6E, 0x60, 0x3C, 0x00, /* @ */ diff --git a/lib/fonts/font_mini_4x6.c b/lib/fonts/font_mini_4x6.c index 8d39fd447952..dc919c160dde 100644 --- a/lib/fonts/font_mini_4x6.c +++ b/lib/fonts/font_mini_4x6.c @@ -18,15 +18,15 @@ s{((0x)?[0-9a-fA-F]+)(.*\[([\*\ ]{4})\])}{ ($num,$pat,$bits) = ($1,$3,$4); - + $bits =~ s/([^\s0])|(.)/ defined($1) + 0 /ge; - + $num = ord(pack("B8", $bits)); $num |= $num >> 4; $num = sprintf("0x%.2x", $num); - + #print "$num,$pat,$bits\n"; - + $num . $pat; }ge; @@ -39,7 +39,7 @@ __END__; MSBit to LSBit = left to right. */ -#include <linux/font.h> +#include "font.h" #define FONTDATAMAX 1536 diff --git a/lib/fonts/font_pearl_8x8.c b/lib/fonts/font_pearl_8x8.c index ae98ca17982e..2438b374acea 100644 --- a/lib/fonts/font_pearl_8x8.c +++ b/lib/fonts/font_pearl_8x8.c @@ -10,7 +10,7 @@ /* */ /**********************************************/ -#include <linux/font.h> +#include "font.h" #define FONTDATAMAX 2048 diff --git a/lib/fonts/font_rotate.c b/lib/fonts/font_rotate.c new file mode 100644 index 000000000000..275406008823 --- /dev/null +++ b/lib/fonts/font_rotate.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Font rotation + * + * Copyright (C) 2005 Antonino Daplas <adaplas @pol.net> + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file COPYING in the main directory of this archive for + * more details. + */ + +#include <linux/errno.h> +#include <linux/export.h> +#include <linux/math.h> +#include <linux/overflow.h> +#include <linux/slab.h> +#include <linux/string.h> + +#include "font.h" + +/* number of bits per line */ +static unsigned int font_glyph_bit_pitch(unsigned int width) +{ + return round_up(width, 8); +} + +static unsigned int __font_glyph_pos(unsigned int x, unsigned int y, unsigned int bit_pitch, + unsigned int *bit) +{ + unsigned int off = y * bit_pitch + x; + unsigned int bit_shift = off % 8; + + *bit = 0x80 >> bit_shift; /* MSB has position 0, LSB has position 7 */ + + return off / 8; +} + +static bool font_glyph_test_bit(const unsigned char *glyph, unsigned int x, unsigned int y, + unsigned int bit_pitch) +{ + unsigned int bit; + unsigned int i = __font_glyph_pos(x, y, bit_pitch, &bit); + + return glyph[i] & bit; +} + +static void font_glyph_set_bit(unsigned char *glyph, unsigned int x, unsigned int y, + unsigned int bit_pitch) +{ + unsigned int bit; + unsigned int i = __font_glyph_pos(x, y, bit_pitch, &bit); + + glyph[i] |= bit; +} + +static void __font_glyph_rotate_90(const unsigned char *glyph, + unsigned int width, unsigned int height, + unsigned char *out) +{ + unsigned int x, y; + unsigned int shift = (8 - (height % 8)) & 7; + unsigned int bit_pitch = font_glyph_bit_pitch(width); + unsigned int out_bit_pitch = font_glyph_bit_pitch(height); + + for (y = 0; y < height; y++) { + for (x = 0; x < width; x++) { + if (font_glyph_test_bit(glyph, x, y, bit_pitch)) { + font_glyph_set_bit(out, out_bit_pitch - 1 - y - shift, x, + out_bit_pitch); + } + } + } +} + +/** + * font_glyph_rotate_90 - Rotate a glyph pattern by 90° in clockwise direction + * @glyph: The glyph to rotate + * @width: The glyph width in bits per scanline + * @height: The number of scanlines in the glyph + * @out: The rotated glyph bitmap + * + * The parameters @width and @height refer to the input glyph given in @glyph. + * The caller has to provide the output buffer @out of sufficient size to hold + * the rotated glyph. Rotating by 90° flips the width and height for the output + * glyph. Depending on the glyph pitch, the size of the output glyph can be + * different than the size of the input. Callers have to take this into account + * when allocating the output memory. + */ +void font_glyph_rotate_90(const unsigned char *glyph, unsigned int width, unsigned int height, + unsigned char *out) +{ + memset(out, 0, font_glyph_size(height, width)); /* flip width/height */ + + __font_glyph_rotate_90(glyph, width, height, out); +} +EXPORT_SYMBOL_GPL(font_glyph_rotate_90); + +static void __font_glyph_rotate_180(const unsigned char *glyph, + unsigned int width, unsigned int height, + unsigned char *out) +{ + unsigned int x, y; + unsigned int shift = (8 - (width % 8)) & 7; + unsigned int bit_pitch = font_glyph_bit_pitch(width); + + for (y = 0; y < height; y++) { + for (x = 0; x < width; x++) { + if (font_glyph_test_bit(glyph, x, y, bit_pitch)) { + font_glyph_set_bit(out, bit_pitch - 1 - x - shift, height - 1 - y, + bit_pitch); + } + } + } +} + +/** + * font_glyph_rotate_180 - Rotate a glyph pattern by 180° + * @glyph: The glyph to rotate + * @width: The glyph width in bits per scanline + * @height: The number of scanlines in the glyph + * @out: The rotated glyph bitmap + * + * The parameters @width and @height refer to the input glyph given in @glyph. + * The caller has to provide the output buffer @out of sufficient size to hold + * the rotated glyph. + */ +void font_glyph_rotate_180(const unsigned char *glyph, unsigned int width, unsigned int height, + unsigned char *out) +{ + memset(out, 0, font_glyph_size(width, height)); + + __font_glyph_rotate_180(glyph, width, height, out); +} +EXPORT_SYMBOL_GPL(font_glyph_rotate_180); + +static void __font_glyph_rotate_270(const unsigned char *glyph, + unsigned int width, unsigned int height, + unsigned char *out) +{ + unsigned int x, y; + unsigned int shift = (8 - (width % 8)) & 7; + unsigned int bit_pitch = font_glyph_bit_pitch(width); + unsigned int out_bit_pitch = font_glyph_bit_pitch(height); + + for (y = 0; y < height; y++) { + for (x = 0; x < width; x++) { + if (font_glyph_test_bit(glyph, x, y, bit_pitch)) + font_glyph_set_bit(out, y, bit_pitch - 1 - x - shift, + out_bit_pitch); + } + } +} + +/** + * font_glyph_rotate_270 - Rotate a glyph pattern by 270° in clockwise direction + * @glyph: The glyph to rotate + * @width: The glyph width in bits per scanline + * @height: The number of scanlines in the glyph + * @out: The rotated glyph bitmap + * + * The parameters @width and @height refer to the input glyph given in @glyph. + * The caller has to provide the output buffer @out of sufficient size to hold + * the rotated glyph. Rotating by 270° flips the width and height for the output + * glyph. Depending on the glyph pitch, the size of the output glyph can be + * different than the size of the input. Callers have to take this into account + * when allocating the output memory. + */ +void font_glyph_rotate_270(const unsigned char *glyph, unsigned int width, unsigned int height, + unsigned char *out) +{ + memset(out, 0, font_glyph_size(height, width)); /* flip width/height */ + + __font_glyph_rotate_270(glyph, width, height, out); +} +EXPORT_SYMBOL_GPL(font_glyph_rotate_270); + +/** + * font_data_rotate - Rotate font data by multiples of 90° + * @fd: The font data to rotate + * @width: The glyph width in bits per scanline + * @height: The number of scanlines in the glyph + * @charcount: The number of glyphs in the font + * @steps: Number of rotation steps of 90° + * @buf: Preallocated output buffer; can be NULL + * @bufsize: The size of @buf in bytes; can be NULL + * + * The parameters @width and @height refer to the visible number of pixels + * and scanlines in a single glyph. The number of glyphs is given in @charcount. + * Rotation happens in steps of 90°. The @steps parameter can have any value, + * but only 0 to 3 produce distinct results. With 4 or higher, a full rotation + * has been performed. You can pass any value for @steps and the helper will + * perform the appropriate rotation. Note that the returned buffer is not + * compatible with font_data_t. It only contains glyph data in the same format + * as returned by font_data_buf(). Callers are responsible to free the returned + * buffer with kfree(). Font rotation typically happens when displays get + * re-oriented. To avoid unnecessary re-allocation of the memory buffer, the + * caller can pass in an earlier result buffer in @buf for reuse. The old and + * new buffer sizes are given and retrieved by the caller in @bufsize. The + * allocation semantics are compatible with krealloc(). + * + * Returns: + * A buffer with rotated glyphs on success, or an error pointer otherwise + */ +unsigned char *font_data_rotate(font_data_t *fd, unsigned int width, unsigned int height, + unsigned int charcount, unsigned int steps, + unsigned char *buf, size_t *bufsize) +{ + const unsigned char *src = font_data_buf(fd); + unsigned int s_cellsize = font_glyph_size(width, height); + unsigned int d_cellsize, i; + unsigned char *dst; + size_t size; + + steps %= 4; + + switch (steps) { + case 0: + case 2: + d_cellsize = s_cellsize; + break; + case 1: + case 3: + d_cellsize = font_glyph_size(height, width); /* flip width/height */ + break; + } + + if (check_mul_overflow(charcount, d_cellsize, &size)) + return ERR_PTR(-EINVAL); + + if (!buf || !bufsize || size > *bufsize) { + dst = kmalloc_array(charcount, d_cellsize, GFP_KERNEL); + if (!dst) + return ERR_PTR(-ENOMEM); + + kfree(buf); + buf = dst; + if (bufsize) + *bufsize = size; + } else { + dst = buf; + } + + switch (steps) { + case 0: + memcpy(dst, src, size); + break; + case 1: + memset(dst, 0, size); + for (i = 0; i < charcount; ++i) { + __font_glyph_rotate_90(src, width, height, dst); + src += s_cellsize; + dst += d_cellsize; + } + break; + case 2: + memset(dst, 0, size); + for (i = 0; i < charcount; ++i) { + __font_glyph_rotate_180(src, width, height, dst); + src += s_cellsize; + dst += d_cellsize; + } + break; + case 3: + memset(dst, 0, size); + for (i = 0; i < charcount; ++i) { + __font_glyph_rotate_270(src, width, height, dst); + src += s_cellsize; + dst += d_cellsize; + } + break; + } + + return buf; +} +EXPORT_SYMBOL_GPL(font_data_rotate); diff --git a/lib/fonts/font_sun12x22.c b/lib/fonts/font_sun12x22.c index 91daf5ab8b6b..2afbc144bea8 100644 --- a/lib/fonts/font_sun12x22.c +++ b/lib/fonts/font_sun12x22.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 -#include <linux/font.h> + +#include "font.h" #define FONTDATAMAX 11264 diff --git a/lib/fonts/font_sun8x16.c b/lib/fonts/font_sun8x16.c index 81bb4eeae04e..2b7b2d8e548a 100644 --- a/lib/fonts/font_sun8x16.c +++ b/lib/fonts/font_sun8x16.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 -#include <linux/font.h> + +#include "font.h" #define FONTDATAMAX 4096 diff --git a/lib/fonts/font_ter10x18.c b/lib/fonts/font_ter10x18.c index 80356e9d56c7..3f30b4a211ab 100644 --- a/lib/fonts/font_ter10x18.c +++ b/lib/fonts/font_ter10x18.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 -#include <linux/font.h> + #include <linux/module.h> +#include "font.h" + #define FONTDATAMAX 9216 static const struct font_data fontdata_ter10x18 = { diff --git a/lib/fonts/font_ter16x32.c b/lib/fonts/font_ter16x32.c index 5baedc573dd6..93616cffe642 100644 --- a/lib/fonts/font_ter16x32.c +++ b/lib/fonts/font_ter16x32.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 -#include <linux/font.h> + #include <linux/module.h> +#include "font.h" + #define FONTDATAMAX 16384 static const struct font_data fontdata_ter16x32 = { diff --git a/lib/fonts/fonts.c b/lib/fonts/fonts.c index a7f118b30171..4fc66722d00d 100644 --- a/lib/fonts/fonts.c +++ b/lib/fonts/fonts.c @@ -12,13 +12,272 @@ * for more details. */ +#include <linux/container_of.h> +#include <linux/kd.h> #include <linux/module.h> -#include <linux/types.h> +#include <linux/overflow.h> +#include <linux/slab.h> #include <linux/string.h> +#include <linux/types.h> + #if defined(__mc68000__) #include <asm/setup.h> #endif -#include <linux/font.h> + +#include "font.h" + +#define console_font_pitch(font) font_glyph_pitch((font)->width) + +/* + * Helpers for font_data_t + */ + +/* Extra word getters */ +#define REFCOUNT(fd) (((int *)(fd))[-1]) +#define FNTSIZE(fd) (((int *)(fd))[-2]) +#define FNTSUM(fd) (((int *)(fd))[-4]) + +static struct font_data *to_font_data_struct(font_data_t *fd) +{ + return container_of(fd, struct font_data, data[0]); +} + +static bool font_data_is_internal(font_data_t *fd) +{ + return !REFCOUNT(fd); /* internal fonts have no reference counting */ +} + +static void font_data_free(font_data_t *fd) +{ + kfree(to_font_data_struct(fd)); +} + +/** + * font_data_import - Allocates and initializes font data from user space + * @font: A font from user space + * @vpitch: The size of a single glyph in @font in bytes + * @calc_csum: An optional helper to calculate a chechsum + * + * Font data from user space must be translated to the kernel's format. The + * font's glyph geometry and data is provided in @font. The parameter @vpitch + * gives the number of bytes per glyph, including trailing bytes. + * + * The parameter @calc_csum is optional. Fbcon passes crc32() to calculate the + * font data's checksum. + * + * Returns: + * Newly initialized font data on success, or a pointer-encoded errno value otherwise. + */ +font_data_t *font_data_import(const struct console_font *font, unsigned int vpitch, + u32 (*calc_csum)(u32, const void *, size_t)) +{ + unsigned int pitch = console_font_pitch(font); + unsigned int h = font->height; + unsigned int charcount = font->charcount; + const unsigned char *data = font->data; + u32 csum = 0; + struct font_data *font_data; + int size, alloc_size; + unsigned int i; + font_data_t *fd; + + /* Check for integer overflow in font-size calculation */ + if (check_mul_overflow(h, pitch, &size) || + check_mul_overflow(size, charcount, &size)) + return ERR_PTR(-EINVAL); + + /* Check for overflow in allocation size calculation */ + if (check_add_overflow(sizeof(*font_data), size, &alloc_size)) + return ERR_PTR(-EINVAL); + + font_data = kmalloc(alloc_size, GFP_USER); + if (!font_data) + return ERR_PTR(-ENOMEM); + memset(font_data->extra, 0, sizeof(font_data->extra)); + + for (i = 0; i < charcount; ++i) + memcpy(font_data->data + i * h * pitch, data + i * vpitch * pitch, h * pitch); + + if (calc_csum) + csum = calc_csum(0, font_data->data, size); + + fd = font_data->data; + REFCOUNT(fd) = 1; /* start with reference acquired */ + FNTSIZE(fd) = size; + FNTSUM(fd) = csum; + + return fd; +} +EXPORT_SYMBOL_GPL(font_data_import); + +/** + * font_data_get - Acquires a reference on font data + * @fd: Font data + * + * Font data from user space is reference counted. The helper + * font_data_get() increases the reference counter by one. Invoke + * font_data_put() to release the reference. + * + * Internal font data is located in read-only memory. In this case + * the helper returns success without modifying the counter field. + * It is still required to call font_data_put() on internal font data. + */ +void font_data_get(font_data_t *fd) +{ + if (font_data_is_internal(fd)) + return; /* never ref static data */ + + if (WARN_ON(!REFCOUNT(fd))) + return; /* should never be 0 */ + ++REFCOUNT(fd); +} +EXPORT_SYMBOL_GPL(font_data_get); + +/** + * font_data_put - Release a reference on font data + * @fd: Font data + * + * Font data from user space is reference counted. The helper + * font_data_put() decreases the reference counter by one. If this was + * the final reference, it frees the allocated memory. + * + * Internal font data is located in read-only memory. In this case + * the helper returns success without modifying the counter field. + * + * Returns: + * True if the font data's memory buffer has been freed, false otherwise. + */ +bool font_data_put(font_data_t *fd) +{ + unsigned int count; + + if (font_data_is_internal(fd)) + return false; /* never unref static data */ + + if (WARN_ON(!REFCOUNT(fd))) + return false; /* should never be 0 */ + + count = --REFCOUNT(fd); + if (!count) + font_data_free(fd); + + return !count; +} +EXPORT_SYMBOL_GPL(font_data_put); + +/** + * font_data_size - Return size of the font data in bytes + * @fd: Font data + * + * Returns: + * The number of bytes in the given font data. + */ +unsigned int font_data_size(font_data_t *fd) +{ + return FNTSIZE(fd); +} +EXPORT_SYMBOL_GPL(font_data_size); + +static unsigned int font_data_num_glyphs(font_data_t *fd, unsigned int width, unsigned int height) +{ + return font_data_size(fd) / font_glyph_size(width, height); +} + +/** + * font_data_glyph_buf() - Returns the glyph for a specific character as raw bytes + * @fd: The font data + * @width: The glyph width in bits per scanline + * @vpitch: The number of scanlines per glyph + * @c: The character + * + * Glyphs start at fixed intervals within the font data. font_data_glyph_buf() + * returns the glyph shape of the specified character. If no such glyph + * exists in the font, it returns NULL. + * + * Returns: + * The character's raw glyph shape, or NULL if no glyph exists for the character. The + * provided buffer is read-only. + */ +const unsigned char *font_data_glyph_buf(font_data_t *fd, + unsigned int width, unsigned int vpitch, + unsigned int c) +{ + if (c >= font_data_num_glyphs(fd, width, vpitch)) + return NULL; + + return font_data_buf(fd) + font_glyph_size(width, vpitch) * c; +} +EXPORT_SYMBOL_GPL(font_data_glyph_buf); + +/** + * font_data_is_equal - Compares font data for equality + * @lhs: Left-hand side font data + * @rhs: Right-hand-size font data + * + * Font data is equal if is constain the same sequence of values. The + * helper also use the checksum, if both arguments contain it. Font data + * coming from different origins, internal or from user space, is never + * equal. Allowing this would break reference counting. + * + * Returns: + * True if the given font data is equal, false otherwise. + */ +bool font_data_is_equal(font_data_t *lhs, font_data_t *rhs) +{ + if (font_data_is_internal(lhs) != font_data_is_internal(rhs)) + return false; + if (font_data_size(lhs) != font_data_size(rhs)) + return false; + if (FNTSUM(lhs) && FNTSUM(rhs) && FNTSUM(lhs) != FNTSUM(rhs)) + return false; + + return !memcmp(lhs, rhs, FNTSIZE(lhs)); +} +EXPORT_SYMBOL_GPL(font_data_is_equal); + +/** + * font_data_export - Stores font data for user space + * @fd: Font data + * @font: A font for user space + * @vpitch: The size of a single glyph in @font in bytes + * + * Store the font data given in @fd to the font in @font. Values and + * pointers in @font are pre-initialized. This helper mostly checks some + * corner cases and translates glyph sizes according to the value given + * @vpitch. + * + * Returns: + * 0 on success, or a negative errno code otherwise. + */ +int font_data_export(font_data_t *fd, struct console_font *font, unsigned int vpitch) +{ + const unsigned char *font_data = font_data_buf(fd); + unsigned char *data = font->data; + unsigned int pitch = console_font_pitch(font); + unsigned int glyphsize, i; + + if (!font->width || !font->height || !font->charcount || !font->data) + return 0; + + glyphsize = font->height * pitch; + + if (font->charcount * glyphsize > font_data_size(fd)) + return -EINVAL; + + for (i = 0; i < font->charcount; i++) { + memcpy(data, font_data, glyphsize); + memset(data + glyphsize, 0, pitch * vpitch - glyphsize); + data += pitch * vpitch; + font_data += glyphsize; + } + + return 0; +} +EXPORT_SYMBOL_GPL(font_data_export); + +/* + * Font lookup + */ static const struct font_desc *fonts[] = { #ifdef CONFIG_FONT_8x8 diff --git a/lib/glob.c b/lib/glob.c index aa57900d2062..c80d9dd736b4 100644 --- a/lib/glob.c +++ b/lib/glob.c @@ -1,5 +1,7 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) #include <linux/module.h> #include <linux/glob.h> +#include <linux/export.h> /* * The only reason this code can be compiled as a module is because the @@ -9,6 +11,9 @@ MODULE_DESCRIPTION("glob(7) matching"); MODULE_LICENSE("Dual MIT/GPL"); +static bool __pure glob_match_str(char const *pat, char const *str, + char const *str_end); + /** * glob_match - Shell-style pattern matching, like !fnmatch(pat, str, 0) * @pat: Shell-style pattern to match, e.g. "*.[ch]". @@ -20,7 +25,7 @@ MODULE_LICENSE("Dual MIT/GPL"); * Pattern metacharacters are ?, *, [ and \. * (And, inside character classes, !, - and ].) * - * This is small and simple implementation intended for device blacklists + * This is a small and simple implementation intended for device denylists * where a string is matched against a number of patterns. Thus, it * does not preprocess the patterns. It is non-recursive, and run-time * is at most quadratic: strlen(@str)*strlen(@pat). @@ -39,13 +44,36 @@ MODULE_LICENSE("Dual MIT/GPL"); */ bool __pure glob_match(char const *pat, char const *str) { + return glob_match_str(pat, str, NULL); +} +EXPORT_SYMBOL(glob_match); + +/** + * glob_match_len - glob match against a length-bounded string + * @pat: Shell-style pattern to match. + * @str: String to match. Need not be NUL-terminated. + * @len: Number of bytes of @str that may be read. + * + * Like glob_match(), but @str is only read up to @len bytes, so it can be + * used on buffers that are not NUL-terminated (e.g. trace event fields). + * A NUL byte within @len still terminates the string. + */ +bool __pure glob_match_len(char const *pat, char const *str, size_t len) +{ + return glob_match_str(pat, str, str + len); +} +EXPORT_SYMBOL(glob_match_len); + +static bool __pure glob_match_str(char const *pat, char const *str, + char const *str_end) +{ /* * Backtrack to previous * on mismatch and retry starting one * character later in the string. Because * matches all characters * (no exception for /), it can be easily proved that there's * never a need to backtrack multiple levels. */ - char const *back_pat = NULL, *back_str; + char const *back_pat = NULL, *back_str = NULL; /* * Loop over each token (character or class) in pat, matching @@ -53,9 +81,11 @@ bool __pure glob_match(char const *pat, char const *str) * on mismatch, or true after matching the trailing nul bytes. */ for (;;) { - unsigned char c = *str++; + unsigned char c = (str_end && str >= str_end) ? '\0' : *str; unsigned char d = *pat++; + str++; + switch (d) { case '?': /* Wildcard: anything but nul */ if (c == '\0') @@ -71,7 +101,7 @@ bool __pure glob_match(char const *pat, char const *str) if (c == '\0') /* No possible match */ return false; bool match = false, inverted = (*pat == '!'); - char const *class = pat + inverted; + char const *class = inverted ? pat + 1 : pat; unsigned char a = *class++; /* @@ -94,7 +124,8 @@ bool __pure glob_match(char const *pat, char const *str) class += 2; /* Any special action if a > b? */ } - match |= (a <= c && c <= b); + if (a <= c && c <= b) + match = true; } while ((a = *class++) != ']'); if (match == inverted) @@ -122,4 +153,3 @@ backtrack: } } } -EXPORT_SYMBOL(glob_match); diff --git a/lib/inflate.c b/lib/inflate.c index eab886baa1b4..44a7da582baa 100644 --- a/lib/inflate.c +++ b/lib/inflate.c @@ -9,7 +9,7 @@ * based on gzip-1.0.3 * * Nicolas Pitre <nico@fluxnic.net>, 1999/04/14 : - * Little mods for all variable to reside either into rodata or bss segments + * Little mods for all variables to reside either into rodata or bss segments * by marking constant variables with 'const' and initializing all the others * at run-time only. This allows for the kernel uncompressor to run * directly from Flash or ROM memory on embedded systems. @@ -286,7 +286,7 @@ static void free(void *where) the longer codes. The time it costs to decode the longer codes is then traded against the time it takes to make longer tables. - This results of this trade are in the variables lbits and dbits + The results of this trade are in the variables lbits and dbits below. lbits is the number of bits the first level table for literal/ length codes can decode in one step, and dbits is the same thing for the distance codes. Subsequent tables are also less than or equal to @@ -811,6 +811,8 @@ DEBG("<fix"); /* decompress until an end-of-block code */ if (inflate_codes(tl, td, bl, bd)) { + huft_free(tl); + huft_free(td); free(l); return 1; } @@ -1007,10 +1009,10 @@ DEBG("dyn5d "); DEBG("dyn6 "); /* decompress until an end-of-block code */ - if (inflate_codes(tl, td, bl, bd)) { + if (inflate_codes(tl, td, bl, bd)) ret = 1; - goto out; - } + else + ret = 0; DEBG("dyn7 "); @@ -1019,7 +1021,6 @@ DEBG("dyn7 "); huft_free(td); DEBG(">"); - ret = 0; out: free(ll); return ret; diff --git a/lib/interval_tree_test.c b/lib/interval_tree_test.c index 16200feacbf3..b0b07270ce7c 100644 --- a/lib/interval_tree_test.c +++ b/lib/interval_tree_test.c @@ -4,6 +4,7 @@ #include <linux/interval_tree.h> #include <linux/prandom.h> #include <linux/slab.h> +#include <linux/printk.h> #include <asm/timex.h> #include <linux/bitmap.h> #include <linux/maple_tree.h> @@ -139,13 +140,13 @@ static int intersection_range_check(void) intxn1 = bitmap_alloc(nnodes, GFP_KERNEL); if (!intxn1) { - WARN_ON_ONCE("Failed to allocate intxn1\n"); + WARN_ONCE(1, "Failed to allocate intxn1\n"); return -ENOMEM; } intxn2 = bitmap_alloc(nnodes, GFP_KERNEL); if (!intxn2) { - WARN_ON_ONCE("Failed to allocate intxn2\n"); + WARN_ONCE(1, "Failed to allocate intxn2\n"); bitmap_free(intxn1); return -ENOMEM; } @@ -311,6 +312,27 @@ static inline int span_iteration_check(void) {return 0; } static int interval_tree_test_init(void) { + if (nnodes <= 0) { + pr_warn("nnodes must be positive\n"); + return -EINVAL; + } + if (nsearches <= 0) { + pr_warn("nsearches must be positive\n"); + return -EINVAL; + } + if (perf_loops <= 0) { + pr_warn("perf_loops must be positive\n"); + return -EINVAL; + } + if (search_loops <= 0) { + pr_warn("search_loops must be positive\n"); + return -EINVAL; + } + if (max_endpoint < 2) { + pr_warn("max_endpoint must be at least 2\n"); + return -EINVAL; + } + nodes = kmalloc_objs(struct interval_tree_node, nnodes); if (!nodes) return -ENOMEM; diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 0a63c7fba313..2072c04e99d0 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -277,7 +277,7 @@ static __always_inline size_t copy_from_user_iter_nocache(void __user *iter_from, size_t progress, size_t len, void *to, void *priv2) { - return __copy_from_user_inatomic_nocache(to + progress, iter_from, len); + return copy_from_user_inatomic_nontemporal(to + progress, iter_from, len); } size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) @@ -296,7 +296,7 @@ static __always_inline size_t copy_from_user_iter_flushcache(void __user *iter_from, size_t progress, size_t len, void *to, void *priv2) { - return __copy_from_user_flushcache(to + progress, iter_from, len); + return copy_from_user_flushcache(to + progress, iter_from, len); } static __always_inline @@ -1224,13 +1224,13 @@ const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags) { *new = *old; if (iov_iter_is_bvec(new)) - return new->bvec = kmemdup(new->bvec, - new->nr_segs * sizeof(struct bio_vec), + return new->bvec = kmemdup_array(new->bvec, + new->nr_segs, sizeof(struct bio_vec), flags); else if (iov_iter_is_kvec(new) || iter_is_iovec(new)) /* iovec and kvec have identical layout */ - return new->__iov = kmemdup(new->__iov, - new->nr_segs * sizeof(struct iovec), + return new->__iov = kmemdup_array(new->__iov, + new->nr_segs, sizeof(struct iovec), flags); return NULL; } @@ -1491,6 +1491,7 @@ void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state) i->__iov -= state->nr_segs - i->nr_segs; i->nr_segs = state->nr_segs; } +EXPORT_SYMBOL_FOR_MODULES(iov_iter_restore, "vmw_vsock_virtio_transport_common"); /* * Extract a list of contiguous pages from an ITER_FOLIOQ iterator. This does @@ -1568,6 +1569,7 @@ static ssize_t iov_iter_extract_xarray_pages(struct iov_iter *i, struct folio *folio; unsigned int nr = 0, offset; loff_t pos = i->xarray_start + i->iov_offset; + bool will_alloc = !*pages; XA_STATE(xas, i->xarray, pos >> PAGE_SHIFT); offset = pos & ~PAGE_MASK; @@ -1595,6 +1597,14 @@ static ssize_t iov_iter_extract_xarray_pages(struct iov_iter *i, } rcu_read_unlock(); + if (!nr) { + if (will_alloc) { + kvfree(*pages); + *pages = NULL; + } + return 0; + } + maxsize = min_t(size_t, nr * PAGE_SIZE - offset, maxsize); iov_iter_advance(i, maxsize); return maxsize; @@ -1625,9 +1635,11 @@ static ssize_t iov_iter_extract_bvec_pages(struct iov_iter *i, } bi.bi_idx = 0; bi.bi_size = maxsize; - bi.bi_bvec_done = skip; + bi.bi_offset = skip; maxpages = want_pages_array(pages, maxsize, skip, maxpages); + if (!maxpages) + return -ENOMEM; while (bi.bi_size && bi.bi_idx < i->nr_segs) { struct bio_vec bv = bvec_iter_bvec(i->bvec, bi); @@ -1745,6 +1757,7 @@ static ssize_t iov_iter_extract_user_pages(struct iov_iter *i, unsigned long addr; unsigned int gup_flags = 0; size_t offset; + bool will_alloc = !*pages; int res; if (i->data_source == ITER_DEST) @@ -1761,8 +1774,14 @@ static ssize_t iov_iter_extract_user_pages(struct iov_iter *i, if (!maxpages) return -ENOMEM; res = pin_user_pages_fast(addr, maxpages, gup_flags, *pages); - if (unlikely(res <= 0)) + if (unlikely(res <= 0)) { + if (will_alloc) { + kvfree(*pages); + *pages = NULL; + } return res; + } + maxsize = min_t(size_t, maxsize, res * PAGE_SIZE - offset); iov_iter_advance(i, maxsize); return maxsize; @@ -1886,6 +1905,8 @@ static unsigned int get_contig_folio_len(struct page **pages, * @max_size: maximum size to extract from @iter * @nr_vecs: number of vectors in @bv (on in and output) * @max_vecs: maximum vectors in @bv, including those filled before calling + * @mem_align_mask: reject with -EINVAL if the source address or + * length is not aligned to this mask * @extraction_flags: flags to qualify request * * Like iov_iter_extract_pages(), but returns physically contiguous ranges @@ -1897,7 +1918,8 @@ static unsigned int get_contig_folio_len(struct page **pages, */ ssize_t iov_iter_extract_bvecs(struct iov_iter *iter, struct bio_vec *bv, size_t max_size, unsigned short *nr_vecs, - unsigned short max_vecs, iov_iter_extraction_t extraction_flags) + unsigned short max_vecs, unsigned mem_align_mask, + iov_iter_extraction_t extraction_flags) { unsigned short entries_left = max_vecs - *nr_vecs; unsigned short nr_pages, i = 0; @@ -1906,6 +1928,24 @@ ssize_t iov_iter_extract_bvecs(struct iov_iter *iter, struct bio_vec *bv, ssize_t size; /* + * DMA engines typically have both memory address and length alignment + * requirements, so check these against the alignment mask. For UBUF, + * IOVEC and KVEC, only the current segment will be extracted from; for + * everything else we might extract from multiple segments, so we need + * to check those too. + */ + if (likely(iter_is_ubuf(iter) || + iter_is_iovec(iter) || + iov_iter_is_kvec(iter))) { + unsigned long start = (unsigned long)iter_iov_addr(iter); + + if ((start | iter_iov_len(iter)) & mem_align_mask) + return -EINVAL; + } else if (iov_iter_alignment(iter) & mem_align_mask) { + return -EINVAL; + } + + /* * Move page array up in the allocated memory for the bio vecs as far as * possible so that we can start filling biovecs from the beginning * without overwriting the temporary page array. diff --git a/lib/kobject.c b/lib/kobject.c index cfdb2c3f20a2..e7b010a989fb 100644 --- a/lib/kobject.c +++ b/lib/kobject.c @@ -27,7 +27,7 @@ * and thus @kobj should have a namespace tag associated with it. Returns * %NULL otherwise. */ -const void *kobject_namespace(const struct kobject *kobj) +const struct ns_common *kobject_namespace(const struct kobject *kobj) { const struct kobj_ns_type_operations *ns_ops = kobj_ns_ops(kobj); @@ -823,9 +823,11 @@ static ssize_t kobj_attr_show(struct kobject *kobj, struct attribute *attr, struct kobj_attribute *kattr; ssize_t ret = -EIO; - kattr = container_of(attr, struct kobj_attribute, attr); + kattr = container_of_const(attr, struct kobj_attribute, attr); if (kattr->show) ret = kattr->show(kobj, kattr, buf); + else if (kattr->show_const) + ret = kattr->show_const(kobj, kattr, buf); return ret; } @@ -835,9 +837,11 @@ static ssize_t kobj_attr_store(struct kobject *kobj, struct attribute *attr, struct kobj_attribute *kattr; ssize_t ret = -EIO; - kattr = container_of(attr, struct kobj_attribute, attr); + kattr = container_of_const(attr, struct kobj_attribute, attr); if (kattr->store) ret = kattr->store(kobj, kattr, buf, count); + else if (kattr->store_const) + ret = kattr->store_const(kobj, kattr, buf, count); return ret; } @@ -1083,9 +1087,9 @@ bool kobj_ns_current_may_mount(enum kobj_ns_type type) return may_mount; } -void *kobj_ns_grab_current(enum kobj_ns_type type) +struct ns_common *kobj_ns_grab_current(enum kobj_ns_type type) { - void *ns = NULL; + struct ns_common *ns = NULL; spin_lock(&kobj_ns_type_lock); if (kobj_ns_type_is_valid(type) && kobj_ns_ops_tbl[type]) @@ -1096,7 +1100,7 @@ void *kobj_ns_grab_current(enum kobj_ns_type type) } EXPORT_SYMBOL_GPL(kobj_ns_grab_current); -void kobj_ns_drop(enum kobj_ns_type type, void *ns) +void kobj_ns_drop(enum kobj_ns_type type, struct ns_common *ns) { spin_lock(&kobj_ns_type_lock); if (kobj_ns_type_is_valid(type) && diff --git a/lib/kobject_uevent.c b/lib/kobject_uevent.c index 871941c9830c..ddbc4d7482d2 100644 --- a/lib/kobject_uevent.c +++ b/lib/kobject_uevent.c @@ -238,7 +238,7 @@ static int kobj_usermode_filter(struct kobject *kobj) ops = kobj_ns_ops(kobj); if (ops) { - const void *init_ns, *ns; + const struct ns_common *init_ns, *ns; ns = kobj->ktype->namespace(kobj); init_ns = ops->initial_ns(); @@ -388,7 +388,7 @@ static int kobject_uevent_net_broadcast(struct kobject *kobj, #ifdef CONFIG_NET const struct kobj_ns_type_operations *ops; - const struct net *net = NULL; + const struct ns_common *ns = NULL; ops = kobj_ns_ops(kobj); if (!ops && kobj->kset) { @@ -404,14 +404,17 @@ static int kobject_uevent_net_broadcast(struct kobject *kobj, */ if (ops && ops->netlink_ns && kobj->ktype->namespace) if (ops->type == KOBJ_NS_TYPE_NET) - net = kobj->ktype->namespace(kobj); + ns = kobj->ktype->namespace(kobj); - if (!net) + if (!ns) ret = uevent_net_broadcast_untagged(env, action_string, devpath); - else + else { + const struct net *net = container_of(ns, struct net, ns); + ret = uevent_net_broadcast_tagged(net->uevent_sock->sk, env, action_string, devpath); + } #endif return ret; diff --git a/lib/kstrtox.c b/lib/kstrtox.c index 97be2a39f537..bac1c057e1b0 100644 --- a/lib/kstrtox.c +++ b/lib/kstrtox.c @@ -17,6 +17,7 @@ #include <linux/export.h> #include <linux/kstrtox.h> #include <linux/math64.h> +#include <linux/overflow.h> #include <linux/types.h> #include <linux/uaccess.h> @@ -39,25 +40,31 @@ const char *_parse_integer_fixup_radix(const char *s, unsigned int *base) return s; } -/* - * Convert non-negative integer string representation in explicitly given radix - * to an integer. A maximum of max_chars characters will be converted. +/** + * _parse_integer_limit - Convert integer string representation to an integer + * @s: Integer string representation + * @base: Radix + * @p: Where to store result + * @max_chars: Maximum amount of characters to convert + * @init: Initial value of the multiply-accumulate result + * + * Convert non-negative integer string representation in explicitly given + * radix to an integer. If overflow occurs, value at @p is set to ULLONG_MAX. * - * Return number of characters consumed maybe or-ed with overflow bit. - * If overflow occurs, result integer (incorrect) is still returned. + * This function is the workhorse of other string conversion functions and it + * is discouraged to use it explicitly. Consider kstrto*() family instead. * - * Don't you dare use this function. + * Return: Number of characters consumed, maybe ORed with overflow bit */ noinline unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned long long *p, - size_t max_chars) + size_t max_chars, unsigned long long init) { + unsigned int rv, overflow = 0; unsigned long long res; - unsigned int rv; - res = 0; - rv = 0; - while (max_chars--) { + res = init; + for (rv = 0; rv < max_chars; rv++, s++) { unsigned int c = *s; unsigned int lc = _tolower(c); unsigned int val; @@ -76,21 +83,17 @@ unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned lon * it in the max base we support (16) */ if (unlikely(res & (~0ull << 60))) { - if (res > div_u64(ULLONG_MAX - val, base)) - rv |= KSTRTOX_OVERFLOW; + if (check_mul_overflow(res, base, &res) || + check_add_overflow(res, val, &res)) { + res = ULLONG_MAX; + overflow = KSTRTOX_OVERFLOW; + } + } else { + res = res * base + val; } - res = res * base + val; - rv++; - s++; } *p = res; - return rv; -} - -noinline -unsigned int _parse_integer(const char *s, unsigned int base, unsigned long long *p) -{ - return _parse_integer_limit(s, base, p, INT_MAX); + return rv | overflow; } static int _kstrtoull(const char *s, unsigned int base, unsigned long long *res) @@ -392,6 +395,109 @@ int kstrtobool(const char *s, bool *res) } EXPORT_SYMBOL(kstrtobool); +static int _kstrtoudec64(const char *s, unsigned int scale, u64 *res) +{ + unsigned int rv_int, rv_frac; + u64 _res = 0; + + rv_int = _parse_integer(s, 10, &_res); + if (rv_int & KSTRTOX_OVERFLOW) + return -ERANGE; + s += rv_int; + + if (*s == '.') + s++; /* skip decimal point */ + + rv_frac = _parse_integer(s, 10, &_res, scale, _res); + if (rv_frac & KSTRTOX_OVERFLOW) + return -ERANGE; + s += rv_frac; + + /* + * Check input beyond rv_int and rv_frac to cover cases like ".5" with + * scale 0, which is considered a valid input, being parsed as 0. + */ + if (!rv_int && !rv_frac && !isdigit(*s)) + return -EINVAL; + + while (isdigit(*s)) /* truncate digits */ + s++; + + if (*s == '\n') + s++; + if (*s) + return -EINVAL; + + if (_res && ((scale - rv_frac) > 19 /* log10(2^64) = 19.26 */ || + check_mul_overflow(_res, int_pow(10, scale - rv_frac), &_res))) + return -ERANGE; + + *res = _res; + return 0; +} + +/** + * kstrtoudec64() - Convert a string to an unsigned 64-bit scaled decimal value. + * @s: The start of the string. The string must be null-terminated, and may also + * include a single newline before its terminating null. The first character + * may also be a plus sign, but not a minus sign. + * @scale: The number of digits to the right of the decimal point. + * @res: Where to write the result of the conversion on success. + * + * For example, a scale of 3 with input "123.45" results in 123450. Note that + * trailing zeros in the fractional part input to match the scale are not + * required. Also, digits beyond the specified scale are ignored. + * + * Return: 0 on success, -ERANGE on overflow and -EINVAL on parsing error. + */ +noinline +int kstrtoudec64(const char *s, unsigned int scale, u64 *res) +{ + if (s[0] == '+') + s++; + return _kstrtoudec64(s, scale, res); +} +EXPORT_SYMBOL(kstrtoudec64); + +/** + * kstrtodec64() - Convert a string to a signed 64-bit scaled decimal value. + * @s: The start of the string. The string must be null-terminated, and may also + * include a single newline before its terminating null. The first character + * may also be a plus sign or a minus sign. + * @scale: The number of digits to the right of the decimal point. + * @res: Where to write the result of the conversion on success. + * + * For example, a scale of 4 with input "-3.141592" results in -31415. Note + * that digits beyond the specified scale are ignored. Also, trailing zeros in + * the fractional part input to match the scale are not required. + * + * Return: 0 on success, -ERANGE on overflow and -EINVAL on parsing error. + */ +noinline +int kstrtodec64(const char *s, unsigned int scale, s64 *res) +{ + u64 tmp; + int rv; + + if (s[0] == '-') { + rv = _kstrtoudec64(s + 1, scale, &tmp); + if (rv < 0) + return rv; + if ((s64)-tmp > 0) + return -ERANGE; + *res = -tmp; + } else { + rv = kstrtoudec64(s, scale, &tmp); + if (rv < 0) + return rv; + if ((s64)tmp < 0) + return -ERANGE; + *res = tmp; + } + return 0; +} +EXPORT_SYMBOL(kstrtodec64); + /* * Since "base" would be a nonsense argument, this open-codes the * _from_user helper instead of using the helper macro below. diff --git a/lib/kstrtox.h b/lib/kstrtox.h index 158c400ca865..73dee79fd8ed 100644 --- a/lib/kstrtox.h +++ b/lib/kstrtox.h @@ -2,10 +2,23 @@ #ifndef _LIB_KSTRTOX_H #define _LIB_KSTRTOX_H +#include <linux/args.h> + #define KSTRTOX_OVERFLOW (1U << 31) const char *_parse_integer_fixup_radix(const char *s, unsigned int *base); unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned long long *res, - size_t max_chars); -unsigned int _parse_integer(const char *s, unsigned int base, unsigned long long *res); + size_t max_chars, unsigned long long init); + +#define _parse_integer0(s, base, res, ...) \ + _parse_integer_limit(s, base, res, INT_MAX, 0) + +#define _parse_integer1(s, base, res, max_chars, ...) \ + _parse_integer_limit(s, base, res, max_chars, 0) + +#define _parse_integer2(s, base, res, max_chars, init, ...) \ + _parse_integer_limit(s, base, res, max_chars, init) + +#define _parse_integer(s, base, res, ...) \ + CONCATENATE(_parse_integer, COUNT_ARGS(__VA_ARGS__))(s, base, res, __VA_ARGS__) #endif diff --git a/lib/kunit/Kconfig b/lib/kunit/Kconfig index 498cc51e493d..94ff8e4089bf 100644 --- a/lib/kunit/Kconfig +++ b/lib/kunit/Kconfig @@ -16,8 +16,9 @@ menuconfig KUNIT if KUNIT config KUNIT_DEBUGFS - bool "KUnit - Enable /sys/kernel/debug/kunit debugfs representation" if !KUNIT_ALL_TESTS - default KUNIT_ALL_TESTS + bool "KUnit - Enable /sys/kernel/debug/kunit debugfs representation" + depends on DEBUG_FS + default y help Enable debugfs representation for kunit. Currently this consists of /sys/kernel/debug/kunit/<test_suite>/results files for each diff --git a/lib/kunit/Makefile b/lib/kunit/Makefile index 656f1fa35abc..204e02b10eba 100644 --- a/lib/kunit/Makefile +++ b/lib/kunit/Makefile @@ -10,7 +10,9 @@ kunit-objs += test.o \ executor.o \ attributes.o \ device.o \ - platform.o + platform.o \ + fwnode.o \ + bug.o ifeq ($(CONFIG_KUNIT_DEBUGFS),y) kunit-objs += debugfs.o @@ -21,6 +23,7 @@ obj-$(if $(CONFIG_KUNIT),y) += hooks.o obj-$(CONFIG_KUNIT_TEST) += kunit-test.o obj-$(CONFIG_KUNIT_TEST) += platform-test.o +obj-$(CONFIG_KUNIT_TEST) += backtrace-suppression-test.o # string-stream-test compiles built-in only. ifeq ($(CONFIG_KUNIT_TEST),y) diff --git a/lib/kunit/backtrace-suppression-test.c b/lib/kunit/backtrace-suppression-test.c new file mode 100644 index 000000000000..7a2a59c6a780 --- /dev/null +++ b/lib/kunit/backtrace-suppression-test.c @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit test for suppressing warning tracebacks. + * + * Copyright (C) 2024, Guenter Roeck + * Author: Guenter Roeck <linux@roeck-us.net> + */ + +#include <kunit/test.h> +#include <linux/bug.h> +#include <linux/completion.h> +#include <linux/kthread.h> + +static void backtrace_suppression_test_warn_direct(struct kunit *test) +{ + if (!IS_ENABLED(CONFIG_BUG)) + kunit_skip(test, "requires CONFIG_BUG"); + + kunit_warning_suppress(test) { + WARN(1, "This backtrace should be suppressed"); + /* + * Count must be checked inside the scope; the handle + * is not accessible after the block exits. + */ + KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 1); + } + KUNIT_EXPECT_FALSE(test, kunit_has_active_suppress_warning()); +} + +static noinline void trigger_backtrace_warn(void) +{ + WARN(1, "This backtrace should be suppressed"); +} + +static void backtrace_suppression_test_warn_indirect(struct kunit *test) +{ + if (!IS_ENABLED(CONFIG_BUG)) + kunit_skip(test, "requires CONFIG_BUG"); + + kunit_warning_suppress(test) { + trigger_backtrace_warn(); + KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 1); + } +} + +static void backtrace_suppression_test_warn_multi(struct kunit *test) +{ + if (!IS_ENABLED(CONFIG_BUG)) + kunit_skip(test, "requires CONFIG_BUG"); + + kunit_warning_suppress(test) { + WARN(1, "This backtrace should be suppressed"); + trigger_backtrace_warn(); + KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 2); + } +} + +static void backtrace_suppression_test_warn_on_direct(struct kunit *test) +{ + if (!IS_ENABLED(CONFIG_BUG)) + kunit_skip(test, "requires CONFIG_BUG"); + if (!IS_ENABLED(CONFIG_DEBUG_BUGVERBOSE) && !IS_ENABLED(CONFIG_KALLSYMS)) + kunit_skip(test, "requires CONFIG_DEBUG_BUGVERBOSE or CONFIG_KALLSYMS"); + + kunit_warning_suppress(test) { + WARN_ON(1); + KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 1); + } +} + +static noinline void trigger_backtrace_warn_on(void) +{ + WARN_ON(1); +} + +static void backtrace_suppression_test_warn_on_indirect(struct kunit *test) +{ + if (!IS_ENABLED(CONFIG_BUG)) + kunit_skip(test, "requires CONFIG_BUG"); + if (!IS_ENABLED(CONFIG_DEBUG_BUGVERBOSE)) + kunit_skip(test, "requires CONFIG_DEBUG_BUGVERBOSE"); + + kunit_warning_suppress(test) { + trigger_backtrace_warn_on(); + KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 1); + } +} + +static void backtrace_suppression_test_count(struct kunit *test) +{ + if (!IS_ENABLED(CONFIG_BUG)) + kunit_skip(test, "requires CONFIG_BUG"); + + kunit_warning_suppress(test) { + KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 0); + + WARN(1, "suppressed"); + KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 1); + + WARN(1, "suppressed again"); + KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 2); + } +} + +static void backtrace_suppression_test_active_state(struct kunit *test) +{ + KUNIT_EXPECT_FALSE(test, kunit_has_active_suppress_warning()); + + kunit_warning_suppress(test) { + KUNIT_EXPECT_TRUE(test, kunit_has_active_suppress_warning()); + } + + KUNIT_EXPECT_FALSE(test, kunit_has_active_suppress_warning()); + + kunit_warning_suppress(test) { + KUNIT_EXPECT_TRUE(test, kunit_has_active_suppress_warning()); + } + + KUNIT_EXPECT_FALSE(test, kunit_has_active_suppress_warning()); +} + +static void backtrace_suppression_test_multi_scope(struct kunit *test) +{ + struct kunit_suppressed_warning *sw1, *sw2; + + if (!IS_ENABLED(CONFIG_BUG)) + kunit_skip(test, "requires CONFIG_BUG"); + if (!IS_ENABLED(CONFIG_DEBUG_BUGVERBOSE)) + kunit_skip(test, "requires CONFIG_DEBUG_BUGVERBOSE"); + + sw1 = kunit_start_suppress_warning(test); + trigger_backtrace_warn_on(); + WARN(1, "suppressed by sw1"); + kunit_end_suppress_warning(test, sw1); + + sw2 = kunit_start_suppress_warning(test); + WARN(1, "suppressed by sw2"); + kunit_end_suppress_warning(test, sw2); + + KUNIT_EXPECT_EQ(test, kunit_suppressed_warning_count(sw1), 2); + KUNIT_EXPECT_EQ(test, kunit_suppressed_warning_count(sw2), 1); +} + +struct cross_kthread_data { + bool was_active; + struct completion done; +}; + +static int cross_kthread_fn(void *data) +{ + struct cross_kthread_data *d = data; + + d->was_active = kunit_has_active_suppress_warning(); + complete(&d->done); + while (!kthread_should_stop()) + schedule(); + return 0; +} + +static void backtrace_suppression_test_cross_kthread(struct kunit *test) +{ + struct cross_kthread_data data; + struct task_struct *task; + + data.was_active = false; + init_completion(&data.done); + + kunit_warning_suppress(test) { + task = kthread_run(cross_kthread_fn, &data, "kunit-cross-test"); + KUNIT_ASSERT_FALSE(test, IS_ERR(task)); + wait_for_completion(&data.done); + kthread_stop(task); + } + + KUNIT_EXPECT_FALSE(test, data.was_active); +} + +static struct kunit_case backtrace_suppression_test_cases[] = { + KUNIT_CASE(backtrace_suppression_test_warn_direct), + KUNIT_CASE(backtrace_suppression_test_warn_indirect), + KUNIT_CASE(backtrace_suppression_test_warn_multi), + KUNIT_CASE(backtrace_suppression_test_warn_on_direct), + KUNIT_CASE(backtrace_suppression_test_warn_on_indirect), + KUNIT_CASE(backtrace_suppression_test_count), + KUNIT_CASE(backtrace_suppression_test_active_state), + KUNIT_CASE(backtrace_suppression_test_multi_scope), + KUNIT_CASE(backtrace_suppression_test_cross_kthread), + {} +}; + +static struct kunit_suite backtrace_suppression_test_suite = { + .name = "backtrace-suppression-test", + .test_cases = backtrace_suppression_test_cases, +}; +kunit_test_suites(&backtrace_suppression_test_suite); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("KUnit test to verify warning backtrace suppression"); diff --git a/lib/kunit/bug.c b/lib/kunit/bug.c new file mode 100644 index 000000000000..8579235c9ca6 --- /dev/null +++ b/lib/kunit/bug.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit helpers for backtrace suppression + * + * Copyright (C) 2025 Alessandro Carminati <acarmina@redhat.com> + * Copyright (C) 2024 Guenter Roeck <linux@roeck-us.net> + */ + +#include <kunit/resource.h> +#include <linux/export.h> +#include <linux/rculist.h> +#include <linux/sched.h> +#include <linux/sched/task.h> +#include <linux/spinlock.h> + +#include "hooks-impl.h" + +struct kunit_suppressed_warning { + struct list_head node; + struct task_struct *task; + struct kunit *test; + atomic_t counter; +}; + +static LIST_HEAD(suppressed_warnings); +static DEFINE_SPINLOCK(suppressed_warnings_lock); + +static void kunit_suppress_warning_remove(struct kunit_suppressed_warning *w) +{ + unsigned long flags; + + spin_lock_irqsave(&suppressed_warnings_lock, flags); + list_del_rcu(&w->node); + spin_unlock_irqrestore(&suppressed_warnings_lock, flags); + put_task_struct(w->task); +} + +KUNIT_DEFINE_ACTION_WRAPPER(kunit_suppress_warning_cleanup, + kunit_suppress_warning_remove, + struct kunit_suppressed_warning *); + +bool kunit_has_active_suppress_warning(void) +{ + return __kunit_is_suppressed_warning_impl(false); +} +EXPORT_SYMBOL_GPL(kunit_has_active_suppress_warning); + +struct kunit_suppressed_warning * +kunit_start_suppress_warning(struct kunit *test) +{ + struct kunit_suppressed_warning *w; + unsigned long flags; + int ret; + + if (kunit_has_active_suppress_warning()) { + KUNIT_FAIL(test, "Another suppression block is already active"); + return NULL; + } + + w = kunit_kzalloc(test, sizeof(*w), GFP_KERNEL); + if (!w) { + KUNIT_FAIL(test, "Failed to allocate suppression handle."); + return NULL; + } + + w->task = get_task_struct(current); + w->test = test; + + spin_lock_irqsave(&suppressed_warnings_lock, flags); + list_add_rcu(&w->node, &suppressed_warnings); + spin_unlock_irqrestore(&suppressed_warnings_lock, flags); + + ret = kunit_add_action_or_reset(test, + kunit_suppress_warning_cleanup, w); + if (ret) { + KUNIT_FAIL(test, "Failed to add suppression cleanup action."); + return NULL; + } + + return w; +} +EXPORT_SYMBOL_GPL(kunit_start_suppress_warning); + +void kunit_end_suppress_warning(struct kunit *test, + struct kunit_suppressed_warning *w) +{ + if (!w) + return; + kunit_release_action(test, kunit_suppress_warning_cleanup, w); +} +EXPORT_SYMBOL_GPL(kunit_end_suppress_warning); + +void __kunit_suppress_auto_cleanup(struct kunit_suppressed_warning **wp) +{ + if (*wp) + kunit_end_suppress_warning((*wp)->test, *wp); +} +EXPORT_SYMBOL_GPL(__kunit_suppress_auto_cleanup); + +int kunit_suppressed_warning_count(struct kunit_suppressed_warning *w) +{ + return w ? atomic_read(&w->counter) : 0; +} +EXPORT_SYMBOL_GPL(kunit_suppressed_warning_count); + +bool __kunit_is_suppressed_warning_impl(bool count) +{ + struct kunit_suppressed_warning *w; + + guard(rcu)(); + list_for_each_entry_rcu(w, &suppressed_warnings, node) { + if (w->task == current) { + if (count) + atomic_inc(&w->counter); + return true; + } + } + + return false; +} diff --git a/lib/kunit/debugfs.c b/lib/kunit/debugfs.c index 9c326f1837bd..442b2ceb955b 100644 --- a/lib/kunit/debugfs.c +++ b/lib/kunit/debugfs.c @@ -76,18 +76,30 @@ static int debugfs_print_results(struct seq_file *seq, void *v) seq_puts(seq, "KTAP version 1\n"); seq_puts(seq, "1..1\n"); - /* Print suite header because it is not stored in the test logs. */ - seq_puts(seq, KUNIT_SUBTEST_INDENT "KTAP version 1\n"); - seq_printf(seq, KUNIT_SUBTEST_INDENT "# Subtest: %s\n", suite->name); - seq_printf(seq, KUNIT_SUBTEST_INDENT "1..%zd\n", kunit_suite_num_test_cases(suite)); - - kunit_suite_for_each_test_case(suite, test_case) - debugfs_print_result(seq, test_case->log); + if (suite->status != KUNIT_SKIPPED) { + /* Print suite header because it is not stored in the test logs. */ + seq_puts(seq, + KUNIT_SUBTEST_INDENT "KTAP version 1\n"); + seq_printf(seq, + KUNIT_SUBTEST_INDENT "# Subtest: %s\n", + suite->name); + seq_printf(seq, + KUNIT_SUBTEST_INDENT "1..%zd\n", + kunit_suite_num_test_cases(suite)); + + kunit_suite_for_each_test_case(suite, test_case) + debugfs_print_result(seq, test_case->log); + } debugfs_print_result(seq, suite->log); - seq_printf(seq, "%s %d %s\n", - kunit_status_to_ok_not_ok(success), 1, suite->name); + if (suite->status != KUNIT_SKIPPED) + seq_printf(seq, "%s %d %s\n", + kunit_status_to_ok_not_ok(success), 1, suite->name); + else + seq_printf(seq, "%s %d %s # SKIP %s\n", + kunit_status_to_ok_not_ok(success), 1, suite->name, + suite->status_comment); return 0; } diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c index 1fef217de11d..b0f8a41d61d3 100644 --- a/lib/kunit/executor.c +++ b/lib/kunit/executor.c @@ -15,6 +15,16 @@ extern struct kunit_suite * const __kunit_suites_end[]; extern struct kunit_suite * const __kunit_init_suites_start[]; extern struct kunit_suite * const __kunit_init_suites_end[]; +static struct kunit_suite_set kunit_boot_suites; + +void kunit_free_boot_suites(void) +{ + if (kunit_boot_suites.start) { + kunit_free_suite_set(kunit_boot_suites); + kunit_boot_suites = (struct kunit_suite_set){ NULL, NULL }; + } +} + static char *action_param; module_param_named(action, action_param, charp, 0400); @@ -411,9 +421,12 @@ int kunit_run_all_tests(void) pr_err("kunit executor: unknown action '%s'\n", action_param); free_out: - if (filter_glob_param || filter_param) - kunit_free_suite_set(suite_set); - else if (init_num_suites > 0) + if (filter_glob_param || filter_param) { + if (err) + kunit_free_suite_set(suite_set); + else + kunit_boot_suites = suite_set; + } else if (init_num_suites > 0) /* Don't use kunit_free_suite_set because suites aren't individually allocated */ kfree(suite_set.start); diff --git a/lib/kunit/fwnode.c b/lib/kunit/fwnode.c new file mode 100644 index 000000000000..a58ce0984d76 --- /dev/null +++ b/lib/kunit/fwnode.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) Qualcomm Technologies, Inc. and/or its subsidiaries + */ + +#include <kunit/fwnode.h> +#include <kunit/test.h> + +#include <linux/fwnode.h> +#include <linux/property.h> + +KUNIT_DEFINE_ACTION_WRAPPER(fwnode_remove_software_node_wrapper, + fwnode_remove_software_node, + struct fwnode_handle *); + +/** + * kunit_fwnode_create_software_node() - Create a kunit-managed software node + * @test: Test context + * @properties: Properties to use to create the new software node + * @parent: Parent of this software node + * + * Create a test-managed software node and return its firmware node handle. + * The software node is removed after the test case completes. + * + * Returns: + * Firmware node handle of the newly created software node or IS_ERR() on + * failure. + */ +struct fwnode_handle * +kunit_fwnode_create_software_node(struct kunit *test, + const struct property_entry *properties, + const struct fwnode_handle *parent) +{ + struct fwnode_handle *fwnode; + int ret; + + fwnode = fwnode_create_software_node(properties, parent); + if (IS_ERR(fwnode)) + return fwnode; + + ret = kunit_add_action_or_reset(test, fwnode_remove_software_node_wrapper, + fwnode); + if (ret) + return ERR_PTR(ret); + + return fwnode; +} +EXPORT_SYMBOL_GPL(kunit_fwnode_create_software_node); + +KUNIT_DEFINE_ACTION_WRAPPER(software_node_unregister_wrapper, + software_node_unregister, + const struct software_node *); + +/** + * kunit_software_node_register() - Register a kunit-managed software node + * @test: Test context + * @swnode: Software node to register + * + * Register a test-managed software node and return its firmware node handle. + * The software node is unregistered after the test case completes. + * + * Returns: + * Firmware node handle of the registered software node or IS_ERR() on failure. + */ +struct fwnode_handle * +kunit_software_node_register(struct kunit *test, + const struct software_node *swnode) +{ + struct fwnode_handle *fwnode; + int ret; + + ret = software_node_register(swnode); + if (ret) + return ERR_PTR(ret); + + fwnode = software_node_fwnode(swnode); + if (WARN_ON(!fwnode)) + return ERR_PTR(-ENOENT); + + ret = kunit_add_action_or_reset(test, software_node_unregister_wrapper, + (void *)swnode); + if (ret) + return ERR_PTR(ret); + + return fwnode; +} +EXPORT_SYMBOL_GPL(kunit_software_node_register); + +KUNIT_DEFINE_ACTION_WRAPPER(software_node_unregister_node_group_wrapper, + software_node_unregister_node_group, + const struct software_node *const *); + +/** + * kunit_software_node_register_node_group() - Register a kunit-managed software node group + * @test: Test context + * @nodes: Software node group to register + * + * Register a test-managed software node group. The nodes are unregistered + * after the test case completes. + * + * Returns: + * 0 on success, negative error number on failure. + */ +int kunit_software_node_register_node_group(struct kunit *test, + const struct software_node *const *nodes) +{ + int ret; + + ret = software_node_register_node_group(nodes); + if (ret) + return ret; + + return kunit_add_action_or_reset(test, software_node_unregister_node_group_wrapper, + (void *)nodes); +} +EXPORT_SYMBOL_GPL(kunit_software_node_register_node_group); + +KUNIT_DEFINE_ACTION_WRAPPER(device_remove_software_node_wrapper, + device_remove_software_node, + struct device *); + +/** + * kunit_device_add_software_node() - Assign a kunit-managed software node to a device + * @test: Test context + * @dev: Device to assign the software node for + * @node: The software node to assign + * + * Make @node the secondary firmware node of @dev. If @dev has no primary + * firmware node, @node will become the primary node. The software node will + * be automatically removed from @dev when the test case completes. + * + * Returns: + * 0 on success, negative error number on failure. + */ +int kunit_device_add_software_node(struct kunit *test, struct device *dev, + const struct software_node *node) +{ + int ret; + + ret = device_add_software_node(dev, node); + if (ret) + return ret; + + return kunit_add_action_or_reset(test, device_remove_software_node_wrapper, dev); +} +EXPORT_SYMBOL_GPL(kunit_device_add_software_node); diff --git a/lib/kunit/hooks-impl.h b/lib/kunit/hooks-impl.h index 4e71b2d0143b..d8720f261692 100644 --- a/lib/kunit/hooks-impl.h +++ b/lib/kunit/hooks-impl.h @@ -19,6 +19,7 @@ void __printf(3, 4) __kunit_fail_current_test_impl(const char *file, int line, const char *fmt, ...); void *__kunit_get_static_stub_address_impl(struct kunit *test, void *real_fn_addr); +bool __kunit_is_suppressed_warning_impl(bool count); /* Code to set all of the function pointers. */ static inline void kunit_install_hooks(void) @@ -26,6 +27,7 @@ static inline void kunit_install_hooks(void) /* Install the KUnit hook functions. */ kunit_hooks.fail_current_test = __kunit_fail_current_test_impl; kunit_hooks.get_static_stub_address = __kunit_get_static_stub_address_impl; + kunit_hooks.is_suppressed_warning = __kunit_is_suppressed_warning_impl; } #endif /* _KUNIT_HOOKS_IMPL_H */ diff --git a/lib/kunit/kunit-example-test.c b/lib/kunit/kunit-example-test.c index 0bae7b7ca0b0..b8ded54fa46d 100644 --- a/lib/kunit/kunit-example-test.c +++ b/lib/kunit/kunit-example-test.c @@ -591,5 +591,34 @@ static struct kunit_suite example_init_test_suite = { */ kunit_test_init_section_suites(&example_init_test_suite); +/* + * This test should always be skipped. + */ +static void example_skip_suite_test(struct kunit *test) +{ + /* This line should never be seen */ + KUNIT_FAIL(test, "You should not see a this."); +} + +static struct kunit_case example_skip_suite_test_cases[] = { + KUNIT_CASE(example_skip_suite_test), + {} +}; + +static int example_skip_suite_init(struct kunit_suite *suite) +{ + kunit_mark_skipped(suite, "Test suite expected to be skipped"); + return 0; +} + +static struct kunit_suite example_test_skip_suite = { + .name = "example_skip_suite", + .suite_init = example_skip_suite_init, + .test_cases = example_skip_suite_test_cases, +}; + +/* This registers a test suite that will be skipped */ +kunit_test_suite(example_test_skip_suite); + MODULE_DESCRIPTION("Example KUnit test suite"); MODULE_LICENSE("GPL v2"); diff --git a/lib/kunit/platform.c b/lib/kunit/platform.c index 0b518de26065..737758d710b2 100644 --- a/lib/kunit/platform.c +++ b/lib/kunit/platform.c @@ -6,6 +6,7 @@ #include <linux/completion.h> #include <linux/device/bus.h> #include <linux/device/driver.h> +#include <linux/err.h> #include <linux/platform_device.h> #include <kunit/platform_device.h> @@ -130,6 +131,69 @@ int kunit_platform_device_add(struct kunit *test, struct platform_device *pdev) } EXPORT_SYMBOL_GPL(kunit_platform_device_add); +/** + * kunit_platform_device_register_full() - Register a KUnit test-managed platform + * device described by platform device info + * @test: test context + * @pdevinfo: platform device information describing the new device + * + * Register a test-managed platform device. The device is unregistered when the + * test completes. + * + * Return: New platform device on success, IS_ERR() on error. + */ +struct platform_device * +kunit_platform_device_register_full(struct kunit *test, + const struct platform_device_info *pdevinfo) +{ + struct platform_device *pdev; + int ret; + + pdev = platform_device_register_full(pdevinfo); + if (IS_ERR(pdev)) + return pdev; + + ret = kunit_add_action_or_reset(test, platform_device_unregister_wrapper, pdev); + if (ret) + return ERR_PTR(ret); + + return pdev; +} +EXPORT_SYMBOL_GPL(kunit_platform_device_register_full); + +static bool +kunit_platform_device_add_match(struct kunit *test, struct kunit_resource *res, + void *match_data) +{ + struct platform_device *pdev = match_data; + + return res->data == pdev && res->free == kunit_platform_device_add_exit; +} + +/** + * kunit_platform_device_unregister() - Unregister a KUnit-managed platform device + * @test: test context + * @pdev: platform device to unregister + * + * Unregister a test-managed platform device and cancel its release action. + */ +void kunit_platform_device_unregister(struct kunit *test, + struct platform_device *pdev) +{ + struct kunit_resource *res; + + res = kunit_find_resource(test, kunit_platform_device_add_match, pdev); + if (res) { + res->free = NULL; + kunit_put_resource(res); + } else { + kunit_remove_action(test, platform_device_unregister_wrapper, pdev); + } + + platform_device_unregister(pdev); +} +EXPORT_SYMBOL_GPL(kunit_platform_device_unregister); + struct kunit_platform_device_probe_nb { struct completion *x; struct device *dev; diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c index 0d8f1b30559b..51ba40ebf19f 100644 --- a/lib/kunit/string-stream.c +++ b/lib/kunit/string-stream.c @@ -9,6 +9,7 @@ #include <kunit/static_stub.h> #include <kunit/test.h> #include <linux/list.h> +#include <linux/seq_buf.h> #include <linux/slab.h> #include "string-stream.h" @@ -74,7 +75,8 @@ int string_stream_vadd(struct string_stream *stream, /* Append newline if necessary. */ if (frag_container->fragment[result_len - 1] != '\n') - result_len = strlcat(frag_container->fragment, "\n", buf_len); + result_len += strscpy(frag_container->fragment + result_len, + "\n", buf_len - result_len); } else { result_len = vsnprintf(frag_container->fragment, buf_len, fmt, args); } @@ -118,15 +120,18 @@ char *string_stream_get_string(struct string_stream *stream) { struct string_stream_fragment *frag_container; size_t buf_len = stream->length + 1; /* +1 for null byte. */ + struct seq_buf sb; char *buf; buf = kzalloc(buf_len, stream->gfp); if (!buf) return NULL; + seq_buf_init(&sb, buf, buf_len); + spin_lock(&stream->lock); list_for_each_entry(frag_container, &stream->fragments, node) - strlcat(buf, frag_container->fragment, buf_len); + seq_buf_puts(&sb, frag_container->fragment); spin_unlock(&stream->lock); return buf; diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 41e1c89799b6..09e3dabfac0c 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -214,12 +214,18 @@ enum kunit_status kunit_suite_has_succeeded(struct kunit_suite *suite) const struct kunit_case *test_case; enum kunit_status status = KUNIT_SKIPPED; + if (suite->status == KUNIT_SKIPPED) + return KUNIT_SKIPPED; + if (suite->suite_init_err) return KUNIT_FAILURE; kunit_suite_for_each_test_case(suite, test_case) { - if (test_case->status == KUNIT_FAILURE) + if (test_case->status == KUNIT_FAILURE) { + /* Update the kunit_suite status also */ + suite->status = KUNIT_FAILURE; return KUNIT_FAILURE; + } else if (test_case->status == KUNIT_SUCCESS) status = KUNIT_SUCCESS; } @@ -795,12 +801,20 @@ int kunit_run_tests(struct kunit_suite *suite) /* Taint the kernel so we know we've run tests. */ add_taint(TAINT_TEST, LOCKDEP_STILL_OK); + if (suite->status == KUNIT_SKIPPED) + goto suite_end; + if (suite->suite_init) { suite->suite_init_err = suite->suite_init(suite); if (suite->suite_init_err) { + suite->status = KUNIT_FAILURE; kunit_err(suite, KUNIT_SUBTEST_INDENT "# failed to initialize (%d)", suite->suite_init_err); goto suite_end; + + } else if (suite->status == KUNIT_SKIPPED) { + /* Skip this kunit suite */ + goto suite_end; } } @@ -825,6 +839,7 @@ static void kunit_init_suite(struct kunit_suite *suite) kunit_debugfs_create_suite(suite); suite->status_comment[0] = '\0'; suite->suite_init_err = 0; + suite->status = KUNIT_SUCCESS; if (suite->log) string_stream_clear(suite->log); @@ -1075,6 +1090,7 @@ static void __exit kunit_exit(void) kunit_bus_shutdown(); kunit_debugfs_cleanup(); + kunit_free_boot_suites(); } module_exit(kunit_exit); diff --git a/lib/linear_ranges.c b/lib/linear_ranges.c index a1a7dfa881de..c85583678f6b 100644 --- a/lib/linear_ranges.c +++ b/lib/linear_ranges.c @@ -242,6 +242,42 @@ int linear_range_get_selector_high(const struct linear_range *r, EXPORT_SYMBOL_GPL(linear_range_get_selector_high); /** + * linear_range_get_selector_high_array - return linear range selector for value + * @r: pointer to array of linear ranges where selector is looked from + * @ranges: amount of ranges to scan from array + * @val: value for which the selector is searched + * @selector: address where found selector value is updated + * @found: flag to indicate that given value was in the range + * + * Scan array of ranges for selector for which range value matches given + * input value. Value is matching if it is equal or higher than given value + * If given value is found to be in a range scanning is stopped and @found is + * set true. If a range with values greater than given value is found + * but the range min is being greater than given value, then the range's + * lowest selector is updated to @selector and scanning is stopped. + * + * Return: 0 on success, -EINVAL if range array is invalid or does not contain + * range with a value greater or equal to given value + */ +int linear_range_get_selector_high_array(const struct linear_range *r, + int ranges, unsigned int val, + unsigned int *selector, bool *found) +{ + int i; + int ret; + + for (i = 0; i < ranges; i++) { + ret = linear_range_get_selector_high(&r[i], val, selector, + found); + if (!ret) + return 0; + } + + return -EINVAL; +} +EXPORT_SYMBOL_GPL(linear_range_get_selector_high_array); + +/** * linear_range_get_selector_within - return linear range selector for value * @r: pointer to linear range where selector is looked from * @val: value for which the selector is searched diff --git a/lib/list_sort.c b/lib/list_sort.c index a310ecb7ccc0..ff99203f208f 100644 --- a/lib/list_sort.c +++ b/lib/list_sort.c @@ -50,7 +50,6 @@ static void merge_final(void *priv, list_cmp_func_t cmp, struct list_head *head, struct list_head *a, struct list_head *b) { struct list_head *tail = head; - u8 count = 0; for (;;) { /* if equal, take 'a' -- important for sort stability */ @@ -76,15 +75,6 @@ static void merge_final(void *priv, list_cmp_func_t cmp, struct list_head *head, /* Finish linking remainder of list b on to tail */ tail->next = b; do { - /* - * If the merge is highly unbalanced (e.g. the input is - * already sorted), this loop may run many iterations. - * Continue callbacks to the client even though no - * element comparison is needed, so the client's cmp() - * routine can invoke cond_resched() periodically. - */ - if (unlikely(!++count)) - cmp(priv, b, b); b->prev = tail; tail = b; b = b->next; diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c index d939403331b5..c3d976c801bb 100644 --- a/lib/locking-selftest.c +++ b/lib/locking-selftest.c @@ -1429,12 +1429,11 @@ static int unexpected_testcase_failures; static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) { - int saved_preempt_count = preempt_count(); + long saved_preempt_count = preempt_count(); #ifdef CONFIG_PREEMPT_RT -#ifdef CONFIG_SMP int saved_mgd_count = current->migration_disabled; -#endif int saved_rcu_count = current->rcu_read_lock_nesting; + int saved_sched_rt_mutex = current->sched_rt_mutex; #endif WARN_ON(irqs_disabled()); @@ -1471,10 +1470,10 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) preempt_count_set(saved_preempt_count); #ifdef CONFIG_PREEMPT_RT -#ifdef CONFIG_SMP + current->sched_rt_mutex = saved_sched_rt_mutex; + while (current->migration_disabled > saved_mgd_count) migrate_enable(); -#endif while (current->rcu_read_lock_nesting > saved_rcu_count) rcu_read_unlock(); diff --git a/lib/lockref.c b/lib/lockref.c index 5d8e3ef3860e..9b3dd688d8cd 100644 --- a/lib/lockref.c +++ b/lib/lockref.c @@ -131,7 +131,7 @@ EXPORT_SYMBOL(lockref_put_or_lock); void lockref_mark_dead(struct lockref *lockref) { assert_spin_locked(&lockref->lock); - lockref->count = -128; + lockref->count = __LOCKREF_DEAD_VAL; } EXPORT_SYMBOL(lockref_mark_dead); diff --git a/lib/maple_tree.c b/lib/maple_tree.c index 5aa4c9500018..1aba6cced713 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -2,7 +2,7 @@ /* * Maple Tree implementation * Copyright (c) 2018-2022 Oracle Corporation - * Authors: Liam R. Howlett <Liam.Howlett@oracle.com> + * Authors: Liam R. Howlett <liam@infradead.org> * Matthew Wilcox <willy@infradead.org> * Copyright (c) 2023 ByteDance * Author: Peng Zhang <zhangpeng.00@bytedance.com> @@ -101,6 +101,7 @@ static const unsigned long mt_max[] = { [maple_leaf_64] = ULONG_MAX, [maple_range_64] = ULONG_MAX, [maple_arange_64] = ULONG_MAX, + [maple_copy] = ULONG_MAX, }; #define mt_node_max(x) mt_max[mte_node_type(x)] #endif @@ -110,6 +111,7 @@ static const unsigned char mt_slots[] = { [maple_leaf_64] = MAPLE_RANGE64_SLOTS, [maple_range_64] = MAPLE_RANGE64_SLOTS, [maple_arange_64] = MAPLE_ARANGE64_SLOTS, + [maple_copy] = 3, }; #define mt_slot_count(x) mt_slots[mte_node_type(x)] @@ -118,6 +120,7 @@ static const unsigned char mt_pivots[] = { [maple_leaf_64] = MAPLE_RANGE64_SLOTS - 1, [maple_range_64] = MAPLE_RANGE64_SLOTS - 1, [maple_arange_64] = MAPLE_ARANGE64_SLOTS - 1, + [maple_copy] = 3, }; #define mt_pivot_count(x) mt_pivots[mte_node_type(x)] @@ -126,48 +129,10 @@ static const unsigned char mt_min_slots[] = { [maple_leaf_64] = (MAPLE_RANGE64_SLOTS / 2) - 2, [maple_range_64] = (MAPLE_RANGE64_SLOTS / 2) - 2, [maple_arange_64] = (MAPLE_ARANGE64_SLOTS / 2) - 1, + [maple_copy] = 1, /* Should never be used */ }; #define mt_min_slot_count(x) mt_min_slots[mte_node_type(x)] -#define MAPLE_BIG_NODE_SLOTS (MAPLE_RANGE64_SLOTS * 2 + 2) -#define MAPLE_BIG_NODE_GAPS (MAPLE_ARANGE64_SLOTS * 2 + 1) - -struct maple_big_node { - unsigned long pivot[MAPLE_BIG_NODE_SLOTS - 1]; - union { - struct maple_enode *slot[MAPLE_BIG_NODE_SLOTS]; - struct { - unsigned long padding[MAPLE_BIG_NODE_GAPS]; - unsigned long gap[MAPLE_BIG_NODE_GAPS]; - }; - }; - unsigned char b_end; - enum maple_type type; -}; - -/* - * The maple_subtree_state is used to build a tree to replace a segment of an - * existing tree in a more atomic way. Any walkers of the older tree will hit a - * dead node and restart on updates. - */ -struct maple_subtree_state { - struct ma_state *orig_l; /* Original left side of subtree */ - struct ma_state *orig_r; /* Original right side of subtree */ - struct ma_state *l; /* New left side of subtree */ - struct ma_state *m; /* New middle of subtree (rare) */ - struct ma_state *r; /* New right side of subtree */ - struct ma_topiary *free; /* nodes to be freed */ - struct ma_topiary *destroy; /* Nodes to be destroyed (walked and freed) */ - struct maple_big_node *bn; -}; - -#ifdef CONFIG_KASAN_STACK -/* Prevent mas_wr_bnode() from exceeding the stack frame limit */ -#define noinline_for_kasan noinline_for_stack -#else -#define noinline_for_kasan inline -#endif - /* Functions */ static inline struct maple_node *mt_alloc_one(gfp_t gfp) { @@ -296,6 +261,12 @@ static inline bool mas_is_underflow(struct ma_state *mas) return mas->status == ma_underflow; } +static inline void mas_make_walkable(struct ma_state *mas) +{ + if (!mas_is_active(mas) && !mas_is_start(mas)) + mas->status = ma_start; +} + static __always_inline struct maple_node *mte_to_node( const struct maple_enode *entry) { @@ -349,6 +320,13 @@ static inline struct maple_enode *mt_mk_node(const struct maple_node *node, (type << MAPLE_ENODE_TYPE_SHIFT) | MAPLE_ENODE_NULL); } +static inline void ma_init_slot(void __rcu **slot, const struct maple_node *mn, + const enum maple_type mt) +{ + /* WARNING: this is unsafe if the slot is exposed to readers. */ + RCU_INIT_POINTER(*slot, (void *)mt_mk_node(mn, mt)); +} + static inline void *mte_mk_root(const struct maple_enode *node) { return (void *)((unsigned long)node | MAPLE_ROOT_NODE); @@ -384,11 +362,6 @@ static __always_inline bool mte_is_root(const struct maple_enode *node) return ma_is_root(mte_to_node(node)); } -static inline bool mas_is_root_limits(const struct ma_state *mas) -{ - return !mas->min && mas->max == ULONG_MAX; -} - static __always_inline bool mt_is_alloc(struct maple_tree *mt) { return (mt->ma_flags & MT_FLAGS_ALLOC_RANGE); @@ -484,46 +457,6 @@ enum maple_type mas_parent_type(struct ma_state *mas, struct maple_enode *enode) } /* - * mas_set_parent() - Set the parent node and encode the slot - * @mas: The maple state - * @enode: The encoded maple node. - * @parent: The encoded maple node that is the parent of @enode. - * @slot: The slot that @enode resides in @parent. - * - * Slot number is encoded in the enode->parent bit 3-6 or 2-6, depending on the - * parent type. - */ -static inline -void mas_set_parent(struct ma_state *mas, struct maple_enode *enode, - const struct maple_enode *parent, unsigned char slot) -{ - unsigned long val = (unsigned long)parent; - unsigned long shift; - unsigned long type; - enum maple_type p_type = mte_node_type(parent); - - MAS_BUG_ON(mas, p_type == maple_dense); - MAS_BUG_ON(mas, p_type == maple_leaf_64); - - switch (p_type) { - case maple_range_64: - case maple_arange_64: - shift = MAPLE_PARENT_SLOT_SHIFT; - type = MAPLE_PARENT_RANGE64; - break; - default: - case maple_dense: - case maple_leaf_64: - shift = type = 0; - break; - } - - val &= ~MAPLE_NODE_MASK; /* Clear all node metadata in parent */ - val |= (slot << shift) | type; - mte_to_node(enode)->parent = ma_parent_ptr(val); -} - -/* * mte_parent_slot() - get the parent slot of @enode. * @enode: The encoded maple node. * @@ -605,6 +538,8 @@ static inline unsigned long *ma_pivots(struct maple_node *node, case maple_range_64: case maple_leaf_64: return node->mr64.pivot; + case maple_copy: + return node->cp.pivot; case maple_dense: return NULL; } @@ -624,6 +559,8 @@ static inline unsigned long *ma_gaps(struct maple_node *node, switch (type) { case maple_arange_64: return node->ma64.gap; + case maple_copy: + return node->cp.gap; case maple_range_64: case maple_leaf_64: case maple_dense: @@ -690,6 +627,7 @@ static inline void mte_set_pivot(struct maple_enode *mn, unsigned char piv, case maple_arange_64: node->ma64.pivot[piv] = val; break; + case maple_copy: case maple_dense: break; } @@ -711,6 +649,8 @@ static inline void __rcu **ma_slots(struct maple_node *mn, enum maple_type mt) case maple_range_64: case maple_leaf_64: return mn->mr64.slot; + case maple_copy: + return mn->cp.slot; case maple_dense: return mn->slot; } @@ -898,6 +838,42 @@ static inline void ma_set_meta_gap(struct maple_node *mn, enum maple_type mt, } /* + * mas_set_parent_slots() - Bulk operation to set many slot parent pointers + * @mas: The maple state + * @parent: The encoded maple node that is the parent of @enode. + * @slot: The slot that of the @enode. + * @start_slot: The offset into @slot + * @count: The number of slots to set (eg: exclusive) + */ +static inline +void mas_set_parent_slots(struct ma_state *mas, struct maple_enode *parent, + void __rcu **slots, unsigned char start_slot, unsigned char count) +{ + unsigned long val; + unsigned long shift; + unsigned long type; + enum maple_type p_type = mte_node_type(parent); + unsigned char i; + + MAS_BUG_ON(mas, p_type != maple_range_64 && + p_type != maple_arange_64); + + shift = MAPLE_PARENT_SLOT_SHIFT; + type = MAPLE_PARENT_RANGE64; + + val = (unsigned long)parent; + val &= ~MAPLE_NODE_MASK; + + for (i = 0; i < count; i++) { + unsigned long pval = val | ((start_slot + i) << shift) | type; + struct maple_enode *child; + + child = mt_slot_locked(mas->tree, slots, i); + mte_to_node(child)->parent = ma_parent_ptr(pval); + } +} + +/* * mat_add() - Add a @dead_enode to the ma_topiary of a list of dead nodes. * @mat: the ma_topiary, a linked list of dead nodes. * @dead_enode: the node to be marked as dead and added to the tail of the list @@ -1174,6 +1150,79 @@ static inline void mas_free(struct ma_state *mas, struct maple_enode *used) ma_free_rcu(mte_to_node(used)); } + +#ifdef CONFIG_LOCKDEP +static struct lockdep_map *mas_lockdep_map(struct ma_state *mas) +{ + struct maple_tree *mt = mas->tree; + + if (mt_external_lock(mt)) + return mt->ma_external_lock; + + return &(mt->ma_lock).dep_map; +} + +#endif + +static void mas_lock_check(struct ma_state *mas) +{ +#ifdef CONFIG_LOCKDEP + struct lockdep_map *map; + u32 seq; + + if (!mas_is_active(mas)) + return; + +#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD + if (!mt_locked(mas->tree)) { + if (mt_in_rcu(mas->tree)) + WARN_ON_ONCE(poll_state_synchronize_rcu(mas->rcu_gp)); + } +#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */ + + map = mas_lockdep_map(mas); + if (!map) + return; + + seq = lock_sequence(map); + if (seq != UINT_MAX && mas->ld_seq != UINT_MAX) + WARN_ON_ONCE(mas->ld_seq != seq); +#endif /* CONFIG_LOCKDEP */ + +} + +static void mas_init_lock_check(struct ma_state *mas) +{ +#ifdef CONFIG_LOCKDEP + struct lockdep_map *map; +#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD + if (!mt_locked(mas->tree)) { + if (mt_in_rcu(mas->tree)) + mas->rcu_gp = get_state_synchronize_rcu(); + return; + } +#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */ + + map = mas_lockdep_map(mas); + if (map) /* Update regardless of lock state */ + mas->ld_seq = lock_sequence(map); +#endif /* CONFIG_LOCKDEP */ + +} + +static void mas_may_init_lock_check(struct ma_state *mas) +{ +#ifdef CONFIG_LOCKDEP +#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD + if (mas_is_start(mas) || mas_is_paused(mas)) { + mas_init_lock_check(mas); + return; + } +#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */ + mas_lock_check(mas); +#endif /* CONFIG_LOCKDEP */ +} + /* * mas_start() - Sets up maple state for operations. * @mas: The maple state. @@ -1192,6 +1241,7 @@ static inline struct maple_enode *mas_start(struct ma_state *mas) if (likely(mas_is_start(mas))) { struct maple_enode *root; + mas_init_lock_check(mas); mas->min = 0; mas->max = ULONG_MAX; @@ -1298,25 +1348,40 @@ static inline unsigned char mas_data_end(struct ma_state *mas) return mt_pivots[type]; } -/* - * mas_leaf_max_gap() - Returns the largest gap in a leaf node - * @mas: the maple state - * - * Return: The maximum gap in the leaf. - */ -static unsigned long mas_leaf_max_gap(struct ma_state *mas) +static inline +void wr_mas_setup(struct ma_wr_state *wr_mas, struct ma_state *mas) +{ + wr_mas->node = mas_mn(mas); + wr_mas->type = mte_node_type(mas->node); + wr_mas->pivots = ma_pivots(wr_mas->node, wr_mas->type); + wr_mas->slots = ma_slots(wr_mas->node, wr_mas->type); + wr_mas->r_min = mas_safe_min(mas, wr_mas->pivots, mas->offset); + wr_mas->r_max = mas_safe_pivot(mas, wr_mas->pivots, mas->offset, + wr_mas->type); +} + +static inline +void wr_mas_ascend(struct ma_wr_state *wr_mas) +{ + struct ma_state *mas = wr_mas->mas; + + mas_ascend(mas); + wr_mas_setup(wr_mas, mas); + mas->end = ma_data_end(wr_mas->node, wr_mas->type, wr_mas->pivots, + mas->max); + /* Careful, this may be wrong.. */ + wr_mas->end_piv = wr_mas->r_max; + wr_mas->offset_end = mas->offset; +} + +static inline unsigned long ma_leaf_max_gap(struct maple_node *mn, + enum maple_type mt, unsigned long min, unsigned long max, + unsigned long *pivots, void __rcu **slots) { - enum maple_type mt; unsigned long pstart, gap, max_gap; - struct maple_node *mn; - unsigned long *pivots; - void __rcu **slots; unsigned char i; unsigned char max_piv; - mt = mte_node_type(mas->node); - mn = mas_mn(mas); - slots = ma_slots(mn, mt); max_gap = 0; if (unlikely(ma_is_dense(mt))) { gap = 0; @@ -1338,26 +1403,25 @@ static unsigned long mas_leaf_max_gap(struct ma_state *mas) * Check the first implied pivot optimizes the loop below and slot 1 may * be skipped if there is a gap in slot 0. */ - pivots = ma_pivots(mn, mt); if (likely(!slots[0])) { - max_gap = pivots[0] - mas->min + 1; + max_gap = pivots[0] - min + 1; i = 2; } else { i = 1; } /* reduce max_piv as the special case is checked before the loop */ - max_piv = ma_data_end(mn, mt, pivots, mas->max) - 1; + max_piv = ma_data_end(mn, mt, pivots, max) - 1; /* * Check end implied pivot which can only be a gap on the right most * node. */ - if (unlikely(mas->max == ULONG_MAX) && !slots[max_piv + 1]) { + if (unlikely(max == ULONG_MAX) && !slots[max_piv + 1]) { gap = ULONG_MAX - pivots[max_piv]; if (gap > max_gap) max_gap = gap; - if (max_gap > pivots[max_piv] - mas->min) + if (max_gap > pivots[max_piv] - min) return max_gap; } @@ -1378,6 +1442,27 @@ static unsigned long mas_leaf_max_gap(struct ma_state *mas) } /* + * mas_leaf_max_gap() - Returns the largest gap in a leaf node + * @mas: the maple state + * + * Return: The maximum gap in the leaf. + */ +static inline unsigned long mas_leaf_max_gap(struct ma_state *mas) +{ + enum maple_type mt; + struct maple_node *mn; + unsigned long *pivots; + void __rcu **slots; + + mn = mas_mn(mas); + mt = mte_node_type(mas->node); + slots = ma_slots(mn, mt); + pivots = ma_pivots(mn, mt); + + return ma_leaf_max_gap(mn, mt, mas->min, mas->max, pivots, slots); +} + +/* * ma_max_gap() - Get the maximum gap in a maple node (non-leaf) * @node: The maple node * @gaps: The pointer to the gaps @@ -1487,14 +1572,26 @@ ascend: goto ascend; } +static __always_inline void mas_update_gap_known(struct ma_state *mas, + unsigned long gap) +{ + unsigned char pslot; + unsigned long p_gap; + + pslot = mte_parent_slot(mas->node); + p_gap = ma_gaps(mte_parent(mas->node), + mas_parent_type(mas, mas->node))[pslot]; + + if (p_gap != gap) + mas_parent_gap(mas, pslot, gap); +} + /* * mas_update_gap() - Update a nodes gaps and propagate up if necessary. * @mas: the maple state. */ static inline void mas_update_gap(struct ma_state *mas) { - unsigned char pslot; - unsigned long p_gap; unsigned long max_gap; if (!mt_is_alloc(mas->tree)) @@ -1504,13 +1601,7 @@ static inline void mas_update_gap(struct ma_state *mas) return; max_gap = mas_max_gap(mas); - - pslot = mte_parent_slot(mas->node); - p_gap = ma_gaps(mte_parent(mas->node), - mas_parent_type(mas, mas->node))[pslot]; - - if (p_gap != max_gap) - mas_parent_gap(mas, pslot, max_gap); + mas_update_gap_known(mas, max_gap); } /* @@ -1526,14 +1617,10 @@ static inline void mas_adopt_children(struct ma_state *mas, struct maple_node *node = mte_to_node(parent); void __rcu **slots = ma_slots(node, type); unsigned long *pivots = ma_pivots(node, type); - struct maple_enode *child; - unsigned char offset; + unsigned char end; - offset = ma_data_end(node, type, pivots, mas->max); - do { - child = mas_slot_locked(mas, slots, offset); - mas_set_parent(mas, child, parent, offset); - } while (offset--); + end = ma_data_end(node, type, pivots, mas->max); + mas_set_parent_slots(mas, parent, slots, 0, end + 1); } /* @@ -1617,169 +1704,6 @@ static inline bool mas_find_child(struct ma_state *mas, struct ma_state *child) } /* - * mab_shift_right() - Shift the data in mab right. Note, does not clean out the - * old data or set b_node->b_end. - * @b_node: the maple_big_node - * @shift: the shift count - */ -static inline void mab_shift_right(struct maple_big_node *b_node, - unsigned char shift) -{ - unsigned long size = b_node->b_end * sizeof(unsigned long); - - memmove(b_node->pivot + shift, b_node->pivot, size); - memmove(b_node->slot + shift, b_node->slot, size); - if (b_node->type == maple_arange_64) - memmove(b_node->gap + shift, b_node->gap, size); -} - -/* - * mab_middle_node() - Check if a middle node is needed (unlikely) - * @b_node: the maple_big_node that contains the data. - * @split: the potential split location - * @slot_count: the size that can be stored in a single node being considered. - * - * Return: true if a middle node is required. - */ -static inline bool mab_middle_node(struct maple_big_node *b_node, int split, - unsigned char slot_count) -{ - unsigned char size = b_node->b_end; - - if (size >= 2 * slot_count) - return true; - - if (!b_node->slot[split] && (size >= 2 * slot_count - 1)) - return true; - - return false; -} - -/* - * mab_no_null_split() - ensure the split doesn't fall on a NULL - * @b_node: the maple_big_node with the data - * @split: the suggested split location - * @slot_count: the number of slots in the node being considered. - * - * Return: the split location. - */ -static inline int mab_no_null_split(struct maple_big_node *b_node, - unsigned char split, unsigned char slot_count) -{ - if (!b_node->slot[split]) { - /* - * If the split is less than the max slot && the right side will - * still be sufficient, then increment the split on NULL. - */ - if ((split < slot_count - 1) && - (b_node->b_end - split) > (mt_min_slots[b_node->type])) - split++; - else - split--; - } - return split; -} - -/* - * mab_calc_split() - Calculate the split location and if there needs to be two - * splits. - * @mas: The maple state - * @bn: The maple_big_node with the data - * @mid_split: The second split, if required. 0 otherwise. - * - * Return: The first split location. The middle split is set in @mid_split. - */ -static inline int mab_calc_split(struct ma_state *mas, - struct maple_big_node *bn, unsigned char *mid_split) -{ - unsigned char b_end = bn->b_end; - int split = b_end / 2; /* Assume equal split. */ - unsigned char slot_count = mt_slots[bn->type]; - - /* - * To support gap tracking, all NULL entries are kept together and a node cannot - * end on a NULL entry, with the exception of the left-most leaf. The - * limitation means that the split of a node must be checked for this condition - * and be able to put more data in one direction or the other. - * - * Although extremely rare, it is possible to enter what is known as the 3-way - * split scenario. The 3-way split comes about by means of a store of a range - * that overwrites the end and beginning of two full nodes. The result is a set - * of entries that cannot be stored in 2 nodes. Sometimes, these two nodes can - * also be located in different parent nodes which are also full. This can - * carry upwards all the way to the root in the worst case. - */ - if (unlikely(mab_middle_node(bn, split, slot_count))) { - split = b_end / 3; - *mid_split = split * 2; - } else { - *mid_split = 0; - } - - /* Avoid ending a node on a NULL entry */ - split = mab_no_null_split(bn, split, slot_count); - - if (unlikely(*mid_split)) - *mid_split = mab_no_null_split(bn, *mid_split, slot_count); - - return split; -} - -/* - * mas_mab_cp() - Copy data from a maple state inclusively to a maple_big_node - * and set @b_node->b_end to the next free slot. - * @mas: The maple state - * @mas_start: The starting slot to copy - * @mas_end: The end slot to copy (inclusively) - * @b_node: The maple_big_node to place the data - * @mab_start: The starting location in maple_big_node to store the data. - */ -static inline void mas_mab_cp(struct ma_state *mas, unsigned char mas_start, - unsigned char mas_end, struct maple_big_node *b_node, - unsigned char mab_start) -{ - enum maple_type mt; - struct maple_node *node; - void __rcu **slots; - unsigned long *pivots, *gaps; - int i = mas_start, j = mab_start; - unsigned char piv_end; - - node = mas_mn(mas); - mt = mte_node_type(mas->node); - pivots = ma_pivots(node, mt); - if (!i) { - b_node->pivot[j] = pivots[i++]; - if (unlikely(i > mas_end)) - goto complete; - j++; - } - - piv_end = min(mas_end, mt_pivots[mt]); - for (; i < piv_end; i++, j++) { - b_node->pivot[j] = pivots[i]; - if (unlikely(!b_node->pivot[j])) - goto complete; - - if (unlikely(mas->max == b_node->pivot[j])) - goto complete; - } - - b_node->pivot[j] = mas_safe_pivot(mas, pivots, i, mt); - -complete: - b_node->b_end = ++j; - j -= mab_start; - slots = ma_slots(node, mt); - memcpy(b_node->slot + mab_start, slots + mas_start, sizeof(void *) * j); - if (!ma_is_leaf(mt) && mt_is_alloc(mas->tree)) { - gaps = ma_gaps(node, mt); - memcpy(b_node->gap + mab_start, gaps + mas_start, - sizeof(unsigned long) * j); - } -} - -/* * mas_leaf_set_meta() - Set the metadata of a leaf if possible. * @node: The maple node * @mt: The maple type @@ -1793,134 +1717,6 @@ static inline void mas_leaf_set_meta(struct maple_node *node, } /* - * mab_mas_cp() - Copy data from maple_big_node to a maple encoded node. - * @b_node: the maple_big_node that has the data - * @mab_start: the start location in @b_node. - * @mab_end: The end location in @b_node (inclusively) - * @mas: The maple state with the maple encoded node. - */ -static inline void mab_mas_cp(struct maple_big_node *b_node, - unsigned char mab_start, unsigned char mab_end, - struct ma_state *mas, bool new_max) -{ - int i, j = 0; - enum maple_type mt = mte_node_type(mas->node); - struct maple_node *node = mte_to_node(mas->node); - void __rcu **slots = ma_slots(node, mt); - unsigned long *pivots = ma_pivots(node, mt); - unsigned long *gaps = NULL; - unsigned char end; - - if (mab_end - mab_start > mt_pivots[mt]) - mab_end--; - - if (!pivots[mt_pivots[mt] - 1]) - slots[mt_pivots[mt]] = NULL; - - i = mab_start; - do { - pivots[j++] = b_node->pivot[i++]; - } while (i <= mab_end && likely(b_node->pivot[i])); - - memcpy(slots, b_node->slot + mab_start, - sizeof(void *) * (i - mab_start)); - - if (new_max) - mas->max = b_node->pivot[i - 1]; - - end = j - 1; - if (likely(!ma_is_leaf(mt) && mt_is_alloc(mas->tree))) { - unsigned long max_gap = 0; - unsigned char offset = 0; - - gaps = ma_gaps(node, mt); - do { - gaps[--j] = b_node->gap[--i]; - if (gaps[j] > max_gap) { - offset = j; - max_gap = gaps[j]; - } - } while (j); - - ma_set_meta(node, mt, offset, end); - } else { - mas_leaf_set_meta(node, mt, end); - } -} - -/* - * mas_store_b_node() - Store an @entry into the b_node while also copying the - * data from a maple encoded node. - * @wr_mas: the maple write state - * @b_node: the maple_big_node to fill with data - * @offset_end: the offset to end copying - * - * Return: The actual end of the data stored in @b_node - */ -static noinline_for_kasan void mas_store_b_node(struct ma_wr_state *wr_mas, - struct maple_big_node *b_node, unsigned char offset_end) -{ - unsigned char slot; - unsigned char b_end; - /* Possible underflow of piv will wrap back to 0 before use. */ - unsigned long piv; - struct ma_state *mas = wr_mas->mas; - - b_node->type = wr_mas->type; - b_end = 0; - slot = mas->offset; - if (slot) { - /* Copy start data up to insert. */ - mas_mab_cp(mas, 0, slot - 1, b_node, 0); - b_end = b_node->b_end; - piv = b_node->pivot[b_end - 1]; - } else - piv = mas->min - 1; - - if (piv + 1 < mas->index) { - /* Handle range starting after old range */ - b_node->slot[b_end] = wr_mas->content; - if (!wr_mas->content) - b_node->gap[b_end] = mas->index - 1 - piv; - b_node->pivot[b_end++] = mas->index - 1; - } - - /* Store the new entry. */ - mas->offset = b_end; - b_node->slot[b_end] = wr_mas->entry; - b_node->pivot[b_end] = mas->last; - - /* Appended. */ - if (mas->last >= mas->max) - goto b_end; - - /* Handle new range ending before old range ends */ - piv = mas_safe_pivot(mas, wr_mas->pivots, offset_end, wr_mas->type); - if (piv > mas->last) { - if (offset_end != slot) - wr_mas->content = mas_slot_locked(mas, wr_mas->slots, - offset_end); - - b_node->slot[++b_end] = wr_mas->content; - if (!wr_mas->content) - b_node->gap[b_end] = piv - mas->last + 1; - b_node->pivot[b_end] = piv; - } - - slot = offset_end + 1; - if (slot > mas->end) - goto b_end; - - /* Copy end data to the end of the node. */ - mas_mab_cp(mas, slot, mas->end + 1, b_node, ++b_end); - b_node->b_end--; - return; - -b_end: - b_node->b_end = b_end; -} - -/* * mas_prev_sibling() - Find the previous node with the same parent. * @mas: the maple state * @@ -1965,25 +1761,6 @@ static inline bool mas_next_sibling(struct ma_state *mas) } /* - * mas_node_or_none() - Set the enode and state. - * @mas: the maple state - * @enode: The encoded maple node. - * - * Set the node to the enode and the status. - */ -static inline void mas_node_or_none(struct ma_state *mas, - struct maple_enode *enode) -{ - if (enode) { - mas->node = enode; - mas->status = ma_active; - } else { - mas->node = NULL; - mas->status = ma_none; - } -} - -/* * mas_wr_node_walk() - Find the correct offset for the index in the @mas. * If @mas->index cannot be found within the containing * node, we traverse to the last entry in the node. @@ -2016,277 +1793,55 @@ static inline void mas_wr_node_walk(struct ma_wr_state *wr_mas) wr_mas->offset_end = mas->offset = offset; } -/* - * mast_rebalance_next() - Rebalance against the next node - * @mast: The maple subtree state - */ -static inline void mast_rebalance_next(struct maple_subtree_state *mast) +static inline void rebalance_sib(struct ma_state *parent, struct ma_state *sib) { - unsigned char b_end = mast->bn->b_end; - - mas_mab_cp(mast->orig_r, 0, mt_slot_count(mast->orig_r->node), - mast->bn, b_end); - mast->orig_r->last = mast->orig_r->max; -} - -/* - * mast_rebalance_prev() - Rebalance against the previous node - * @mast: The maple subtree state - */ -static inline void mast_rebalance_prev(struct maple_subtree_state *mast) -{ - unsigned char end = mas_data_end(mast->orig_l) + 1; - unsigned char b_end = mast->bn->b_end; + *sib = *parent; + /* Prioritize move right to pull data left */ + if (sib->offset < sib->end) + sib->offset++; + else + sib->offset--; - mab_shift_right(mast->bn, end); - mas_mab_cp(mast->orig_l, 0, end - 1, mast->bn, 0); - mast->l->min = mast->orig_l->min; - mast->orig_l->index = mast->orig_l->min; - mast->bn->b_end = end + b_end; - mast->l->offset += end; + mas_descend(sib); + sib->end = mas_data_end(sib); } -/* - * mast_spanning_rebalance() - Rebalance nodes with nearest neighbour favouring - * the node to the right. Checking the nodes to the right then the left at each - * level upwards until root is reached. - * Data is copied into the @mast->bn. - * @mast: The maple_subtree_state. - */ static inline -bool mast_spanning_rebalance(struct maple_subtree_state *mast) +void spanning_sib(struct ma_wr_state *l_wr_mas, + struct ma_wr_state *r_wr_mas, struct ma_state *nneighbour) { - struct ma_state r_tmp = *mast->orig_r; - struct ma_state l_tmp = *mast->orig_l; + struct ma_state l_tmp = *l_wr_mas->mas; + struct ma_state r_tmp = *r_wr_mas->mas; unsigned char depth = 0; do { - mas_ascend(mast->orig_r); - mas_ascend(mast->orig_l); + mas_ascend(&r_tmp); + mas_ascend(&l_tmp); depth++; - if (mast->orig_r->offset < mas_data_end(mast->orig_r)) { - mast->orig_r->offset++; - do { - mas_descend(mast->orig_r); - mast->orig_r->offset = 0; - } while (--depth); - - mast_rebalance_next(mast); - *mast->orig_l = l_tmp; - return true; - } else if (mast->orig_l->offset != 0) { - mast->orig_l->offset--; + if (r_tmp.offset < mas_data_end(&r_tmp)) { + r_tmp.offset++; + mas_descend(&r_tmp); + r_tmp.offset = 0; + while (--depth) + mas_descend(&r_tmp); + + r_tmp.end = mas_data_end(&r_tmp); + *nneighbour = r_tmp; + return; + } else if (l_tmp.offset) { + l_tmp.offset--; do { - mas_descend(mast->orig_l); - mast->orig_l->offset = - mas_data_end(mast->orig_l); + mas_descend(&l_tmp); + l_tmp.offset = mas_data_end(&l_tmp); } while (--depth); - mast_rebalance_prev(mast); - *mast->orig_r = r_tmp; - return true; + l_tmp.end = l_tmp.offset; + *nneighbour = l_tmp; + return; } - } while (!mte_is_root(mast->orig_r->node)); + } while (!mte_is_root(r_tmp.node)); - *mast->orig_r = r_tmp; - *mast->orig_l = l_tmp; - return false; -} - -/* - * mast_ascend() - Ascend the original left and right maple states. - * @mast: the maple subtree state. - * - * Ascend the original left and right sides. Set the offsets to point to the - * data already in the new tree (@mast->l and @mast->r). - */ -static inline void mast_ascend(struct maple_subtree_state *mast) -{ - MA_WR_STATE(wr_mas, mast->orig_r, NULL); - mas_ascend(mast->orig_l); - mas_ascend(mast->orig_r); - - mast->orig_r->offset = 0; - mast->orig_r->index = mast->r->max; - /* last should be larger than or equal to index */ - if (mast->orig_r->last < mast->orig_r->index) - mast->orig_r->last = mast->orig_r->index; - - wr_mas.type = mte_node_type(mast->orig_r->node); - mas_wr_node_walk(&wr_mas); - /* Set up the left side of things */ - mast->orig_l->offset = 0; - mast->orig_l->index = mast->l->min; - wr_mas.mas = mast->orig_l; - wr_mas.type = mte_node_type(mast->orig_l->node); - mas_wr_node_walk(&wr_mas); - - mast->bn->type = wr_mas.type; -} - -/* - * mas_new_ma_node() - Create and return a new maple node. Helper function. - * @mas: the maple state with the allocations. - * @b_node: the maple_big_node with the type encoding. - * - * Use the node type from the maple_big_node to allocate a new node from the - * ma_state. This function exists mainly for code readability. - * - * Return: A new maple encoded node - */ -static inline struct maple_enode -*mas_new_ma_node(struct ma_state *mas, struct maple_big_node *b_node) -{ - return mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)), b_node->type); -} - -/* - * mas_mab_to_node() - Set up right and middle nodes - * - * @mas: the maple state that contains the allocations. - * @b_node: the node which contains the data. - * @left: The pointer which will have the left node - * @right: The pointer which may have the right node - * @middle: the pointer which may have the middle node (rare) - * @mid_split: the split location for the middle node - * - * Return: the split of left. - */ -static inline unsigned char mas_mab_to_node(struct ma_state *mas, - struct maple_big_node *b_node, struct maple_enode **left, - struct maple_enode **right, struct maple_enode **middle, - unsigned char *mid_split) -{ - unsigned char split = 0; - unsigned char slot_count = mt_slots[b_node->type]; - - *left = mas_new_ma_node(mas, b_node); - *right = NULL; - *middle = NULL; - *mid_split = 0; - - if (b_node->b_end < slot_count) { - split = b_node->b_end; - } else { - split = mab_calc_split(mas, b_node, mid_split); - *right = mas_new_ma_node(mas, b_node); - } - - if (*mid_split) - *middle = mas_new_ma_node(mas, b_node); - - return split; - -} - -/* - * mab_set_b_end() - Add entry to b_node at b_node->b_end and increment the end - * pointer. - * @b_node: the big node to add the entry - * @mas: the maple state to get the pivot (mas->max) - * @entry: the entry to add, if NULL nothing happens. - */ -static inline void mab_set_b_end(struct maple_big_node *b_node, - struct ma_state *mas, - void *entry) -{ - if (!entry) - return; - - b_node->slot[b_node->b_end] = entry; - if (mt_is_alloc(mas->tree)) - b_node->gap[b_node->b_end] = mas_max_gap(mas); - b_node->pivot[b_node->b_end++] = mas->max; -} - -/* - * mas_set_split_parent() - combine_then_separate helper function. Sets the parent - * of @mas->node to either @left or @right, depending on @slot and @split - * - * @mas: the maple state with the node that needs a parent - * @left: possible parent 1 - * @right: possible parent 2 - * @slot: the slot the mas->node was placed - * @split: the split location between @left and @right - */ -static inline void mas_set_split_parent(struct ma_state *mas, - struct maple_enode *left, - struct maple_enode *right, - unsigned char *slot, unsigned char split) -{ - if (mas_is_none(mas)) - return; - - if ((*slot) <= split) - mas_set_parent(mas, mas->node, left, *slot); - else if (right) - mas_set_parent(mas, mas->node, right, (*slot) - split - 1); - - (*slot)++; -} - -/* - * mte_mid_split_check() - Check if the next node passes the mid-split - * @l: Pointer to left encoded maple node. - * @m: Pointer to middle encoded maple node. - * @r: Pointer to right encoded maple node. - * @slot: The offset - * @split: The split location. - * @mid_split: The middle split. - */ -static inline void mte_mid_split_check(struct maple_enode **l, - struct maple_enode **r, - struct maple_enode *right, - unsigned char slot, - unsigned char *split, - unsigned char mid_split) -{ - if (*r == right) - return; - - if (slot < mid_split) - return; - - *l = *r; - *r = right; - *split = mid_split; -} - -/* - * mast_set_split_parents() - Helper function to set three nodes parents. Slot - * is taken from @mast->l. - * @mast: the maple subtree state - * @left: the left node - * @right: the right node - * @split: the split location. - */ -static inline void mast_set_split_parents(struct maple_subtree_state *mast, - struct maple_enode *left, - struct maple_enode *middle, - struct maple_enode *right, - unsigned char split, - unsigned char mid_split) -{ - unsigned char slot; - struct maple_enode *l = left; - struct maple_enode *r = right; - - if (mas_is_none(mast->l)) - return; - - if (middle) - r = middle; - - slot = mast->l->offset; - - mte_mid_split_check(&l, &r, right, slot, &split, mid_split); - mas_set_split_parent(mast->l, l, r, &slot, split); - - mte_mid_split_check(&l, &r, right, slot, &split, mid_split); - mas_set_split_parent(mast->m, l, r, &slot, split); - - mte_mid_split_check(&l, &r, right, slot, &split, mid_split); - mas_set_split_parent(mast->r, l, r, &slot, split); + WARN_ON_ONCE(1); } /* @@ -2419,120 +1974,104 @@ static inline void mas_topiary_replace(struct ma_state *mas, } /* - * mas_wmb_replace() - Write memory barrier and replace - * @mas: The maple state - * @old_enode: The old maple encoded node that is being replaced. - * @new_height: The new height of the tree as a result of the operation + * node_copy() - Copy from one node to another. * - * Updates gap as necessary. - */ -static inline void mas_wmb_replace(struct ma_state *mas, - struct maple_enode *old_enode, unsigned char new_height) -{ - /* Insert the new data in the tree */ - mas_topiary_replace(mas, old_enode, new_height); - - if (mte_is_leaf(mas->node)) - return; - - mas_update_gap(mas); -} - -/* - * mast_cp_to_nodes() - Copy data out to nodes. - * @mast: The maple subtree state - * @left: The left encoded maple node - * @middle: The middle encoded maple node - * @right: The right encoded maple node - * @split: The location to split between left and (middle ? middle : right) - * @mid_split: The location to split between middle and right. + * @mas: The maple state + * @src: The source node + * @start: The offset into the src to start copying + * @size: The size to copy (non-zero) + * @s_max: The source node max + * @s_mt: The source maple node type + * @dst: The destination + * @d_start: The start location in the destination node + * @d_mt: The destination maple node type */ -static inline void mast_cp_to_nodes(struct maple_subtree_state *mast, - struct maple_enode *left, struct maple_enode *middle, - struct maple_enode *right, unsigned char split, unsigned char mid_split) -{ - bool new_lmax = true; - - mas_node_or_none(mast->l, left); - mas_node_or_none(mast->m, middle); - mas_node_or_none(mast->r, right); - - mast->l->min = mast->orig_l->min; - if (split == mast->bn->b_end) { - mast->l->max = mast->orig_r->max; - new_lmax = false; - } +static inline +unsigned long node_copy(struct ma_state *mas, struct maple_node *src, + unsigned char start, unsigned char size, unsigned long s_max, + enum maple_type s_mt, struct maple_node *dst, unsigned char d_start, + enum maple_type d_mt) +{ + unsigned long *s_pivots, *d_pivots; + void __rcu **s_slots, **d_slots; + unsigned long *s_gaps, *d_gaps; + unsigned long d_max; + + d_slots = ma_slots(dst, d_mt) + d_start; + d_pivots = ma_pivots(dst, d_mt) + d_start; + s_slots = ma_slots(src, s_mt) + start; + s_pivots = ma_pivots(src, s_mt) + start; + memcpy(d_slots, s_slots, size * sizeof(void __rcu *)); + + if (!ma_is_leaf(d_mt) && s_mt == maple_copy) + mas_set_parent_slots(mas, mt_mk_node(dst, d_mt), + d_slots, d_start, size); + + d_gaps = ma_gaps(dst, d_mt); + if (d_gaps) { + s_gaps = ma_gaps(src, s_mt) + start; + d_gaps += d_start; + memcpy(d_gaps, s_gaps, size * sizeof(unsigned long)); + } + + if (start + size - 1 < mt_pivots[s_mt]) + d_max = s_pivots[size - 1]; + else + d_max = s_max; - mab_mas_cp(mast->bn, 0, split, mast->l, new_lmax); + if (d_start + size <= mt_pivots[d_mt]) + d_pivots[size - 1] = d_max; - if (middle) { - mab_mas_cp(mast->bn, 1 + split, mid_split, mast->m, true); - mast->m->min = mast->bn->pivot[split] + 1; - split = mid_split; - } + size--; + if (size) + memcpy(d_pivots, s_pivots, size * sizeof(unsigned long)); - mast->r->max = mast->orig_r->max; - if (right) { - mab_mas_cp(mast->bn, 1 + split, mast->bn->b_end, mast->r, false); - mast->r->min = mast->bn->pivot[split] + 1; - } + return d_max; } /* - * mast_combine_cp_left - Copy in the original left side of the tree into the - * combined data set in the maple subtree state big node. - * @mast: The maple subtree state + * node_finalise() - Zero out unused area and populate metadata + * @node: The maple node + * @mt: The maple node type + * @end: The end of the used area */ -static inline void mast_combine_cp_left(struct maple_subtree_state *mast) +static inline +void node_finalise(struct maple_node *node, enum maple_type mt, + unsigned char end) { - unsigned char l_slot = mast->orig_l->offset; - - if (!l_slot) - return; - - mas_mab_cp(mast->orig_l, 0, l_slot - 1, mast->bn, 0); -} + unsigned char max_end = mt_slots[mt]; + unsigned char size; + unsigned long *gaps; + unsigned char gap_slot; -/* - * mast_combine_cp_right: Copy in the original right side of the tree into the - * combined data set in the maple subtree state big node. - * @mast: The maple subtree state - */ -static inline void mast_combine_cp_right(struct maple_subtree_state *mast) -{ - if (mast->bn->pivot[mast->bn->b_end - 1] >= mast->orig_r->max) - return; + gaps = ma_gaps(node, mt); + if (end < max_end - 1) { + size = max_end - end; + memset(ma_slots(node, mt) + end, 0, size * sizeof(void *)); - mas_mab_cp(mast->orig_r, mast->orig_r->offset + 1, - mt_slot_count(mast->orig_r->node), mast->bn, - mast->bn->b_end); - mast->orig_r->last = mast->orig_r->max; -} + if (gaps) + memset(gaps + end, 0, size * sizeof(unsigned long)); -/* - * mast_sufficient: Check if the maple subtree state has enough data in the big - * node to create at least one sufficient node - * @mast: the maple subtree state - */ -static inline bool mast_sufficient(struct maple_subtree_state *mast) -{ - if (mast->bn->b_end > mt_min_slot_count(mast->orig_l->node)) - return true; + if (--size) + memset(ma_pivots(node, mt) + end, 0, size * sizeof(unsigned long)); + } - return false; -} + gap_slot = 0; + if (gaps && !ma_is_leaf(mt)) { + unsigned long max_gap; -/* - * mast_overflow: Check if there is too much data in the subtree state for a - * single node. - * @mast: The maple subtree state - */ -static inline bool mast_overflow(struct maple_subtree_state *mast) -{ - if (mast->bn->b_end > mt_slot_count(mast->orig_l->node)) - return true; + max_gap = 0; + for (int i = 0; i <= end; i++) + if (gaps[i] > max_gap) { + gap_slot = i; + max_gap = gaps[i]; + } + } - return false; + if (mt == maple_arange_64) + ma_set_meta(node, mt, gap_slot, end - 1); + else if (end <= max_end - 1) + ma_set_meta(node, mt, gap_slot, end - 1); } static inline void *mtree_range_walk(struct ma_state *mas) @@ -2596,480 +2135,662 @@ dead_node: } /* - * mas_spanning_rebalance() - Rebalance across two nodes which may not be peers. - * @mas: The starting maple state - * @mast: The maple_subtree_state, keeps track of 4 maple states. - * @count: The estimated count of iterations needed. - * - * Follow the tree upwards from @l_mas and @r_mas for @count, or until the root - * is hit. First @b_node is split into two entries which are inserted into the - * next iteration of the loop. @b_node is returned populated with the final - * iteration. @mas is used to obtain allocations. orig_l_mas keeps track of the - * nodes that will remain active by using orig_l_mas->index and orig_l_mas->last - * to account of what has been copied into the new sub-tree. The update of - * orig_l_mas->last is used in mas_consume to find the slots that will need to - * be either freed or destroyed. orig_l_mas->depth keeps track of the height of - * the new sub-tree in case the sub-tree becomes the full tree. - */ -static void mas_spanning_rebalance(struct ma_state *mas, - struct maple_subtree_state *mast, unsigned char count) -{ - unsigned char split, mid_split; - unsigned char slot = 0; - unsigned char new_height = 0; /* used if node is a new root */ - struct maple_enode *left = NULL, *middle = NULL, *right = NULL; + * mas_wmb_replace() - Write memory barrier and replace + * @mas: The maple state + * @cp: The maple copy node + * + * Updates gap as necessary. + */ +static inline void mas_wmb_replace(struct ma_state *mas, struct maple_copy *cp) +{ struct maple_enode *old_enode; - MA_STATE(l_mas, mas->tree, mas->index, mas->index); - MA_STATE(r_mas, mas->tree, mas->index, mas->last); - MA_STATE(m_mas, mas->tree, mas->index, mas->index); + old_enode = mas->node; + mas->node = mt_slot_locked(mas->tree, cp->slot, 0); + /* Insert the new data in the tree */ + mas_topiary_replace(mas, old_enode, cp->height); + if (mt_is_alloc(mas->tree) && !mte_is_root(mas->node)) + mas_update_gap_known(mas, cp->gap[0]); + + mtree_range_walk(mas); +} + + +/* + * cp_leaf_init() - Initialize a maple_copy node for the leaf level of a + * spanning store + * @cp: The maple copy node + * @mas: The maple state + * @l_wr_mas: The left write state of the spanning store + * @r_wr_mas: The right write state of the spanning store + */ +static inline void cp_leaf_init(struct maple_copy *cp, + struct ma_state *mas, struct ma_wr_state *l_wr_mas, + struct ma_wr_state *r_wr_mas) +{ + unsigned char end = 0; /* - * The tree needs to be rebalanced and leaves need to be kept at the same level. - * Rebalancing is done by use of the ``struct maple_topiary``. + * WARNING: The use of RCU_INIT_POINTER() makes it extremely important + * to not expose the maple_copy node to any readers. Exposure may + * result in buggy code when a compiler reorders the instructions. */ - mast->l = &l_mas; - mast->m = &m_mas; - mast->r = &r_mas; - l_mas.status = r_mas.status = m_mas.status = ma_none; - /* Check if this is not root and has sufficient data. */ - if (((mast->orig_l->min != 0) || (mast->orig_r->max != ULONG_MAX)) && - unlikely(mast->bn->b_end <= mt_min_slots[mast->bn->type])) - mast_spanning_rebalance(mast); + cp->height = 1; + /* Create entries to insert including split entries to left and right */ + if (l_wr_mas->r_min < mas->index) { + end++; + RCU_INIT_POINTER(cp->slot[0], l_wr_mas->content); + cp->pivot[0] = mas->index - 1; + } + RCU_INIT_POINTER(cp->slot[end], l_wr_mas->entry); + cp->pivot[end] = mas->last; + + if (r_wr_mas->end_piv > mas->last) { + end++; + RCU_INIT_POINTER(cp->slot[end], + r_wr_mas->slots[r_wr_mas->offset_end]); + cp->pivot[end] = r_wr_mas->end_piv; + } + + cp->min = l_wr_mas->r_min; + cp->max = cp->pivot[end]; + cp->end = end; +} + +/* + * cp_data_calc() - Calculate the size of the data (1 indexed). + * @cp: The maple copy struct with the new data populated. + * @l_wr_mas: The maple write state containing the data to the left of the write + * @r_wr_mas: The maple write state containing the data to the right of the + * write + * + * cp->data is a size (not indexed by 0). + */ +static inline void cp_data_calc(struct maple_copy *cp, + struct ma_wr_state *l_wr_mas, struct ma_wr_state *r_wr_mas) +{ + + /* Add 1 every time for the 0th element */ + cp->data = l_wr_mas->mas->offset; + /* Add the new data and any partial overwrites */ + cp->data += cp->end + 1; + /* Data from right (offset + 1 to end), +1 for zero */ + cp->data += r_wr_mas->mas->end - r_wr_mas->offset_end; +} + +static bool data_fits(struct ma_state *sib, struct ma_state *mas, + struct maple_copy *cp) +{ + unsigned char new_data; + enum maple_type type; + unsigned char space; + unsigned char end; + + type = mte_node_type(mas->node); + space = 2 * mt_slots[type]; + end = sib->end; + + new_data = end + 1 + cp->data; + if (new_data > space) + return false; /* - * Each level of the tree is examined and balanced, pushing data to the left or - * right, or rebalancing against left or right nodes is employed to avoid - * rippling up the tree to limit the amount of churn. Once a new sub-section of - * the tree is created, there may be a mix of new and old nodes. The old nodes - * will have the incorrect parent pointers and currently be in two trees: the - * original tree and the partially new tree. To remedy the parent pointers in - * the old tree, the new data is swapped into the active tree and a walk down - * the tree is performed and the parent pointers are updated. - * See mas_topiary_replace() for more information. + * This is off by one by design. The extra space is left to reduce + * jitter in operations that add then remove two entries. + * + * end is an index while new space and data are both sizes. Adding one + * to end to convert the index to a size means that the below + * calculation should be <=, but we want to keep an extra space in nodes + * to reduce jitter. + * + * Note that it is still possible to get a full node on the left by the + * NULL landing exactly on the split. The NULL ending of a node happens + * in the dst_setup() function, where we will either increase the split + * by one or decrease it by one, if possible. In the case of split + * (this case), it is always possible to shift the spilt by one - again + * because there is at least one slot free by the below checking. */ - while (count--) { - mast->bn->b_end--; - mast->bn->type = mte_node_type(mast->orig_l->node); - split = mas_mab_to_node(mas, mast->bn, &left, &right, &middle, - &mid_split); - mast_set_split_parents(mast, left, middle, right, split, - mid_split); - mast_cp_to_nodes(mast, left, middle, right, split, mid_split); - new_height++; + if (new_data < space) + return true; - /* - * Copy data from next level in the tree to mast->bn from next - * iteration - */ - memset(mast->bn, 0, sizeof(struct maple_big_node)); - mast->bn->type = mte_node_type(left); - - /* Root already stored in l->node. */ - if (mas_is_root_limits(mast->l)) - goto new_root; - - mast_ascend(mast); - mast_combine_cp_left(mast); - l_mas.offset = mast->bn->b_end; - mab_set_b_end(mast->bn, &l_mas, left); - mab_set_b_end(mast->bn, &m_mas, middle); - mab_set_b_end(mast->bn, &r_mas, right); - - /* Copy anything necessary out of the right node. */ - mast_combine_cp_right(mast); - mast->orig_l->last = mast->orig_l->max; - - if (mast_sufficient(mast)) { - if (mast_overflow(mast)) - continue; + return false; +} - if (mast->orig_l->node == mast->orig_r->node) { - /* - * The data in b_node should be stored in one - * node and in the tree - */ - slot = mast->l->offset; - break; - } +static inline void push_data_sib(struct maple_copy *cp, struct ma_state *mas, + struct ma_state *sib, struct ma_state *parent) +{ - continue; - } + if (mte_is_root(mas->node)) + goto no_push; - /* May be a new root stored in mast->bn */ - if (mas_is_root_limits(mast->orig_l)) - break; - mast_spanning_rebalance(mast); + *sib = *parent; + if (sib->offset) { + sib->offset--; + mas_descend(sib); + sib->end = mas_data_end(sib); + if (data_fits(sib, mas, cp)) /* Push left */ + return; - /* rebalancing from other nodes may require another loop. */ - if (!count) - count++; + *sib = *parent; } - l_mas.node = mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)), - mte_node_type(mast->orig_l->node)); + if (sib->offset >= sib->end) + goto no_push; - mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, &l_mas, true); - new_height++; - mas_set_parent(mas, left, l_mas.node, slot); - if (middle) - mas_set_parent(mas, middle, l_mas.node, ++slot); + sib->offset++; + mas_descend(sib); + sib->end = mas_data_end(sib); + if (data_fits(sib, mas, cp)) /* Push right*/ + return; - if (right) - mas_set_parent(mas, right, l_mas.node, ++slot); +no_push: + sib->end = 0; +} - if (mas_is_root_limits(mast->l)) { -new_root: - mas_mn(mast->l)->parent = ma_parent_ptr(mas_tree_parent(mas)); - while (!mte_is_root(mast->orig_l->node)) - mast_ascend(mast); - } else { - mas_mn(&l_mas)->parent = mas_mn(mast->orig_l)->parent; +/* + * rebalance_data() - Calculate the @cp data, populate @sib if insufficient or + * if the data can be pushed into a sibling. + * @cp: The maple copy node + * @wr_mas: The left write maple state + * @sib: The maple state of the sibling. + * + * Note: @cp->data is a size and not indexed by 0. @sib->end may be set to 0 to + * indicate it will not be used. + * + */ +static inline void rebalance_data(struct maple_copy *cp, + struct ma_wr_state *wr_mas, struct ma_state *sib, + struct ma_state *parent) +{ + cp_data_calc(cp, wr_mas, wr_mas); + sib->end = 0; + if (cp->data > mt_slots[wr_mas->type]) { + push_data_sib(cp, wr_mas->mas, sib, parent); + if (sib->end) + goto use_sib; + } else if (cp->data <= mt_min_slots[wr_mas->type]) { + if ((wr_mas->mas->min != 0) || + (wr_mas->mas->max != ULONG_MAX)) { + rebalance_sib(parent, sib); + goto use_sib; + } } - old_enode = mast->orig_l->node; - mas->depth = l_mas.depth; - mas->node = l_mas.node; - mas->min = l_mas.min; - mas->max = l_mas.max; - mas->offset = l_mas.offset; - mas_wmb_replace(mas, old_enode, new_height); - mtree_range_walk(mas); return; + +use_sib: + + cp->data += sib->end + 1; } /* - * mas_rebalance() - Rebalance a given node. + * spanning_data() - Calculate the @cp data and populate @sib if insufficient + * @cp: The maple copy node + * @l_wr_mas: The left write maple state + * @r_wr_mas: The right write maple state + * @sib: The maple state of the sibling. + * + * Note: @cp->data is a size and not indexed by 0. @sib->end may be set to 0 to + * indicate it will not be used. + */ +static inline void spanning_data(struct maple_copy *cp, + struct ma_wr_state *l_wr_mas, struct ma_wr_state *r_wr_mas, + struct ma_state *sib) +{ + cp_data_calc(cp, l_wr_mas, r_wr_mas); + if (((l_wr_mas->mas->min != 0) || (r_wr_mas->mas->max != ULONG_MAX)) && + (cp->data <= mt_min_slots[l_wr_mas->type])) { + spanning_sib(l_wr_mas, r_wr_mas, sib); + cp->data += sib->end + 1; + } else { + sib->end = 0; + } +} + +/* + * dst_setup() - Set up one or more destinations for the new data. + * @cp: The maple copy node * @mas: The maple state - * @b_node: The big maple node. - * - * Rebalance two nodes into a single node or two new nodes that are sufficient. - * Continue upwards until tree is sufficient. + * @mt: The source node type */ -static inline void mas_rebalance(struct ma_state *mas, - struct maple_big_node *b_node) +static inline +void dst_setup(struct maple_copy *cp, struct ma_state *mas, enum maple_type mt) { - char empty_count = mas_mt_height(mas); - struct maple_subtree_state mast; - unsigned char shift, b_end = ++b_node->b_end; + /* Data is 1 indexed, every src has +1 added. */ - MA_STATE(l_mas, mas->tree, mas->index, mas->last); - MA_STATE(r_mas, mas->tree, mas->index, mas->last); + if (cp->data <= mt_slots[mt]) { + cp->split = cp->data - 1; + cp->d_count = 1; + goto node_setup; + } - trace_ma_op(TP_FCT, mas); + cp->split = (cp->data - 1) / 2; + cp->d_count = 2; + if (cp->data < mt_slots[mt] * 2) + goto node_setup; - /* - * Rebalancing occurs if a node is insufficient. Data is rebalanced - * against the node to the right if it exists, otherwise the node to the - * left of this node is rebalanced against this node. If rebalancing - * causes just one node to be produced instead of two, then the parent - * is also examined and rebalanced if it is insufficient. Every level - * tries to combine the data in the same way. If one node contains the - * entire range of the tree, then that node is used as a new root node. - */ + if (cp->data == mt_slots[mt] * 2) { + unsigned char off; + unsigned char s; - mast.orig_l = &l_mas; - mast.orig_r = &r_mas; - mast.bn = b_node; - mast.bn->type = mte_node_type(mas->node); + if (!ma_is_leaf(mt)) + goto node_setup; - l_mas = r_mas = *mas; + /* + * Leaf nodes are a bit tricky because we cannot assume the data + * can fit due to the NULL limitation on node ends. + */ + off = cp->split; + for (s = 0; s < cp->s_count; s++) { + unsigned char s_off; - if (mas_next_sibling(&r_mas)) { - mas_mab_cp(&r_mas, 0, mt_slot_count(r_mas.node), b_node, b_end); - r_mas.last = r_mas.index = r_mas.max; - } else { - mas_prev_sibling(&l_mas); - shift = mas_data_end(&l_mas) + 1; - mab_shift_right(b_node, shift); - mas->offset += shift; - mas_mab_cp(&l_mas, 0, shift - 1, b_node, 0); - b_node->b_end = shift + b_end; - l_mas.index = l_mas.last = l_mas.min; - } + s_off = cp->src[s].end - cp->src[s].start; + if (s_off >= off) + break; - return mas_spanning_rebalance(mas, &mast, empty_count); -} + s_off++; + off -= s_off; + } -/* - * mas_split_final_node() - Split the final node in a subtree operation. - * @mast: the maple subtree state - * @mas: The maple state - */ -static inline void mas_split_final_node(struct maple_subtree_state *mast, - struct ma_state *mas) -{ - struct maple_enode *ancestor; + off += cp->src[s].start; + if (ma_slots(cp->src[s].node, cp->src[s].mt)[off]) + goto node_setup; + + cp->split++; + if (cp->split < mt_slots[mt]) + goto node_setup; + + cp->split -= 2; + if (cp->data - 2 - cp->split < mt_slots[mt]) + goto node_setup; - if (mte_is_root(mas->node)) { - if (mt_is_alloc(mas->tree)) - mast->bn->type = maple_arange_64; - else - mast->bn->type = maple_range_64; } - /* - * Only a single node is used here, could be root. - * The Big_node data should just fit in a single node. - */ - ancestor = mas_new_ma_node(mas, mast->bn); - mas_set_parent(mas, mast->l->node, ancestor, mast->l->offset); - mas_set_parent(mas, mast->r->node, ancestor, mast->r->offset); - mte_to_node(ancestor)->parent = mas_mn(mas)->parent; - mast->l->node = ancestor; - mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, mast->l, true); - mas->offset = mast->bn->b_end - 1; + /* No other choice but to 3-way split the data */ + cp->split = (cp->data + 2) / 3; + cp->d_count = 3; + +node_setup: + for (int i = 0; i < cp->d_count; i++) { + cp->dst[i].mt = mt; + cp->dst[i].node = ma_mnode_ptr(mas_pop_node(mas)); + } } -/* - * mast_fill_bnode() - Copy data into the big node in the subtree state - * @mast: The maple subtree state - * @mas: the maple state - * @skip: The number of entries to skip for new nodes insertion. - */ -static inline void mast_fill_bnode(struct maple_subtree_state *mast, - struct ma_state *mas, - unsigned char skip) +static inline void append_mas_cp(struct maple_copy *cp, + struct ma_state *mas, unsigned char start, unsigned char end) { - bool cp = true; - unsigned char split; + struct maple_node *node; + enum maple_type mt; + unsigned char count; - memset(mast->bn, 0, sizeof(struct maple_big_node)); + count = cp->s_count; + node = mas_mn(mas); + mt = mte_node_type(mas->node); + cp->src[count].node = node; + cp->src[count].mt = mt; + if (mas->end <= end) + cp->src[count].max = mas->max; + else + cp->src[count].max = ma_pivots(node, mt)[end]; - if (mte_is_root(mas->node)) { - cp = false; - } else { - mas_ascend(mas); - mas->offset = mte_parent_slot(mas->node); - } + cp->src[count].start = start; + cp->src[count].end = end; + cp->s_count++; +} - if (cp && mast->l->offset) - mas_mab_cp(mas, 0, mast->l->offset - 1, mast->bn, 0); +static inline void append_wr_mas_cp(struct maple_copy *cp, + struct ma_wr_state *wr_mas, unsigned char start, unsigned char end) +{ + unsigned char count; - split = mast->bn->b_end; - mab_set_b_end(mast->bn, mast->l, mast->l->node); - mast->r->offset = mast->bn->b_end; - mab_set_b_end(mast->bn, mast->r, mast->r->node); - if (mast->bn->pivot[mast->bn->b_end - 1] == mas->max) - cp = false; + count = cp->s_count; + cp->src[count].node = wr_mas->node; + cp->src[count].mt = wr_mas->type; + if (wr_mas->mas->end <= end) + cp->src[count].max = wr_mas->mas->max; + else + cp->src[count].max = wr_mas->pivots[end]; - if (cp) - mas_mab_cp(mas, split + skip, mt_slot_count(mas->node) - 1, - mast->bn, mast->bn->b_end); + cp->src[count].start = start; + cp->src[count].end = end; + cp->s_count++; +} - mast->bn->b_end--; - mast->bn->type = mte_node_type(mas->node); +static inline void init_cp_src(struct maple_copy *cp) +{ + cp->src[cp->s_count].node = ma_mnode_ptr(cp); + cp->src[cp->s_count].mt = maple_copy; + cp->src[cp->s_count].max = cp->max; + cp->src[cp->s_count].start = 0; + cp->src[cp->s_count].end = cp->end; + cp->s_count++; } /* - * mast_split_data() - Split the data in the subtree state big node into regular - * nodes. - * @mast: The maple subtree state - * @mas: The maple state - * @split: The location to split the big node + * multi_src_setup() - Set the @cp node up with multiple sources to copy from. + * @cp: The maple copy node + * @l_wr_mas: The left write maple state + * @r_wr_mas: The right write maple state + * @sib: The sibling maple state + * + * Note: @sib->end == 0 indicates no sibling will be used. */ -static inline void mast_split_data(struct maple_subtree_state *mast, - struct ma_state *mas, unsigned char split) +static inline +void multi_src_setup(struct maple_copy *cp, struct ma_wr_state *l_wr_mas, + struct ma_wr_state *r_wr_mas, struct ma_state *sib) { - unsigned char p_slot; + cp->s_count = 0; + if (sib->end && sib->max < l_wr_mas->mas->min) + append_mas_cp(cp, sib, 0, sib->end); - mab_mas_cp(mast->bn, 0, split, mast->l, true); - mte_set_pivot(mast->r->node, 0, mast->r->max); - mab_mas_cp(mast->bn, split + 1, mast->bn->b_end, mast->r, false); - mast->l->offset = mte_parent_slot(mas->node); - mast->l->max = mast->bn->pivot[split]; - mast->r->min = mast->l->max + 1; - if (mte_is_leaf(mas->node)) - return; + /* Copy left 0 - offset */ + if (l_wr_mas->mas->offset) { + unsigned char off = l_wr_mas->mas->offset - 1; + + append_wr_mas_cp(cp, l_wr_mas, 0, off); + cp->src[cp->s_count - 1].max = cp->min - 1; + } + + init_cp_src(cp); + + /* Copy right either from offset or offset + 1 pending on r_max */ + if (r_wr_mas->mas->end != r_wr_mas->offset_end) + append_wr_mas_cp(cp, r_wr_mas, r_wr_mas->offset_end + 1, + r_wr_mas->mas->end); - p_slot = mast->orig_l->offset; - mas_set_split_parent(mast->orig_l, mast->l->node, mast->r->node, - &p_slot, split); - mas_set_split_parent(mast->orig_r, mast->l->node, mast->r->node, - &p_slot, split); + if (sib->end && sib->min > r_wr_mas->mas->max) + append_mas_cp(cp, sib, 0, sib->end); } -/* - * mas_push_data() - Instead of splitting a node, it is beneficial to push the - * data to the right or left node if there is room. - * @mas: The maple state - * @mast: The maple subtree state - * @left: Push left or not. - * - * Keeping the height of the tree low means faster lookups. - * - * Return: True if pushed, false otherwise. - */ -static inline bool mas_push_data(struct ma_state *mas, - struct maple_subtree_state *mast, bool left) +static inline +void cp_data_write(struct maple_copy *cp, struct ma_state *mas) { - unsigned char slot_total = mast->bn->b_end; - unsigned char end, space, split; + struct maple_node *dst, *src; + unsigned char s, d; + unsigned char dst_offset; + unsigned char data_offset; + unsigned char src_end, s_offset; + unsigned char split; + unsigned long s_max, d_max; + unsigned char dst_size; + enum maple_type s_mt, d_mt; + + data_offset = 0; + s = d = 0; + /* Readability help */ + src = cp->src[s].node; + dst = cp->dst[d].node; + s_offset = cp->src[s].start; + src_end = cp->src[s].end; + split = cp->split; + s_max = cp->src[s].max; + s_mt = cp->src[s].mt; + d_mt = cp->dst[d].mt; + do { + dst_offset = 0; + d_max = 0; + dst = cp->dst[d].node; + d_mt = cp->dst[d].mt; + dst_size = split + 1; - MA_STATE(tmp_mas, mas->tree, mas->index, mas->last); - tmp_mas = *mas; - tmp_mas.depth = mast->l->depth; + while (dst_size) { + unsigned char size; - if (left && !mas_prev_sibling(&tmp_mas)) - return false; - else if (!left && !mas_next_sibling(&tmp_mas)) - return false; + if (src_end - s_offset + 1 < dst_size) + size = src_end - s_offset + 1; + else + size = dst_size; + + d_max = node_copy(mas, src, s_offset, size, s_max, s_mt, + dst, dst_offset, d_mt); + + dst_offset += size; + s_offset += size; + if (s_offset > src_end) { + /* This source is exhausted */ + s++; + if (s >= cp->s_count) { + cp->dst[d].max = d_max; + node_finalise(dst, d_mt, dst_offset); + return; + } + /* Reset local src */ + src = cp->src[s].node; + s_offset = cp->src[s].start; + src_end = cp->src[s].end; + s_max = cp->src[s].max; + s_mt = cp->src[s].mt; + } - end = mas_data_end(&tmp_mas); - slot_total += end; - space = 2 * mt_slot_count(mas->node) - 2; - /* -2 instead of -1 to ensure there isn't a triple split */ - if (ma_is_leaf(mast->bn->type)) - space--; + dst_size -= size; + data_offset += size; + } - if (mas->max == ULONG_MAX) - space--; + split = cp->split; + cp->dst[d].max = d_max; + /* Handle null entries */ + if (cp->dst[d].max != ULONG_MAX && + !ma_slots(dst, d_mt)[dst_offset - 1]) { + if (s_offset == cp->src[s].start) { + s--; + src = cp->src[s].node; + src_end = cp->src[s].end; + s_max = cp->src[s].max; + s_mt = cp->src[s].mt; + s_offset = src_end; + } else { + s_offset--; + } + /* Set dst max and clear pivot */ + split++; + data_offset--; + dst_offset--; + cp->dst[d].max = ma_pivots(dst, d_mt)[dst_offset - 1]; + } - if (slot_total >= space) - return false; + node_finalise(dst, d_mt, dst_offset); + ++d; /* Next destination */ + if (d == cp->d_count - 1) + split = cp->data - data_offset; - /* Get the data; Fill mast->bn */ - mast->bn->b_end++; - if (left) { - mab_shift_right(mast->bn, end + 1); - mas_mab_cp(&tmp_mas, 0, end, mast->bn, 0); - mast->bn->b_end = slot_total + 1; - } else { - mas_mab_cp(&tmp_mas, 0, end, mast->bn, mast->bn->b_end); - } + if (d >= cp->d_count) { + WARN_ON(data_offset < cp->data); + return; + } - /* Configure mast for splitting of mast->bn */ - split = mt_slots[mast->bn->type] - 2; - if (left) { - /* Switch mas to prev node */ - *mas = tmp_mas; - /* Start using mast->l for the left side. */ - tmp_mas.node = mast->l->node; - *mast->l = tmp_mas; - } else { - tmp_mas.node = mast->r->node; - *mast->r = tmp_mas; - split = slot_total - split; - } - split = mab_no_null_split(mast->bn, split, mt_slots[mast->bn->type]); - /* Update parent slot for split calculation. */ - if (left) - mast->orig_l->offset += end + 1; - - mast_split_data(mast, mas, split); - mast_fill_bnode(mast, mas, 2); - mas_split_final_node(mast, mas); - return true; + } while (data_offset <= cp->data); } /* - * mas_split() - Split data that is too big for one node into two. + * cp_dst_to_slots() - Migrate the maple copy destination to the maple copy + * slots + * @cp: The maple copy node + * @min: The minimal value represented + * @max: The maximum value represented * @mas: The maple state - * @b_node: The maple big node */ -static void mas_split(struct ma_state *mas, struct maple_big_node *b_node) +static inline void cp_dst_to_slots(struct maple_copy *cp, unsigned long min, + unsigned long max, struct ma_state *mas) { - struct maple_subtree_state mast; - int height = 0; - unsigned int orig_height = mas_mt_height(mas); - unsigned char mid_split, split = 0; - struct maple_enode *old; + unsigned char d; + unsigned long slot_min = min; - /* - * Splitting is handled differently from any other B-tree; the Maple - * Tree splits upwards. Splitting up means that the split operation - * occurs when the walk of the tree hits the leaves and not on the way - * down. The reason for splitting up is that it is impossible to know - * how much space will be needed until the leaf is (or leaves are) - * reached. Since overwriting data is allowed and a range could - * overwrite more than one range or result in changing one entry into 3 - * entries, it is impossible to know if a split is required until the - * data is examined. - * - * Splitting is a balancing act between keeping allocations to a minimum - * and avoiding a 'jitter' event where a tree is expanded to make room - * for an entry followed by a contraction when the entry is removed. To - * accomplish the balance, there are empty slots remaining in both left - * and right nodes after a split. - */ - MA_STATE(l_mas, mas->tree, mas->index, mas->last); - MA_STATE(r_mas, mas->tree, mas->index, mas->last); - MA_STATE(prev_l_mas, mas->tree, mas->index, mas->last); - MA_STATE(prev_r_mas, mas->tree, mas->index, mas->last); + for (d = 0; d < cp->d_count; d++) { + struct maple_node *mn = cp->dst[d].node; + enum maple_type mt = cp->dst[d].mt; + unsigned long slot_max = cp->dst[d].max; - trace_ma_op(TP_FCT, mas); + /* + * Warning, see cp_leaf_init() comment and rcu_assign_pointer() + * documentation. Since these are new nodes, there are no + * read-side operations that can view them until they are + * inserted into the tree after an rcu_assign_pointer() call. + */ + ma_init_slot(&cp->slot[d], mn, mt); + cp->pivot[d] = slot_max; + if (mt_is_alloc(mas->tree)) { + if (ma_is_leaf(mt)) { + cp->gap[d] = ma_leaf_max_gap(mn, mt, slot_min, + slot_max, ma_pivots(mn, mt), + ma_slots(mn, mt)); + } else { + unsigned long *gaps = ma_gaps(mn, mt); - mast.l = &l_mas; - mast.r = &r_mas; - mast.orig_l = &prev_l_mas; - mast.orig_r = &prev_r_mas; - mast.bn = b_node; + if (gaps) { + unsigned char gap_slot; - while (height++ <= orig_height) { - if (mt_slots[b_node->type] > b_node->b_end) { - mas_split_final_node(&mast, mas); - break; + gap_slot = ma_meta_gap(mn); + cp->gap[d] = gaps[gap_slot]; + } + } } + slot_min = slot_max + 1; + } - l_mas = r_mas = *mas; - l_mas.node = mas_new_ma_node(mas, b_node); - r_mas.node = mas_new_ma_node(mas, b_node); - /* - * Another way that 'jitter' is avoided is to terminate a split up early if the - * left or right node has space to spare. This is referred to as "pushing left" - * or "pushing right" and is similar to the B* tree, except the nodes left or - * right can rarely be reused due to RCU, but the ripple upwards is halted which - * is a significant savings. - */ - /* Try to push left. */ - if (mas_push_data(mas, &mast, true)) { - height++; - break; - } - /* Try to push right. */ - if (mas_push_data(mas, &mast, false)) { - height++; - break; - } + cp->end = cp->d_count - 1; + cp->min = min; + cp->max = max; +} - split = mab_calc_split(mas, b_node, &mid_split); - mast_split_data(&mast, mas, split); +static inline bool cp_is_new_root(struct maple_copy *cp, struct ma_state *mas) +{ + if (cp->min || cp->max != ULONG_MAX) + return false; + + if (cp->d_count != 1) { + enum maple_type mt = maple_arange_64; + + if (!mt_is_alloc(mas->tree)) + mt = maple_range_64; + + cp->data = cp->d_count; + cp->s_count = 0; + dst_setup(cp, mas, mt); + init_cp_src(cp); + node_copy(mas, cp->src[0].node, 0, cp->data, cp->max, maple_copy, + cp->dst[0].node, 0, mt); + node_finalise(cp->dst[0].node, mt, cp->end + 1); /* - * Usually correct, mab_mas_cp in the above call overwrites - * r->max. + * Warning, see cp_leaf_init() comment and rcu_assign_pointer() + * documentation. Since this is a new root, there are no + * read-side operations that can view it until it is insert into + * the tree after an rcu_assign_pointer() call. */ - mast.r->max = mas->max; - mast_fill_bnode(&mast, mas, 1); - prev_l_mas = *mast.l; - prev_r_mas = *mast.r; + ma_init_slot(&cp->slot[0], cp->dst[0].node, mt); + cp->height++; + } + WARN_ON_ONCE(cp->dst[0].node != mte_to_node( + mt_slot_locked(mas->tree, cp->slot, 0))); + cp->dst[0].node->parent = ma_parent_ptr(mas_tree_parent(mas)); + mas->min = 0; + mas->max = ULONG_MAX; + mas->depth = 0; + mas->node = mas_root_locked(mas); + return true; +} + +static inline bool cp_converged(struct maple_copy *cp, struct ma_state *mas, + struct ma_state *sib) +{ + if (cp->d_count != 1 || sib->end) + return false; + + cp->dst[0].node->parent = ma_parent_ptr(mas_mn(mas)->parent); + return true; +} + +/* + * spanning_ascend() - See if a spanning store operation has to keep walking up + * the tree + * @cp: The maple_copy node + * @l_wr_mas: The left maple write state + * @r_wr_mas: The right maple write state + * @sib: the maple state of the sibling + * + * Returns: True if another iteration is necessary. + */ +static bool spanning_ascend(struct maple_copy *cp, struct ma_state *mas, + struct ma_wr_state *l_wr_mas, struct ma_wr_state *r_wr_mas, + struct ma_state *sib) +{ + if (sib->end) { + if (sib->max < l_wr_mas->mas->min) + *l_wr_mas->mas = *sib; + else + *r_wr_mas->mas = *sib; } - /* Set the original node as dead */ - old = mas->node; - mas->node = l_mas.node; - mas_wmb_replace(mas, old, height); - mtree_range_walk(mas); - return; + cp_dst_to_slots(cp, l_wr_mas->mas->min, r_wr_mas->mas->max, mas); + if (cp_is_new_root(cp, mas)) + return false; + + /* Converged and has a single destination */ + if ((cp->d_count == 1) && + (l_wr_mas->mas->node == r_wr_mas->mas->node)) { + cp->dst[0].node->parent = ma_parent_ptr(mas_mn(mas)->parent); + return false; + } + + cp->height++; + wr_mas_ascend(l_wr_mas); + wr_mas_ascend(r_wr_mas); + return true; +} + +static inline +void copy_tree_location(const struct ma_state *src, struct ma_state *dst) +{ + dst->node = src->node; + dst->offset = src->offset; + dst->min = src->min; + dst->max = src->max; + dst->end = src->end; + dst->depth = src->depth; } /* - * mas_commit_b_node() - Commit the big node into the tree. - * @wr_mas: The maple write state - * @b_node: The maple big node + * rebalance_ascend() - Ascend the tree and set up for the next loop - if + * necessary + * + * Return: True if there another rebalancing operation on the next level is + * needed, false otherwise. */ -static noinline_for_kasan void mas_commit_b_node(struct ma_wr_state *wr_mas, - struct maple_big_node *b_node) +static inline bool rebalance_ascend(struct maple_copy *cp, + struct ma_wr_state *wr_mas, struct ma_state *sib, + struct ma_state *parent) { - enum store_type type = wr_mas->mas->store_type; + struct ma_state *mas; + unsigned long min, max; - WARN_ON_ONCE(type != wr_rebalance && type != wr_split_store); + mas = wr_mas->mas; + if (!sib->end) { + min = mas->min; + max = mas->max; + } else if (sib->min > mas->max) { /* Move right succeeded */ + min = mas->min; + max = sib->max; + wr_mas->offset_end = parent->offset + 1; + } else { + min = sib->min; + max = mas->max; + wr_mas->offset_end = parent->offset; + parent->offset--; + } - if (type == wr_rebalance) - return mas_rebalance(wr_mas->mas, b_node); + cp_dst_to_slots(cp, min, max, mas); + if (cp_is_new_root(cp, mas)) + return false; - return mas_split(wr_mas->mas, b_node); + if (cp_converged(cp, mas, sib)) + return false; + + cp->height++; + copy_tree_location(parent, mas); + wr_mas_setup(wr_mas, mas); + return true; } /* @@ -3112,7 +2833,6 @@ static inline void mas_root_expand(struct ma_state *mas, void *entry) ma_set_meta(node, maple_leaf_64, 0, slot); /* swap the new root into the tree */ rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node)); - return; } /* @@ -3269,18 +2989,13 @@ static inline void mas_extend_spanning_null(struct ma_wr_state *l_wr_mas, l_mas->index = l_mas->min; l_mas->offset = l_slot - 1; + l_wr_mas->r_min = l_mas->index; } if (!r_wr_mas->content) { if (r_mas->last < r_wr_mas->r_max) r_mas->last = r_wr_mas->r_max; r_mas->offset++; - } else if ((r_mas->last == r_wr_mas->r_max) && - (r_mas->last < r_mas->max) && - !mas_slot_locked(r_mas, r_wr_mas->slots, r_mas->offset + 1)) { - r_mas->last = mas_safe_pivot(r_mas, r_wr_mas->pivots, - r_wr_mas->type, r_mas->offset + 1); - r_mas->offset++; } } @@ -3382,8 +3097,6 @@ static inline void mas_new_root(struct ma_state *mas, void *entry) done: if (xa_is_node(root)) mte_destroy_walk(root, mas->tree); - - return; } /* * mas_wr_spanning_store() - Create a subtree with the store operation completed @@ -3392,18 +3105,15 @@ done: * span. * @wr_mas: The maple write state */ -static noinline void mas_wr_spanning_store(struct ma_wr_state *wr_mas) +static void mas_wr_spanning_store(struct ma_wr_state *wr_mas) { - struct maple_subtree_state mast; - struct maple_big_node b_node; + struct maple_copy cp; struct ma_state *mas; - unsigned char height; + struct ma_state sib; /* Left and Right side of spanning store */ - MA_STATE(l_mas, NULL, 0, 0); MA_STATE(r_mas, NULL, 0, 0); MA_WR_STATE(r_wr_mas, &r_mas, wr_mas->entry); - MA_WR_STATE(l_wr_mas, &l_mas, wr_mas->entry); /* * A store operation that spans multiple nodes is called a spanning @@ -3426,7 +3136,6 @@ static noinline void mas_wr_spanning_store(struct ma_wr_state *wr_mas) * Node rebalancing may occur due to this store, so there may be three new * entries per level plus a new root. */ - height = mas_mt_height(mas); /* * Set up right side. Need to get to the next offset after the spanning @@ -3441,42 +3150,31 @@ static noinline void mas_wr_spanning_store(struct ma_wr_state *wr_mas) r_mas.index = r_mas.last; mas_wr_walk_index(&r_wr_mas); r_mas.last = r_mas.index = mas->last; + r_wr_mas.end_piv = r_wr_mas.r_max; /* Set up left side. */ - l_mas = *mas; - mas_wr_walk_index(&l_wr_mas); + mas_wr_walk_index(wr_mas); if (!wr_mas->entry) { - mas_extend_spanning_null(&l_wr_mas, &r_wr_mas); - mas->offset = l_mas.offset; - mas->index = l_mas.index; - mas->last = l_mas.last = r_mas.last; + mas_extend_spanning_null(wr_mas, &r_wr_mas); + mas->last = r_mas.last; } /* expanding NULLs may make this cover the entire range */ - if (!l_mas.index && r_mas.last == ULONG_MAX) { + if (!mas->index && r_mas.last == ULONG_MAX) { mas_set_range(mas, 0, ULONG_MAX); return mas_new_root(mas, wr_mas->entry); } - memset(&b_node, 0, sizeof(struct maple_big_node)); - /* Copy l_mas and store the value in b_node. */ - mas_store_b_node(&l_wr_mas, &b_node, l_mas.end); - /* Copy r_mas into b_node if there is anything to copy. */ - if (r_mas.max > r_mas.last) - mas_mab_cp(&r_mas, r_mas.offset, r_mas.end, - &b_node, b_node.b_end + 1); - else - b_node.b_end++; - - /* Stop spanning searches by searching for just index. */ - l_mas.index = l_mas.last = mas->index; + cp_leaf_init(&cp, mas, wr_mas, &r_wr_mas); + do { + spanning_data(&cp, wr_mas, &r_wr_mas, &sib); + multi_src_setup(&cp, wr_mas, &r_wr_mas, &sib); + dst_setup(&cp, mas, wr_mas->type); + cp_data_write(&cp, mas); + } while (spanning_ascend(&cp, mas, wr_mas, &r_wr_mas, &sib)); - mast.bn = &b_node; - mast.orig_l = &l_mas; - mast.orig_r = &r_mas; - /* Combine l_mas and r_mas and split them up evenly again. */ - return mas_spanning_rebalance(mas, &mast, height + 1); + mas_wmb_replace(mas, &cp); } /* @@ -3485,26 +3183,34 @@ static noinline void mas_wr_spanning_store(struct ma_wr_state *wr_mas) * * Attempts to reuse the node, but may allocate. */ -static inline void mas_wr_node_store(struct ma_wr_state *wr_mas, - unsigned char new_end) +static inline void mas_wr_node_store(struct ma_wr_state *wr_mas) { - struct ma_state *mas = wr_mas->mas; - void __rcu **dst_slots; - unsigned long *dst_pivots; - unsigned char dst_offset, offset_end = wr_mas->offset_end; + unsigned char dst_offset, offset_end; + unsigned char copy_size, node_pivots, node_slots; struct maple_node reuse, *newnode; - unsigned char copy_size, node_pivots = mt_pivots[wr_mas->type]; - bool in_rcu = mt_in_rcu(mas->tree); - unsigned char height = mas_mt_height(mas); + unsigned long *dst_pivots; + void __rcu **dst_slots; + unsigned char new_end; + struct ma_state *mas; + bool in_rcu; - if (mas->last == wr_mas->end_piv) + mas = wr_mas->mas; + trace_ma_op(TP_FCT, mas); + in_rcu = mt_in_rcu(mas->tree); + offset_end = wr_mas->offset_end; + node_pivots = mt_pivots[wr_mas->type]; + node_slots = mt_slots[wr_mas->type]; + /* Assume last adds an entry */ + new_end = mas->end + 1 - offset_end + mas->offset; + if (mas->last == wr_mas->end_piv) { offset_end++; /* don't copy this offset */ + new_end--; + } /* set up node. */ if (in_rcu) { newnode = mas_pop_node(mas); } else { - memset(&reuse, 0, sizeof(struct maple_node)); newnode = &reuse; } @@ -3512,13 +3218,16 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas, dst_pivots = ma_pivots(newnode, wr_mas->type); dst_slots = ma_slots(newnode, wr_mas->type); /* Copy from start to insert point */ - memcpy(dst_pivots, wr_mas->pivots, sizeof(unsigned long) * mas->offset); - memcpy(dst_slots, wr_mas->slots, sizeof(void *) * mas->offset); + if (mas->offset) { + memcpy(dst_pivots, wr_mas->pivots, sizeof(unsigned long) * mas->offset); + memcpy(dst_slots, wr_mas->slots, sizeof(void __rcu *) * mas->offset); + } /* Handle insert of new range starting after old range */ if (wr_mas->r_min < mas->index) { rcu_assign_pointer(dst_slots[mas->offset], wr_mas->content); dst_pivots[mas->offset++] = mas->index - 1; + new_end++; } /* Store the new entry and range end. */ @@ -3537,7 +3246,7 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas, /* Copy to the end of node if necessary. */ copy_size = mas->end - offset_end + 1; memcpy(dst_slots + dst_offset, wr_mas->slots + offset_end, - sizeof(void *) * copy_size); + sizeof(void __rcu *) * copy_size); memcpy(dst_pivots + dst_offset, wr_mas->pivots + offset_end, sizeof(unsigned long) * (copy_size - 1)); @@ -3545,19 +3254,32 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas, dst_pivots[new_end] = mas->max; done: - mas_leaf_set_meta(newnode, maple_leaf_64, new_end); + if (!in_rcu && new_end + 2 < node_slots) { + unsigned char clear_from = new_end + 1; + + /* + * Note that the last slot is never cleared, since the metadata + * will be stored there or it has a value. + */ + memset(dst_slots + clear_from, 0, + sizeof(void __rcu *) * (node_slots - clear_from)); + if (clear_from < node_pivots) + memset(dst_pivots + clear_from, 0, + sizeof(unsigned long) * (node_pivots - clear_from)); + } + + mas_leaf_set_meta(newnode, wr_mas->type, new_end); if (in_rcu) { struct maple_enode *old_enode = mas->node; mas->node = mt_mk_node(newnode, wr_mas->type); - mas_replace_node(mas, old_enode, height); + mas_replace_node(mas, old_enode, mas_mt_height(mas)); } else { memcpy(wr_mas->node, newnode, sizeof(struct maple_node)); } trace_ma_write(TP_FCT, mas, 0, wr_mas->entry); mas_update_gap(mas); mas->end = new_end; - return; } /* @@ -3571,7 +3293,7 @@ static inline void mas_wr_slot_store(struct ma_wr_state *wr_mas) void __rcu **slots = wr_mas->slots; bool gap = false; - gap |= !mt_slot_locked(mas->tree, slots, offset); + gap |= !wr_mas->content; gap |= !mt_slot_locked(mas->tree, slots, offset + 1); if (wr_mas->offset_end - offset == 1) { @@ -3605,8 +3327,6 @@ static inline void mas_wr_slot_store(struct ma_wr_state *wr_mas) */ if (!wr_mas->entry || gap) mas_update_gap(mas); - - return; } static inline void mas_wr_extend_null(struct ma_wr_state *wr_mas) @@ -3675,18 +3395,17 @@ static inline unsigned char mas_wr_new_end(struct ma_wr_state *wr_mas) /* * mas_wr_append: Attempt to append * @wr_mas: the maple write state - * @new_end: The end of the node after the modification * * This is currently unsafe in rcu mode since the end of the node may be cached * by readers while the node contents may be updated which could result in * inaccurate information. */ -static inline void mas_wr_append(struct ma_wr_state *wr_mas, - unsigned char new_end) +static inline void mas_wr_append(struct ma_wr_state *wr_mas) { struct ma_state *mas = wr_mas->mas; void __rcu **slots; unsigned char end = mas->end; + unsigned char new_end = mas_wr_new_end(wr_mas); if (new_end < mt_pivots[wr_mas->type]) { wr_mas->pivots[new_end] = wr_mas->pivots[end]; @@ -3720,23 +3439,147 @@ static inline void mas_wr_append(struct ma_wr_state *wr_mas, mas->end = new_end; trace_ma_write(TP_FCT, mas, new_end, wr_mas->entry); - return; } /* - * mas_wr_bnode() - Slow path for a modification. - * @wr_mas: The write maple state + * split_ascend() - See if a split operation has to keep walking up the tree + * @cp: The maple_copy node + * @wr_mas: The maple write state + * @sib: the maple state of the sibling * - * This is where split, rebalance end up. + * Return: true if another split operation on the next level is needed, false + * otherwise */ -static void mas_wr_bnode(struct ma_wr_state *wr_mas) +static inline bool split_ascend(struct maple_copy *cp, + struct ma_wr_state *wr_mas, struct ma_state *sib, + struct ma_state *parent) { - struct maple_big_node b_node; + struct ma_state *mas; + unsigned long min, max; + mas = wr_mas->mas; + min = mas->min; /* push right, or normal split */ + max = mas->max; + wr_mas->offset_end = parent->offset; + if (sib->end) { + if (sib->max < mas->min) { + min = sib->min; /* push left */ + parent->offset--; + } else { + max = sib->max; /* push right */ + wr_mas->offset_end++; + } + } + + cp_dst_to_slots(cp, min, max, mas); + if (cp_is_new_root(cp, mas)) + return false; + + if (cp_converged(cp, mas, sib)) + return false; + + cp->height++; + copy_tree_location(parent, mas); + wr_mas_setup(wr_mas, mas); + return true; +} + +/* + * split_data() - Calculate the @cp data, populate @sib if the data can be + * pushed into a sibling. + * @cp: The maple copy node + * @wr_mas: The left write maple state + * @sib: The maple state of the sibling. + * + * Note: @cp->data is a size and not indexed by 0. @sib->end may be set to 0 to + * indicate it will not be used. + * + */ +static inline void split_data(struct maple_copy *cp, + struct ma_wr_state *wr_mas, struct ma_state *sib, + struct ma_state *parent) +{ + cp_data_calc(cp, wr_mas, wr_mas); + if (cp->data <= mt_slots[wr_mas->type]) { + sib->end = 0; + return; + } + + push_data_sib(cp, wr_mas->mas, sib, parent); + if (sib->end) + cp->data += sib->end + 1; +} + +/* + * mas_wr_split() - Expand one node into two + * @wr_mas: The write maple state + */ +static void mas_wr_split(struct ma_wr_state *wr_mas) +{ + struct ma_state parent; + struct ma_state *mas; + struct maple_copy cp; + struct ma_state sib; + + mas = wr_mas->mas; trace_ma_write(TP_FCT, wr_mas->mas, 0, wr_mas->entry); - memset(&b_node, 0, sizeof(struct maple_big_node)); - mas_store_b_node(wr_mas, &b_node, wr_mas->offset_end); - mas_commit_b_node(wr_mas, &b_node); + parent = *mas; + cp_leaf_init(&cp, mas, wr_mas, wr_mas); + do { + if (!mte_is_root(parent.node)) { + mas_ascend(&parent); + parent.end = mas_data_end(&parent); + } + split_data(&cp, wr_mas, &sib, &parent); + multi_src_setup(&cp, wr_mas, wr_mas, &sib); + dst_setup(&cp, mas, wr_mas->type); + cp_data_write(&cp, mas); + } while (split_ascend(&cp, wr_mas, &sib, &parent)); + + mas_wmb_replace(mas, &cp); +} + +/* + * mas_wr_rebalance() - Insufficient data in one node needs to either get data + * from a sibling or absorb a sibling all together. + * @wr_mas: The write maple state + * + * Rebalance is different than a spanning store in that the write state is + * already at the leaf node that's being altered. + */ +static void mas_wr_rebalance(struct ma_wr_state *wr_mas) +{ + struct ma_state parent; + struct ma_state *mas; + struct maple_copy cp; + struct ma_state sib; + + /* + * Rebalancing occurs if a node is insufficient. Data is rebalanced + * against the node to the right if it exists, otherwise the node to the + * left of this node is rebalanced against this node. If rebalancing + * causes just one node to be produced instead of two, then the parent + * is also examined and rebalanced if it is insufficient. Every level + * tries to combine the data in the same way. If one node contains the + * entire range of the tree, then that node is used as a new root node. + */ + + mas = wr_mas->mas; + trace_ma_op(TP_FCT, mas); + parent = *mas; + cp_leaf_init(&cp, mas, wr_mas, wr_mas); + do { + if (!mte_is_root(parent.node)) { + mas_ascend(&parent); + parent.end = mas_data_end(&parent); + } + rebalance_data(&cp, wr_mas, &sib, &parent); + multi_src_setup(&cp, wr_mas, wr_mas, &sib); + dst_setup(&cp, mas, wr_mas->type); + cp_data_write(&cp, mas); + } while (rebalance_ascend(&cp, wr_mas, &sib, &parent)); + + mas_wmb_replace(mas, &cp); } /* @@ -3746,7 +3589,6 @@ static void mas_wr_bnode(struct ma_wr_state *wr_mas) static inline void mas_wr_store_entry(struct ma_wr_state *wr_mas) { struct ma_state *mas = wr_mas->mas; - unsigned char new_end = mas_wr_new_end(wr_mas); switch (mas->store_type) { case wr_exact_fit: @@ -3755,20 +3597,22 @@ static inline void mas_wr_store_entry(struct ma_wr_state *wr_mas) mas_update_gap(mas); break; case wr_append: - mas_wr_append(wr_mas, new_end); + mas_wr_append(wr_mas); break; case wr_slot_store: mas_wr_slot_store(wr_mas); break; case wr_node_store: - mas_wr_node_store(wr_mas, new_end); + mas_wr_node_store(wr_mas); break; case wr_spanning_store: mas_wr_spanning_store(wr_mas); break; case wr_split_store: + mas_wr_split(wr_mas); + break; case wr_rebalance: - mas_wr_bnode(wr_mas); + mas_wr_rebalance(wr_mas); break; case wr_new_root: mas_new_root(mas, wr_mas->entry); @@ -3779,8 +3623,6 @@ static inline void mas_wr_store_entry(struct ma_wr_state *wr_mas) case wr_invalid: MT_BUG_ON(mas->tree, 1); } - - return; } static inline void mas_wr_prealloc_setup(struct ma_wr_state *wr_mas) @@ -3828,7 +3670,7 @@ set_content: /** * mas_prealloc_calc() - Calculate number of nodes needed for a - * given store oepration + * given store operation * @wr_mas: The maple write state * @entry: The entry to store into the tree * @@ -3894,6 +3736,9 @@ static inline enum store_type mas_wr_store_type(struct ma_wr_state *wr_mas) { struct ma_state *mas = wr_mas->mas; unsigned char new_end; + bool appending; + bool one_slot; + bool in_rcu; if (unlikely(mas_is_none(mas) || mas_is_ptr(mas))) return wr_store_root; @@ -3913,21 +3758,30 @@ static inline enum store_type mas_wr_store_type(struct ma_wr_state *wr_mas) return wr_new_root; new_end = mas_wr_new_end(wr_mas); + in_rcu = mt_in_rcu(mas->tree); + appending = mas->offset == mas->end; + one_slot = wr_mas->offset_end - mas->offset == 1; + /* Potential spanning rebalance collapsing a node */ if (new_end < mt_min_slots[wr_mas->type]) { if (!mte_is_root(mas->node)) return wr_rebalance; + if (!in_rcu) { + if (appending) + return wr_append; + else if (mas->end == new_end && one_slot) + return wr_slot_store; + } return wr_node_store; } if (new_end >= mt_slots[wr_mas->type]) return wr_split_store; - if (!mt_in_rcu(mas->tree) && (mas->offset == mas->end)) + if (!in_rcu && appending) return wr_append; - if ((new_end == mas->end) && (!mt_in_rcu(mas->tree) || - (wr_mas->offset_end - mas->offset == 1))) + if (new_end == mas->end && (!in_rcu || one_slot)) return wr_slot_store; return wr_node_store; @@ -4026,35 +3880,40 @@ int mas_alloc_cyclic(struct ma_state *mas, unsigned long *startp, void *entry, unsigned long range_lo, unsigned long range_hi, unsigned long *next, gfp_t gfp) { - unsigned long min = range_lo; - int ret = 0; - - range_lo = max(min, *next); - ret = mas_empty_area(mas, range_lo, range_hi, 1); - if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) { - mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED; - ret = 1; - } - if (ret < 0 && range_lo > min) { - mas_reset(mas); - ret = mas_empty_area(mas, min, range_hi, 1); - if (ret == 0) - ret = 1; - } - if (ret < 0) - return ret; + int ret; + unsigned long min; + min = range_lo; do { + range_lo = max(min, *next); + ret = mas_empty_area(mas, range_lo, range_hi, 1); + if (ret < 0 && range_lo > min) { + mas_reset(mas); + ret = mas_empty_area(mas, min, range_hi, 1); + if (ret == 0) + ret = 1; + } + if (ret < 0) + goto out; + mas_insert(mas, entry); } while (mas_nomem(mas, gfp)); - if (mas_is_err(mas)) - return xa_err(mas->node); + if (mas_is_err(mas)) { + ret = xa_err(mas->node); + goto out; + } + + if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) { + mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED; + ret = 1; + } *startp = mas->index; *next = *startp + 1; if (*next == 0) mas->tree->ma_flags |= MT_FLAGS_ALLOC_WRAPPED; +out: mas_destroy(mas); return ret; } @@ -4593,8 +4452,8 @@ void *mas_walk(struct ma_state *mas) { void *entry; - if (!mas_is_active(mas) && !mas_is_start(mas)) - mas->status = ma_start; + mas_may_init_lock_check(mas); + mas_make_walkable(mas); retry: entry = mas_state_walk(mas); if (mas_is_start(mas)) { @@ -5064,6 +4923,7 @@ void *mas_store(struct ma_state *mas, void *entry) { MA_WR_STATE(wr_mas, mas, entry); + mas_may_init_lock_check(mas); trace_ma_write(TP_FCT, mas, 0, entry); #ifdef CONFIG_DEBUG_MAPLE_TREE if (MAS_WARN_ON(mas, mas->index > mas->last)) @@ -5122,6 +4982,7 @@ int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp) MA_WR_STATE(wr_mas, mas, entry); int ret = 0; + mas_may_init_lock_check(mas); retry: mas_wr_preallocate(&wr_mas, entry); if (unlikely(mas_nomem(mas, gfp))) { @@ -5152,6 +5013,7 @@ void mas_store_prealloc(struct ma_state *mas, void *entry) { MA_WR_STATE(wr_mas, mas, entry); + mas_lock_check(mas); if (mas->store_type == wr_store_root) { mas_wr_prealloc_setup(&wr_mas); goto store; @@ -5184,6 +5046,7 @@ int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp) { MA_WR_STATE(wr_mas, mas, entry); + mas_may_init_lock_check(mas); mas_wr_prealloc_setup(&wr_mas); mas->store_type = mas_wr_store_type(&wr_mas); mas_prealloc_calc(&wr_mas, entry); @@ -5230,6 +5093,7 @@ static void mas_may_activate(struct ma_state *mas) mas->status = ma_start; } else { mas->status = ma_active; + mas_lock_check(mas); } } @@ -5307,6 +5171,7 @@ void *mas_next(struct ma_state *mas, unsigned long max) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_next_setup(mas, max, &entry)) return entry; @@ -5330,6 +5195,7 @@ void *mas_next_range(struct ma_state *mas, unsigned long max) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_next_setup(mas, max, &entry)) return entry; @@ -5438,6 +5304,7 @@ void *mas_prev(struct ma_state *mas, unsigned long min) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_prev_setup(mas, min, &entry)) return entry; @@ -5461,6 +5328,7 @@ void *mas_prev_range(struct ma_state *mas, unsigned long min) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_prev_setup(mas, min, &entry)) return entry; @@ -5507,6 +5375,7 @@ EXPORT_SYMBOL_GPL(mt_prev); */ void mas_pause(struct ma_state *mas) { + mas_lock_check(mas); mas->status = ma_pause; mas->node = NULL; } @@ -5615,6 +5484,7 @@ void *mas_find(struct ma_state *mas, unsigned long max) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_find_setup(mas, max, &entry)) return entry; @@ -5642,6 +5512,7 @@ void *mas_find_range(struct ma_state *mas, unsigned long max) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_find_setup(mas, max, &entry)) return entry; @@ -5661,7 +5532,6 @@ EXPORT_SYMBOL_GPL(mas_find_range); static bool mas_find_rev_setup(struct ma_state *mas, unsigned long min, void **entry) { - switch (mas->status) { case ma_active: goto active; @@ -5754,6 +5624,7 @@ void *mas_find_rev(struct ma_state *mas, unsigned long min) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_find_rev_setup(mas, min, &entry)) return entry; @@ -5780,6 +5651,7 @@ void *mas_find_range_rev(struct ma_state *mas, unsigned long min) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_find_rev_setup(mas, min, &entry)) return entry; @@ -5797,6 +5669,11 @@ EXPORT_SYMBOL_GPL(mas_find_range_rev); * Searches for @mas->index, sets @mas->index and @mas->last to the range and * erases that range. * + * Note that erase requires allocations and will use GFP_KERNEL | __GFP_NOFAIL + * to do so if necessary. If the allocation fails, the internal lock will be + * dropped to retry. An externally locked tree must be protected by a lock that + * allows blocking for this API. + * * Return: the entry that was erased or %NULL, @mas->index and @mas->last are updated. */ void *mas_erase(struct ma_state *mas) @@ -5805,25 +5682,25 @@ void *mas_erase(struct ma_state *mas) unsigned long index = mas->index; MA_WR_STATE(wr_mas, mas, NULL); - if (!mas_is_active(mas) || !mas_is_start(mas)) - mas->status = ma_start; + /* + * In low memory situations, the allocation is retried with the gfp flag + * GFP_KERNEL | __GFP_NOFAIL. The internal spinlock is dropped in + * mas_nomem_nofail(), however the external lock is not dropped. + */ + if (mt_external_lock(mas->tree)) + might_alloc(GFP_KERNEL); + mas_make_walkable(mas); write_retry: entry = mas_state_walk(mas); if (!entry) - return NULL; + goto out; /* Must reset to ensure spanning writes of last slot are detected */ mas_reset(mas); mas_wr_preallocate(&wr_mas, NULL); - if (mas_nomem(mas, GFP_KERNEL)) { - /* in case the range of entry changed when unlocked */ - mas->index = mas->last = index; + if (mas_nomem_nofail(mas, index, index)) goto write_retry; - } - - if (mas_is_err(mas)) - goto out; mas_wr_store_entry(&wr_mas); out: @@ -5834,10 +5711,11 @@ EXPORT_SYMBOL_GPL(mas_erase); /** * mas_nomem() - Check if there was an error allocating and do the allocation - * if necessary If there are allocations, then free them. + * if necessary. + * * @mas: The maple state * @gfp: The GFP_FLAGS to use for allocations - * Return: true on allocation, false otherwise. + * Return: False on no memory. True otherwise (partial success as well) */ bool mas_nomem(struct ma_state *mas, gfp_t gfp) __must_hold(mas->tree->ma_lock) @@ -5845,6 +5723,10 @@ bool mas_nomem(struct ma_state *mas, gfp_t gfp) if (likely(mas->node != MA_ERROR(-ENOMEM))) return false; + /* Allocations can fail, don't do this. */ + WARN_ON_ONCE(!gfpflags_allow_blocking(gfp) && + mt_external_lock(mas->tree)); + if (gfpflags_allow_blocking(gfp) && !mt_external_lock(mas->tree)) { mtree_unlock(mas->tree); mas_alloc_nodes(mas, gfp); @@ -5853,10 +5735,49 @@ bool mas_nomem(struct ma_state *mas, gfp_t gfp) mas_alloc_nodes(mas, gfp); } + /* + * Return false on zero forward progress. Partial allocations are kept + * so the retry path will attempt to get the rest. The failure should + * not happen as we try our best to reclaim. The user would need an + * external lock with a non-blocking gfp in a low memory situation. + */ if (!mas->sheaf && !mas->alloc) return false; - mas->status = ma_start; + mas_reset(mas); + return true; +} + +/** + * mas_nomem_nofail() - Retry allocations with __GFP_NOFAIL, if the maple state + * has stored the -ENOMEM error. + * @mas: The maple state + * @index: The start of the range for the @mas reset + * @last: The end of the range for the @mas reset + * + * Return: false if @mas isn't in an -ENOMEM state. True if the allocation + * happens, the state is reset. The internal lock will be dropped and external + * locks must allow blocking. + */ +bool mas_nomem_nofail(struct ma_state *mas, unsigned long index, + unsigned long last) + __must_hold(mas->tree->ma_lock) +{ + gfp_t gfp; + + if (likely(mas->node != MA_ERROR(-ENOMEM))) + return false; + + gfp = GFP_KERNEL | __GFP_NOFAIL; + if (!mt_external_lock(mas->tree)) { + mtree_unlock(mas->tree); + mas_alloc_nodes(mas, gfp); + mtree_lock(mas->tree); + } else { + mas_alloc_nodes(mas, gfp); + } + + mas_set_range(mas, index, last); return true; } @@ -5960,13 +5881,16 @@ int mtree_store(struct maple_tree *mt, unsigned long index, void *entry, EXPORT_SYMBOL(mtree_store); /** - * mtree_insert_range() - Insert an entry at a given range if there is no value. + * mtree_insert_range() - Insert an entry from [first, last] at a given range + * if there is no value. * @mt: The maple tree * @first: The start of the range - * @last: The end of the range + * @last: The end of the range (inclusive) * @entry: The entry to store * @gfp: The GFP_FLAGS to use for allocations. * + * Note that @last is inclusive. That is, @last = @first + length - 1; + * * Return: 0 on success, -EEXISTS if the range is occupied, -EINVAL on invalid * request, -ENOMEM if memory could not be allocated. */ @@ -6143,6 +6067,10 @@ EXPORT_SYMBOL(mtree_alloc_rrange); * Erasing is the same as a walk to an entry then a store of a NULL to that * ENTIRE range. In fact, it is implemented as such using the advanced API. * + * Note that erase requires allocations and will use GFP_KERNEL | __GFP_NOFAIL + * to do so if necessary. If the allocation fails, the internal lock will be + * dropped to retry. + * * Return: The entry stored at the @index or %NULL */ void *mtree_erase(struct maple_tree *mt, unsigned long index) @@ -6152,6 +6080,7 @@ void *mtree_erase(struct maple_tree *mt, unsigned long index) MA_STATE(mas, mt, index, index); trace_ma_op(TP_FCT, &mas); + might_alloc(GFP_KERNEL); mtree_lock(mt); entry = mas_erase(&mas); mtree_unlock(mt); @@ -6260,8 +6189,15 @@ static inline void mas_dup_alloc(struct ma_state *mas, struct ma_state *new_mas, for (i = 0; i < count; i++) { val = (unsigned long)mt_slot_locked(mas->tree, slots, i); val &= MAPLE_NODE_MASK; - new_slots[i] = ma_mnode_ptr((unsigned long)mas_pop_node(mas) | - val); + /* + * Warning, see rcu_assign_pointer() documentation. Since this + * is a duplication of a tree, there are no readers walking the + * tree until after the rcu_assign_pointer() call in + * mas_dup_build(). + */ + RCU_INIT_POINTER(new_slots[i], + ma_mnode_ptr((unsigned long)mas_pop_node(mas) | + val)); } } @@ -6670,7 +6606,7 @@ static void mt_dump_range(unsigned long min, unsigned long max, { static const char spaces[] = " "; - switch(format) { + switch (format) { case mt_dump_hex: if (min == max) pr_info("%.*s%lx: ", depth * 2, spaces, min); @@ -6712,7 +6648,7 @@ static void mt_dump_range64(const struct maple_tree *mt, void *entry, pr_cont(" contents: "); for (i = 0; i < MAPLE_RANGE64_SLOTS - 1; i++) { - switch(format) { + switch (format) { case mt_dump_hex: pr_cont(PTR_FMT " %lX ", node->slot[i], node->pivot[i]); break; @@ -6740,7 +6676,7 @@ static void mt_dump_range64(const struct maple_tree *mt, void *entry, if (last == max) break; if (last > max) { - switch(format) { + switch (format) { case mt_dump_hex: pr_err("node " PTR_FMT " last (%lx) > max (%lx) at pivot %d!\n", node, last, max, i); @@ -6799,7 +6735,7 @@ static void mt_dump_arange64(const struct maple_tree *mt, void *entry, if (last == max) break; if (last > max) { - switch(format) { + switch (format) { case mt_dump_hex: pr_err("node " PTR_FMT " last (%lx) > max (%lx) at pivot %d!\n", node, last, max, i); diff --git a/lib/math/Kconfig b/lib/math/Kconfig index 0634b428d0cb..0e6d9cffc5d6 100644 --- a/lib/math/Kconfig +++ b/lib/math/Kconfig @@ -5,6 +5,9 @@ config CORDIC This option provides an implementation of the CORDIC algorithm; calculations are in fixed point. Module will be called cordic. +config POLYNOMIAL + tristate + config PRIME_NUMBERS tristate "Simple prime number generator for testing" help diff --git a/lib/math/Makefile b/lib/math/Makefile index d1caba23baa0..9a3850d55b79 100644 --- a/lib/math/Makefile +++ b/lib/math/Makefile @@ -2,6 +2,7 @@ obj-y += div64.o gcd.o lcm.o int_log.o int_pow.o int_sqrt.o reciprocal_div.o obj-$(CONFIG_CORDIC) += cordic.o +obj-$(CONFIG_POLYNOMIAL) += polynomial.o obj-$(CONFIG_PRIME_NUMBERS) += prime_numbers.o obj-$(CONFIG_RATIONAL) += rational.o diff --git a/lib/math/div64.c b/lib/math/div64.c index d1e92ea24fce..0b10ded09a9b 100644 --- a/lib/math/div64.c +++ b/lib/math/div64.c @@ -158,6 +158,21 @@ u64 div64_u64(u64 dividend, u64 divisor) EXPORT_SYMBOL(div64_u64); #endif +#ifndef div64_s64_rem +s64 div64_s64_rem(s64 dividend, s64 divisor, s64 *remainder) +{ + s64 quot, t, rem; + + quot = div64_u64_rem(abs(dividend), abs(divisor), (u64 *)&rem); + t = dividend >> 63; + *remainder = (rem ^ t) - t; + t = (dividend ^ divisor) >> 63; + + return (quot ^ t) - t; +} +EXPORT_SYMBOL(div64_s64_rem); +#endif + #ifndef div64_s64 s64 div64_s64(s64 dividend, s64 divisor) { diff --git a/lib/polynomial.c b/lib/math/polynomial.c index 66d383445fec..f26677cfeeff 100644 --- a/lib/polynomial.c +++ b/lib/math/polynomial.c @@ -10,21 +10,19 @@ * */ -#include <linux/kernel.h> +#include <linux/export.h> +#include <linux/math.h> #include <linux/module.h> #include <linux/polynomial.h> /* - * Originally this was part of drivers/hwmon/bt1-pvt.c. - * There the following conversion is used and should serve as an example here: + * The following conversion is an example: * * The original translation formulae of the temperature (in degrees of Celsius) * to PVT data and vice-versa are following: * - * N = 1.8322e-8*(T^4) + 2.343e-5*(T^3) + 8.7018e-3*(T^2) + 3.9269*(T^1) + - * 1.7204e2 - * T = -1.6743e-11*(N^4) + 8.1542e-8*(N^3) + -1.8201e-4*(N^2) + - * 3.1020e-1*(N^1) - 4.838e1 + * N = 1.8322e-8*(T^4) + 2.343e-5*(T^3) + 8.7018e-3*(T^2) + 3.9269*(T^1) + 1.7204e2 + * T = -1.6743e-11*(N^4) + 8.1542e-8*(N^3) + -1.8201e-4*(N^2) + 3.1020e-1*(N^1) - 4.838e1 * * where T = [-48.380, 147.438]C and N = [0, 1023]. * @@ -35,10 +33,9 @@ * formulae to accept millidegrees of Celsius. Here what they look like after * the alterations: * - * N = (18322e-20*(T^4) + 2343e-13*(T^3) + 87018e-9*(T^2) + 39269e-3*T + - * 17204e2) / 1e4 - * T = -16743e-12*(D^4) + 81542e-9*(D^3) - 182010e-6*(D^2) + 310200e-3*D - - * 48380 + * N = (18322e-20*(T^4) + 2343e-13*(T^3) + 87018e-9*(T^2) + 39269e-3*T + 17204e2) / 1e4 + * T = -16743e-12*(D^4) + 81542e-9*(D^3) - 182010e-6*(D^2) + 310200e-3*D - 48380 + * * where T = [-48380, 147438] mC and N = [0, 1023]. * * static const struct polynomial poly_temp_to_N = { @@ -68,13 +65,13 @@ * polynomial_calc - calculate a polynomial using integer arithmetic * * @poly: pointer to the descriptor of the polynomial - * @data: input value of the polynimal + * @data: input value of the polynomial * * Calculate the result of a polynomial using only integer arithmetic. For * this to work without too much loss of precision the coefficients has to * be altered. This is called factor redistribution. * - * Returns the result of the polynomial calculation. + * Return: the result of the polynomial calculation. */ long polynomial_calc(const struct polynomial *poly, long data) { diff --git a/lib/math/test_mul_u64_u64_div_u64.c b/lib/math/test_mul_u64_u64_div_u64.c index 338d014f0c73..d12dc05938fb 100644 --- a/lib/math/test_mul_u64_u64_div_u64.c +++ b/lib/math/test_mul_u64_u64_div_u64.c @@ -157,6 +157,7 @@ static void __exit test_exit(void) #define __div64_32 __div64_32 #define div_s64_rem div_s64_rem #define div64_u64_rem div64_u64_rem +#define div64_s64_rem div64_s64_rem #define div64_u64 div64_u64 #define div64_s64 div64_s64 #define iter_div_u64_rem iter_div_u64_rem diff --git a/lib/math/tests/Makefile b/lib/math/tests/Makefile index 13dc96e48408..85e1ad59f29d 100644 --- a/lib/math/tests/Makefile +++ b/lib/math/tests/Makefile @@ -4,5 +4,6 @@ obj-$(CONFIG_GCD_KUNIT_TEST) += gcd_kunit.o obj-$(CONFIG_INT_LOG_KUNIT_TEST) += int_log_kunit.o obj-$(CONFIG_INT_POW_KUNIT_TEST) += int_pow_kunit.o obj-$(CONFIG_INT_SQRT_KUNIT_TEST) += int_sqrt_kunit.o +obj-$(CONFIG_POLYNOMIAL_KUNIT_TEST) += polynomial_kunit.o obj-$(CONFIG_PRIME_NUMBERS_KUNIT_TEST) += prime_numbers_kunit.o obj-$(CONFIG_RATIONAL_KUNIT_TEST) += rational_kunit.o diff --git a/lib/math/tests/polynomial_kunit.c b/lib/math/tests/polynomial_kunit.c new file mode 100644 index 000000000000..ef443b57fc12 --- /dev/null +++ b/lib/math/tests/polynomial_kunit.c @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <kunit/test.h> +#include <linux/polynomial.h> + +struct polynomial_test_param { + const struct polynomial *poly; + long data; + long expected; + const char *name; +}; + +/* f(x) = 5 */ +static const struct polynomial poly_constant = { + .total_divider = 1, + .terms = { + {0, 5, 1, 1}, + } +}; + +/* f(x) = 2x^2 + 3x + 5 */ +static const struct polynomial poly_simple = { + .total_divider = 1, + .terms = { + {2, 2, 1, 1}, + {1, 3, 1, 1}, + {0, 5, 1, 1}, + } +}; + +/* f(x) = -5x + 100 */ +static const struct polynomial poly_negative_coef = { + .total_divider = 1, + .terms = { + {1, -5, 1, 1}, + {0, 100, 1, 1}, + } +}; + +/* f(x) = (150x + 50) / 10 */ +static const struct polynomial poly_total_divider = { + .total_divider = 10, + .terms = { + {1, 150, 1, 1}, + {0, 50, 1, 1}, + } +}; + +/* + * f(x) = x / 2 + * divider=2 applied once per multiply: mult_frac(coef, data, 2) = coef*data/2 + */ +static const struct polynomial poly_step_divider = { + .total_divider = 1, + .terms = { + {1, 1, 2, 1}, + {0, 0, 1, 1}, + } +}; + +/* + * f(x) = (100/500) * x^2 = 0.2 * x^2 + * Encoded as coef=100, divider=10, divider_leftover=5: + * denom = 10^2 * 5 = 500 + */ +static const struct polynomial poly_leftover = { + .total_divider = 1, + .terms = { + {2, 100, 10, 5}, + {0, 0, 1, 1}, + } +}; + +/* + * f(x) = 2x^3 (single high-degree term, no constant) + * Used to exercise the power loop alone. + */ +static const struct polynomial poly_cubic = { + .total_divider = 1, + .terms = { + {3, 2, 1, 1}, + {0, 0, 1, 1}, + } +}; + +/* + * f(x) = 4x + 1 with a zero-coefficient quadratic term. + * The deg-2 term contributes nothing regardless of input. + */ +static const struct polynomial poly_zero_coef = { + .total_divider = 1, + .terms = { + {2, 0, 1, 1}, + {1, 4, 1, 1}, + {0, 1, 1, 1}, + } +}; + +/* + * f(x) = 9 with total_divider = 0. + * The implementation treats 0 as 1 via `total_divider ?: 1`, so the + * result must equal the constant term unchanged. + */ +static const struct polynomial poly_zero_total_divider = { + .total_divider = 0, + .terms = { + {0, 9, 1, 1}, + } +}; + + +static const struct polynomial_test_param test_params[] = { + { + .poly = &poly_constant, + .data = 0, + .expected = 5, + .name = "Constant polynomial at x=0", + }, + { + .poly = &poly_constant, + .data = 42, + .expected = 5, + .name = "Constant polynomial is independent of input", + }, + { + .poly = &poly_simple, + .data = 0, + .expected = 5, /* zero input collapses all power terms */ + .name = "Zero input yields constant term only", + }, + { + .poly = &poly_simple, + .data = 10, + .expected = 235, /* 2*100 + 3*10 + 5 */ + .name = "Simple quadratic at x=10", + }, + { + .poly = &poly_negative_coef, + .data = 10, + .expected = 50, /* -5*10 + 100 */ + .name = "Negative coefficient at x=10", + }, + { + .poly = &poly_negative_coef, + .data = 20, + .expected = 0, /* -5*20 + 100 = 0 */ + .name = "Negative coefficient result is zero", + }, + { + .poly = &poly_total_divider, + .data = 3, + .expected = 50, /* (150*3 + 50) / 10 = 500/10 */ + .name = "total_divider scales the final sum", + }, + { + .poly = &poly_step_divider, + .data = 100, + .expected = 50, /* 1*100/2 */ + .name = "Per-step divider halves input", + }, + { + .poly = &poly_leftover, + .data = 30, + .expected = 180, /* 100*30^2 / (10^2 * 5) = 90000/500 */ + .name = "divider_leftover with quadratic term", + }, + /* Boundary: unit and negative-unit input */ + { + /* + * data=1: each mult_frac(tmp, 1, divider) strips one factor of + * divider from coef per degree, so coef is left-shifted right + * until intermediate precision is exhausted. + * 2*1 + 3*1 + 5 = 10 + */ + .poly = &poly_simple, + .data = 1, + .expected = 10, + .name = "Boundary: data=1 (unit input)", + }, + { + /* + * data=-1: even degrees produce positive contributions, + * odd degrees produce negative ones. + * 2*(-1)^2 + 3*(-1) + 5 = 2 - 3 + 5 = 4 + */ + .poly = &poly_simple, + .data = -1, + .expected = 4, + .name = "Boundary: data=-1 (negative unit input)", + }, + + /* Boundary: negative non-trivial input */ + { + /* + * 2*(-3)^2 + 3*(-3) + 5 = 18 - 9 + 5 = 14 + * Verifies sign handling for negative data across all degrees. + */ + .poly = &poly_simple, + .data = -3, + .expected = 14, + .name = "Boundary: negative data with quadratic", + }, + + /* Boundary: total_divider = 0 is treated as 1 */ + { + .poly = &poly_zero_total_divider, + .data = 42, + .expected = 9, + .name = "Boundary: total_divider=0 defaults to 1", + }, + + /* Boundary: zero-coefficient high-degree term */ + { + /* + * The deg-2 term has coef=0, so it contributes 0 regardless + * of data. Result: 0 + 4*10 + 1 = 41 + */ + .poly = &poly_zero_coef, + .data = 10, + .expected = 41, + .name = "Boundary: zero-coefficient term is inert", + }, + + /* Boundary: single high-degree term, no constant */ + { + /* 2 * 5^3 = 250; also verifies the loop terminates on deg-0 */ + .poly = &poly_cubic, + .data = 5, + .expected = 250, + .name = "Boundary: single cubic term", + }, + { + /* 2 * (-2)^3 = -16; odd power preserves sign of negative data */ + .poly = &poly_cubic, + .data = -2, + .expected = -16, + .name = "Boundary: single cubic term, negative data", + }, + +}; + +static void get_desc(const struct polynomial_test_param *param, char *desc) +{ + strscpy(desc, param->name, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(polynomial, test_params, get_desc); + +static void polynomial_calc_test(struct kunit *test) +{ + const struct polynomial_test_param *param = test->param_value; + + KUNIT_EXPECT_EQ(test, polynomial_calc(param->poly, param->data), + param->expected); +} + +static struct kunit_case polynomial_test_cases[] = { + KUNIT_CASE_PARAM(polynomial_calc_test, polynomial_gen_params), + {} +}; + +static struct kunit_suite polynomial_test_suite = { + .name = "math-polynomial", + .test_cases = polynomial_test_cases, +}; + +kunit_test_suites(&polynomial_test_suite); + +MODULE_DESCRIPTION("math.polynomial_calc KUnit test suite"); +MODULE_LICENSE("GPL"); diff --git a/lib/math/tests/prime_numbers_kunit.c b/lib/math/tests/prime_numbers_kunit.c index 2f1643208c66..55ac160c6dfa 100644 --- a/lib/math/tests/prime_numbers_kunit.c +++ b/lib/math/tests/prime_numbers_kunit.c @@ -8,12 +8,10 @@ static void dump_primes(void *ctx, const struct primes *p) { - static char buf[PAGE_SIZE]; struct kunit_suite *suite = ctx; - bitmap_print_to_pagebuf(true, buf, p->primes, p->sz); - kunit_info(suite, "primes.{last=%lu, .sz=%lu, .primes[]=...x%lx} = %s", - p->last, p->sz, p->primes[BITS_TO_LONGS(p->sz) - 1], buf); + kunit_info(suite, "primes.{last=%lu, .sz=%lu, .primes[]=...x%lx} = %*pbl", + p->last, p->sz, p->primes[BITS_TO_LONGS(p->sz) - 1], (int)p->sz, p->primes); } static void prime_numbers_test(struct kunit *test) diff --git a/lib/nmi_backtrace.c b/lib/nmi_backtrace.c index 33c154264bfe..f39c9eca8888 100644 --- a/lib/nmi_backtrace.c +++ b/lib/nmi_backtrace.c @@ -16,8 +16,10 @@ #include <linux/cpumask.h> #include <linux/delay.h> #include <linux/kprobes.h> +#include <linux/stringify.h> #include <linux/nmi.h> #include <linux/cpu.h> +#include <linux/export.h> #include <linux/sched/debug.h> #ifdef arch_trigger_cpumask_backtrace @@ -27,6 +29,8 @@ static DECLARE_BITMAP(backtrace_mask, NR_CPUS) __read_mostly; /* "in progress" flag of arch_trigger_cpumask_backtrace */ static unsigned long backtrace_flag; +#define NMI_BT_TIMEOUT_SEC 10 + /* * When raise() is called it will be passed a pointer to the * backtrace_mask. Architectures that call nmi_cpu_backtrace() @@ -68,14 +72,20 @@ void nmi_trigger_cpumask_backtrace(const cpumask_t *mask, raise(to_cpumask(backtrace_mask)); } - /* Wait for up to 10 seconds for all CPUs to do the backtrace */ - for (i = 0; i < 10 * 1000; i++) { + /* Wait for up to NMI_BT_TIMEOUT_SEC seconds for all CPUs to do the backtrace */ + for (i = 0; i < NMI_BT_TIMEOUT_SEC * 1000; i++) { if (cpumask_empty(to_cpumask(backtrace_mask))) break; mdelay(1); touch_softlockup_watchdog(); } - nmi_backtrace_stall_check(to_cpumask(backtrace_mask)); + + if (!cpumask_empty(to_cpumask(backtrace_mask))) { + pr_warn("After " __stringify(NMI_BT_TIMEOUT_SEC) " seconds, these CPUS still haven't responded to the NMI: %*pbl\n", + cpumask_pr_args(to_cpumask(backtrace_mask))); + + nmi_backtrace_stall_check(to_cpumask(backtrace_mask)); + } /* * Force flush any remote buffers that might be stuck in IRQ context @@ -120,4 +130,10 @@ bool nmi_cpu_backtrace(struct pt_regs *regs) return false; } NOKPROBE_SYMBOL(nmi_cpu_backtrace); + +void cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) +{ + arch_trigger_cpumask_backtrace(mask, exclude_cpu); +} +EXPORT_SYMBOL_GPL(cpumask_backtrace); #endif diff --git a/lib/once.c b/lib/once.c index d801bfa945e6..0a0a919156e0 100644 --- a/lib/once.c +++ b/lib/once.c @@ -93,6 +93,6 @@ void __do_once_sleepable_done(bool *done, struct static_key_true *once_key, { *done = true; mutex_unlock(&once_mutex); - static_branch_disable(once_key); + once_disable_jump(once_key, mod); } EXPORT_SYMBOL(__do_once_sleepable_done); diff --git a/lib/parser.c b/lib/parser.c index 73e8f8e5be73..62da0ac0d438 100644 --- a/lib/parser.c +++ b/lib/parser.c @@ -315,7 +315,7 @@ bool match_wildcard(const char *pattern, const char *str) } } - if (*p == '*') + while (*p == '*') ++p; return !*p; } diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c index 97772e42b9b2..f8d90689af9f 100644 --- a/lib/percpu-refcount.c +++ b/lib/percpu-refcount.c @@ -289,7 +289,7 @@ static void __percpu_ref_switch_mode(struct percpu_ref *ref, * @confirm_switch: optional confirmation callback * * There's no reason to use this function for the usual reference counting. - * Use percpu_ref_kill[_and_confirm](). + * Use percpu_ref_kill() or percpu_ref_kill_and_confirm(). * * Schedule switching of @ref to atomic mode. All its percpu counts will * be collected to the main atomic counter. On completion, when all CPUs diff --git a/lib/raid/.kunitconfig b/lib/raid/.kunitconfig new file mode 100644 index 000000000000..351d22ed1954 --- /dev/null +++ b/lib/raid/.kunitconfig @@ -0,0 +1,3 @@ +CONFIG_KUNIT=y +CONFIG_BTRFS_FS=y +CONFIG_XOR_KUNIT_TEST=y diff --git a/lib/raid/Kconfig b/lib/raid/Kconfig new file mode 100644 index 000000000000..01f007b2522c --- /dev/null +++ b/lib/raid/Kconfig @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: GPL-2.0 + +config XOR_BLOCKS + tristate + +# selected by architectures that provide an optimized XOR implementation +config XOR_BLOCKS_ARCH + depends on XOR_BLOCKS + default y if ALPHA + default y if ARM + default y if ARM64 + default y if CPU_HAS_LSX # loongarch + default y if ALTIVEC # powerpc + default y if RISCV_ISA_V + default y if SPARC + default y if S390 + default y if X86_32 + default y if X86_64 + bool + +config XOR_KUNIT_TEST + tristate "KUnit tests for xor_gen" if !KUNIT_ALL_TESTS + depends on KUNIT + depends on XOR_BLOCKS + default KUNIT_ALL_TESTS + help + Unit tests for the XOR library functions. + + This is intended to help people writing architecture-specific + optimized versions. If unsure, say N. + +config XOR_BENCHMARK + bool "Benchmark for xor_gen" + depends on XOR_KUNIT_TEST + help + Include benchmarks in the KUnit test suite for xor_gen. + +config RAID6_PQ + tristate + +# selected by architectures that provide an optimized PQ implementation +config RAID6_PQ_ARCH + depends on RAID6_PQ + default y if KERNEL_MODE_NEON # arm32/arm64 + default y if LOONGARCH + default y if ALTIVEC # powerpc + default y if RISCV_ISA_V + default y if S390 + default y if X86 + bool + +config RAID6_PQ_KUNIT_TEST + tristate "KUnit tests for RAID6 PQ functions" if !KUNIT_ALL_TESTS + depends on KUNIT + depends on RAID6_PQ + default KUNIT_ALL_TESTS + help + Unit tests for the RAID6 PQ library functions. + + This is intended to help people writing architecture-specific + optimized versions. If unsure, say N. + +config RAID6_PQ_KUNIT_BENCHMARK + bool "Benchmark for RAID6 PQ" + depends on RAID6_PQ_KUNIT_TEST + help + Include benchmarks in the KUnit test suite for raid P/Q generation. + +config RAID6_PQ_BENCHMARK + bool "Automatically choose fastest RAID6 PQ functions" + depends on RAID6_PQ + default y + help + Benchmark all available RAID6 PQ functions on init and choose the + fastest one. diff --git a/lib/raid/Makefile b/lib/raid/Makefile new file mode 100644 index 000000000000..6fc5eeb53df0 --- /dev/null +++ b/lib/raid/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-y += xor/ raid6/ diff --git a/lib/raid6/.gitignore b/lib/raid/raid6/.gitignore index 6be57745afd1..6be57745afd1 100644 --- a/lib/raid6/.gitignore +++ b/lib/raid/raid6/.gitignore diff --git a/lib/raid/raid6/Makefile b/lib/raid/raid6/Makefile new file mode 100644 index 000000000000..5cdb7223de2a --- /dev/null +++ b/lib/raid/raid6/Makefile @@ -0,0 +1,128 @@ +# SPDX-License-Identifier: GPL-2.0 + +CONTEXT_ANALYSIS := y + +ccflags-y += -I $(src) + +ifeq ($(CONFIG_RAID6_PQ_ARCH),y) +CFLAGS_algos.o += -I$(src)/$(SRCARCH) +endif + +obj-$(CONFIG_RAID6_PQ) += raid6_pq.o tests/ + +raid6_pq-y += algos.o tables.o + +# generic integer generation and recovery implementation +raid6_pq-y += int1.o int2.o int4.o int8.o +raid6_pq-y += recov.o + +# architecture-specific generation and recovery implementations: +raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += arm/neon.o \ + arm/neon1.o \ + arm/neon2.o \ + arm/neon4.o \ + arm/neon8.o \ + arm/recov_neon.o \ + arm/recov_neon_inner.o +raid6_pq-$(CONFIG_LOONGARCH) += loongarch/loongarch_simd.o \ + loongarch/recov_loongarch_simd.o +raid6_pq-$(CONFIG_ALTIVEC) += powerpc/altivec1.o \ + powerpc/altivec2.o \ + powerpc/altivec4.o \ + powerpc/altivec8.o \ + powerpc/vpermxor1.o \ + powerpc/vpermxor2.o \ + powerpc/vpermxor4.o \ + powerpc/vpermxor8.o +raid6_pq-$(CONFIG_RISCV_ISA_V) += riscv/rvv.o \ + riscv/recov_rvv.o +raid6_pq-$(CONFIG_S390) += s390/s390vx8.o \ + s390/recov_s390xc.o +ifeq ($(CONFIG_X86),y) +raid6_pq-$(CONFIG_X86_32) += x86/mmx.o \ + x86/sse1.o +endif +raid6_pq-$(CONFIG_X86) += x86/sse2.o \ + x86/avx2.o \ + x86/avx512.o \ + x86/recov_ssse3.o \ + x86/recov_avx2.o \ + x86/recov_avx512.o + +hostprogs += mktables + +CFLAGS_arm/neon1.o += $(CC_FLAGS_FPU) +CFLAGS_arm/neon2.o += $(CC_FLAGS_FPU) +CFLAGS_arm/neon4.o += $(CC_FLAGS_FPU) +CFLAGS_arm/neon8.o += $(CC_FLAGS_FPU) +CFLAGS_arm/recov_neon_inner.o += $(CC_FLAGS_FPU) +CFLAGS_REMOVE_arm/neon1.o += $(CC_FLAGS_NO_FPU) +CFLAGS_REMOVE_arm/neon2.o += $(CC_FLAGS_NO_FPU) +CFLAGS_REMOVE_arm/neon4.o += $(CC_FLAGS_NO_FPU) +CFLAGS_REMOVE_arm/neon8.o += $(CC_FLAGS_NO_FPU) +CFLAGS_REMOVE_arm/recov_neon_inner.o += $(CC_FLAGS_NO_FPU) + +ifeq ($(CONFIG_ALTIVEC),y) +altivec_flags := -maltivec $(call cc-option,-mabi=altivec) +# Enable <altivec.h> +altivec_flags += -isystem $(shell $(CC) -print-file-name=include) + +CFLAGS_powerpc/altivec1.o += $(altivec_flags) +CFLAGS_powerpc/altivec2.o += $(altivec_flags) +CFLAGS_powerpc/altivec4.o += $(altivec_flags) +CFLAGS_powerpc/altivec8.o += $(altivec_flags) +CFLAGS_powerpc/vpermxor1.o += $(altivec_flags) +CFLAGS_powerpc/vpermxor2.o += $(altivec_flags) +CFLAGS_powerpc/vpermxor4.o += $(altivec_flags) +CFLAGS_powerpc/vpermxor8.o += $(altivec_flags) + +ifdef CONFIG_CC_IS_CLANG +# clang ppc port does not yet support -maltivec when -msoft-float is +# enabled. A future release of clang will resolve this +# https://llvm.org/pr31177 +CFLAGS_REMOVE_powerpc/altivec1.o += -msoft-float +CFLAGS_REMOVE_powerpc/altivec2.o += -msoft-float +CFLAGS_REMOVE_powerpc/altivec4.o += -msoft-float +CFLAGS_REMOVE_powerpc/altivec8.o += -msoft-float +CFLAGS_REMOVE_powerpc/vpermxor1.o += -msoft-float +CFLAGS_REMOVE_powerpc/vpermxor2.o += -msoft-float +CFLAGS_REMOVE_powerpc/vpermxor4.o += -msoft-float +CFLAGS_REMOVE_powerpc/vpermxor8.o += -msoft-float +endif # CONFIG_CC_IS_CLANG +endif # CONFIG_ALTIVEC + +quiet_cmd_mktable = TABLE $@ + cmd_mktable = $(obj)/mktables > $@ + +targets += tables.c +$(obj)/tables.c: $(obj)/mktables FORCE + $(call if_changed,mktable) + +quiet_cmd_unroll = UNROLL $@ + cmd_unroll = $(AWK) -v N=$* -f $(src)/unroll.awk < $< > $@ + +targets += int1.c int2.c int4.c int8.c +$(obj)/int%.c: $(src)/int.uc $(src)/unroll.awk FORCE + $(call if_changed,unroll) + +targets += arm/neon1.c arm/neon2.c arm/neon4.c arm/neon8.c +$(obj)/arm/neon%.c: $(src)/arm/neon.uc $(src)/unroll.awk FORCE + $(call if_changed,unroll) + +targets += powerpc/altivec1.c \ + powerpc/altivec2.c \ + powerpc/altivec4.c \ + powerpc/altivec8.c +$(obj)/powerpc/altivec%.c: $(src)/powerpc/altivec.uc $(src)/unroll.awk FORCE + $(call if_changed,unroll) + +targets += powerpc/vpermxor1.c \ + powerpc/vpermxor2.c \ + powerpc/vpermxor4.c \ + powerpc/vpermxor8.c +$(obj)/powerpc/vpermxor%.c: $(src)/powerpc/vpermxor.uc $(src)/unroll.awk FORCE + $(call if_changed,unroll) + +targets += s390/s390vx8.c +$(obj)/s390/s390vx%.c: $(src)/s390/s390vx.uc $(src)/unroll.awk FORCE + $(call if_changed,unroll) diff --git a/lib/raid/raid6/algos.c b/lib/raid/raid6/algos.c new file mode 100644 index 000000000000..011aa9d0eb3c --- /dev/null +++ b/lib/raid/raid6/algos.c @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2002 H. Peter Anvin - All Rights Reserved + * + * Algorithm list and algorithm selection for RAID-6 + */ + +#include <linux/module.h> +#include <linux/gfp.h> +#include <linux/raid/pq.h> +#include <linux/slab.h> +#include <linux/static_call.h> +#include <kunit/visibility.h> +#include "algos.h" + +#define RAID6_MAX_ALGOS 16 +static const struct raid6_calls *raid6_algos[RAID6_MAX_ALGOS]; +static unsigned int raid6_nr_algos; +static const struct raid6_recov_calls *raid6_recov_algo; + +/* Selected algorithm */ +DEFINE_STATIC_CALL_NULL(raid6_gen_syndrome_impl, *raid6_intx1.gen_syndrome); +DEFINE_STATIC_CALL_NULL(raid6_xor_syndrome_impl, *raid6_intx1.xor_syndrome); +DEFINE_STATIC_CALL_NULL(raid6_recov_2data_impl, *raid6_recov_intx1.data2); +DEFINE_STATIC_CALL_NULL(raid6_recov_datap_impl, *raid6_recov_intx1.datap); + +/** + * raid6_gen_syndrome - generate RAID6 P/Q parity + * @disks: number of "disks" to operate on including parity + * @bytes: length in bytes of each vector + * @ptrs: @disks size array of memory pointers + * + * Generate @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and + * @ptrs[@disks - 1] respectively from the memory pointed to by @ptrs[0] to + * @ptrs[@disks - 3]. + * + * @disks must be at least 4, and the memory pointed to by each member of @ptrs + * must be at least 64-byte aligned. @bytes must be non-zero and a multiple of + * 512. + * + * See https://kernel.org/pub/linux/kernel/people/hpa/raid6.pdf for underlying + * algorithm. + */ +void raid6_gen_syndrome(int disks, size_t bytes, void **ptrs) +{ + WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); + WARN_ON_ONCE(bytes & 511); + WARN_ON_ONCE(disks < RAID6_MIN_DISKS); + + static_call(raid6_gen_syndrome_impl)(disks, bytes, ptrs); +} +EXPORT_SYMBOL_GPL(raid6_gen_syndrome); + +/** + * raid6_xor_syndrome - update RAID6 P/Q parity + * @disks: number of "disks" to operate on including parity + * @start: first index into @disk to update + * @stop: last index into @disk to update + * @bytes: length in bytes of each vector + * @ptrs: @disks size array of memory pointers + * + * Update @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and + * @ptrs[@disks - 1] respectively for the memory pointed to by + * @ptrs[@start..@stop]. + * + * This is used to update parity in place using the following sequence: + * + * 1) call raid6_xor_syndrome(disk, start, stop, ...) for the existing data. + * 2) update the the data in @ptrs[@start..@stop]. + * 3) call raid6_xor_syndrome(disk, start, stop, ...) for the new data. + * + * Data between @start and @stop that is not changed should be filled + * with a pointer to the kernel zero page. + * + * @disks must be at least 4, and the memory pointed to by each member of @ptrs + * must be at least 64-byte aligned. @bytes must be non-zero and a multiple of + * 512. @stop must be larger or equal to @start. + */ +void raid6_xor_syndrome(int disks, int start, int stop, size_t bytes, + void **ptrs) +{ + WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); + WARN_ON_ONCE(bytes & 511); + WARN_ON_ONCE(disks < RAID6_MIN_DISKS); + WARN_ON_ONCE(stop < start); + + static_call(raid6_xor_syndrome_impl)(disks, start, stop, bytes, ptrs); +} +EXPORT_SYMBOL_GPL(raid6_xor_syndrome); + +/* + * raid6_can_xor_syndrome - check if raid6_xor_syndrome() can be used + * + * Returns %true if raid6_can_xor_syndrome() can be used, else %false. + */ +bool raid6_can_xor_syndrome(void) +{ + return !!static_call_query(raid6_xor_syndrome_impl); +} +EXPORT_SYMBOL_GPL(raid6_can_xor_syndrome); + +/** + * raid6_recov_2data - recover two missing data disks + * @disks: number of "disks" to operate on including parity + * @bytes: length in bytes of each vector + * @faila: first failed data disk index + * @failb: second failed data disk index + * @ptrs: @disks size array of memory pointers + * + * Rebuild @bytes of missing data in @ptrs[@faila] and @ptrs[@failb] from the + * data in the remaining disks and the two parities pointed to by the other + * indices between 0 and @disks - 1 in @ptrs. @disks includes the data disks + * and the two parities. @faila must be smaller than @failb. + * + * Memory pointed to by each pointer in @ptrs must be page aligned and is + * limited to %PAGE_SIZE. + */ +void raid6_recov_2data(int disks, size_t bytes, int faila, int failb, + void **ptrs) +{ + WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); + WARN_ON_ONCE(bytes & 511); + WARN_ON_ONCE(bytes > PAGE_SIZE); + WARN_ON_ONCE(failb <= faila); + + static_call(raid6_recov_2data_impl)(disks, bytes, faila, failb, ptrs); +} +EXPORT_SYMBOL_GPL(raid6_recov_2data); + +/** + * raid6_recov_datap - recover a missing data disk and missing P-parity + * @disks: number of "disks" to operate on including parity + * @bytes: length in bytes of each vector + * @faila: failed data disk index + * @ptrs: @disks size array of memory pointers + * + * Rebuild @bytes of missing data in @ptrs[@faila] and the missing P-parity in + * @ptrs[@disks - 2] from the data in the remaining disks and the Q-parity + * pointed to by the other indices between 0 and @disks - 1 in @ptrs. @disks + * includes the data disks and the two parities. + * + * Memory pointed to by each pointer in @ptrs must be page aligned and is + * limited to %PAGE_SIZE. + */ +void raid6_recov_datap(int disks, size_t bytes, int faila, void **ptrs) +{ + WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); + WARN_ON_ONCE(bytes & 511); + WARN_ON_ONCE(bytes > PAGE_SIZE); + + static_call(raid6_recov_datap_impl)(disks, bytes, faila, ptrs); +} +EXPORT_SYMBOL_GPL(raid6_recov_datap); + +#define BENCH_SIZE SZ_4K +#define NR_SRCS 8 +#define NR_DISKS (NR_SRCS + 2) +#define REPS 800U + +static int raid6_choose_gen(void *dptrs[NR_DISKS], const int disks) +{ + const struct raid6_calls *best = NULL; + unsigned long bestgenperf = 0; + unsigned int i; + + for (i = 0; i < raid6_nr_algos; i++) { + const struct raid6_calls *algo = raid6_algos[i]; + unsigned long perf = 0; + u64 t; + int i; + + preempt_disable(); + t = ktime_get_ns(); + for (i = 0; i < REPS; i++) + algo->gen_syndrome(disks, BENCH_SIZE, dptrs); + t = max(ktime_get_ns() - t, 1); + preempt_enable(); + + /* bytes/ns == GB/s, multiply by 1000 to get MB/s [not MiB/s] */ + perf = div64_u64((u64)BENCH_SIZE * REPS * NR_SRCS * 1000, t); + if (perf > bestgenperf) { + bestgenperf = perf; + best = algo; + } + pr_info("raid6: %-8s gen() %5lu MB/s\n", algo->name, perf); + } + + if (!best) { + pr_err("raid6: Yikes! No algorithm found!\n"); + return -EINVAL; + } + + static_call_update(raid6_gen_syndrome_impl, best->gen_syndrome); + static_call_update(raid6_xor_syndrome_impl, best->xor_syndrome); + + pr_info("raid6: using algorithm %s gen() %ld MB/s\n", + best->name, bestgenperf); + + if (best->xor_syndrome) { + /* work on the second half of the disks */ + int start = (disks / 2) - 1, stop = disks - 3; + u64 t; + + preempt_disable(); + t = ktime_get_ns(); + for (i = 0; i < REPS; i++) + best->xor_syndrome(disks, start, stop, BENCH_SIZE, + dptrs); + t = max(ktime_get_ns() - t, 1); + preempt_enable(); + + pr_info("raid6: .... xor() %llu MB/s, rmw enabled\n", + div64_u64((u64)BENCH_SIZE * REPS * NR_SRCS / 2 * 1000, + t)); + } + + return 0; +} + + +/* Try to pick the best algorithm */ +/* This code uses the gfmul table as convenient data set to abuse */ + +static int __init raid6_select_algo(void) +{ + const int disks = NR_DISKS; + void *dptrs[NR_DISKS]; + char *disk_ptr, *p; + int i, cycle; + int error; + + if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK) || raid6_nr_algos == 1) { + pr_info("raid6: skipped pq benchmark and selected %s\n", + raid6_algos[raid6_nr_algos - 1]->name); + return 0; + } + + /* prepare the buffer and fill it circularly with gfmul table */ + disk_ptr = kmalloc_array(NR_DISKS, BENCH_SIZE, GFP_KERNEL); + if (!disk_ptr) { + pr_err("raid6: Yikes! No memory available.\n"); + return -ENOMEM; + } + + p = disk_ptr; + for (i = 0; i < disks; i++) + dptrs[i] = p + BENCH_SIZE * i; + + cycle = ((disks - 2) * BENCH_SIZE) / 65536; + for (i = 0; i < cycle; i++) { + memcpy(p, raid6_gfmul, 65536); + p += 65536; + } + + if ((disks - 2) * BENCH_SIZE % 65536) + memcpy(p, raid6_gfmul, (disks - 2) * BENCH_SIZE % 65536); + + /* select raid gen_syndrome function */ + error = raid6_choose_gen(dptrs, disks); + + kfree(disk_ptr); + + return error; +} + +/* + * Register a RAID6 P/Q generation algorithm. The most optimized/unrolled + * implementation should be registered last so it will be selected when the + * boot-time benchmark is disabled. + */ +void __init raid6_algo_add(const struct raid6_calls *algo) +{ + if (WARN_ON_ONCE(raid6_nr_algos == RAID6_MAX_ALGOS)) + return; + raid6_algos[raid6_nr_algos++] = algo; +} + +void __init raid6_algo_add_default(void) +{ + raid6_algo_add(&raid6_intx1); + raid6_algo_add(&raid6_intx2); + raid6_algo_add(&raid6_intx4); + raid6_algo_add(&raid6_intx8); +} + +void __init raid6_recov_algo_add(const struct raid6_recov_calls *algo) +{ + if (WARN_ON_ONCE(raid6_recov_algo)) + return; + raid6_recov_algo = algo; +} + +#ifdef CONFIG_RAID6_PQ_ARCH +#include "pq_arch.h" +#else +static inline void arch_raid6_init(void) +{ + raid6_algo_add_default(); +} +#endif /* CONFIG_RAID6_PQ_ARCH */ + +static int __init raid6_init(void) +{ + /* + * Architectures providing arch_raid6_init must add all PQ generation + * algorithms they want to consider in arch_raid6_init(), including + * the generic ones using raid6_algo_add_default() if wanted. + */ + arch_raid6_init(); + + /* + * Architectures don't have to set a recovery algorithm, we'll just pick + * the generic integer one if none was set. + */ + if (!raid6_recov_algo) + raid6_recov_algo = &raid6_recov_intx1; + static_call_update(raid6_recov_2data_impl, raid6_recov_algo->data2); + static_call_update(raid6_recov_datap_impl, raid6_recov_algo->datap); + pr_info("raid6: using %s recovery algorithm\n", raid6_recov_algo->name); + + /* + * Pick the last registered implementation as the temporary default until + * calibration happens. + */ + static_call_update(raid6_gen_syndrome_impl, + raid6_algos[raid6_nr_algos - 1]->gen_syndrome); + static_call_update(raid6_xor_syndrome_impl, + raid6_algos[raid6_nr_algos - 1]->xor_syndrome); + +#ifdef MODULE + return raid6_select_algo(); +#else + return 0; +#endif +} + +static void __exit raid6_exit(void) +{ +} + +/* + * When built-in we must register the default implementation before md + * initializes, but we don't want calibration to run that early as that + * would delay the boot process. + */ +#ifndef MODULE +device_initcall(raid6_select_algo); +#endif +subsys_initcall(raid6_init); +module_exit(raid6_exit); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("RAID6 Q-syndrome calculations"); + +#if IS_ENABLED(CONFIG_RAID6_PQ_KUNIT_TEST) +const struct raid6_calls *raid6_algo_find(unsigned int idx) +{ + if (idx >= raid6_nr_algos) { + /* + * Always include the simplest generic integer implementation in + * the unit tests as a baseline. + */ + if (idx == raid6_nr_algos && + raid6_algos[0] != &raid6_intx1) + return &raid6_intx1; + return NULL; + } + return raid6_algos[idx]; +} +EXPORT_SYMBOL_IF_KUNIT(raid6_algo_find); + +const struct raid6_recov_calls *raid6_recov_algo_find(unsigned int idx) +{ + switch (idx) { + case 0: + /* always test the generic integer implementation */ + return &raid6_recov_intx1; + case 1: + /* test the optimized implementation if there is one */ + if (raid6_recov_algo != &raid6_recov_intx1) + return raid6_recov_algo; + return NULL; + default: + return NULL; + } +} +EXPORT_SYMBOL_IF_KUNIT(raid6_recov_algo_find); +#endif /* CONFIG_RAID6_PQ_KUNIT_TEST */ diff --git a/lib/raid/raid6/algos.h b/lib/raid/raid6/algos.h new file mode 100644 index 000000000000..43f636be183f --- /dev/null +++ b/lib/raid/raid6/algos.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright 2003 H. Peter Anvin - All Rights Reserved + */ +#ifndef _PQ_IMPL_H +#define _PQ_IMPL_H + +#include <linux/init.h> +#include <linux/raid/pq_tables.h> + +/* Routine choices */ +struct raid6_calls { + const char *name; + void (*gen_syndrome)(int disks, size_t bytes, void **ptrs); + void (*xor_syndrome)(int disks, int start, int stop, size_t bytes, + void **ptrs); +}; + +struct raid6_recov_calls { + const char *name; + void (*data2)(int disks, size_t bytes, int faila, int failb, + void **ptrs); + void (*datap)(int disks, size_t bytes, int faila, void **ptrs); +}; + +void __init raid6_algo_add(const struct raid6_calls *algo); +void __init raid6_algo_add_default(void); +void __init raid6_recov_algo_add(const struct raid6_recov_calls *algo); + +/* for the kunit test */ +const struct raid6_calls *raid6_algo_find(unsigned int idx); +const struct raid6_recov_calls *raid6_recov_algo_find(unsigned int idx); + +/* generic implementations */ +extern const struct raid6_calls raid6_intx1; +extern const struct raid6_calls raid6_intx2; +extern const struct raid6_calls raid6_intx4; +extern const struct raid6_calls raid6_intx8; +extern const struct raid6_recov_calls raid6_recov_intx1; + +#endif /* _PQ_IMPL_H */ diff --git a/lib/raid6/neon.c b/lib/raid/raid6/arm/neon.c index 6d9474ce6da9..af90869aaffc 100644 --- a/lib/raid6/neon.c +++ b/lib/raid/raid6/arm/neon.c @@ -1,18 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * linux/lib/raid6/neon.c - RAID6 syndrome calculation using ARM NEON intrinsics + * RAID6 syndrome calculation using ARM NEON intrinsics * * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org> */ -#include <linux/raid/pq.h> - -#ifdef __KERNEL__ #include <asm/simd.h> -#else -#define scoped_ksimd() -#define cpu_has_neon() (1) -#endif +#include "algos.h" /* * There are 2 reasons these wrappers are kept in a separate compilation unit @@ -46,18 +40,11 @@ start, stop, (unsigned long)bytes, ptrs);\ } \ struct raid6_calls const raid6_neonx ## _n = { \ - raid6_neon ## _n ## _gen_syndrome, \ - raid6_neon ## _n ## _xor_syndrome, \ - raid6_have_neon, \ - "neonx" #_n, \ - 0 \ + .gen_syndrome = raid6_neon ## _n ## _gen_syndrome, \ + .xor_syndrome = raid6_neon ## _n ## _xor_syndrome, \ + .name = "neonx" #_n, \ } -static int raid6_have_neon(void) -{ - return cpu_has_neon(); -} - RAID6_NEON_WRAPPER(1); RAID6_NEON_WRAPPER(2); RAID6_NEON_WRAPPER(4); diff --git a/lib/raid6/neon.h b/lib/raid/raid6/arm/neon.h index 2ca41ee9b499..2ca41ee9b499 100644 --- a/lib/raid6/neon.h +++ b/lib/raid/raid6/arm/neon.h diff --git a/lib/raid6/neon.uc b/lib/raid/raid6/arm/neon.uc index 355270af0cd6..14a9fc2c60fa 100644 --- a/lib/raid6/neon.uc +++ b/lib/raid/raid6/arm/neon.uc @@ -25,7 +25,7 @@ */ #include <arm_neon.h> -#include "neon.h" +#include "arm/neon.h" typedef uint8x16_t unative_t; diff --git a/lib/raid/raid6/arm/pq_arch.h b/lib/raid/raid6/arm/pq_arch.h new file mode 100644 index 000000000000..3f876ea6749c --- /dev/null +++ b/lib/raid/raid6/arm/pq_arch.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include <asm/neon.h> + +extern const struct raid6_calls raid6_neonx1; +extern const struct raid6_calls raid6_neonx2; +extern const struct raid6_calls raid6_neonx4; +extern const struct raid6_calls raid6_neonx8; +extern const struct raid6_recov_calls raid6_recov_neon; + +static __always_inline void __init arch_raid6_init(void) +{ + raid6_algo_add_default(); + if (cpu_has_neon()) { + raid6_algo_add(&raid6_neonx1); + raid6_algo_add(&raid6_neonx2); + raid6_algo_add(&raid6_neonx4); + raid6_algo_add(&raid6_neonx8); + raid6_recov_algo_add(&raid6_recov_neon); + } +} diff --git a/lib/raid6/recov_neon.c b/lib/raid/raid6/arm/recov_neon.c index 9d99aeabd31a..1524050d09b7 100644 --- a/lib/raid6/recov_neon.c +++ b/lib/raid/raid6/arm/recov_neon.c @@ -4,20 +4,11 @@ * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org> */ +#include <linux/mm.h> #include <linux/raid/pq.h> - -#ifdef __KERNEL__ #include <asm/simd.h> -#include "neon.h" -#else -#define scoped_ksimd() -#define cpu_has_neon() (1) -#endif - -static int raid6_has_neon(void) -{ - return cpu_has_neon(); -} +#include "algos.h" +#include "arm/neon.h" static void raid6_2data_recov_neon(int disks, size_t bytes, int faila, int failb, void **ptrs) @@ -35,13 +26,13 @@ static void raid6_2data_recov_neon(int disks, size_t bytes, int faila, * delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -72,10 +63,10 @@ static void raid6_datap_recov_neon(int disks, size_t bytes, int faila, * Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -91,7 +82,5 @@ static void raid6_datap_recov_neon(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_neon = { .data2 = raid6_2data_recov_neon, .datap = raid6_datap_recov_neon, - .valid = raid6_has_neon, .name = "neon", - .priority = 10, }; diff --git a/lib/raid6/recov_neon_inner.c b/lib/raid/raid6/arm/recov_neon_inner.c index f9e7e8f5a151..53c355efa7ff 100644 --- a/lib/raid6/recov_neon_inner.c +++ b/lib/raid/raid6/arm/recov_neon_inner.c @@ -5,7 +5,7 @@ */ #include <arm_neon.h> -#include "neon.h" +#include "arm/neon.h" #ifdef CONFIG_ARM /* diff --git a/lib/raid/raid6/arm64/pq_arch.h b/lib/raid/raid6/arm64/pq_arch.h new file mode 100644 index 000000000000..27ff564d7594 --- /dev/null +++ b/lib/raid/raid6/arm64/pq_arch.h @@ -0,0 +1 @@ +#include "arm/pq_arch.h" diff --git a/lib/raid6/int.uc b/lib/raid/raid6/int.uc index 1ba56c3fa482..e63bd5a9c2ed 100644 --- a/lib/raid6/int.uc +++ b/lib/raid/raid6/int.uc @@ -18,7 +18,7 @@ * This file is postprocessed using unroll.awk */ -#include <linux/raid/pq.h> +#include "algos.h" /* * This is the C data type to use @@ -139,9 +139,7 @@ static void raid6_int$#_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_intx$# = { - raid6_int$#_gen_syndrome, - raid6_int$#_xor_syndrome, - NULL, /* always valid */ - "int" NSTRING "x$#", - 0 + .gen_syndrome = raid6_int$#_gen_syndrome, + .xor_syndrome = raid6_int$#_xor_syndrome, + .name = "int" NSTRING "x$#", }; diff --git a/lib/raid6/loongarch_simd.c b/lib/raid/raid6/loongarch/loongarch_simd.c index aa5d9f924ca3..c1eb53fafd27 100644 --- a/lib/raid6/loongarch_simd.c +++ b/lib/raid/raid6/loongarch/loongarch_simd.c @@ -9,8 +9,9 @@ * Copyright 2002-2004 H. Peter Anvin */ -#include <linux/raid/pq.h> -#include "loongarch.h" +#include <asm/cpu-features.h> +#include <asm/fpu.h> +#include "algos.h" /* * The vector algorithms are currently priority 0, which means the generic @@ -25,11 +26,6 @@ #ifdef CONFIG_CPU_HAS_LSX #define NSIZE 16 -static int raid6_has_lsx(void) -{ - return cpu_has_lsx; -} - static void raid6_lsx_gen_syndrome(int disks, size_t bytes, void **ptrs) { u8 **dptr = (u8 **)ptrs; @@ -243,11 +239,9 @@ static void raid6_lsx_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_lsx = { - raid6_lsx_gen_syndrome, - raid6_lsx_xor_syndrome, - raid6_has_lsx, - "lsx", - .priority = 0 /* see the comment near the top of the file for reason */ + .gen_syndrome = raid6_lsx_gen_syndrome, + .xor_syndrome = raid6_lsx_xor_syndrome, + .name = "lsx", }; #undef NSIZE @@ -256,11 +250,6 @@ const struct raid6_calls raid6_lsx = { #ifdef CONFIG_CPU_HAS_LASX #define NSIZE 32 -static int raid6_has_lasx(void) -{ - return cpu_has_lasx; -} - static void raid6_lasx_gen_syndrome(int disks, size_t bytes, void **ptrs) { u8 **dptr = (u8 **)ptrs; @@ -412,11 +401,9 @@ static void raid6_lasx_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_lasx = { - raid6_lasx_gen_syndrome, - raid6_lasx_xor_syndrome, - raid6_has_lasx, - "lasx", - .priority = 0 /* see the comment near the top of the file for reason */ + .gen_syndrome = raid6_lasx_gen_syndrome, + .xor_syndrome = raid6_lasx_xor_syndrome, + .name = "lasx", }; #undef NSIZE #endif /* CONFIG_CPU_HAS_LASX */ diff --git a/lib/raid/raid6/loongarch/pq_arch.h b/lib/raid/raid6/loongarch/pq_arch.h new file mode 100644 index 000000000000..ae443a4d7b69 --- /dev/null +++ b/lib/raid/raid6/loongarch/pq_arch.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include <asm/cpu-features.h> + +extern const struct raid6_calls raid6_lsx; +extern const struct raid6_calls raid6_lasx; + +extern const struct raid6_recov_calls raid6_recov_lsx; +extern const struct raid6_recov_calls raid6_recov_lasx; + +static __always_inline void __init arch_raid6_init(void) +{ + raid6_algo_add_default(); + if (IS_ENABLED(CONFIG_CPU_HAS_LSX) && cpu_has_lsx) + raid6_algo_add(&raid6_lsx); + if (IS_ENABLED(CONFIG_CPU_HAS_LASX) && cpu_has_lasx) + raid6_algo_add(&raid6_lasx); + + if (IS_ENABLED(CONFIG_CPU_HAS_LASX) && cpu_has_lasx) + raid6_recov_algo_add(&raid6_recov_lasx); + else if (IS_ENABLED(CONFIG_CPU_HAS_LSX) && cpu_has_lsx) + raid6_recov_algo_add(&raid6_recov_lsx); +} diff --git a/lib/raid6/recov_loongarch_simd.c b/lib/raid/raid6/loongarch/recov_loongarch_simd.c index 93dc515997a1..87a2313bbb4f 100644 --- a/lib/raid6/recov_loongarch_simd.c +++ b/lib/raid/raid6/loongarch/recov_loongarch_simd.c @@ -10,8 +10,11 @@ * Author: Jim Kukunas <james.t.kukunas@linux.intel.com> */ +#include <linux/mm.h> #include <linux/raid/pq.h> -#include "loongarch.h" +#include <asm/cpu-features.h> +#include <asm/fpu.h> +#include "algos.h" /* * Unlike with the syndrome calculation algorithms, there's no boot-time @@ -21,11 +24,6 @@ */ #ifdef CONFIG_CPU_HAS_LSX -static int raid6_has_lsx(void) -{ - return cpu_has_lsx; -} - static void raid6_2data_recov_lsx(int disks, size_t bytes, int faila, int failb, void **ptrs) { @@ -42,13 +40,13 @@ static void raid6_2data_recov_lsx(int disks, size_t bytes, int faila, * delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -197,10 +195,10 @@ static void raid6_datap_recov_lsx(int disks, size_t bytes, int faila, * Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -288,18 +286,11 @@ static void raid6_datap_recov_lsx(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_lsx = { .data2 = raid6_2data_recov_lsx, .datap = raid6_datap_recov_lsx, - .valid = raid6_has_lsx, .name = "lsx", - .priority = 1, }; #endif /* CONFIG_CPU_HAS_LSX */ #ifdef CONFIG_CPU_HAS_LASX -static int raid6_has_lasx(void) -{ - return cpu_has_lasx; -} - static void raid6_2data_recov_lasx(int disks, size_t bytes, int faila, int failb, void **ptrs) { @@ -316,13 +307,13 @@ static void raid6_2data_recov_lasx(int disks, size_t bytes, int faila, * delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -436,10 +427,10 @@ static void raid6_datap_recov_lasx(int disks, size_t bytes, int faila, * Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -506,8 +497,6 @@ static void raid6_datap_recov_lasx(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_lasx = { .data2 = raid6_2data_recov_lasx, .datap = raid6_datap_recov_lasx, - .valid = raid6_has_lasx, .name = "lasx", - .priority = 2, }; #endif /* CONFIG_CPU_HAS_LASX */ diff --git a/lib/raid6/mktables.c b/lib/raid/raid6/mktables.c index 3be03793237c..b6327b562fdb 100644 --- a/lib/raid6/mktables.c +++ b/lib/raid/raid6/mktables.c @@ -1,15 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002-2007 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - /* - * mktables.c + * Copyright 2002-2007 H. Peter Anvin - All Rights Reserved * - * Make RAID-6 tables. This is a host user space program to be run at - * compile time. + * Make RAID-6 tables. This is a host user space program to be run at compile + * time. */ #include <stdio.h> @@ -56,10 +50,8 @@ int main(int argc, char *argv[]) uint8_t v; uint8_t exptbl[256], invtbl[256]; - printf("#ifdef __KERNEL__\n"); printf("#include <linux/export.h>\n"); - printf("#endif\n"); - printf("#include <linux/raid/pq.h>\n"); + printf("#include \"algos.h\"\n"); /* Compute multiplication table */ printf("\nconst u8 __attribute__((aligned(256)))\n" @@ -76,9 +68,7 @@ int main(int argc, char *argv[]) printf("\t},\n"); } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_gfmul);\n"); - printf("#endif\n"); /* Compute vector multiplication table */ printf("\nconst u8 __attribute__((aligned(256)))\n" @@ -101,9 +91,7 @@ int main(int argc, char *argv[]) printf("\t},\n"); } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_vgfmul);\n"); - printf("#endif\n"); /* Compute power-of-2 table (exponent) */ v = 1; @@ -120,9 +108,7 @@ int main(int argc, char *argv[]) } } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_gfexp);\n"); - printf("#endif\n"); /* Compute log-of-2 table */ printf("\nconst u8 __attribute__((aligned(256)))\n" @@ -140,9 +126,7 @@ int main(int argc, char *argv[]) } } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_gflog);\n"); - printf("#endif\n"); /* Compute inverse table x^-1 == x^254 */ printf("\nconst u8 __attribute__((aligned(256)))\n" @@ -155,9 +139,7 @@ int main(int argc, char *argv[]) } } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_gfinv);\n"); - printf("#endif\n"); /* Compute inv(2^x + 1) (exponent-xor-inverse) table */ printf("\nconst u8 __attribute__((aligned(256)))\n" @@ -169,9 +151,7 @@ int main(int argc, char *argv[]) (j == 7) ? '\n' : ' '); } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_gfexi);\n"); - printf("#endif\n"); return 0; } diff --git a/lib/raid6/altivec.uc b/lib/raid/raid6/powerpc/altivec.uc index d20ed0d11411..c5429fb71dd6 100644 --- a/lib/raid6/altivec.uc +++ b/lib/raid/raid6/powerpc/altivec.uc @@ -22,15 +22,11 @@ * bracked this with preempt_disable/enable or in a lock) */ -#include <linux/raid/pq.h> - -#ifdef CONFIG_ALTIVEC +#include "algos.h" #include <altivec.h> -#ifdef __KERNEL__ -# include <asm/cputable.h> -# include <asm/switch_to.h> -#endif /* __KERNEL__ */ +#include <asm/cputable.h> +#include <asm/switch_to.h> /* * This is the C data type to use. We use a vector of @@ -108,25 +104,7 @@ static void raid6_altivec$#_gen_syndrome(int disks, size_t bytes, void **ptrs) preempt_enable(); } -int raid6_have_altivec(void); -#if $# == 1 -int raid6_have_altivec(void) -{ - /* This assumes either all CPUs have Altivec or none does */ -# ifdef __KERNEL__ - return cpu_has_feature(CPU_FTR_ALTIVEC); -# else - return 1; -# endif -} -#endif - const struct raid6_calls raid6_altivec$# = { - raid6_altivec$#_gen_syndrome, - NULL, /* XOR not yet implemented */ - raid6_have_altivec, - "altivecx$#", - 0 + .gen_syndrome = raid6_altivec$#_gen_syndrome, + .name = "altivecx$#", }; - -#endif /* CONFIG_ALTIVEC */ diff --git a/lib/raid/raid6/powerpc/pq_arch.h b/lib/raid/raid6/powerpc/pq_arch.h new file mode 100644 index 000000000000..ea1878777ff2 --- /dev/null +++ b/lib/raid/raid6/powerpc/pq_arch.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include <asm/cputable.h> + +extern const struct raid6_calls raid6_altivec1; +extern const struct raid6_calls raid6_altivec2; +extern const struct raid6_calls raid6_altivec4; +extern const struct raid6_calls raid6_altivec8; +extern const struct raid6_calls raid6_vpermxor1; +extern const struct raid6_calls raid6_vpermxor2; +extern const struct raid6_calls raid6_vpermxor4; +extern const struct raid6_calls raid6_vpermxor8; + +static __always_inline void __init arch_raid6_init(void) +{ + raid6_algo_add_default(); + + /* This assumes either all CPUs have Altivec or none does */ + if (cpu_has_feature(CPU_FTR_ALTIVEC)) { + raid6_algo_add(&raid6_altivec1); + raid6_algo_add(&raid6_altivec2); + raid6_algo_add(&raid6_altivec4); + raid6_algo_add(&raid6_altivec8); + } + if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP) && + cpu_has_feature(CPU_FTR_ARCH_207S)) { + raid6_algo_add(&raid6_vpermxor1); + raid6_algo_add(&raid6_vpermxor2); + raid6_algo_add(&raid6_vpermxor4); + raid6_algo_add(&raid6_vpermxor8); + } +} diff --git a/lib/raid6/vpermxor.uc b/lib/raid/raid6/powerpc/vpermxor.uc index 1bfb127fbfe8..e8964361aaef 100644 --- a/lib/raid6/vpermxor.uc +++ b/lib/raid/raid6/powerpc/vpermxor.uc @@ -20,15 +20,11 @@ * This instruction was introduced in POWER8 - ISA v2.07. */ -#include <linux/raid/pq.h> -#ifdef CONFIG_ALTIVEC - #include <altivec.h> #include <asm/ppc-opcode.h> -#ifdef __KERNEL__ #include <asm/cputable.h> #include <asm/switch_to.h> -#endif +#include "algos.h" typedef vector unsigned char unative_t; #define NSIZE sizeof(unative_t) @@ -80,26 +76,7 @@ static void raid6_vpermxor$#_gen_syndrome(int disks, size_t bytes, void **ptrs) preempt_enable(); } -int raid6_have_altivec_vpermxor(void); -#if $# == 1 -int raid6_have_altivec_vpermxor(void) -{ - /* Check if arch has both altivec and the vpermxor instructions */ -# ifdef __KERNEL__ - return (cpu_has_feature(CPU_FTR_ALTIVEC_COMP) && - cpu_has_feature(CPU_FTR_ARCH_207S)); -# else - return 1; -#endif - -} -#endif - const struct raid6_calls raid6_vpermxor$# = { - raid6_vpermxor$#_gen_syndrome, - NULL, - raid6_have_altivec_vpermxor, - "vpermxor$#", - 0 + .gen_syndrome = raid6_vpermxor$#_gen_syndrome, + .name = "vpermxor$#", }; -#endif diff --git a/lib/raid6/recov.c b/lib/raid/raid6/recov.c index b5e47c008b41..3fa53bc3fde4 100644 --- a/lib/raid6/recov.c +++ b/lib/raid/raid6/recov.c @@ -1,19 +1,15 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - /* - * raid6/recov.c + * Copyright 2002 H. Peter Anvin - All Rights Reserved * - * RAID-6 data recovery in dual failure mode. In single failure mode, - * use the RAID-5 algorithm (or, in the case of Q failure, just reconstruct - * the syndrome.) + * RAID-6 data recovery in dual failure mode. In single failure mode, use the + * RAID-5 algorithm (or, in the case of Q failure, just reconstruct the + * syndrome.) */ +#include <linux/mm.h> #include <linux/raid/pq.h> +#include "algos.h" /* Recover two failed data blocks. */ static void raid6_2data_recov_intx1(int disks, size_t bytes, int faila, @@ -31,13 +27,13 @@ static void raid6_2data_recov_intx1(int disks, size_t bytes, int faila, Use the dead data pages as temporary storage for delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -72,10 +68,10 @@ static void raid6_datap_recov_intx1(int disks, size_t bytes, int faila, /* Compute syndrome with zero for the missing data page Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -95,41 +91,5 @@ static void raid6_datap_recov_intx1(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_intx1 = { .data2 = raid6_2data_recov_intx1, .datap = raid6_datap_recov_intx1, - .valid = NULL, .name = "intx1", - .priority = 0, }; - -#ifndef __KERNEL__ -/* Testing only */ - -/* Recover two failed blocks. */ -void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, void **ptrs) -{ - if ( faila > failb ) { - int tmp = faila; - faila = failb; - failb = tmp; - } - - if ( failb == disks-1 ) { - if ( faila == disks-2 ) { - /* P+Q failure. Just rebuild the syndrome. */ - raid6_call.gen_syndrome(disks, bytes, ptrs); - } else { - /* data+Q failure. Reconstruct data from P, - then rebuild syndrome. */ - /* NOT IMPLEMENTED - equivalent to RAID-5 */ - } - } else { - if ( failb == disks-2 ) { - /* data+P failure. */ - raid6_datap_recov(disks, bytes, faila, ptrs); - } else { - /* data+data failure. */ - raid6_2data_recov(disks, bytes, faila, failb, ptrs); - } - } -} - -#endif diff --git a/lib/raid/raid6/riscv/pq_arch.h b/lib/raid/raid6/riscv/pq_arch.h new file mode 100644 index 000000000000..82f1a188f8c4 --- /dev/null +++ b/lib/raid/raid6/riscv/pq_arch.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include <asm/vector.h> + +extern const struct raid6_calls raid6_rvvx1; +extern const struct raid6_calls raid6_rvvx2; +extern const struct raid6_calls raid6_rvvx4; +extern const struct raid6_calls raid6_rvvx8; +extern const struct raid6_recov_calls raid6_recov_rvv; + +static __always_inline void __init arch_raid6_init(void) +{ + raid6_algo_add_default(); + if (has_vector()) { + raid6_algo_add(&raid6_rvvx1); + raid6_algo_add(&raid6_rvvx2); + raid6_algo_add(&raid6_rvvx4); + raid6_algo_add(&raid6_rvvx8); + raid6_recov_algo_add(&raid6_recov_rvv); + } +} diff --git a/lib/raid6/recov_rvv.c b/lib/raid/raid6/riscv/recov_rvv.c index 40c393206b6a..78e158a3e332 100644 --- a/lib/raid6/recov_rvv.c +++ b/lib/raid/raid6/riscv/recov_rvv.c @@ -4,8 +4,11 @@ * Author: Chunyan Zhang <zhangchunyan@iscas.ac.cn> */ +#include <linux/mm.h> #include <linux/raid/pq.h> +#include "algos.h" #include "rvv.h" +#include "pq_arch.h" static void __raid6_2data_recov_rvv(int bytes, u8 *p, u8 *q, u8 *dp, u8 *dq, const u8 *pbmul, @@ -158,13 +161,13 @@ static void raid6_2data_recov_rvv(int disks, size_t bytes, int faila, * delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -196,10 +199,10 @@ static void raid6_datap_recov_rvv(int disks, size_t bytes, int faila, * Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -216,7 +219,5 @@ static void raid6_datap_recov_rvv(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_rvv = { .data2 = raid6_2data_recov_rvv, .datap = raid6_datap_recov_rvv, - .valid = rvv_has_vector, .name = "rvv", - .priority = 1, }; diff --git a/lib/raid6/rvv.c b/lib/raid/raid6/riscv/rvv.c index 75c9dafedb28..4ac50606f3dc 100644 --- a/lib/raid6/rvv.c +++ b/lib/raid/raid6/riscv/rvv.c @@ -10,6 +10,7 @@ */ #include "rvv.h" +#include "pq_arch.h" #ifdef __riscv_vector #error "This code must be built without compiler support for vector" diff --git a/lib/raid6/rvv.h b/lib/raid/raid6/riscv/rvv.h index 6d0708a2c8a4..df0e3637cae8 100644 --- a/lib/raid6/rvv.h +++ b/lib/raid/raid6/riscv/rvv.h @@ -2,27 +2,11 @@ /* * Copyright 2024 Institute of Software, CAS. * - * raid6/rvv.h - * * Definitions for RISC-V RAID-6 code */ -#ifdef __KERNEL__ #include <asm/vector.h> -#else -#define kernel_vector_begin() -#define kernel_vector_end() -#include <sys/auxv.h> -#include <asm/hwcap.h> -#define has_vector() (getauxval(AT_HWCAP) & COMPAT_HWCAP_ISA_V) -#endif - -#include <linux/raid/pq.h> - -static int rvv_has_vector(void) -{ - return has_vector(); -} +#include "algos.h" #define RAID6_RVV_WRAPPER(_n) \ static void raid6_rvv ## _n ## _gen_syndrome(int disks, \ @@ -48,9 +32,7 @@ static int rvv_has_vector(void) kernel_vector_end(); \ } \ struct raid6_calls const raid6_rvvx ## _n = { \ - raid6_rvv ## _n ## _gen_syndrome, \ - raid6_rvv ## _n ## _xor_syndrome, \ - rvv_has_vector, \ - "rvvx" #_n, \ - 0 \ + .gen_syndrome = raid6_rvv ## _n ## _gen_syndrome, \ + .xor_syndrome = raid6_rvv ## _n ## _xor_syndrome, \ + .name = "rvvx" #_n, \ } diff --git a/lib/raid/raid6/s390/pq_arch.h b/lib/raid/raid6/s390/pq_arch.h new file mode 100644 index 000000000000..95d14c342306 --- /dev/null +++ b/lib/raid/raid6/s390/pq_arch.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include <linux/cpufeature.h> + +extern const struct raid6_calls raid6_s390vx8; +extern const struct raid6_recov_calls raid6_recov_s390xc; + +static __always_inline void __init arch_raid6_init(void) +{ + if (cpu_has_vx()) + raid6_algo_add(&raid6_s390vx8); + else + raid6_algo_add_default(); + raid6_recov_algo_add(&raid6_recov_s390xc); +} diff --git a/lib/raid6/recov_s390xc.c b/lib/raid/raid6/s390/recov_s390xc.c index 487018f81192..08d56896e5ea 100644 --- a/lib/raid6/recov_s390xc.c +++ b/lib/raid/raid6/s390/recov_s390xc.c @@ -6,7 +6,9 @@ * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> */ +#include <linux/mm.h> #include <linux/raid/pq.h> +#include "algos.h" static inline void xor_block(u8 *p1, u8 *p2) { @@ -34,13 +36,13 @@ static void raid6_2data_recov_s390xc(int disks, size_t bytes, int faila, Use the dead data pages as temporary storage for delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -81,10 +83,10 @@ static void raid6_datap_recov_s390xc(int disks, size_t bytes, int faila, /* Compute syndrome with zero for the missing data page Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -110,7 +112,5 @@ static void raid6_datap_recov_s390xc(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_s390xc = { .data2 = raid6_2data_recov_s390xc, .datap = raid6_datap_recov_s390xc, - .valid = NULL, .name = "s390xc", - .priority = 1, }; diff --git a/lib/raid6/s390vx.uc b/lib/raid/raid6/s390/s390vx.uc index 8aa53eb2f395..e5cf9054be2a 100644 --- a/lib/raid6/s390vx.uc +++ b/lib/raid/raid6/s390/s390vx.uc @@ -12,8 +12,8 @@ */ #include <linux/cpufeature.h> -#include <linux/raid/pq.h> #include <asm/fpu.h> +#include "algos.h" #define NSIZE 16 @@ -121,15 +121,8 @@ static void raid6_s390vx$#_xor_syndrome(int disks, int start, int stop, kernel_fpu_end(&vxstate, KERNEL_VXR); } -static int raid6_s390vx$#_valid(void) -{ - return cpu_has_vx(); -} - const struct raid6_calls raid6_s390vx$# = { - raid6_s390vx$#_gen_syndrome, - raid6_s390vx$#_xor_syndrome, - raid6_s390vx$#_valid, - "vx128x$#", - 1 + .gen_syndrome = raid6_s390vx$#_gen_syndrome, + .xor_syndrome = raid6_s390vx$#_xor_syndrome, + .name = "vx128x$#", }; diff --git a/lib/raid/raid6/tests/Makefile b/lib/raid/raid6/tests/Makefile new file mode 100644 index 000000000000..87a001b22847 --- /dev/null +++ b/lib/raid/raid6/tests/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_RAID6_PQ_KUNIT_TEST) += raid6_kunit.o diff --git a/lib/raid/raid6/tests/raid6_kunit.c b/lib/raid/raid6/tests/raid6_kunit.c new file mode 100644 index 000000000000..7d0a1ec98001 --- /dev/null +++ b/lib/raid/raid6/tests/raid6_kunit.c @@ -0,0 +1,385 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2002-2007 H. Peter Anvin - All Rights Reserved + * + * Test RAID-6 recovery algorithms. + */ + +#include <kunit/test.h> +#include <linux/prandom.h> +#include <linux/vmalloc.h> +#include <linux/raid/pq.h> +#include "../algos.h" + +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); + +#define RAID6_KUNIT_SEED 42 +#define RAID6_KUNIT_NUM_TEST_ITERS 10 +#define RAID6_KUNIT_MAX_BUFFERS 64 /* Including P and Q */ +#define RAID6_KUNIT_MAX_FAILURES 2 +#define RAID6_KUNIT_MAX_BYTES PAGE_SIZE +#define RAID6_KUNIT_ALLOC_BYTES SZ_16K + +static struct rnd_state rng; +static void *test_buffers[RAID6_KUNIT_MAX_BUFFERS]; +static void *aligned_buffers[RAID6_KUNIT_MAX_BUFFERS]; +static void *test_recov_buffers[RAID6_KUNIT_MAX_FAILURES]; +static size_t test_buflen; + +struct test_args { + unsigned int recov_idx; + const struct raid6_recov_calls *recov; + unsigned int gen_idx; + const struct raid6_calls *gen; +}; + +static struct test_args args; + +static u32 rand32(void) +{ + return prandom_u32_state(&rng); +} + +/* Generate a random length that is a multiple of 512. */ +static unsigned int random_length(unsigned int max_length) +{ + return round_up((rand32() % max_length) + 1, 512); +} + +static unsigned int random_nr_buffers(void) +{ + return (rand32() % (RAID6_KUNIT_MAX_BUFFERS - (RAID6_MIN_DISKS - 1))) + + RAID6_MIN_DISKS; +} + +/* Generate a random alignment that is a multiple of 64. */ +static unsigned int random_alignment(unsigned int max_alignment) +{ + if (max_alignment == 0) + return 0; + return (rand32() % (max_alignment + 1)) & ~63; +} + +static void makedata(int start, int stop) +{ + int i; + + for (i = start; i <= stop; i++) + prandom_bytes_state(&rng, test_buffers[i], test_buflen); +} + +static char member_type(unsigned int nr_buffers, int d) +{ + if (d == nr_buffers - 2) + return 'P'; + if (d == nr_buffers - 1) + return 'Q'; + return 'D'; +} + +static void test_recover_one(struct kunit *test, unsigned int nr_buffers, + unsigned int len, int faila, int failb) +{ + const struct test_args *ta = test->param_value; + void *dataptrs[RAID6_KUNIT_MAX_BUFFERS]; + int i; + + if (faila > failb) + swap(faila, failb); + + for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++) + memset(test_recov_buffers[i], 0xf0, test_buflen); + + memcpy(dataptrs, aligned_buffers, sizeof(dataptrs)); + dataptrs[faila] = test_recov_buffers[0]; + dataptrs[failb] = test_recov_buffers[1]; + + if (failb == nr_buffers - 1) { + /* + * We don't implement the data+Q failure scenario, since it + * is equivalent to a RAID-5 failure (XOR, then recompute Q). + */ + if (WARN_ON_ONCE(faila != nr_buffers - 2)) + return; + + /* P+Q failure. Just rebuild the syndrome. */ + ta->gen->gen_syndrome(nr_buffers, len, dataptrs); + } else if (failb == nr_buffers - 2) { + /* data+P failure. */ + ta->recov->datap(nr_buffers, len, faila, dataptrs); + } else { + /* data+data failure. */ + ta->recov->data2(nr_buffers, len, faila, failb, dataptrs); + } + + KUNIT_EXPECT_MEMEQ_MSG(test, aligned_buffers[faila], dataptrs[faila], + len, + "faila miscompared: %3d[%c] buffers %u len %u (failb=%3d[%c])\n", + faila, member_type(nr_buffers, faila), + nr_buffers, len, + failb, member_type(nr_buffers, failb)); + KUNIT_EXPECT_MEMEQ_MSG(test, aligned_buffers[failb], dataptrs[failb], + len, + "failb miscompared: %3d[%c] buffers %u len %u (faila=%3d[%c])\n", + failb, member_type(nr_buffers, failb), + nr_buffers, len, + faila, member_type(nr_buffers, faila)); +} + +static void test_recover(struct kunit *test, unsigned int nr_buffers, + unsigned int len) +{ + unsigned int nr_data = nr_buffers - 2; + int iterations, i; + + /* Test P+Q recovery */ + test_recover_one(test, nr_buffers, len, nr_data, nr_buffers - 1); + + /* Test data+P recovery */ + for (i = 0; i < nr_buffers - 2; i++) + test_recover_one(test, nr_buffers, len, i, nr_data); + + /* Double data failure is impossible with a single data disk */ + if (nr_data == 1) + return; + + /* Test data+data recovery using random sampling */ + iterations = nr_buffers * 2; /* should provide good enough coverage */ + for (i = 0; i < iterations; i++) { + int faila = rand32() % nr_data, failb; + + do { + failb = rand32() % nr_data; + } while (failb == faila); + + test_recover_one(test, nr_buffers, len, faila, failb); + } +} + +/* Simulate rmw run */ +static void test_rmw_one(struct kunit *test, unsigned int nr_buffers, + unsigned int len, int p1, int p2) +{ + const struct test_args *ta = test->param_value; + + ta->gen->xor_syndrome(nr_buffers, p1, p2, len, aligned_buffers); + makedata(p1, p2); + ta->gen->xor_syndrome(nr_buffers, p1, p2, len, aligned_buffers); + test_recover(test, nr_buffers, len); +} + +static void test_rmw(struct kunit *test, unsigned int nr_buffers, + unsigned int len) +{ + int iterations = nr_buffers / 2, i; + + for (i = 0; i < iterations; i++) { + int p1 = rand32() % (nr_buffers - 2); + int p2 = rand32() % (nr_buffers - 2); + + if (p2 < p1) + swap(p1, p2); + test_rmw_one(test, nr_buffers, len, p1, p2); + } +} + +static void raid6_test_one(struct kunit *test) +{ + const struct test_args *ta = test->param_value; + unsigned int nr_buffers = random_nr_buffers(); + unsigned int len = random_length(RAID6_KUNIT_MAX_BYTES); + unsigned int max_alignment; + int i; + + /* Nuke syndromes */ + memset(test_buffers[nr_buffers - 2], 0xee, test_buflen); + memset(test_buffers[nr_buffers - 1], 0xee, test_buflen); + + /* + * If we're not using the entire buffer size, inject randomize alignment + * into the buffer. + */ + max_alignment = RAID6_KUNIT_MAX_BYTES - len; + if (rand32() % 2 == 0) { + /* Use random alignments mod 64 */ + for (i = 0; i < nr_buffers; i++) + aligned_buffers[i] = test_buffers[i] + + random_alignment(max_alignment); + } else { + /* Go up to the guard page, to catch buffer overreads */ + unsigned int align = test_buflen - len; + + for (i = 0; i < nr_buffers; i++) + aligned_buffers[i] = test_buffers[i] + align; + } + + /* Generate assumed good syndrome */ + ta->gen->gen_syndrome(nr_buffers, len, aligned_buffers); + + test_recover(test, nr_buffers, len); + + if (ta->gen->xor_syndrome) + test_rmw(test, nr_buffers, len); +} + +static void raid6_test(struct kunit *test) +{ + int i; + + for (i = 0; i < RAID6_KUNIT_NUM_TEST_ITERS; i++) + raid6_test_one(test); +} + +static void raid6_benchmark(struct kunit *test) +{ + static const unsigned int nr_to_test[] = { + 4, 5, 6, 7, 8, 10, 12, 15, 16, 32, + }; + static const unsigned int len_to_test[] = { + SZ_4K, SZ_16K, + }; + unsigned int i, j, l; + u64 t; + + if (!IS_ENABLED(CONFIG_RAID6_PQ_KUNIT_BENCHMARK)) + kunit_skip(test, "not enabled"); + + /* warm-up */ + for (i = 0; i < ARRAY_SIZE(nr_to_test); i++) { + for (j = 0; j < ARRAY_SIZE(len_to_test); j++) { + for (l = 0; l < 10; l++) { + raid6_gen_syndrome(nr_to_test[i], + len_to_test[j], test_buffers); + } + } + } + + /* + * Preferably this would be a loop over len_to_test, but the kunit + * logging always adds a newline to each logged format string. + */ + static_assert(ARRAY_SIZE(len_to_test) == 2); + kunit_info(test, " \t%5u bytes\t%5u bytes\n", + len_to_test[0], len_to_test[1]); + + for (i = 0; i < ARRAY_SIZE(nr_to_test); i++) { + unsigned int nr = nr_to_test[i]; + u64 speed[ARRAY_SIZE(len_to_test)]; + + KUNIT_ASSERT_LE(test, nr, RAID6_KUNIT_MAX_BUFFERS); + + for (j = 0; j < ARRAY_SIZE(len_to_test); j++) { + unsigned int len = len_to_test[j]; + const unsigned long num_iters = 1000; + + KUNIT_ASSERT_GT(test, len, 0); + KUNIT_ASSERT_LE(test, len, RAID6_KUNIT_ALLOC_BYTES); + + preempt_disable(); + t = ktime_get_ns(); + for (l = 0; l < num_iters; l++) + raid6_gen_syndrome(nr_to_test[i], + len_to_test[j], test_buffers); + t = max(ktime_get_ns() - t, 1); + preempt_enable(); + + speed[j] = div64_u64((u64)len * num_iters * nr, t); + } + + static_assert(ARRAY_SIZE(len_to_test) == 2); + kunit_info(test, "%3u disks:\t%5llu GB/s\t%5llu GB/s\n", + nr, speed[0], speed[1]); + } +} + +static const void *raid6_gen_params(struct kunit *test, const void *prev, + char *desc) +{ + if (!prev) { + memset(&args, 0, sizeof(args)); +next_algo: + args.recov_idx = 0; + args.gen = raid6_algo_find(args.gen_idx); + if (!args.gen) + return NULL; + } + + if (args.recov) + args.recov_idx++; + args.recov = raid6_recov_algo_find(args.recov_idx); + if (!args.recov) { + args.gen_idx++; + goto next_algo; + } + + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "gen=%s recov=%s", + args.gen->name, args.recov->name); + return &args; +} + +static struct kunit_case raid6_test_cases[] = { + KUNIT_CASE_PARAM(raid6_test, raid6_gen_params), + KUNIT_CASE(raid6_benchmark), + {}, +}; + +static int raid6_suite_init(struct kunit_suite *suite) +{ + int i; + + prandom_seed_state(&rng, RAID6_KUNIT_SEED); + + /* + * Allocate the test buffer using vmalloc() with a page-aligned length + * so that it is immediately followed by a guard page. This allows + * buffer overreads to be detected, even in assembly code. + */ + test_buflen = round_up(RAID6_KUNIT_ALLOC_BYTES, PAGE_SIZE); + for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++) { + test_recov_buffers[i] = vmalloc(test_buflen); + if (!test_recov_buffers[i]) + goto out_free_recov_buffers; + } + for (i = 0; i < RAID6_KUNIT_MAX_BUFFERS; i++) { + test_buffers[i] = vmalloc(test_buflen); + if (!test_buffers[i]) + goto out_free_buffers; + } + + makedata(0, RAID6_KUNIT_MAX_BUFFERS - 1); + + return 0; + +out_free_buffers: + for (i = 0; i < RAID6_KUNIT_MAX_BUFFERS; i++) + vfree(test_buffers[i]); + memset(test_buffers, 0, sizeof(test_buffers)); +out_free_recov_buffers: + for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++) + vfree(test_recov_buffers[i]); + memset(test_recov_buffers, 0, sizeof(test_recov_buffers)); + return -ENOMEM; +} + +static void raid6_suite_exit(struct kunit_suite *suite) +{ + int i; + + for (i = 0; i < RAID6_KUNIT_MAX_BUFFERS; i++) + vfree(test_buffers[i]); + memset(test_buffers, 0, sizeof(test_buffers)); + for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++) + vfree(test_recov_buffers[i]); + memset(test_recov_buffers, 0, sizeof(test_recov_buffers)); +} + +static struct kunit_suite raid6_test_suite = { + .name = "raid6", + .test_cases = raid6_test_cases, + .suite_init = raid6_suite_init, + .suite_exit = raid6_suite_exit, +}; +kunit_test_suite(raid6_test_suite); + +MODULE_DESCRIPTION("Unit test for the RAID P/Q library functions"); +MODULE_LICENSE("GPL"); diff --git a/lib/raid6/unroll.awk b/lib/raid/raid6/unroll.awk index 0809805a7e23..0809805a7e23 100644 --- a/lib/raid6/unroll.awk +++ b/lib/raid/raid6/unroll.awk diff --git a/lib/raid6/avx2.c b/lib/raid/raid6/x86/avx2.c index 059024234dce..7d829c669ea7 100644 --- a/lib/raid6/avx2.c +++ b/lib/raid/raid6/x86/avx2.c @@ -1,20 +1,16 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright (C) 2012 Intel Corporation - * Author: Yuanhan Liu <yuanhan.liu@linux.intel.com> +/* + * Copyright (C) 2012 Intel Corporation + * Author: Yuanhan Liu <yuanhan.liu@linux.intel.com> * - * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved + * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved * - * ----------------------------------------------------------------------- */ - -/* * AVX2 implementation of RAID-6 syndrome functions - * */ -#include <linux/raid/pq.h> -#include "x86.h" +#include <asm/cpufeature.h> +#include <asm/fpu/api.h> +#include "algos.h" static const struct raid6_avx2_constants { u64 x1d[4]; @@ -23,11 +19,6 @@ static const struct raid6_avx2_constants { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,}, }; -static int raid6_have_avx2(void) -{ - return boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX); -} - /* * Plain AVX2 implementation */ @@ -128,11 +119,9 @@ static void raid6_avx21_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_avx2x1 = { - raid6_avx21_gen_syndrome, - raid6_avx21_xor_syndrome, - raid6_have_avx2, - "avx2x1", - .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx21_gen_syndrome, + .xor_syndrome = raid6_avx21_xor_syndrome, + .name = "avx2x1", }; /* @@ -258,11 +247,9 @@ static void raid6_avx22_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_avx2x2 = { - raid6_avx22_gen_syndrome, - raid6_avx22_xor_syndrome, - raid6_have_avx2, - "avx2x2", - .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx22_gen_syndrome, + .xor_syndrome = raid6_avx22_xor_syndrome, + .name = "avx2x2", }; #ifdef CONFIG_X86_64 @@ -461,10 +448,8 @@ static void raid6_avx24_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_avx2x4 = { - raid6_avx24_gen_syndrome, - raid6_avx24_xor_syndrome, - raid6_have_avx2, - "avx2x4", - .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx24_gen_syndrome, + .xor_syndrome = raid6_avx24_xor_syndrome, + .name = "avx2x4", }; #endif /* CONFIG_X86_64 */ diff --git a/lib/raid6/avx512.c b/lib/raid/raid6/x86/avx512.c index 009bd0adeebf..e671eb5bde63 100644 --- a/lib/raid6/avx512.c +++ b/lib/raid/raid6/x86/avx512.c @@ -1,24 +1,19 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- -------------------------------------------------------- - * - * Copyright (C) 2016 Intel Corporation +/* + * Copyright (C) 2016 Intel Corporation * - * Author: Gayatri Kammela <gayatri.kammela@intel.com> - * Author: Megha Dey <megha.dey@linux.intel.com> + * Author: Gayatri Kammela <gayatri.kammela@intel.com> + * Author: Megha Dey <megha.dey@linux.intel.com> * - * Based on avx2.c: Copyright 2012 Yuanhan Liu All Rights Reserved - * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved + * Based on avx2.c: Copyright 2012 Yuanhan Liu All Rights Reserved + * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved * - * ----------------------------------------------------------------------- - */ - -/* * AVX512 implementation of RAID-6 syndrome functions - * */ -#include <linux/raid/pq.h> -#include "x86.h" +#include <asm/cpufeature.h> +#include <asm/fpu/api.h> +#include "algos.h" static const struct raid6_avx512_constants { u64 x1d[8]; @@ -29,16 +24,6 @@ static const struct raid6_avx512_constants { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,}, }; -static int raid6_have_avx512(void) -{ - return boot_cpu_has(X86_FEATURE_AVX2) && - boot_cpu_has(X86_FEATURE_AVX) && - boot_cpu_has(X86_FEATURE_AVX512F) && - boot_cpu_has(X86_FEATURE_AVX512BW) && - boot_cpu_has(X86_FEATURE_AVX512VL) && - boot_cpu_has(X86_FEATURE_AVX512DQ); -} - static void raid6_avx5121_gen_syndrome(int disks, size_t bytes, void **ptrs) { u8 **dptr = (u8 **)ptrs; @@ -156,11 +141,9 @@ static void raid6_avx5121_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_avx512x1 = { - raid6_avx5121_gen_syndrome, - raid6_avx5121_xor_syndrome, - raid6_have_avx512, - "avx512x1", - .priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx5121_gen_syndrome, + .xor_syndrome = raid6_avx5121_xor_syndrome, + .name = "avx512x1", }; /* @@ -313,11 +296,9 @@ static void raid6_avx5122_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_avx512x2 = { - raid6_avx5122_gen_syndrome, - raid6_avx5122_xor_syndrome, - raid6_have_avx512, - "avx512x2", - .priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx5122_gen_syndrome, + .xor_syndrome = raid6_avx5122_xor_syndrome, + .name = "avx512x2", }; #ifdef CONFIG_X86_64 @@ -551,10 +532,8 @@ static void raid6_avx5124_xor_syndrome(int disks, int start, int stop, kernel_fpu_end(); } const struct raid6_calls raid6_avx512x4 = { - raid6_avx5124_gen_syndrome, - raid6_avx5124_xor_syndrome, - raid6_have_avx512, - "avx512x4", - .priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx5124_gen_syndrome, + .xor_syndrome = raid6_avx5124_xor_syndrome, + .name = "avx512x4", }; #endif diff --git a/lib/raid6/mmx.c b/lib/raid/raid6/x86/mmx.c index 3a5bf53a297b..afa82536142d 100644 --- a/lib/raid6/mmx.c +++ b/lib/raid/raid6/x86/mmx.c @@ -1,20 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - /* - * raid6/mmx.c + * Copyright 2002 H. Peter Anvin - All Rights Reserved * - * MMX implementation of RAID-6 syndrome functions + * MMX implementation of RAID-6 syndrome functions. */ -#ifdef CONFIG_X86_32 - -#include <linux/raid/pq.h> -#include "x86.h" +#include <asm/cpufeature.h> +#include <asm/fpu/api.h> +#include "algos.h" /* Shared with raid6/sse1.c */ const struct raid6_mmx_constants { @@ -23,12 +16,6 @@ const struct raid6_mmx_constants { 0x1d1d1d1d1d1d1d1dULL, }; -static int raid6_have_mmx(void) -{ - /* Not really "boot_cpu" but "all_cpus" */ - return boot_cpu_has(X86_FEATURE_MMX); -} - /* * Plain MMX implementation */ @@ -70,11 +57,8 @@ static void raid6_mmx1_gen_syndrome(int disks, size_t bytes, void **ptrs) } const struct raid6_calls raid6_mmxx1 = { - raid6_mmx1_gen_syndrome, - NULL, /* XOR not yet implemented */ - raid6_have_mmx, - "mmxx1", - 0 + .gen_syndrome = raid6_mmx1_gen_syndrome, + .name = "mmxx1", }; /* @@ -129,11 +113,6 @@ static void raid6_mmx2_gen_syndrome(int disks, size_t bytes, void **ptrs) } const struct raid6_calls raid6_mmxx2 = { - raid6_mmx2_gen_syndrome, - NULL, /* XOR not yet implemented */ - raid6_have_mmx, - "mmxx2", - 0 + .gen_syndrome = raid6_mmx2_gen_syndrome, + .name = "mmxx2", }; - -#endif diff --git a/lib/raid/raid6/x86/pq_arch.h b/lib/raid/raid6/x86/pq_arch.h new file mode 100644 index 000000000000..02f8843b0537 --- /dev/null +++ b/lib/raid/raid6/x86/pq_arch.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include <asm/cpufeature.h> + +extern const struct raid6_calls raid6_mmxx1; +extern const struct raid6_calls raid6_mmxx2; +extern const struct raid6_calls raid6_sse1x1; +extern const struct raid6_calls raid6_sse1x2; +extern const struct raid6_calls raid6_sse2x1; +extern const struct raid6_calls raid6_sse2x2; +extern const struct raid6_calls raid6_sse2x4; +extern const struct raid6_calls raid6_avx2x1; +extern const struct raid6_calls raid6_avx2x2; +extern const struct raid6_calls raid6_avx2x4; +extern const struct raid6_calls raid6_avx512x1; +extern const struct raid6_calls raid6_avx512x2; +extern const struct raid6_calls raid6_avx512x4; + +extern const struct raid6_recov_calls raid6_recov_ssse3; +extern const struct raid6_recov_calls raid6_recov_avx2; +extern const struct raid6_recov_calls raid6_recov_avx512; + +static inline int raid6_has_avx512(void) +{ + return boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX512F) && + boot_cpu_has(X86_FEATURE_AVX512BW) && + boot_cpu_has(X86_FEATURE_AVX512VL) && + boot_cpu_has(X86_FEATURE_AVX512DQ); +} + +static inline bool raid6_has_avx2(void) +{ + return boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX); +} + +static inline bool raid6_has_ssse3(void) +{ + return boot_cpu_has(X86_FEATURE_XMM) && + boot_cpu_has(X86_FEATURE_XMM2) && + boot_cpu_has(X86_FEATURE_SSSE3); +} + +static inline bool raid6_has_sse2(void) +{ + return boot_cpu_has(X86_FEATURE_MMX) && + boot_cpu_has(X86_FEATURE_FXSR) && + boot_cpu_has(X86_FEATURE_XMM) && + boot_cpu_has(X86_FEATURE_XMM2); +} + +static inline bool raid6_has_sse1_or_mmxext(void) +{ + return boot_cpu_has(X86_FEATURE_MMX) && + (boot_cpu_has(X86_FEATURE_XMM) || + boot_cpu_has(X86_FEATURE_MMXEXT)); +} + +static __always_inline void __init arch_raid6_init(void) +{ + if (raid6_has_avx2()) { + raid6_algo_add(&raid6_avx2x1); + raid6_algo_add(&raid6_avx2x2); + if (IS_ENABLED(CONFIG_X86_64)) + raid6_algo_add(&raid6_avx2x4); + if (raid6_has_avx512()) { + raid6_algo_add(&raid6_avx512x1); + raid6_algo_add(&raid6_avx512x2); + if (IS_ENABLED(CONFIG_X86_64)) + raid6_algo_add(&raid6_avx512x4); + } + } else if (IS_ENABLED(CONFIG_X86_64) || raid6_has_sse2()) { + /* x86_64 can assume SSE2 as baseline */ + raid6_algo_add(&raid6_sse2x1); + raid6_algo_add(&raid6_sse2x2); + if (IS_ENABLED(CONFIG_X86_64)) + raid6_algo_add(&raid6_sse2x4); + } else { + raid6_algo_add_default(); + if (raid6_has_sse1_or_mmxext()) { + raid6_algo_add(&raid6_sse1x1); + raid6_algo_add(&raid6_sse1x2); + } else if (boot_cpu_has(X86_FEATURE_MMX)) { + raid6_algo_add(&raid6_mmxx1); + raid6_algo_add(&raid6_mmxx2); + } + } + + if (raid6_has_avx512()) + raid6_recov_algo_add(&raid6_recov_avx512); + else if (raid6_has_avx2()) + raid6_recov_algo_add(&raid6_recov_avx2); + else if (raid6_has_ssse3()) + raid6_recov_algo_add(&raid6_recov_ssse3); +} diff --git a/lib/raid6/recov_avx2.c b/lib/raid/raid6/x86/recov_avx2.c index 97d598d2535c..a714a780a2d8 100644 --- a/lib/raid6/recov_avx2.c +++ b/lib/raid/raid6/x86/recov_avx2.c @@ -4,14 +4,10 @@ * Author: Jim Kukunas <james.t.kukunas@linux.intel.com> */ +#include <linux/mm.h> #include <linux/raid/pq.h> -#include "x86.h" - -static int raid6_has_avx2(void) -{ - return boot_cpu_has(X86_FEATURE_AVX2) && - boot_cpu_has(X86_FEATURE_AVX); -} +#include <asm/fpu/api.h> +#include "algos.h" static void raid6_2data_recov_avx2(int disks, size_t bytes, int faila, int failb, void **ptrs) @@ -28,13 +24,13 @@ static void raid6_2data_recov_avx2(int disks, size_t bytes, int faila, Use the dead data pages as temporary storage for delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -196,10 +192,10 @@ static void raid6_datap_recov_avx2(int disks, size_t bytes, int faila, /* Compute syndrome with zero for the missing data page Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -303,11 +299,9 @@ static void raid6_datap_recov_avx2(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_avx2 = { .data2 = raid6_2data_recov_avx2, .datap = raid6_datap_recov_avx2, - .valid = raid6_has_avx2, #ifdef CONFIG_X86_64 .name = "avx2x2", #else .name = "avx2x1", #endif - .priority = 2, }; diff --git a/lib/raid6/recov_avx512.c b/lib/raid/raid6/x86/recov_avx512.c index 7986120ca444..ec72d5a30c01 100644 --- a/lib/raid6/recov_avx512.c +++ b/lib/raid/raid6/x86/recov_avx512.c @@ -6,18 +6,10 @@ * Author: Megha Dey <megha.dey@linux.intel.com> */ +#include <linux/mm.h> #include <linux/raid/pq.h> -#include "x86.h" - -static int raid6_has_avx512(void) -{ - return boot_cpu_has(X86_FEATURE_AVX2) && - boot_cpu_has(X86_FEATURE_AVX) && - boot_cpu_has(X86_FEATURE_AVX512F) && - boot_cpu_has(X86_FEATURE_AVX512BW) && - boot_cpu_has(X86_FEATURE_AVX512VL) && - boot_cpu_has(X86_FEATURE_AVX512DQ); -} +#include <asm/fpu/api.h> +#include "algos.h" static void raid6_2data_recov_avx512(int disks, size_t bytes, int faila, int failb, void **ptrs) @@ -37,13 +29,13 @@ static void raid6_2data_recov_avx512(int disks, size_t bytes, int faila, */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -238,10 +230,10 @@ static void raid6_datap_recov_avx512(int disks, size_t bytes, int faila, */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -367,11 +359,9 @@ static void raid6_datap_recov_avx512(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_avx512 = { .data2 = raid6_2data_recov_avx512, .datap = raid6_datap_recov_avx512, - .valid = raid6_has_avx512, #ifdef CONFIG_X86_64 .name = "avx512x2", #else .name = "avx512x1", #endif - .priority = 3, }; diff --git a/lib/raid6/recov_ssse3.c b/lib/raid/raid6/x86/recov_ssse3.c index 2e849185c32b..700bd2c865ec 100644 --- a/lib/raid6/recov_ssse3.c +++ b/lib/raid/raid6/x86/recov_ssse3.c @@ -3,15 +3,10 @@ * Copyright (C) 2012 Intel Corporation */ +#include <linux/mm.h> #include <linux/raid/pq.h> -#include "x86.h" - -static int raid6_has_ssse3(void) -{ - return boot_cpu_has(X86_FEATURE_XMM) && - boot_cpu_has(X86_FEATURE_XMM2) && - boot_cpu_has(X86_FEATURE_SSSE3); -} +#include <asm/fpu/api.h> +#include "algos.h" static void raid6_2data_recov_ssse3(int disks, size_t bytes, int faila, int failb, void **ptrs) @@ -30,13 +25,13 @@ static void raid6_2data_recov_ssse3(int disks, size_t bytes, int faila, Use the dead data pages as temporary storage for delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -203,10 +198,10 @@ static void raid6_datap_recov_ssse3(int disks, size_t bytes, int faila, /* Compute syndrome with zero for the missing data page Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -318,11 +313,9 @@ static void raid6_datap_recov_ssse3(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_ssse3 = { .data2 = raid6_2data_recov_ssse3, .datap = raid6_datap_recov_ssse3, - .valid = raid6_has_ssse3, #ifdef CONFIG_X86_64 .name = "ssse3x2", #else .name = "ssse3x1", #endif - .priority = 1, }; diff --git a/lib/raid6/sse1.c b/lib/raid/raid6/x86/sse1.c index 692fa3a93bf0..f4b260df522a 100644 --- a/lib/raid6/sse1.c +++ b/lib/raid/raid6/x86/sse1.c @@ -1,39 +1,24 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - /* - * raid6/sse1.c + * Copyright 2002 H. Peter Anvin - All Rights Reserved * - * SSE-1/MMXEXT implementation of RAID-6 syndrome functions + * SSE-1/MMXEXT implementation of RAID-6 syndrome functions. * - * This is really an MMX implementation, but it requires SSE-1 or - * AMD MMXEXT for prefetch support and a few other features. The - * support for nontemporal memory accesses is enough to make this - * worthwhile as a separate implementation. + * This is really an MMX implementation, but it requires SSE-1 or AMD MMXEXT for + * prefetch support and a few other features. The support for nontemporal + * memory accesses is enough to make this worthwhile as a separate + * implementation. */ -#ifdef CONFIG_X86_32 - -#include <linux/raid/pq.h> -#include "x86.h" +#include <asm/cpufeature.h> +#include <asm/fpu/api.h> +#include "algos.h" /* Defined in raid6/mmx.c */ extern const struct raid6_mmx_constants { u64 x1d; } raid6_mmx_constants; -static int raid6_have_sse1_or_mmxext(void) -{ - /* Not really boot_cpu but "all_cpus" */ - return boot_cpu_has(X86_FEATURE_MMX) && - (boot_cpu_has(X86_FEATURE_XMM) || - boot_cpu_has(X86_FEATURE_MMXEXT)); -} - /* * Plain SSE1 implementation */ @@ -86,11 +71,8 @@ static void raid6_sse11_gen_syndrome(int disks, size_t bytes, void **ptrs) } const struct raid6_calls raid6_sse1x1 = { - raid6_sse11_gen_syndrome, - NULL, /* XOR not yet implemented */ - raid6_have_sse1_or_mmxext, - "sse1x1", - 1 /* Has cache hints */ + .gen_syndrome = raid6_sse11_gen_syndrome, + .name = "sse1x1", }; /* @@ -149,11 +131,6 @@ static void raid6_sse12_gen_syndrome(int disks, size_t bytes, void **ptrs) } const struct raid6_calls raid6_sse1x2 = { - raid6_sse12_gen_syndrome, - NULL, /* XOR not yet implemented */ - raid6_have_sse1_or_mmxext, - "sse1x2", - 1 /* Has cache hints */ + .gen_syndrome = raid6_sse12_gen_syndrome, + .name = "sse1x2", }; - -#endif diff --git a/lib/raid6/sse2.c b/lib/raid/raid6/x86/sse2.c index 2930220249c9..43b09ce58270 100644 --- a/lib/raid6/sse2.c +++ b/lib/raid/raid6/x86/sse2.c @@ -1,19 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - /* - * raid6/sse2.c + * Copyright 2002 H. Peter Anvin - All Rights Reserved * * SSE-2 implementation of RAID-6 syndrome functions - * */ -#include <linux/raid/pq.h> -#include "x86.h" +#include <asm/cpufeature.h> +#include <asm/fpu/api.h> +#include "algos.h" static const struct raid6_sse_constants { u64 x1d[2]; @@ -21,15 +15,6 @@ static const struct raid6_sse_constants { { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL }, }; -static int raid6_have_sse2(void) -{ - /* Not really boot_cpu but "all_cpus" */ - return boot_cpu_has(X86_FEATURE_MMX) && - boot_cpu_has(X86_FEATURE_FXSR) && - boot_cpu_has(X86_FEATURE_XMM) && - boot_cpu_has(X86_FEATURE_XMM2); -} - /* * Plain SSE2 implementation */ @@ -133,11 +118,9 @@ static void raid6_sse21_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_sse2x1 = { - raid6_sse21_gen_syndrome, - raid6_sse21_xor_syndrome, - raid6_have_sse2, - "sse2x1", - 1 /* Has cache hints */ + .gen_syndrome = raid6_sse21_gen_syndrome, + .xor_syndrome = raid6_sse21_xor_syndrome, + .name = "sse2x1", }; /* @@ -263,11 +246,9 @@ static void raid6_sse22_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_sse2x2 = { - raid6_sse22_gen_syndrome, - raid6_sse22_xor_syndrome, - raid6_have_sse2, - "sse2x2", - 1 /* Has cache hints */ + .gen_syndrome = raid6_sse22_gen_syndrome, + .xor_syndrome = raid6_sse22_xor_syndrome, + .name = "sse2x2", }; #ifdef CONFIG_X86_64 @@ -470,11 +451,9 @@ static void raid6_sse24_xor_syndrome(int disks, int start, int stop, const struct raid6_calls raid6_sse2x4 = { - raid6_sse24_gen_syndrome, - raid6_sse24_xor_syndrome, - raid6_have_sse2, - "sse2x4", - 1 /* Has cache hints */ + .gen_syndrome = raid6_sse24_gen_syndrome, + .xor_syndrome = raid6_sse24_xor_syndrome, + .name = "sse2x4", }; #endif /* CONFIG_X86_64 */ diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile new file mode 100644 index 000000000000..9b0fad459cdb --- /dev/null +++ b/lib/raid/xor/Makefile @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: GPL-2.0 + +CONTEXT_ANALYSIS := y + +ccflags-y += -I $(src) + +obj-$(CONFIG_XOR_BLOCKS) += xor.o + +xor-y += xor-core.o +xor-y += xor-8regs.o +xor-y += xor-32regs.o +xor-y += xor-8regs-prefetch.o +xor-y += xor-32regs-prefetch.o + +ifeq ($(CONFIG_XOR_BLOCKS_ARCH),y) +CFLAGS_xor-core.o += -I$(src)/$(SRCARCH) +endif + +xor-$(CONFIG_ALPHA) += alpha/xor.o +xor-$(CONFIG_ARM) += arm/xor.o +ifeq ($(CONFIG_ARM),y) +xor-$(CONFIG_KERNEL_MODE_NEON) += xor-neon.o arm/xor-neon-glue.o +endif +xor-$(CONFIG_ARM64) += xor-neon.o arm64/xor-eor3.o \ + arm64/xor-neon-glue.o +xor-$(CONFIG_CPU_HAS_LSX) += loongarch/xor_simd.o +xor-$(CONFIG_CPU_HAS_LSX) += loongarch/xor_simd_glue.o +xor-$(CONFIG_ALTIVEC) += powerpc/xor_vmx.o powerpc/xor_vmx_glue.o +xor-$(CONFIG_RISCV_ISA_V) += riscv/xor.o riscv/xor-glue.o +xor-$(CONFIG_SPARC32) += sparc/xor-sparc32.o +xor-$(CONFIG_SPARC64) += sparc/xor-sparc64.o sparc/xor-sparc64-glue.o +xor-$(CONFIG_S390) += s390/xor.o +xor-$(CONFIG_X86_32) += x86/xor-avx.o x86/xor-sse.o x86/xor-mmx.o +xor-$(CONFIG_X86_64) += x86/xor-avx.o x86/xor-sse.o +obj-y += tests/ + +CFLAGS_xor-neon.o += $(CC_FLAGS_FPU) -I$(src)/$(SRCARCH) +CFLAGS_REMOVE_xor-neon.o += $(CC_FLAGS_NO_FPU) + +CFLAGS_arm64/xor-eor3.o += $(CC_FLAGS_FPU) +CFLAGS_REMOVE_arm64/xor-eor3.o += $(CC_FLAGS_NO_FPU) + +CFLAGS_powerpc/xor_vmx.o += -mhard-float -maltivec \ + $(call cc-option,-mabi=altivec) \ + -isystem $(shell $(CC) -print-file-name=include) diff --git a/lib/raid/xor/alpha/xor.c b/lib/raid/xor/alpha/xor.c new file mode 100644 index 000000000000..a8f72f2dd3a5 --- /dev/null +++ b/lib/raid/xor/alpha/xor.c @@ -0,0 +1,848 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Optimized XOR parity functions for alpha EV5 and EV6 + */ +#include "xor_impl.h" +#include "xor_arch.h" + +extern void +xor_alpha_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2); +extern void +xor_alpha_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3); +extern void +xor_alpha_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4); +extern void +xor_alpha_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5); + +extern void +xor_alpha_prefetch_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2); +extern void +xor_alpha_prefetch_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3); +extern void +xor_alpha_prefetch_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4); +extern void +xor_alpha_prefetch_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5); + +asm(" \n\ + .text \n\ + .align 3 \n\ + .ent xor_alpha_2 \n\ +xor_alpha_2: \n\ + .prologue 0 \n\ + srl $16, 6, $16 \n\ + .align 4 \n\ +2: \n\ + ldq $0,0($17) \n\ + ldq $1,0($18) \n\ + ldq $2,8($17) \n\ + ldq $3,8($18) \n\ + \n\ + ldq $4,16($17) \n\ + ldq $5,16($18) \n\ + ldq $6,24($17) \n\ + ldq $7,24($18) \n\ + \n\ + ldq $19,32($17) \n\ + ldq $20,32($18) \n\ + ldq $21,40($17) \n\ + ldq $22,40($18) \n\ + \n\ + ldq $23,48($17) \n\ + ldq $24,48($18) \n\ + ldq $25,56($17) \n\ + xor $0,$1,$0 # 7 cycles from $1 load \n\ + \n\ + ldq $27,56($18) \n\ + xor $2,$3,$2 \n\ + stq $0,0($17) \n\ + xor $4,$5,$4 \n\ + \n\ + stq $2,8($17) \n\ + xor $6,$7,$6 \n\ + stq $4,16($17) \n\ + xor $19,$20,$19 \n\ + \n\ + stq $6,24($17) \n\ + xor $21,$22,$21 \n\ + stq $19,32($17) \n\ + xor $23,$24,$23 \n\ + \n\ + stq $21,40($17) \n\ + xor $25,$27,$25 \n\ + stq $23,48($17) \n\ + subq $16,1,$16 \n\ + \n\ + stq $25,56($17) \n\ + addq $17,64,$17 \n\ + addq $18,64,$18 \n\ + bgt $16,2b \n\ + \n\ + ret \n\ + .end xor_alpha_2 \n\ + \n\ + .align 3 \n\ + .ent xor_alpha_3 \n\ +xor_alpha_3: \n\ + .prologue 0 \n\ + srl $16, 6, $16 \n\ + .align 4 \n\ +3: \n\ + ldq $0,0($17) \n\ + ldq $1,0($18) \n\ + ldq $2,0($19) \n\ + ldq $3,8($17) \n\ + \n\ + ldq $4,8($18) \n\ + ldq $6,16($17) \n\ + ldq $7,16($18) \n\ + ldq $21,24($17) \n\ + \n\ + ldq $22,24($18) \n\ + ldq $24,32($17) \n\ + ldq $25,32($18) \n\ + ldq $5,8($19) \n\ + \n\ + ldq $20,16($19) \n\ + ldq $23,24($19) \n\ + ldq $27,32($19) \n\ + nop \n\ + \n\ + xor $0,$1,$1 # 8 cycles from $0 load \n\ + xor $3,$4,$4 # 6 cycles from $4 load \n\ + xor $6,$7,$7 # 6 cycles from $7 load \n\ + xor $21,$22,$22 # 5 cycles from $22 load \n\ + \n\ + xor $1,$2,$2 # 9 cycles from $2 load \n\ + xor $24,$25,$25 # 5 cycles from $25 load \n\ + stq $2,0($17) \n\ + xor $4,$5,$5 # 6 cycles from $5 load \n\ + \n\ + stq $5,8($17) \n\ + xor $7,$20,$20 # 7 cycles from $20 load \n\ + stq $20,16($17) \n\ + xor $22,$23,$23 # 7 cycles from $23 load \n\ + \n\ + stq $23,24($17) \n\ + xor $25,$27,$27 # 7 cycles from $27 load \n\ + stq $27,32($17) \n\ + nop \n\ + \n\ + ldq $0,40($17) \n\ + ldq $1,40($18) \n\ + ldq $3,48($17) \n\ + ldq $4,48($18) \n\ + \n\ + ldq $6,56($17) \n\ + ldq $7,56($18) \n\ + ldq $2,40($19) \n\ + ldq $5,48($19) \n\ + \n\ + ldq $20,56($19) \n\ + xor $0,$1,$1 # 4 cycles from $1 load \n\ + xor $3,$4,$4 # 5 cycles from $4 load \n\ + xor $6,$7,$7 # 5 cycles from $7 load \n\ + \n\ + xor $1,$2,$2 # 4 cycles from $2 load \n\ + xor $4,$5,$5 # 5 cycles from $5 load \n\ + stq $2,40($17) \n\ + xor $7,$20,$20 # 4 cycles from $20 load \n\ + \n\ + stq $5,48($17) \n\ + subq $16,1,$16 \n\ + stq $20,56($17) \n\ + addq $19,64,$19 \n\ + \n\ + addq $18,64,$18 \n\ + addq $17,64,$17 \n\ + bgt $16,3b \n\ + ret \n\ + .end xor_alpha_3 \n\ + \n\ + .align 3 \n\ + .ent xor_alpha_4 \n\ +xor_alpha_4: \n\ + .prologue 0 \n\ + srl $16, 6, $16 \n\ + .align 4 \n\ +4: \n\ + ldq $0,0($17) \n\ + ldq $1,0($18) \n\ + ldq $2,0($19) \n\ + ldq $3,0($20) \n\ + \n\ + ldq $4,8($17) \n\ + ldq $5,8($18) \n\ + ldq $6,8($19) \n\ + ldq $7,8($20) \n\ + \n\ + ldq $21,16($17) \n\ + ldq $22,16($18) \n\ + ldq $23,16($19) \n\ + ldq $24,16($20) \n\ + \n\ + ldq $25,24($17) \n\ + xor $0,$1,$1 # 6 cycles from $1 load \n\ + ldq $27,24($18) \n\ + xor $2,$3,$3 # 6 cycles from $3 load \n\ + \n\ + ldq $0,24($19) \n\ + xor $1,$3,$3 \n\ + ldq $1,24($20) \n\ + xor $4,$5,$5 # 7 cycles from $5 load \n\ + \n\ + stq $3,0($17) \n\ + xor $6,$7,$7 \n\ + xor $21,$22,$22 # 7 cycles from $22 load \n\ + xor $5,$7,$7 \n\ + \n\ + stq $7,8($17) \n\ + xor $23,$24,$24 # 7 cycles from $24 load \n\ + ldq $2,32($17) \n\ + xor $22,$24,$24 \n\ + \n\ + ldq $3,32($18) \n\ + ldq $4,32($19) \n\ + ldq $5,32($20) \n\ + xor $25,$27,$27 # 8 cycles from $27 load \n\ + \n\ + ldq $6,40($17) \n\ + ldq $7,40($18) \n\ + ldq $21,40($19) \n\ + ldq $22,40($20) \n\ + \n\ + stq $24,16($17) \n\ + xor $0,$1,$1 # 9 cycles from $1 load \n\ + xor $2,$3,$3 # 5 cycles from $3 load \n\ + xor $27,$1,$1 \n\ + \n\ + stq $1,24($17) \n\ + xor $4,$5,$5 # 5 cycles from $5 load \n\ + ldq $23,48($17) \n\ + ldq $24,48($18) \n\ + \n\ + ldq $25,48($19) \n\ + xor $3,$5,$5 \n\ + ldq $27,48($20) \n\ + ldq $0,56($17) \n\ + \n\ + ldq $1,56($18) \n\ + ldq $2,56($19) \n\ + xor $6,$7,$7 # 8 cycles from $6 load \n\ + ldq $3,56($20) \n\ + \n\ + stq $5,32($17) \n\ + xor $21,$22,$22 # 8 cycles from $22 load \n\ + xor $7,$22,$22 \n\ + xor $23,$24,$24 # 5 cycles from $24 load \n\ + \n\ + stq $22,40($17) \n\ + xor $25,$27,$27 # 5 cycles from $27 load \n\ + xor $24,$27,$27 \n\ + xor $0,$1,$1 # 5 cycles from $1 load \n\ + \n\ + stq $27,48($17) \n\ + xor $2,$3,$3 # 4 cycles from $3 load \n\ + xor $1,$3,$3 \n\ + subq $16,1,$16 \n\ + \n\ + stq $3,56($17) \n\ + addq $20,64,$20 \n\ + addq $19,64,$19 \n\ + addq $18,64,$18 \n\ + \n\ + addq $17,64,$17 \n\ + bgt $16,4b \n\ + ret \n\ + .end xor_alpha_4 \n\ + \n\ + .align 3 \n\ + .ent xor_alpha_5 \n\ +xor_alpha_5: \n\ + .prologue 0 \n\ + srl $16, 6, $16 \n\ + .align 4 \n\ +5: \n\ + ldq $0,0($17) \n\ + ldq $1,0($18) \n\ + ldq $2,0($19) \n\ + ldq $3,0($20) \n\ + \n\ + ldq $4,0($21) \n\ + ldq $5,8($17) \n\ + ldq $6,8($18) \n\ + ldq $7,8($19) \n\ + \n\ + ldq $22,8($20) \n\ + ldq $23,8($21) \n\ + ldq $24,16($17) \n\ + ldq $25,16($18) \n\ + \n\ + ldq $27,16($19) \n\ + xor $0,$1,$1 # 6 cycles from $1 load \n\ + ldq $28,16($20) \n\ + xor $2,$3,$3 # 6 cycles from $3 load \n\ + \n\ + ldq $0,16($21) \n\ + xor $1,$3,$3 \n\ + ldq $1,24($17) \n\ + xor $3,$4,$4 # 7 cycles from $4 load \n\ + \n\ + stq $4,0($17) \n\ + xor $5,$6,$6 # 7 cycles from $6 load \n\ + xor $7,$22,$22 # 7 cycles from $22 load \n\ + xor $6,$23,$23 # 7 cycles from $23 load \n\ + \n\ + ldq $2,24($18) \n\ + xor $22,$23,$23 \n\ + ldq $3,24($19) \n\ + xor $24,$25,$25 # 8 cycles from $25 load \n\ + \n\ + stq $23,8($17) \n\ + xor $25,$27,$27 # 8 cycles from $27 load \n\ + ldq $4,24($20) \n\ + xor $28,$0,$0 # 7 cycles from $0 load \n\ + \n\ + ldq $5,24($21) \n\ + xor $27,$0,$0 \n\ + ldq $6,32($17) \n\ + ldq $7,32($18) \n\ + \n\ + stq $0,16($17) \n\ + xor $1,$2,$2 # 6 cycles from $2 load \n\ + ldq $22,32($19) \n\ + xor $3,$4,$4 # 4 cycles from $4 load \n\ + \n\ + ldq $23,32($20) \n\ + xor $2,$4,$4 \n\ + ldq $24,32($21) \n\ + ldq $25,40($17) \n\ + \n\ + ldq $27,40($18) \n\ + ldq $28,40($19) \n\ + ldq $0,40($20) \n\ + xor $4,$5,$5 # 7 cycles from $5 load \n\ + \n\ + stq $5,24($17) \n\ + xor $6,$7,$7 # 7 cycles from $7 load \n\ + ldq $1,40($21) \n\ + ldq $2,48($17) \n\ + \n\ + ldq $3,48($18) \n\ + xor $7,$22,$22 # 7 cycles from $22 load \n\ + ldq $4,48($19) \n\ + xor $23,$24,$24 # 6 cycles from $24 load \n\ + \n\ + ldq $5,48($20) \n\ + xor $22,$24,$24 \n\ + ldq $6,48($21) \n\ + xor $25,$27,$27 # 7 cycles from $27 load \n\ + \n\ + stq $24,32($17) \n\ + xor $27,$28,$28 # 8 cycles from $28 load \n\ + ldq $7,56($17) \n\ + xor $0,$1,$1 # 6 cycles from $1 load \n\ + \n\ + ldq $22,56($18) \n\ + ldq $23,56($19) \n\ + ldq $24,56($20) \n\ + ldq $25,56($21) \n\ + \n\ + xor $28,$1,$1 \n\ + xor $2,$3,$3 # 9 cycles from $3 load \n\ + xor $3,$4,$4 # 9 cycles from $4 load \n\ + xor $5,$6,$6 # 8 cycles from $6 load \n\ + \n\ + stq $1,40($17) \n\ + xor $4,$6,$6 \n\ + xor $7,$22,$22 # 7 cycles from $22 load \n\ + xor $23,$24,$24 # 6 cycles from $24 load \n\ + \n\ + stq $6,48($17) \n\ + xor $22,$24,$24 \n\ + subq $16,1,$16 \n\ + xor $24,$25,$25 # 8 cycles from $25 load \n\ + \n\ + stq $25,56($17) \n\ + addq $21,64,$21 \n\ + addq $20,64,$20 \n\ + addq $19,64,$19 \n\ + \n\ + addq $18,64,$18 \n\ + addq $17,64,$17 \n\ + bgt $16,5b \n\ + ret \n\ + .end xor_alpha_5 \n\ + \n\ + .align 3 \n\ + .ent xor_alpha_prefetch_2 \n\ +xor_alpha_prefetch_2: \n\ + .prologue 0 \n\ + srl $16, 6, $16 \n\ + \n\ + ldq $31, 0($17) \n\ + ldq $31, 0($18) \n\ + \n\ + ldq $31, 64($17) \n\ + ldq $31, 64($18) \n\ + \n\ + ldq $31, 128($17) \n\ + ldq $31, 128($18) \n\ + \n\ + ldq $31, 192($17) \n\ + ldq $31, 192($18) \n\ + .align 4 \n\ +2: \n\ + ldq $0,0($17) \n\ + ldq $1,0($18) \n\ + ldq $2,8($17) \n\ + ldq $3,8($18) \n\ + \n\ + ldq $4,16($17) \n\ + ldq $5,16($18) \n\ + ldq $6,24($17) \n\ + ldq $7,24($18) \n\ + \n\ + ldq $19,32($17) \n\ + ldq $20,32($18) \n\ + ldq $21,40($17) \n\ + ldq $22,40($18) \n\ + \n\ + ldq $23,48($17) \n\ + ldq $24,48($18) \n\ + ldq $25,56($17) \n\ + ldq $27,56($18) \n\ + \n\ + ldq $31,256($17) \n\ + xor $0,$1,$0 # 8 cycles from $1 load \n\ + ldq $31,256($18) \n\ + xor $2,$3,$2 \n\ + \n\ + stq $0,0($17) \n\ + xor $4,$5,$4 \n\ + stq $2,8($17) \n\ + xor $6,$7,$6 \n\ + \n\ + stq $4,16($17) \n\ + xor $19,$20,$19 \n\ + stq $6,24($17) \n\ + xor $21,$22,$21 \n\ + \n\ + stq $19,32($17) \n\ + xor $23,$24,$23 \n\ + stq $21,40($17) \n\ + xor $25,$27,$25 \n\ + \n\ + stq $23,48($17) \n\ + subq $16,1,$16 \n\ + stq $25,56($17) \n\ + addq $17,64,$17 \n\ + \n\ + addq $18,64,$18 \n\ + bgt $16,2b \n\ + ret \n\ + .end xor_alpha_prefetch_2 \n\ + \n\ + .align 3 \n\ + .ent xor_alpha_prefetch_3 \n\ +xor_alpha_prefetch_3: \n\ + .prologue 0 \n\ + srl $16, 6, $16 \n\ + \n\ + ldq $31, 0($17) \n\ + ldq $31, 0($18) \n\ + ldq $31, 0($19) \n\ + \n\ + ldq $31, 64($17) \n\ + ldq $31, 64($18) \n\ + ldq $31, 64($19) \n\ + \n\ + ldq $31, 128($17) \n\ + ldq $31, 128($18) \n\ + ldq $31, 128($19) \n\ + \n\ + ldq $31, 192($17) \n\ + ldq $31, 192($18) \n\ + ldq $31, 192($19) \n\ + .align 4 \n\ +3: \n\ + ldq $0,0($17) \n\ + ldq $1,0($18) \n\ + ldq $2,0($19) \n\ + ldq $3,8($17) \n\ + \n\ + ldq $4,8($18) \n\ + ldq $6,16($17) \n\ + ldq $7,16($18) \n\ + ldq $21,24($17) \n\ + \n\ + ldq $22,24($18) \n\ + ldq $24,32($17) \n\ + ldq $25,32($18) \n\ + ldq $5,8($19) \n\ + \n\ + ldq $20,16($19) \n\ + ldq $23,24($19) \n\ + ldq $27,32($19) \n\ + nop \n\ + \n\ + xor $0,$1,$1 # 8 cycles from $0 load \n\ + xor $3,$4,$4 # 7 cycles from $4 load \n\ + xor $6,$7,$7 # 6 cycles from $7 load \n\ + xor $21,$22,$22 # 5 cycles from $22 load \n\ + \n\ + xor $1,$2,$2 # 9 cycles from $2 load \n\ + xor $24,$25,$25 # 5 cycles from $25 load \n\ + stq $2,0($17) \n\ + xor $4,$5,$5 # 6 cycles from $5 load \n\ + \n\ + stq $5,8($17) \n\ + xor $7,$20,$20 # 7 cycles from $20 load \n\ + stq $20,16($17) \n\ + xor $22,$23,$23 # 7 cycles from $23 load \n\ + \n\ + stq $23,24($17) \n\ + xor $25,$27,$27 # 7 cycles from $27 load \n\ + stq $27,32($17) \n\ + nop \n\ + \n\ + ldq $0,40($17) \n\ + ldq $1,40($18) \n\ + ldq $3,48($17) \n\ + ldq $4,48($18) \n\ + \n\ + ldq $6,56($17) \n\ + ldq $7,56($18) \n\ + ldq $2,40($19) \n\ + ldq $5,48($19) \n\ + \n\ + ldq $20,56($19) \n\ + ldq $31,256($17) \n\ + ldq $31,256($18) \n\ + ldq $31,256($19) \n\ + \n\ + xor $0,$1,$1 # 6 cycles from $1 load \n\ + xor $3,$4,$4 # 5 cycles from $4 load \n\ + xor $6,$7,$7 # 5 cycles from $7 load \n\ + xor $1,$2,$2 # 4 cycles from $2 load \n\ + \n\ + xor $4,$5,$5 # 5 cycles from $5 load \n\ + xor $7,$20,$20 # 4 cycles from $20 load \n\ + stq $2,40($17) \n\ + subq $16,1,$16 \n\ + \n\ + stq $5,48($17) \n\ + addq $19,64,$19 \n\ + stq $20,56($17) \n\ + addq $18,64,$18 \n\ + \n\ + addq $17,64,$17 \n\ + bgt $16,3b \n\ + ret \n\ + .end xor_alpha_prefetch_3 \n\ + \n\ + .align 3 \n\ + .ent xor_alpha_prefetch_4 \n\ +xor_alpha_prefetch_4: \n\ + .prologue 0 \n\ + srl $16, 6, $16 \n\ + \n\ + ldq $31, 0($17) \n\ + ldq $31, 0($18) \n\ + ldq $31, 0($19) \n\ + ldq $31, 0($20) \n\ + \n\ + ldq $31, 64($17) \n\ + ldq $31, 64($18) \n\ + ldq $31, 64($19) \n\ + ldq $31, 64($20) \n\ + \n\ + ldq $31, 128($17) \n\ + ldq $31, 128($18) \n\ + ldq $31, 128($19) \n\ + ldq $31, 128($20) \n\ + \n\ + ldq $31, 192($17) \n\ + ldq $31, 192($18) \n\ + ldq $31, 192($19) \n\ + ldq $31, 192($20) \n\ + .align 4 \n\ +4: \n\ + ldq $0,0($17) \n\ + ldq $1,0($18) \n\ + ldq $2,0($19) \n\ + ldq $3,0($20) \n\ + \n\ + ldq $4,8($17) \n\ + ldq $5,8($18) \n\ + ldq $6,8($19) \n\ + ldq $7,8($20) \n\ + \n\ + ldq $21,16($17) \n\ + ldq $22,16($18) \n\ + ldq $23,16($19) \n\ + ldq $24,16($20) \n\ + \n\ + ldq $25,24($17) \n\ + xor $0,$1,$1 # 6 cycles from $1 load \n\ + ldq $27,24($18) \n\ + xor $2,$3,$3 # 6 cycles from $3 load \n\ + \n\ + ldq $0,24($19) \n\ + xor $1,$3,$3 \n\ + ldq $1,24($20) \n\ + xor $4,$5,$5 # 7 cycles from $5 load \n\ + \n\ + stq $3,0($17) \n\ + xor $6,$7,$7 \n\ + xor $21,$22,$22 # 7 cycles from $22 load \n\ + xor $5,$7,$7 \n\ + \n\ + stq $7,8($17) \n\ + xor $23,$24,$24 # 7 cycles from $24 load \n\ + ldq $2,32($17) \n\ + xor $22,$24,$24 \n\ + \n\ + ldq $3,32($18) \n\ + ldq $4,32($19) \n\ + ldq $5,32($20) \n\ + xor $25,$27,$27 # 8 cycles from $27 load \n\ + \n\ + ldq $6,40($17) \n\ + ldq $7,40($18) \n\ + ldq $21,40($19) \n\ + ldq $22,40($20) \n\ + \n\ + stq $24,16($17) \n\ + xor $0,$1,$1 # 9 cycles from $1 load \n\ + xor $2,$3,$3 # 5 cycles from $3 load \n\ + xor $27,$1,$1 \n\ + \n\ + stq $1,24($17) \n\ + xor $4,$5,$5 # 5 cycles from $5 load \n\ + ldq $23,48($17) \n\ + xor $3,$5,$5 \n\ + \n\ + ldq $24,48($18) \n\ + ldq $25,48($19) \n\ + ldq $27,48($20) \n\ + ldq $0,56($17) \n\ + \n\ + ldq $1,56($18) \n\ + ldq $2,56($19) \n\ + ldq $3,56($20) \n\ + xor $6,$7,$7 # 8 cycles from $6 load \n\ + \n\ + ldq $31,256($17) \n\ + xor $21,$22,$22 # 8 cycles from $22 load \n\ + ldq $31,256($18) \n\ + xor $7,$22,$22 \n\ + \n\ + ldq $31,256($19) \n\ + xor $23,$24,$24 # 6 cycles from $24 load \n\ + ldq $31,256($20) \n\ + xor $25,$27,$27 # 6 cycles from $27 load \n\ + \n\ + stq $5,32($17) \n\ + xor $24,$27,$27 \n\ + xor $0,$1,$1 # 7 cycles from $1 load \n\ + xor $2,$3,$3 # 6 cycles from $3 load \n\ + \n\ + stq $22,40($17) \n\ + xor $1,$3,$3 \n\ + stq $27,48($17) \n\ + subq $16,1,$16 \n\ + \n\ + stq $3,56($17) \n\ + addq $20,64,$20 \n\ + addq $19,64,$19 \n\ + addq $18,64,$18 \n\ + \n\ + addq $17,64,$17 \n\ + bgt $16,4b \n\ + ret \n\ + .end xor_alpha_prefetch_4 \n\ + \n\ + .align 3 \n\ + .ent xor_alpha_prefetch_5 \n\ +xor_alpha_prefetch_5: \n\ + .prologue 0 \n\ + srl $16, 6, $16 \n\ + \n\ + ldq $31, 0($17) \n\ + ldq $31, 0($18) \n\ + ldq $31, 0($19) \n\ + ldq $31, 0($20) \n\ + ldq $31, 0($21) \n\ + \n\ + ldq $31, 64($17) \n\ + ldq $31, 64($18) \n\ + ldq $31, 64($19) \n\ + ldq $31, 64($20) \n\ + ldq $31, 64($21) \n\ + \n\ + ldq $31, 128($17) \n\ + ldq $31, 128($18) \n\ + ldq $31, 128($19) \n\ + ldq $31, 128($20) \n\ + ldq $31, 128($21) \n\ + \n\ + ldq $31, 192($17) \n\ + ldq $31, 192($18) \n\ + ldq $31, 192($19) \n\ + ldq $31, 192($20) \n\ + ldq $31, 192($21) \n\ + .align 4 \n\ +5: \n\ + ldq $0,0($17) \n\ + ldq $1,0($18) \n\ + ldq $2,0($19) \n\ + ldq $3,0($20) \n\ + \n\ + ldq $4,0($21) \n\ + ldq $5,8($17) \n\ + ldq $6,8($18) \n\ + ldq $7,8($19) \n\ + \n\ + ldq $22,8($20) \n\ + ldq $23,8($21) \n\ + ldq $24,16($17) \n\ + ldq $25,16($18) \n\ + \n\ + ldq $27,16($19) \n\ + xor $0,$1,$1 # 6 cycles from $1 load \n\ + ldq $28,16($20) \n\ + xor $2,$3,$3 # 6 cycles from $3 load \n\ + \n\ + ldq $0,16($21) \n\ + xor $1,$3,$3 \n\ + ldq $1,24($17) \n\ + xor $3,$4,$4 # 7 cycles from $4 load \n\ + \n\ + stq $4,0($17) \n\ + xor $5,$6,$6 # 7 cycles from $6 load \n\ + xor $7,$22,$22 # 7 cycles from $22 load \n\ + xor $6,$23,$23 # 7 cycles from $23 load \n\ + \n\ + ldq $2,24($18) \n\ + xor $22,$23,$23 \n\ + ldq $3,24($19) \n\ + xor $24,$25,$25 # 8 cycles from $25 load \n\ + \n\ + stq $23,8($17) \n\ + xor $25,$27,$27 # 8 cycles from $27 load \n\ + ldq $4,24($20) \n\ + xor $28,$0,$0 # 7 cycles from $0 load \n\ + \n\ + ldq $5,24($21) \n\ + xor $27,$0,$0 \n\ + ldq $6,32($17) \n\ + ldq $7,32($18) \n\ + \n\ + stq $0,16($17) \n\ + xor $1,$2,$2 # 6 cycles from $2 load \n\ + ldq $22,32($19) \n\ + xor $3,$4,$4 # 4 cycles from $4 load \n\ + \n\ + ldq $23,32($20) \n\ + xor $2,$4,$4 \n\ + ldq $24,32($21) \n\ + ldq $25,40($17) \n\ + \n\ + ldq $27,40($18) \n\ + ldq $28,40($19) \n\ + ldq $0,40($20) \n\ + xor $4,$5,$5 # 7 cycles from $5 load \n\ + \n\ + stq $5,24($17) \n\ + xor $6,$7,$7 # 7 cycles from $7 load \n\ + ldq $1,40($21) \n\ + ldq $2,48($17) \n\ + \n\ + ldq $3,48($18) \n\ + xor $7,$22,$22 # 7 cycles from $22 load \n\ + ldq $4,48($19) \n\ + xor $23,$24,$24 # 6 cycles from $24 load \n\ + \n\ + ldq $5,48($20) \n\ + xor $22,$24,$24 \n\ + ldq $6,48($21) \n\ + xor $25,$27,$27 # 7 cycles from $27 load \n\ + \n\ + stq $24,32($17) \n\ + xor $27,$28,$28 # 8 cycles from $28 load \n\ + ldq $7,56($17) \n\ + xor $0,$1,$1 # 6 cycles from $1 load \n\ + \n\ + ldq $22,56($18) \n\ + ldq $23,56($19) \n\ + ldq $24,56($20) \n\ + ldq $25,56($21) \n\ + \n\ + ldq $31,256($17) \n\ + xor $28,$1,$1 \n\ + ldq $31,256($18) \n\ + xor $2,$3,$3 # 9 cycles from $3 load \n\ + \n\ + ldq $31,256($19) \n\ + xor $3,$4,$4 # 9 cycles from $4 load \n\ + ldq $31,256($20) \n\ + xor $5,$6,$6 # 8 cycles from $6 load \n\ + \n\ + stq $1,40($17) \n\ + xor $4,$6,$6 \n\ + xor $7,$22,$22 # 7 cycles from $22 load \n\ + xor $23,$24,$24 # 6 cycles from $24 load \n\ + \n\ + stq $6,48($17) \n\ + xor $22,$24,$24 \n\ + ldq $31,256($21) \n\ + xor $24,$25,$25 # 8 cycles from $25 load \n\ + \n\ + stq $25,56($17) \n\ + subq $16,1,$16 \n\ + addq $21,64,$21 \n\ + addq $20,64,$20 \n\ + \n\ + addq $19,64,$19 \n\ + addq $18,64,$18 \n\ + addq $17,64,$17 \n\ + bgt $16,5b \n\ + \n\ + ret \n\ + .end xor_alpha_prefetch_5 \n\ +"); + +DO_XOR_BLOCKS(alpha, xor_alpha_2, xor_alpha_3, xor_alpha_4, xor_alpha_5); + +struct xor_block_template xor_block_alpha = { + .name = "alpha", + .xor_gen = xor_gen_alpha, +}; + +DO_XOR_BLOCKS(alpha_prefetch, xor_alpha_prefetch_2, xor_alpha_prefetch_3, + xor_alpha_prefetch_4, xor_alpha_prefetch_5); + +struct xor_block_template xor_block_alpha_prefetch = { + .name = "alpha prefetch", + .xor_gen = xor_gen_alpha_prefetch, +}; diff --git a/lib/raid/xor/alpha/xor_arch.h b/lib/raid/xor/alpha/xor_arch.h new file mode 100644 index 000000000000..0dcfea578a48 --- /dev/null +++ b/lib/raid/xor/alpha/xor_arch.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include <asm/special_insns.h> + +extern struct xor_block_template xor_block_alpha; +extern struct xor_block_template xor_block_alpha_prefetch; + +/* + * Force the use of alpha_prefetch if EV6, as it is significantly faster in the + * cold cache case. + */ +static __always_inline void __init arch_xor_init(void) +{ + if (implver() == IMPLVER_EV6) { + xor_force(&xor_block_alpha_prefetch); + } else { + xor_register(&xor_block_8regs); + xor_register(&xor_block_32regs); + xor_register(&xor_block_alpha); + xor_register(&xor_block_alpha_prefetch); + } +} diff --git a/lib/raid/xor/arm/xor-neon-glue.c b/lib/raid/xor/arm/xor-neon-glue.c new file mode 100644 index 000000000000..cea39e019904 --- /dev/null +++ b/lib/raid/xor/arm/xor-neon-glue.c @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2001 Russell King + */ +#include "xor_impl.h" +#include "xor_arch.h" + +static void xor_gen_neon(void *dest, void **srcs, unsigned int src_cnt, + unsigned int bytes) +{ + kernel_neon_begin(); + xor_gen_neon_inner(dest, srcs, src_cnt, bytes); + kernel_neon_end(); +} + +struct xor_block_template xor_block_neon = { + .name = "neon", + .xor_gen = xor_gen_neon, +}; diff --git a/lib/raid/xor/arm/xor-neon.h b/lib/raid/xor/arm/xor-neon.h new file mode 100644 index 000000000000..406e0356f05b --- /dev/null +++ b/lib/raid/xor/arm/xor-neon.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +extern struct xor_block_template xor_block_arm4regs; +extern struct xor_block_template xor_block_neon; + +void xor_gen_neon_inner(void *dest, void **srcs, unsigned int src_cnt, + unsigned int bytes); diff --git a/lib/raid/xor/arm/xor.c b/lib/raid/xor/arm/xor.c new file mode 100644 index 000000000000..45139b6c55ea --- /dev/null +++ b/lib/raid/xor/arm/xor.c @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2001 Russell King + */ +#include "xor_impl.h" +#include "xor_arch.h" + +#define __XOR(a1, a2) a1 ^= a2 + +#define GET_BLOCK_2(dst) \ + __asm__("ldmia %0, {%1, %2}" \ + : "=r" (dst), "=r" (a1), "=r" (a2) \ + : "0" (dst)) + +#define GET_BLOCK_4(dst) \ + __asm__("ldmia %0, {%1, %2, %3, %4}" \ + : "=r" (dst), "=r" (a1), "=r" (a2), "=r" (a3), "=r" (a4) \ + : "0" (dst)) + +#define XOR_BLOCK_2(src) \ + __asm__("ldmia %0!, {%1, %2}" \ + : "=r" (src), "=r" (b1), "=r" (b2) \ + : "0" (src)); \ + __XOR(a1, b1); __XOR(a2, b2); + +#define XOR_BLOCK_4(src) \ + __asm__("ldmia %0!, {%1, %2, %3, %4}" \ + : "=r" (src), "=r" (b1), "=r" (b2), "=r" (b3), "=r" (b4) \ + : "0" (src)); \ + __XOR(a1, b1); __XOR(a2, b2); __XOR(a3, b3); __XOR(a4, b4) + +#define PUT_BLOCK_2(dst) \ + __asm__ __volatile__("stmia %0!, {%2, %3}" \ + : "=r" (dst) \ + : "0" (dst), "r" (a1), "r" (a2)) + +#define PUT_BLOCK_4(dst) \ + __asm__ __volatile__("stmia %0!, {%2, %3, %4, %5}" \ + : "=r" (dst) \ + : "0" (dst), "r" (a1), "r" (a2), "r" (a3), "r" (a4)) + +static void +xor_arm4regs_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) +{ + unsigned int lines = bytes / sizeof(unsigned long) / 4; + register unsigned int a1 __asm__("r4"); + register unsigned int a2 __asm__("r5"); + register unsigned int a3 __asm__("r6"); + register unsigned int a4 __asm__("r10"); + register unsigned int b1 __asm__("r8"); + register unsigned int b2 __asm__("r9"); + register unsigned int b3 __asm__("ip"); + register unsigned int b4 __asm__("lr"); + + do { + GET_BLOCK_4(p1); + XOR_BLOCK_4(p2); + PUT_BLOCK_4(p1); + } while (--lines); +} + +static void +xor_arm4regs_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) +{ + unsigned int lines = bytes / sizeof(unsigned long) / 4; + register unsigned int a1 __asm__("r4"); + register unsigned int a2 __asm__("r5"); + register unsigned int a3 __asm__("r6"); + register unsigned int a4 __asm__("r10"); + register unsigned int b1 __asm__("r8"); + register unsigned int b2 __asm__("r9"); + register unsigned int b3 __asm__("ip"); + register unsigned int b4 __asm__("lr"); + + do { + GET_BLOCK_4(p1); + XOR_BLOCK_4(p2); + XOR_BLOCK_4(p3); + PUT_BLOCK_4(p1); + } while (--lines); +} + +static void +xor_arm4regs_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) +{ + unsigned int lines = bytes / sizeof(unsigned long) / 2; + register unsigned int a1 __asm__("r8"); + register unsigned int a2 __asm__("r9"); + register unsigned int b1 __asm__("ip"); + register unsigned int b2 __asm__("lr"); + + do { + GET_BLOCK_2(p1); + XOR_BLOCK_2(p2); + XOR_BLOCK_2(p3); + XOR_BLOCK_2(p4); + PUT_BLOCK_2(p1); + } while (--lines); +} + +static void +xor_arm4regs_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) +{ + unsigned int lines = bytes / sizeof(unsigned long) / 2; + register unsigned int a1 __asm__("r8"); + register unsigned int a2 __asm__("r9"); + register unsigned int b1 __asm__("ip"); + register unsigned int b2 __asm__("lr"); + + do { + GET_BLOCK_2(p1); + XOR_BLOCK_2(p2); + XOR_BLOCK_2(p3); + XOR_BLOCK_2(p4); + XOR_BLOCK_2(p5); + PUT_BLOCK_2(p1); + } while (--lines); +} + +DO_XOR_BLOCKS(arm4regs, xor_arm4regs_2, xor_arm4regs_3, xor_arm4regs_4, + xor_arm4regs_5); + +struct xor_block_template xor_block_arm4regs = { + .name = "arm4regs", + .xor_gen = xor_gen_arm4regs, +}; diff --git a/lib/raid/xor/arm/xor_arch.h b/lib/raid/xor/arm/xor_arch.h new file mode 100644 index 000000000000..f1ddb64fe62a --- /dev/null +++ b/lib/raid/xor/arm/xor_arch.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2001 Russell King + */ +#include <asm/neon.h> +#include "xor-neon.h" + +static __always_inline void __init arch_xor_init(void) +{ + xor_register(&xor_block_arm4regs); + xor_register(&xor_block_8regs); + xor_register(&xor_block_32regs); +#ifdef CONFIG_KERNEL_MODE_NEON + if (cpu_has_neon()) + xor_register(&xor_block_neon); +#endif +} diff --git a/lib/raid/xor/arm64/xor-eor3.c b/lib/raid/xor/arm64/xor-eor3.c new file mode 100644 index 000000000000..e44016c363f1 --- /dev/null +++ b/lib/raid/xor/arm64/xor-eor3.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <linux/cache.h> +#include <asm/neon-intrinsics.h> +#include "xor_impl.h" +#include "xor_arch.h" +#include "xor-neon.h" + +extern void __xor_eor3_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2); + +static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r) +{ + uint64x2_t res; + + asm(ARM64_ASM_PREAMBLE ".arch_extension sha3\n" + "eor3 %0.16b, %1.16b, %2.16b, %3.16b" + : "=w"(res) : "w"(p), "w"(q), "w"(r)); + return res; +} + +static void __xor_eor3_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) +{ + uint64_t *dp1 = (uint64_t *)p1; + uint64_t *dp2 = (uint64_t *)p2; + uint64_t *dp3 = (uint64_t *)p3; + + register uint64x2_t v0, v1, v2, v3; + long lines = bytes / (sizeof(uint64x2_t) * 4); + + do { + /* p1 ^= p2 ^ p3 */ + v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0), + vld1q_u64(dp3 + 0)); + v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2), + vld1q_u64(dp3 + 2)); + v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4), + vld1q_u64(dp3 + 4)); + v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6), + vld1q_u64(dp3 + 6)); + + /* store */ + vst1q_u64(dp1 + 0, v0); + vst1q_u64(dp1 + 2, v1); + vst1q_u64(dp1 + 4, v2); + vst1q_u64(dp1 + 6, v3); + + dp1 += 8; + dp2 += 8; + dp3 += 8; + } while (--lines > 0); +} + +static void __xor_eor3_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) +{ + uint64_t *dp1 = (uint64_t *)p1; + uint64_t *dp2 = (uint64_t *)p2; + uint64_t *dp3 = (uint64_t *)p3; + uint64_t *dp4 = (uint64_t *)p4; + + register uint64x2_t v0, v1, v2, v3; + long lines = bytes / (sizeof(uint64x2_t) * 4); + + do { + /* p1 ^= p2 ^ p3 */ + v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0), + vld1q_u64(dp3 + 0)); + v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2), + vld1q_u64(dp3 + 2)); + v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4), + vld1q_u64(dp3 + 4)); + v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6), + vld1q_u64(dp3 + 6)); + + /* p1 ^= p4 */ + v0 = veorq_u64(v0, vld1q_u64(dp4 + 0)); + v1 = veorq_u64(v1, vld1q_u64(dp4 + 2)); + v2 = veorq_u64(v2, vld1q_u64(dp4 + 4)); + v3 = veorq_u64(v3, vld1q_u64(dp4 + 6)); + + /* store */ + vst1q_u64(dp1 + 0, v0); + vst1q_u64(dp1 + 2, v1); + vst1q_u64(dp1 + 4, v2); + vst1q_u64(dp1 + 6, v3); + + dp1 += 8; + dp2 += 8; + dp3 += 8; + dp4 += 8; + } while (--lines > 0); +} + +static void __xor_eor3_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) +{ + uint64_t *dp1 = (uint64_t *)p1; + uint64_t *dp2 = (uint64_t *)p2; + uint64_t *dp3 = (uint64_t *)p3; + uint64_t *dp4 = (uint64_t *)p4; + uint64_t *dp5 = (uint64_t *)p5; + + register uint64x2_t v0, v1, v2, v3; + long lines = bytes / (sizeof(uint64x2_t) * 4); + + do { + /* p1 ^= p2 ^ p3 */ + v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0), + vld1q_u64(dp3 + 0)); + v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2), + vld1q_u64(dp3 + 2)); + v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4), + vld1q_u64(dp3 + 4)); + v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6), + vld1q_u64(dp3 + 6)); + + /* p1 ^= p4 ^ p5 */ + v0 = eor3(v0, vld1q_u64(dp4 + 0), vld1q_u64(dp5 + 0)); + v1 = eor3(v1, vld1q_u64(dp4 + 2), vld1q_u64(dp5 + 2)); + v2 = eor3(v2, vld1q_u64(dp4 + 4), vld1q_u64(dp5 + 4)); + v3 = eor3(v3, vld1q_u64(dp4 + 6), vld1q_u64(dp5 + 6)); + + /* store */ + vst1q_u64(dp1 + 0, v0); + vst1q_u64(dp1 + 2, v1); + vst1q_u64(dp1 + 4, v2); + vst1q_u64(dp1 + 6, v3); + + dp1 += 8; + dp2 += 8; + dp3 += 8; + dp4 += 8; + dp5 += 8; + } while (--lines > 0); +} + +__DO_XOR_BLOCKS(eor3_inner, __xor_eor3_2, __xor_eor3_3, __xor_eor3_4, + __xor_eor3_5); diff --git a/lib/raid/xor/arm64/xor-neon-glue.c b/lib/raid/xor/arm64/xor-neon-glue.c new file mode 100644 index 000000000000..f0284f86feb4 --- /dev/null +++ b/lib/raid/xor/arm64/xor-neon-glue.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Authors: Jackie Liu <liuyun01@kylinos.cn> + * Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd. + */ + +#include <asm/simd.h> +#include "xor_impl.h" +#include "xor_arch.h" +#include "xor-neon.h" + +#define XOR_TEMPLATE(_name) \ +static void xor_gen_##_name(void *dest, void **srcs, unsigned int src_cnt, \ + unsigned int bytes) \ +{ \ + scoped_ksimd() \ + xor_gen_##_name##_inner(dest, srcs, src_cnt, bytes); \ +} \ + \ +struct xor_block_template xor_block_##_name = { \ + .name = __stringify(_name), \ + .xor_gen = xor_gen_##_name, \ +}; + +XOR_TEMPLATE(neon); +XOR_TEMPLATE(eor3); diff --git a/lib/raid/xor/arm64/xor-neon.h b/lib/raid/xor/arm64/xor-neon.h new file mode 100644 index 000000000000..514699ba8f5f --- /dev/null +++ b/lib/raid/xor/arm64/xor-neon.h @@ -0,0 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +void xor_gen_neon_inner(void *dest, void **srcs, unsigned int src_cnt, + unsigned int bytes); +void xor_gen_eor3_inner(void *dest, void **srcs, unsigned int src_cnt, + unsigned int bytes); diff --git a/lib/raid/xor/arm64/xor_arch.h b/lib/raid/xor/arm64/xor_arch.h new file mode 100644 index 000000000000..5dbb40319501 --- /dev/null +++ b/lib/raid/xor/arm64/xor_arch.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Authors: Jackie Liu <liuyun01@kylinos.cn> + * Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd. + */ +#include <asm/simd.h> + +extern struct xor_block_template xor_block_neon; +extern struct xor_block_template xor_block_eor3; + +static __always_inline void __init arch_xor_init(void) +{ + xor_register(&xor_block_8regs); + xor_register(&xor_block_32regs); + if (cpu_has_neon()) { + if (cpu_have_named_feature(SHA3)) + xor_register(&xor_block_eor3); + else + xor_register(&xor_block_neon); + } +} diff --git a/lib/raid/xor/loongarch/xor_arch.h b/lib/raid/xor/loongarch/xor_arch.h new file mode 100644 index 000000000000..fe5e8244fd0e --- /dev/null +++ b/lib/raid/xor/loongarch/xor_arch.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2023 WANG Xuerui <git@xen0n.name> + */ +#include <asm/cpu-features.h> + +/* + * For grins, also test the generic routines. + * + * More importantly: it cannot be ruled out at this point of time, that some + * future (maybe reduced) models could run the vector algorithms slower than + * the scalar ones, maybe for errata or micro-op reasons. It may be + * appropriate to revisit this after one or two more uarch generations. + */ + +extern struct xor_block_template xor_block_lsx; +extern struct xor_block_template xor_block_lasx; + +static __always_inline void __init arch_xor_init(void) +{ + xor_register(&xor_block_8regs); + xor_register(&xor_block_8regs_p); + xor_register(&xor_block_32regs); + xor_register(&xor_block_32regs_p); +#ifdef CONFIG_CPU_HAS_LSX + if (cpu_has_lsx) + xor_register(&xor_block_lsx); +#endif +#ifdef CONFIG_CPU_HAS_LASX + if (cpu_has_lasx) + xor_register(&xor_block_lasx); +#endif +} diff --git a/lib/raid/xor/loongarch/xor_simd.c b/lib/raid/xor/loongarch/xor_simd.c new file mode 100644 index 000000000000..84cd24b728c4 --- /dev/null +++ b/lib/raid/xor/loongarch/xor_simd.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * LoongArch SIMD XOR operations + * + * Copyright (C) 2023 WANG Xuerui <git@xen0n.name> + */ + +#include "xor_simd.h" + +/* + * Process one cache line (64 bytes) per loop. This is assuming all future + * popular LoongArch cores are similar performance-characteristics-wise to the + * current models. + */ +#define LINE_WIDTH 64 + +#ifdef CONFIG_CPU_HAS_LSX + +#define LD(reg, base, offset) \ + "vld $vr" #reg ", %[" #base "], " #offset "\n\t" +#define ST(reg, base, offset) \ + "vst $vr" #reg ", %[" #base "], " #offset "\n\t" +#define XOR(dj, k) "vxor.v $vr" #dj ", $vr" #dj ", $vr" #k "\n\t" + +#define LD_INOUT_LINE(base) \ + LD(0, base, 0) \ + LD(1, base, 16) \ + LD(2, base, 32) \ + LD(3, base, 48) + +#define LD_AND_XOR_LINE(base) \ + LD(4, base, 0) \ + LD(5, base, 16) \ + LD(6, base, 32) \ + LD(7, base, 48) \ + XOR(0, 4) \ + XOR(1, 5) \ + XOR(2, 6) \ + XOR(3, 7) + +#define ST_LINE(base) \ + ST(0, base, 0) \ + ST(1, base, 16) \ + ST(2, base, 32) \ + ST(3, base, 48) + +#define XOR_FUNC_NAME(nr) __xor_lsx_##nr +#include "xor_template.c" + +#undef LD +#undef ST +#undef XOR +#undef LD_INOUT_LINE +#undef LD_AND_XOR_LINE +#undef ST_LINE +#undef XOR_FUNC_NAME + +#endif /* CONFIG_CPU_HAS_LSX */ + +#ifdef CONFIG_CPU_HAS_LASX + +#define LD(reg, base, offset) \ + "xvld $xr" #reg ", %[" #base "], " #offset "\n\t" +#define ST(reg, base, offset) \ + "xvst $xr" #reg ", %[" #base "], " #offset "\n\t" +#define XOR(dj, k) "xvxor.v $xr" #dj ", $xr" #dj ", $xr" #k "\n\t" + +#define LD_INOUT_LINE(base) \ + LD(0, base, 0) \ + LD(1, base, 32) + +#define LD_AND_XOR_LINE(base) \ + LD(2, base, 0) \ + LD(3, base, 32) \ + XOR(0, 2) \ + XOR(1, 3) + +#define ST_LINE(base) \ + ST(0, base, 0) \ + ST(1, base, 32) + +#define XOR_FUNC_NAME(nr) __xor_lasx_##nr +#include "xor_template.c" + +#undef LD +#undef ST +#undef XOR +#undef LD_INOUT_LINE +#undef LD_AND_XOR_LINE +#undef ST_LINE +#undef XOR_FUNC_NAME + +#endif /* CONFIG_CPU_HAS_LASX */ diff --git a/lib/raid/xor/loongarch/xor_simd.h b/lib/raid/xor/loongarch/xor_simd.h new file mode 100644 index 000000000000..f50f32514d80 --- /dev/null +++ b/lib/raid/xor/loongarch/xor_simd.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Simple interface to link xor_simd.c and xor_simd_glue.c + * + * Separating these files ensures that no SIMD instructions are run outside of + * the kfpu critical section. + */ + +#ifndef __LOONGARCH_LIB_XOR_SIMD_H +#define __LOONGARCH_LIB_XOR_SIMD_H + +#ifdef CONFIG_CPU_HAS_LSX +void __xor_lsx_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2); +void __xor_lsx_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, const unsigned long * __restrict p3); +void __xor_lsx_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, const unsigned long * __restrict p3, + const unsigned long * __restrict p4); +void __xor_lsx_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, const unsigned long * __restrict p3, + const unsigned long * __restrict p4, const unsigned long * __restrict p5); +#endif /* CONFIG_CPU_HAS_LSX */ + +#ifdef CONFIG_CPU_HAS_LASX +void __xor_lasx_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2); +void __xor_lasx_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, const unsigned long * __restrict p3); +void __xor_lasx_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, const unsigned long * __restrict p3, + const unsigned long * __restrict p4); +void __xor_lasx_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, const unsigned long * __restrict p3, + const unsigned long * __restrict p4, const unsigned long * __restrict p5); +#endif /* CONFIG_CPU_HAS_LASX */ + +#endif /* __LOONGARCH_LIB_XOR_SIMD_H */ diff --git a/lib/raid/xor/loongarch/xor_simd_glue.c b/lib/raid/xor/loongarch/xor_simd_glue.c new file mode 100644 index 000000000000..7f324d924f87 --- /dev/null +++ b/lib/raid/xor/loongarch/xor_simd_glue.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * LoongArch SIMD XOR operations + * + * Copyright (C) 2023 WANG Xuerui <git@xen0n.name> + */ + +#include <linux/sched.h> +#include <asm/fpu.h> +#include "xor_impl.h" +#include "xor_arch.h" +#include "xor_simd.h" + +#define MAKE_XOR_GLUES(flavor) \ +DO_XOR_BLOCKS(flavor##_inner, __xor_##flavor##_2, __xor_##flavor##_3, \ + __xor_##flavor##_4, __xor_##flavor##_5); \ + \ +static void xor_gen_##flavor(void *dest, void **srcs, unsigned int src_cnt, \ + unsigned int bytes) \ +{ \ + kernel_fpu_begin(); \ + xor_gen_##flavor##_inner(dest, srcs, src_cnt, bytes); \ + kernel_fpu_end(); \ +} \ + \ +struct xor_block_template xor_block_##flavor = { \ + .name = __stringify(flavor), \ + .xor_gen = xor_gen_##flavor \ +} + +#ifdef CONFIG_CPU_HAS_LSX +MAKE_XOR_GLUES(lsx); +#endif /* CONFIG_CPU_HAS_LSX */ + +#ifdef CONFIG_CPU_HAS_LASX +MAKE_XOR_GLUES(lasx); +#endif /* CONFIG_CPU_HAS_LASX */ diff --git a/lib/raid/xor/loongarch/xor_template.c b/lib/raid/xor/loongarch/xor_template.c new file mode 100644 index 000000000000..0358ced7fe33 --- /dev/null +++ b/lib/raid/xor/loongarch/xor_template.c @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2023 WANG Xuerui <git@xen0n.name> + * + * Template for XOR operations, instantiated in xor_simd.c. + * + * Expected preprocessor definitions: + * + * - LINE_WIDTH + * - XOR_FUNC_NAME(nr) + * - LD_INOUT_LINE(buf) + * - LD_AND_XOR_LINE(buf) + * - ST_LINE(buf) + */ + +void XOR_FUNC_NAME(2)(unsigned long bytes, + unsigned long * __restrict v1, + const unsigned long * __restrict v2) +{ + unsigned long lines = bytes / LINE_WIDTH; + + do { + __asm__ __volatile__ ( + LD_INOUT_LINE(v1) + LD_AND_XOR_LINE(v2) + ST_LINE(v1) + : : [v1] "r"(v1), [v2] "r"(v2) : "memory" + ); + + v1 += LINE_WIDTH / sizeof(unsigned long); + v2 += LINE_WIDTH / sizeof(unsigned long); + } while (--lines > 0); +} + +void XOR_FUNC_NAME(3)(unsigned long bytes, + unsigned long * __restrict v1, + const unsigned long * __restrict v2, + const unsigned long * __restrict v3) +{ + unsigned long lines = bytes / LINE_WIDTH; + + do { + __asm__ __volatile__ ( + LD_INOUT_LINE(v1) + LD_AND_XOR_LINE(v2) + LD_AND_XOR_LINE(v3) + ST_LINE(v1) + : : [v1] "r"(v1), [v2] "r"(v2), [v3] "r"(v3) : "memory" + ); + + v1 += LINE_WIDTH / sizeof(unsigned long); + v2 += LINE_WIDTH / sizeof(unsigned long); + v3 += LINE_WIDTH / sizeof(unsigned long); + } while (--lines > 0); +} + +void XOR_FUNC_NAME(4)(unsigned long bytes, + unsigned long * __restrict v1, + const unsigned long * __restrict v2, + const unsigned long * __restrict v3, + const unsigned long * __restrict v4) +{ + unsigned long lines = bytes / LINE_WIDTH; + + do { + __asm__ __volatile__ ( + LD_INOUT_LINE(v1) + LD_AND_XOR_LINE(v2) + LD_AND_XOR_LINE(v3) + LD_AND_XOR_LINE(v4) + ST_LINE(v1) + : : [v1] "r"(v1), [v2] "r"(v2), [v3] "r"(v3), [v4] "r"(v4) + : "memory" + ); + + v1 += LINE_WIDTH / sizeof(unsigned long); + v2 += LINE_WIDTH / sizeof(unsigned long); + v3 += LINE_WIDTH / sizeof(unsigned long); + v4 += LINE_WIDTH / sizeof(unsigned long); + } while (--lines > 0); +} + +void XOR_FUNC_NAME(5)(unsigned long bytes, + unsigned long * __restrict v1, + const unsigned long * __restrict v2, + const unsigned long * __restrict v3, + const unsigned long * __restrict v4, + const unsigned long * __restrict v5) +{ + unsigned long lines = bytes / LINE_WIDTH; + + do { + __asm__ __volatile__ ( + LD_INOUT_LINE(v1) + LD_AND_XOR_LINE(v2) + LD_AND_XOR_LINE(v3) + LD_AND_XOR_LINE(v4) + LD_AND_XOR_LINE(v5) + ST_LINE(v1) + : : [v1] "r"(v1), [v2] "r"(v2), [v3] "r"(v3), [v4] "r"(v4), + [v5] "r"(v5) : "memory" + ); + + v1 += LINE_WIDTH / sizeof(unsigned long); + v2 += LINE_WIDTH / sizeof(unsigned long); + v3 += LINE_WIDTH / sizeof(unsigned long); + v4 += LINE_WIDTH / sizeof(unsigned long); + v5 += LINE_WIDTH / sizeof(unsigned long); + } while (--lines > 0); +} diff --git a/lib/raid/xor/powerpc/xor_arch.h b/lib/raid/xor/powerpc/xor_arch.h new file mode 100644 index 000000000000..3b00a4a2fd67 --- /dev/null +++ b/lib/raid/xor/powerpc/xor_arch.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * + * Copyright (C) IBM Corporation, 2012 + * + * Author: Anton Blanchard <anton@au.ibm.com> + */ +#include <asm/cpu_has_feature.h> + +extern struct xor_block_template xor_block_altivec; + +static __always_inline void __init arch_xor_init(void) +{ + xor_register(&xor_block_8regs); + xor_register(&xor_block_8regs_p); + xor_register(&xor_block_32regs); + xor_register(&xor_block_32regs_p); +#ifdef CONFIG_ALTIVEC + if (cpu_has_feature(CPU_FTR_ALTIVEC)) + xor_register(&xor_block_altivec); +#endif +} diff --git a/lib/raid/xor/powerpc/xor_vmx.c b/lib/raid/xor/powerpc/xor_vmx.c new file mode 100644 index 000000000000..09bed98c1bc7 --- /dev/null +++ b/lib/raid/xor/powerpc/xor_vmx.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * + * Copyright (C) IBM Corporation, 2012 + * + * Author: Anton Blanchard <anton@au.ibm.com> + */ + +/* + * Sparse (as at v0.5.0) gets very, very confused by this file. + * Make it a bit simpler for it. + */ +#include "xor_impl.h" +#if !defined(__CHECKER__) +#include <altivec.h> +#else +#define vec_xor(a, b) a ^ b +#define vector __attribute__((vector_size(16))) +#endif + +#include "xor_vmx.h" + +typedef vector signed char unative_t; + +#define DEFINE(V) \ + unative_t *V = (unative_t *)V##_in; \ + unative_t V##_0, V##_1, V##_2, V##_3 + +#define LOAD(V) \ + do { \ + V##_0 = V[0]; \ + V##_1 = V[1]; \ + V##_2 = V[2]; \ + V##_3 = V[3]; \ + } while (0) + +#define STORE(V) \ + do { \ + V[0] = V##_0; \ + V[1] = V##_1; \ + V[2] = V##_2; \ + V[3] = V##_3; \ + } while (0) + +#define XOR(V1, V2) \ + do { \ + V1##_0 = vec_xor(V1##_0, V2##_0); \ + V1##_1 = vec_xor(V1##_1, V2##_1); \ + V1##_2 = vec_xor(V1##_2, V2##_2); \ + V1##_3 = vec_xor(V1##_3, V2##_3); \ + } while (0) + +static void __xor_altivec_2(unsigned long bytes, + unsigned long * __restrict v1_in, + const unsigned long * __restrict v2_in) +{ + DEFINE(v1); + DEFINE(v2); + unsigned long lines = bytes / (sizeof(unative_t)) / 4; + + do { + LOAD(v1); + LOAD(v2); + XOR(v1, v2); + STORE(v1); + + v1 += 4; + v2 += 4; + } while (--lines > 0); +} + +static void __xor_altivec_3(unsigned long bytes, + unsigned long * __restrict v1_in, + const unsigned long * __restrict v2_in, + const unsigned long * __restrict v3_in) +{ + DEFINE(v1); + DEFINE(v2); + DEFINE(v3); + unsigned long lines = bytes / (sizeof(unative_t)) / 4; + + do { + LOAD(v1); + LOAD(v2); + LOAD(v3); + XOR(v1, v2); + XOR(v1, v3); + STORE(v1); + + v1 += 4; + v2 += 4; + v3 += 4; + } while (--lines > 0); +} + +static void __xor_altivec_4(unsigned long bytes, + unsigned long * __restrict v1_in, + const unsigned long * __restrict v2_in, + const unsigned long * __restrict v3_in, + const unsigned long * __restrict v4_in) +{ + DEFINE(v1); + DEFINE(v2); + DEFINE(v3); + DEFINE(v4); + unsigned long lines = bytes / (sizeof(unative_t)) / 4; + + do { + LOAD(v1); + LOAD(v2); + LOAD(v3); + LOAD(v4); + XOR(v1, v2); + XOR(v3, v4); + XOR(v1, v3); + STORE(v1); + + v1 += 4; + v2 += 4; + v3 += 4; + v4 += 4; + } while (--lines > 0); +} + +static void __xor_altivec_5(unsigned long bytes, + unsigned long * __restrict v1_in, + const unsigned long * __restrict v2_in, + const unsigned long * __restrict v3_in, + const unsigned long * __restrict v4_in, + const unsigned long * __restrict v5_in) +{ + DEFINE(v1); + DEFINE(v2); + DEFINE(v3); + DEFINE(v4); + DEFINE(v5); + unsigned long lines = bytes / (sizeof(unative_t)) / 4; + + do { + LOAD(v1); + LOAD(v2); + LOAD(v3); + LOAD(v4); + LOAD(v5); + XOR(v1, v2); + XOR(v3, v4); + XOR(v1, v5); + XOR(v1, v3); + STORE(v1); + + v1 += 4; + v2 += 4; + v3 += 4; + v4 += 4; + v5 += 4; + } while (--lines > 0); +} + +__DO_XOR_BLOCKS(altivec_inner, __xor_altivec_2, __xor_altivec_3, + __xor_altivec_4, __xor_altivec_5); diff --git a/lib/raid/xor/powerpc/xor_vmx.h b/lib/raid/xor/powerpc/xor_vmx.h new file mode 100644 index 000000000000..1d26c1133a86 --- /dev/null +++ b/lib/raid/xor/powerpc/xor_vmx.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Simple interface to link xor_vmx.c and xor_vmx_glue.c + * + * Separating these file ensures that no altivec instructions are run + * outside of the enable/disable altivec block. + */ + +void xor_gen_altivec_inner(void *dest, void **srcs, unsigned int src_cnt, + unsigned int bytes); diff --git a/lib/raid/xor/powerpc/xor_vmx_glue.c b/lib/raid/xor/powerpc/xor_vmx_glue.c new file mode 100644 index 000000000000..dbfbb5cadc36 --- /dev/null +++ b/lib/raid/xor/powerpc/xor_vmx_glue.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Altivec XOR operations + * + * Copyright 2017 IBM Corp. + */ + +#include <linux/preempt.h> +#include <linux/sched.h> +#include <asm/switch_to.h> +#include "xor_impl.h" +#include "xor_arch.h" +#include "xor_vmx.h" + +static void xor_gen_altivec(void *dest, void **srcs, unsigned int src_cnt, + unsigned int bytes) +{ + preempt_disable(); + enable_kernel_altivec(); + xor_gen_altivec_inner(dest, srcs, src_cnt, bytes); + disable_kernel_altivec(); + preempt_enable(); +} + +struct xor_block_template xor_block_altivec = { + .name = "altivec", + .xor_gen = xor_gen_altivec, +}; diff --git a/lib/raid/xor/riscv/xor-glue.c b/lib/raid/xor/riscv/xor-glue.c new file mode 100644 index 000000000000..2e4c1b05d998 --- /dev/null +++ b/lib/raid/xor/riscv/xor-glue.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2021 SiFive + */ + +#include <asm/vector.h> +#include <asm/switch_to.h> +#include <asm/asm-prototypes.h> +#include "xor_impl.h" +#include "xor_arch.h" + +DO_XOR_BLOCKS(vector_inner, xor_regs_2_, xor_regs_3_, xor_regs_4_, xor_regs_5_); + +static void xor_gen_vector(void *dest, void **srcs, unsigned int src_cnt, + unsigned int bytes) +{ + kernel_vector_begin(); + xor_gen_vector_inner(dest, srcs, src_cnt, bytes); + kernel_vector_end(); +} + +struct xor_block_template xor_block_rvv = { + .name = "rvv", + .xor_gen = xor_gen_vector, +}; diff --git a/lib/raid/xor/riscv/xor.S b/lib/raid/xor/riscv/xor.S new file mode 100644 index 000000000000..56fb7fc1e2cd --- /dev/null +++ b/lib/raid/xor/riscv/xor.S @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2021 SiFive + */ +#include <linux/linkage.h> +#include <linux/export.h> +#include <asm/asm.h> + +SYM_FUNC_START(xor_regs_2_) + vsetvli a3, a0, e8, m8, ta, ma + vle8.v v0, (a1) + vle8.v v8, (a2) + sub a0, a0, a3 + vxor.vv v16, v0, v8 + add a2, a2, a3 + vse8.v v16, (a1) + add a1, a1, a3 + bnez a0, xor_regs_2_ + ret +SYM_FUNC_END(xor_regs_2_) + +SYM_FUNC_START(xor_regs_3_) + vsetvli a4, a0, e8, m8, ta, ma + vle8.v v0, (a1) + vle8.v v8, (a2) + sub a0, a0, a4 + vxor.vv v0, v0, v8 + vle8.v v16, (a3) + add a2, a2, a4 + vxor.vv v16, v0, v16 + add a3, a3, a4 + vse8.v v16, (a1) + add a1, a1, a4 + bnez a0, xor_regs_3_ + ret +SYM_FUNC_END(xor_regs_3_) + +SYM_FUNC_START(xor_regs_4_) + vsetvli a5, a0, e8, m8, ta, ma + vle8.v v0, (a1) + vle8.v v8, (a2) + sub a0, a0, a5 + vxor.vv v0, v0, v8 + vle8.v v16, (a3) + add a2, a2, a5 + vxor.vv v0, v0, v16 + vle8.v v24, (a4) + add a3, a3, a5 + vxor.vv v16, v0, v24 + add a4, a4, a5 + vse8.v v16, (a1) + add a1, a1, a5 + bnez a0, xor_regs_4_ + ret +SYM_FUNC_END(xor_regs_4_) + +SYM_FUNC_START(xor_regs_5_) + vsetvli a6, a0, e8, m8, ta, ma + vle8.v v0, (a1) + vle8.v v8, (a2) + sub a0, a0, a6 + vxor.vv v0, v0, v8 + vle8.v v16, (a3) + add a2, a2, a6 + vxor.vv v0, v0, v16 + vle8.v v24, (a4) + add a3, a3, a6 + vxor.vv v0, v0, v24 + vle8.v v8, (a5) + add a4, a4, a6 + vxor.vv v16, v0, v8 + add a5, a5, a6 + vse8.v v16, (a1) + add a1, a1, a6 + bnez a0, xor_regs_5_ + ret +SYM_FUNC_END(xor_regs_5_) diff --git a/lib/raid/xor/riscv/xor_arch.h b/lib/raid/xor/riscv/xor_arch.h new file mode 100644 index 000000000000..9240857d760b --- /dev/null +++ b/lib/raid/xor/riscv/xor_arch.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2021 SiFive + */ +#include <asm/vector.h> + +extern struct xor_block_template xor_block_rvv; + +static __always_inline void __init arch_xor_init(void) +{ + xor_register(&xor_block_8regs); + xor_register(&xor_block_32regs); +#ifdef CONFIG_RISCV_ISA_V + if (has_vector()) + xor_register(&xor_block_rvv); +#endif +} diff --git a/lib/raid/xor/s390/xor.c b/lib/raid/xor/s390/xor.c new file mode 100644 index 000000000000..0c478678a129 --- /dev/null +++ b/lib/raid/xor/s390/xor.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Optimized xor_block operation for RAID4/5 + * + * Copyright IBM Corp. 2016 + * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> + */ + +#include <linux/types.h> +#include "xor_impl.h" +#include "xor_arch.h" + +static void xor_xc_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) +{ + asm volatile( + " aghi %0,-1\n" + " jm 3f\n" + " srlg 0,%0,8\n" + " ltgr 0,0\n" + " jz 1f\n" + "0: xc 0(256,%1),0(%2)\n" + " la %1,256(%1)\n" + " la %2,256(%2)\n" + " brctg 0,0b\n" + "1: exrl %0,2f\n" + " j 3f\n" + "2: xc 0(1,%1),0(%2)\n" + "3:" + : "+a" (bytes), "+a" (p1), "+a" (p2) + : : "0", "cc", "memory"); +} + +static void xor_xc_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) +{ + asm volatile( + " aghi %0,-1\n" + " jm 4f\n" + " srlg 0,%0,8\n" + " ltgr 0,0\n" + " jz 1f\n" + "0: xc 0(256,%1),0(%2)\n" + " xc 0(256,%1),0(%3)\n" + " la %1,256(%1)\n" + " la %2,256(%2)\n" + " la %3,256(%3)\n" + " brctg 0,0b\n" + "1: exrl %0,2f\n" + " exrl %0,3f\n" + " j 4f\n" + "2: xc 0(1,%1),0(%2)\n" + "3: xc 0(1,%1),0(%3)\n" + "4:" + : "+a" (bytes), "+a" (p1), "+a" (p2), "+a" (p3) + : : "0", "cc", "memory"); +} + +static void xor_xc_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) +{ + asm volatile( + " aghi %0,-1\n" + " jm 5f\n" + " srlg 0,%0,8\n" + " ltgr 0,0\n" + " jz 1f\n" + "0: xc 0(256,%1),0(%2)\n" + " xc 0(256,%1),0(%3)\n" + " xc 0(256,%1),0(%4)\n" + " la %1,256(%1)\n" + " la %2,256(%2)\n" + " la %3,256(%3)\n" + " la %4,256(%4)\n" + " brctg 0,0b\n" + "1: exrl %0,2f\n" + " exrl %0,3f\n" + " exrl %0,4f\n" + " j 5f\n" + "2: xc 0(1,%1),0(%2)\n" + "3: xc 0(1,%1),0(%3)\n" + "4: xc 0(1,%1),0(%4)\n" + "5:" + : "+a" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4) + : : "0", "cc", "memory"); +} + +static void xor_xc_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) +{ + asm volatile( + " aghi %0,-1\n" + " jm 6f\n" + " srlg 0,%0,8\n" + " ltgr 0,0\n" + " jz 1f\n" + "0: xc 0(256,%1),0(%2)\n" + " xc 0(256,%1),0(%3)\n" + " xc 0(256,%1),0(%4)\n" + " xc 0(256,%1),0(%5)\n" + " la %1,256(%1)\n" + " la %2,256(%2)\n" + " la %3,256(%3)\n" + " la %4,256(%4)\n" + " la %5,256(%5)\n" + " brctg 0,0b\n" + "1: exrl %0,2f\n" + " exrl %0,3f\n" + " exrl %0,4f\n" + " exrl %0,5f\n" + " j 6f\n" + "2: xc 0(1,%1),0(%2)\n" + "3: xc 0(1,%1),0(%3)\n" + "4: xc 0(1,%1),0(%4)\n" + "5: xc 0(1,%1),0(%5)\n" + "6:" + : "+a" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4), + "+a" (p5) + : : "0", "cc", "memory"); +} + +DO_XOR_BLOCKS(xc, xor_xc_2, xor_xc_3, xor_xc_4, xor_xc_5); + +struct xor_block_template xor_block_xc = { + .name = "xc", + .xor_gen = xor_gen_xc, +}; diff --git a/lib/raid/xor/s390/xor_arch.h b/lib/raid/xor/s390/xor_arch.h new file mode 100644 index 000000000000..4a233ed2b97a --- /dev/null +++ b/lib/raid/xor/s390/xor_arch.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Optimited xor routines + * + * Copyright IBM Corp. 2016 + * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> + */ +extern struct xor_block_template xor_block_xc; + +static __always_inline void __init arch_xor_init(void) +{ + xor_force(&xor_block_xc); +} diff --git a/lib/raid/xor/sparc/xor-sparc32.c b/lib/raid/xor/sparc/xor-sparc32.c new file mode 100644 index 000000000000..fb37631e90e6 --- /dev/null +++ b/lib/raid/xor/sparc/xor-sparc32.c @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * High speed xor_block operation for RAID4/5 utilizing the + * ldd/std SPARC instructions. + * + * Copyright (C) 1999 Jakub Jelinek (jj@ultra.linux.cz) + */ +#include "xor_impl.h" +#include "xor_arch.h" + +static void +sparc_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) +{ + int lines = bytes / (sizeof (long)) / 8; + + do { + __asm__ __volatile__( + "ldd [%0 + 0x00], %%g2\n\t" + "ldd [%0 + 0x08], %%g4\n\t" + "ldd [%0 + 0x10], %%o0\n\t" + "ldd [%0 + 0x18], %%o2\n\t" + "ldd [%1 + 0x00], %%o4\n\t" + "ldd [%1 + 0x08], %%l0\n\t" + "ldd [%1 + 0x10], %%l2\n\t" + "ldd [%1 + 0x18], %%l4\n\t" + "xor %%g2, %%o4, %%g2\n\t" + "xor %%g3, %%o5, %%g3\n\t" + "xor %%g4, %%l0, %%g4\n\t" + "xor %%g5, %%l1, %%g5\n\t" + "xor %%o0, %%l2, %%o0\n\t" + "xor %%o1, %%l3, %%o1\n\t" + "xor %%o2, %%l4, %%o2\n\t" + "xor %%o3, %%l5, %%o3\n\t" + "std %%g2, [%0 + 0x00]\n\t" + "std %%g4, [%0 + 0x08]\n\t" + "std %%o0, [%0 + 0x10]\n\t" + "std %%o2, [%0 + 0x18]\n" + : + : "r" (p1), "r" (p2) + : "g2", "g3", "g4", "g5", + "o0", "o1", "o2", "o3", "o4", "o5", + "l0", "l1", "l2", "l3", "l4", "l5"); + p1 += 8; + p2 += 8; + } while (--lines > 0); +} + +static void +sparc_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) +{ + int lines = bytes / (sizeof (long)) / 8; + + do { + __asm__ __volatile__( + "ldd [%0 + 0x00], %%g2\n\t" + "ldd [%0 + 0x08], %%g4\n\t" + "ldd [%0 + 0x10], %%o0\n\t" + "ldd [%0 + 0x18], %%o2\n\t" + "ldd [%1 + 0x00], %%o4\n\t" + "ldd [%1 + 0x08], %%l0\n\t" + "ldd [%1 + 0x10], %%l2\n\t" + "ldd [%1 + 0x18], %%l4\n\t" + "xor %%g2, %%o4, %%g2\n\t" + "xor %%g3, %%o5, %%g3\n\t" + "ldd [%2 + 0x00], %%o4\n\t" + "xor %%g4, %%l0, %%g4\n\t" + "xor %%g5, %%l1, %%g5\n\t" + "ldd [%2 + 0x08], %%l0\n\t" + "xor %%o0, %%l2, %%o0\n\t" + "xor %%o1, %%l3, %%o1\n\t" + "ldd [%2 + 0x10], %%l2\n\t" + "xor %%o2, %%l4, %%o2\n\t" + "xor %%o3, %%l5, %%o3\n\t" + "ldd [%2 + 0x18], %%l4\n\t" + "xor %%g2, %%o4, %%g2\n\t" + "xor %%g3, %%o5, %%g3\n\t" + "xor %%g4, %%l0, %%g4\n\t" + "xor %%g5, %%l1, %%g5\n\t" + "xor %%o0, %%l2, %%o0\n\t" + "xor %%o1, %%l3, %%o1\n\t" + "xor %%o2, %%l4, %%o2\n\t" + "xor %%o3, %%l5, %%o3\n\t" + "std %%g2, [%0 + 0x00]\n\t" + "std %%g4, [%0 + 0x08]\n\t" + "std %%o0, [%0 + 0x10]\n\t" + "std %%o2, [%0 + 0x18]\n" + : + : "r" (p1), "r" (p2), "r" (p3) + : "g2", "g3", "g4", "g5", + "o0", "o1", "o2", "o3", "o4", "o5", + "l0", "l1", "l2", "l3", "l4", "l5"); + p1 += 8; + p2 += 8; + p3 += 8; + } while (--lines > 0); +} + +static void +sparc_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) +{ + int lines = bytes / (sizeof (long)) / 8; + + do { + __asm__ __volatile__( + "ldd [%0 + 0x00], %%g2\n\t" + "ldd [%0 + 0x08], %%g4\n\t" + "ldd [%0 + 0x10], %%o0\n\t" + "ldd [%0 + 0x18], %%o2\n\t" + "ldd [%1 + 0x00], %%o4\n\t" + "ldd [%1 + 0x08], %%l0\n\t" + "ldd [%1 + 0x10], %%l2\n\t" + "ldd [%1 + 0x18], %%l4\n\t" + "xor %%g2, %%o4, %%g2\n\t" + "xor %%g3, %%o5, %%g3\n\t" + "ldd [%2 + 0x00], %%o4\n\t" + "xor %%g4, %%l0, %%g4\n\t" + "xor %%g5, %%l1, %%g5\n\t" + "ldd [%2 + 0x08], %%l0\n\t" + "xor %%o0, %%l2, %%o0\n\t" + "xor %%o1, %%l3, %%o1\n\t" + "ldd [%2 + 0x10], %%l2\n\t" + "xor %%o2, %%l4, %%o2\n\t" + "xor %%o3, %%l5, %%o3\n\t" + "ldd [%2 + 0x18], %%l4\n\t" + "xor %%g2, %%o4, %%g2\n\t" + "xor %%g3, %%o5, %%g3\n\t" + "ldd [%3 + 0x00], %%o4\n\t" + "xor %%g4, %%l0, %%g4\n\t" + "xor %%g5, %%l1, %%g5\n\t" + "ldd [%3 + 0x08], %%l0\n\t" + "xor %%o0, %%l2, %%o0\n\t" + "xor %%o1, %%l3, %%o1\n\t" + "ldd [%3 + 0x10], %%l2\n\t" + "xor %%o2, %%l4, %%o2\n\t" + "xor %%o3, %%l5, %%o3\n\t" + "ldd [%3 + 0x18], %%l4\n\t" + "xor %%g2, %%o4, %%g2\n\t" + "xor %%g3, %%o5, %%g3\n\t" + "xor %%g4, %%l0, %%g4\n\t" + "xor %%g5, %%l1, %%g5\n\t" + "xor %%o0, %%l2, %%o0\n\t" + "xor %%o1, %%l3, %%o1\n\t" + "xor %%o2, %%l4, %%o2\n\t" + "xor %%o3, %%l5, %%o3\n\t" + "std %%g2, [%0 + 0x00]\n\t" + "std %%g4, [%0 + 0x08]\n\t" + "std %%o0, [%0 + 0x10]\n\t" + "std %%o2, [%0 + 0x18]\n" + : + : "r" (p1), "r" (p2), "r" (p3), "r" (p4) + : "g2", "g3", "g4", "g5", + "o0", "o1", "o2", "o3", "o4", "o5", + "l0", "l1", "l2", "l3", "l4", "l5"); + p1 += 8; + p2 += 8; + p3 += 8; + p4 += 8; + } while (--lines > 0); +} + +static void +sparc_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) +{ + int lines = bytes / (sizeof (long)) / 8; + + do { + __asm__ __volatile__( + "ldd [%0 + 0x00], %%g2\n\t" + "ldd [%0 + 0x08], %%g4\n\t" + "ldd [%0 + 0x10], %%o0\n\t" + "ldd [%0 + 0x18], %%o2\n\t" + "ldd [%1 + 0x00], %%o4\n\t" + "ldd [%1 + 0x08], %%l0\n\t" + "ldd [%1 + 0x10], %%l2\n\t" + "ldd [%1 + 0x18], %%l4\n\t" + "xor %%g2, %%o4, %%g2\n\t" + "xor %%g3, %%o5, %%g3\n\t" + "ldd [%2 + 0x00], %%o4\n\t" + "xor %%g4, %%l0, %%g4\n\t" + "xor %%g5, %%l1, %%g5\n\t" + "ldd [%2 + 0x08], %%l0\n\t" + "xor %%o0, %%l2, %%o0\n\t" + "xor %%o1, %%l3, %%o1\n\t" + "ldd [%2 + 0x10], %%l2\n\t" + "xor %%o2, %%l4, %%o2\n\t" + "xor %%o3, %%l5, %%o3\n\t" + "ldd [%2 + 0x18], %%l4\n\t" + "xor %%g2, %%o4, %%g2\n\t" + "xor %%g3, %%o5, %%g3\n\t" + "ldd [%3 + 0x00], %%o4\n\t" + "xor %%g4, %%l0, %%g4\n\t" + "xor %%g5, %%l1, %%g5\n\t" + "ldd [%3 + 0x08], %%l0\n\t" + "xor %%o0, %%l2, %%o0\n\t" + "xor %%o1, %%l3, %%o1\n\t" + "ldd [%3 + 0x10], %%l2\n\t" + "xor %%o2, %%l4, %%o2\n\t" + "xor %%o3, %%l5, %%o3\n\t" + "ldd [%3 + 0x18], %%l4\n\t" + "xor %%g2, %%o4, %%g2\n\t" + "xor %%g3, %%o5, %%g3\n\t" + "ldd [%4 + 0x00], %%o4\n\t" + "xor %%g4, %%l0, %%g4\n\t" + "xor %%g5, %%l1, %%g5\n\t" + "ldd [%4 + 0x08], %%l0\n\t" + "xor %%o0, %%l2, %%o0\n\t" + "xor %%o1, %%l3, %%o1\n\t" + "ldd [%4 + 0x10], %%l2\n\t" + "xor %%o2, %%l4, %%o2\n\t" + "xor %%o3, %%l5, %%o3\n\t" + "ldd [%4 + 0x18], %%l4\n\t" + "xor %%g2, %%o4, %%g2\n\t" + "xor %%g3, %%o5, %%g3\n\t" + "xor %%g4, %%l0, %%g4\n\t" + "xor %%g5, %%l1, %%g5\n\t" + "xor %%o0, %%l2, %%o0\n\t" + "xor %%o1, %%l3, %%o1\n\t" + "xor %%o2, %%l4, %%o2\n\t" + "xor %%o3, %%l5, %%o3\n\t" + "std %%g2, [%0 + 0x00]\n\t" + "std %%g4, [%0 + 0x08]\n\t" + "std %%o0, [%0 + 0x10]\n\t" + "std %%o2, [%0 + 0x18]\n" + : + : "r" (p1), "r" (p2), "r" (p3), "r" (p4), "r" (p5) + : "g2", "g3", "g4", "g5", + "o0", "o1", "o2", "o3", "o4", "o5", + "l0", "l1", "l2", "l3", "l4", "l5"); + p1 += 8; + p2 += 8; + p3 += 8; + p4 += 8; + p5 += 8; + } while (--lines > 0); +} + +DO_XOR_BLOCKS(sparc32, sparc_2, sparc_3, sparc_4, sparc_5); + +struct xor_block_template xor_block_SPARC = { + .name = "SPARC", + .xor_gen = xor_gen_sparc32, +}; diff --git a/lib/raid/xor/sparc/xor-sparc64-glue.c b/lib/raid/xor/sparc/xor-sparc64-glue.c new file mode 100644 index 000000000000..a8a686e0d258 --- /dev/null +++ b/lib/raid/xor/sparc/xor-sparc64-glue.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * High speed xor_block operation for RAID4/5 utilizing the + * UltraSparc Visual Instruction Set and Niagara block-init + * twin-load instructions. + * + * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz) + * Copyright (C) 2006 David S. Miller <davem@davemloft.net> + */ + +#include "xor_impl.h" +#include "xor_arch.h" + +void xor_vis_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2); +void xor_vis_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3); +void xor_vis_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4); +void xor_vis_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5); + +/* XXX Ugh, write cheetah versions... -DaveM */ + +DO_XOR_BLOCKS(vis, xor_vis_2, xor_vis_3, xor_vis_4, xor_vis_5); + +struct xor_block_template xor_block_VIS = { + .name = "VIS", + .xor_gen = xor_gen_vis, +}; + +void xor_niagara_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2); +void xor_niagara_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3); +void xor_niagara_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4); +void xor_niagara_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5); + +DO_XOR_BLOCKS(niagara, xor_niagara_2, xor_niagara_3, xor_niagara_4, + xor_niagara_5); + +struct xor_block_template xor_block_niagara = { + .name = "Niagara", + .xor_gen = xor_gen_niagara, +}; diff --git a/lib/raid/xor/sparc/xor-sparc64.S b/lib/raid/xor/sparc/xor-sparc64.S new file mode 100644 index 000000000000..a7b74d473bd4 --- /dev/null +++ b/lib/raid/xor/sparc/xor-sparc64.S @@ -0,0 +1,636 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * High speed xor_block operation for RAID4/5 utilizing the + * UltraSparc Visual Instruction Set and Niagara store-init/twin-load. + * + * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz) + * Copyright (C) 2006 David S. Miller <davem@davemloft.net> + */ + +#include <linux/export.h> +#include <linux/linkage.h> +#include <asm/visasm.h> +#include <asm/asi.h> +#include <asm/dcu.h> +#include <asm/spitfire.h> + +/* + * Requirements: + * !(((long)dest | (long)sourceN) & (64 - 1)) && + * !(len & 127) && len >= 256 + */ + .text + + /* VIS versions. */ +ENTRY(xor_vis_2) + rd %fprs, %o5 + andcc %o5, FPRS_FEF|FPRS_DU, %g0 + be,pt %icc, 0f + sethi %hi(VISenter), %g1 + jmpl %g1 + %lo(VISenter), %g7 + add %g7, 8, %g7 +0: wr %g0, FPRS_FEF, %fprs + rd %asi, %g1 + wr %g0, ASI_BLK_P, %asi + membar #LoadStore|#StoreLoad|#StoreStore + sub %o0, 128, %o0 + ldda [%o1] %asi, %f0 + ldda [%o2] %asi, %f16 + +2: ldda [%o1 + 64] %asi, %f32 + fxor %f0, %f16, %f16 + fxor %f2, %f18, %f18 + fxor %f4, %f20, %f20 + fxor %f6, %f22, %f22 + fxor %f8, %f24, %f24 + fxor %f10, %f26, %f26 + fxor %f12, %f28, %f28 + fxor %f14, %f30, %f30 + stda %f16, [%o1] %asi + ldda [%o2 + 64] %asi, %f48 + ldda [%o1 + 128] %asi, %f0 + fxor %f32, %f48, %f48 + fxor %f34, %f50, %f50 + add %o1, 128, %o1 + fxor %f36, %f52, %f52 + add %o2, 128, %o2 + fxor %f38, %f54, %f54 + subcc %o0, 128, %o0 + fxor %f40, %f56, %f56 + fxor %f42, %f58, %f58 + fxor %f44, %f60, %f60 + fxor %f46, %f62, %f62 + stda %f48, [%o1 - 64] %asi + bne,pt %xcc, 2b + ldda [%o2] %asi, %f16 + + ldda [%o1 + 64] %asi, %f32 + fxor %f0, %f16, %f16 + fxor %f2, %f18, %f18 + fxor %f4, %f20, %f20 + fxor %f6, %f22, %f22 + fxor %f8, %f24, %f24 + fxor %f10, %f26, %f26 + fxor %f12, %f28, %f28 + fxor %f14, %f30, %f30 + stda %f16, [%o1] %asi + ldda [%o2 + 64] %asi, %f48 + membar #Sync + fxor %f32, %f48, %f48 + fxor %f34, %f50, %f50 + fxor %f36, %f52, %f52 + fxor %f38, %f54, %f54 + fxor %f40, %f56, %f56 + fxor %f42, %f58, %f58 + fxor %f44, %f60, %f60 + fxor %f46, %f62, %f62 + stda %f48, [%o1 + 64] %asi + membar #Sync|#StoreStore|#StoreLoad + wr %g1, %g0, %asi + retl + wr %g0, 0, %fprs +ENDPROC(xor_vis_2) + +ENTRY(xor_vis_3) + rd %fprs, %o5 + andcc %o5, FPRS_FEF|FPRS_DU, %g0 + be,pt %icc, 0f + sethi %hi(VISenter), %g1 + jmpl %g1 + %lo(VISenter), %g7 + add %g7, 8, %g7 +0: wr %g0, FPRS_FEF, %fprs + rd %asi, %g1 + wr %g0, ASI_BLK_P, %asi + membar #LoadStore|#StoreLoad|#StoreStore + sub %o0, 64, %o0 + ldda [%o1] %asi, %f0 + ldda [%o2] %asi, %f16 + +3: ldda [%o3] %asi, %f32 + fxor %f0, %f16, %f48 + fxor %f2, %f18, %f50 + add %o1, 64, %o1 + fxor %f4, %f20, %f52 + fxor %f6, %f22, %f54 + add %o2, 64, %o2 + fxor %f8, %f24, %f56 + fxor %f10, %f26, %f58 + fxor %f12, %f28, %f60 + fxor %f14, %f30, %f62 + ldda [%o1] %asi, %f0 + fxor %f48, %f32, %f48 + fxor %f50, %f34, %f50 + fxor %f52, %f36, %f52 + fxor %f54, %f38, %f54 + add %o3, 64, %o3 + fxor %f56, %f40, %f56 + fxor %f58, %f42, %f58 + subcc %o0, 64, %o0 + fxor %f60, %f44, %f60 + fxor %f62, %f46, %f62 + stda %f48, [%o1 - 64] %asi + bne,pt %xcc, 3b + ldda [%o2] %asi, %f16 + + ldda [%o3] %asi, %f32 + fxor %f0, %f16, %f48 + fxor %f2, %f18, %f50 + fxor %f4, %f20, %f52 + fxor %f6, %f22, %f54 + fxor %f8, %f24, %f56 + fxor %f10, %f26, %f58 + fxor %f12, %f28, %f60 + fxor %f14, %f30, %f62 + membar #Sync + fxor %f48, %f32, %f48 + fxor %f50, %f34, %f50 + fxor %f52, %f36, %f52 + fxor %f54, %f38, %f54 + fxor %f56, %f40, %f56 + fxor %f58, %f42, %f58 + fxor %f60, %f44, %f60 + fxor %f62, %f46, %f62 + stda %f48, [%o1] %asi + membar #Sync|#StoreStore|#StoreLoad + wr %g1, %g0, %asi + retl + wr %g0, 0, %fprs +ENDPROC(xor_vis_3) + +ENTRY(xor_vis_4) + rd %fprs, %o5 + andcc %o5, FPRS_FEF|FPRS_DU, %g0 + be,pt %icc, 0f + sethi %hi(VISenter), %g1 + jmpl %g1 + %lo(VISenter), %g7 + add %g7, 8, %g7 +0: wr %g0, FPRS_FEF, %fprs + rd %asi, %g1 + wr %g0, ASI_BLK_P, %asi + membar #LoadStore|#StoreLoad|#StoreStore + sub %o0, 64, %o0 + ldda [%o1] %asi, %f0 + ldda [%o2] %asi, %f16 + +4: ldda [%o3] %asi, %f32 + fxor %f0, %f16, %f16 + fxor %f2, %f18, %f18 + add %o1, 64, %o1 + fxor %f4, %f20, %f20 + fxor %f6, %f22, %f22 + add %o2, 64, %o2 + fxor %f8, %f24, %f24 + fxor %f10, %f26, %f26 + fxor %f12, %f28, %f28 + fxor %f14, %f30, %f30 + ldda [%o4] %asi, %f48 + fxor %f16, %f32, %f32 + fxor %f18, %f34, %f34 + fxor %f20, %f36, %f36 + fxor %f22, %f38, %f38 + add %o3, 64, %o3 + fxor %f24, %f40, %f40 + fxor %f26, %f42, %f42 + fxor %f28, %f44, %f44 + fxor %f30, %f46, %f46 + ldda [%o1] %asi, %f0 + fxor %f32, %f48, %f48 + fxor %f34, %f50, %f50 + fxor %f36, %f52, %f52 + add %o4, 64, %o4 + fxor %f38, %f54, %f54 + fxor %f40, %f56, %f56 + fxor %f42, %f58, %f58 + subcc %o0, 64, %o0 + fxor %f44, %f60, %f60 + fxor %f46, %f62, %f62 + stda %f48, [%o1 - 64] %asi + bne,pt %xcc, 4b + ldda [%o2] %asi, %f16 + + ldda [%o3] %asi, %f32 + fxor %f0, %f16, %f16 + fxor %f2, %f18, %f18 + fxor %f4, %f20, %f20 + fxor %f6, %f22, %f22 + fxor %f8, %f24, %f24 + fxor %f10, %f26, %f26 + fxor %f12, %f28, %f28 + fxor %f14, %f30, %f30 + ldda [%o4] %asi, %f48 + fxor %f16, %f32, %f32 + fxor %f18, %f34, %f34 + fxor %f20, %f36, %f36 + fxor %f22, %f38, %f38 + fxor %f24, %f40, %f40 + fxor %f26, %f42, %f42 + fxor %f28, %f44, %f44 + fxor %f30, %f46, %f46 + membar #Sync + fxor %f32, %f48, %f48 + fxor %f34, %f50, %f50 + fxor %f36, %f52, %f52 + fxor %f38, %f54, %f54 + fxor %f40, %f56, %f56 + fxor %f42, %f58, %f58 + fxor %f44, %f60, %f60 + fxor %f46, %f62, %f62 + stda %f48, [%o1] %asi + membar #Sync|#StoreStore|#StoreLoad + wr %g1, %g0, %asi + retl + wr %g0, 0, %fprs +ENDPROC(xor_vis_4) + +ENTRY(xor_vis_5) + save %sp, -192, %sp + rd %fprs, %o5 + andcc %o5, FPRS_FEF|FPRS_DU, %g0 + be,pt %icc, 0f + sethi %hi(VISenter), %g1 + jmpl %g1 + %lo(VISenter), %g7 + add %g7, 8, %g7 +0: wr %g0, FPRS_FEF, %fprs + rd %asi, %g1 + wr %g0, ASI_BLK_P, %asi + membar #LoadStore|#StoreLoad|#StoreStore + sub %i0, 64, %i0 + ldda [%i1] %asi, %f0 + ldda [%i2] %asi, %f16 + +5: ldda [%i3] %asi, %f32 + fxor %f0, %f16, %f48 + fxor %f2, %f18, %f50 + add %i1, 64, %i1 + fxor %f4, %f20, %f52 + fxor %f6, %f22, %f54 + add %i2, 64, %i2 + fxor %f8, %f24, %f56 + fxor %f10, %f26, %f58 + fxor %f12, %f28, %f60 + fxor %f14, %f30, %f62 + ldda [%i4] %asi, %f16 + fxor %f48, %f32, %f48 + fxor %f50, %f34, %f50 + fxor %f52, %f36, %f52 + fxor %f54, %f38, %f54 + add %i3, 64, %i3 + fxor %f56, %f40, %f56 + fxor %f58, %f42, %f58 + fxor %f60, %f44, %f60 + fxor %f62, %f46, %f62 + ldda [%i5] %asi, %f32 + fxor %f48, %f16, %f48 + fxor %f50, %f18, %f50 + add %i4, 64, %i4 + fxor %f52, %f20, %f52 + fxor %f54, %f22, %f54 + add %i5, 64, %i5 + fxor %f56, %f24, %f56 + fxor %f58, %f26, %f58 + fxor %f60, %f28, %f60 + fxor %f62, %f30, %f62 + ldda [%i1] %asi, %f0 + fxor %f48, %f32, %f48 + fxor %f50, %f34, %f50 + fxor %f52, %f36, %f52 + fxor %f54, %f38, %f54 + fxor %f56, %f40, %f56 + fxor %f58, %f42, %f58 + subcc %i0, 64, %i0 + fxor %f60, %f44, %f60 + fxor %f62, %f46, %f62 + stda %f48, [%i1 - 64] %asi + bne,pt %xcc, 5b + ldda [%i2] %asi, %f16 + + ldda [%i3] %asi, %f32 + fxor %f0, %f16, %f48 + fxor %f2, %f18, %f50 + fxor %f4, %f20, %f52 + fxor %f6, %f22, %f54 + fxor %f8, %f24, %f56 + fxor %f10, %f26, %f58 + fxor %f12, %f28, %f60 + fxor %f14, %f30, %f62 + ldda [%i4] %asi, %f16 + fxor %f48, %f32, %f48 + fxor %f50, %f34, %f50 + fxor %f52, %f36, %f52 + fxor %f54, %f38, %f54 + fxor %f56, %f40, %f56 + fxor %f58, %f42, %f58 + fxor %f60, %f44, %f60 + fxor %f62, %f46, %f62 + ldda [%i5] %asi, %f32 + fxor %f48, %f16, %f48 + fxor %f50, %f18, %f50 + fxor %f52, %f20, %f52 + fxor %f54, %f22, %f54 + fxor %f56, %f24, %f56 + fxor %f58, %f26, %f58 + fxor %f60, %f28, %f60 + fxor %f62, %f30, %f62 + membar #Sync + fxor %f48, %f32, %f48 + fxor %f50, %f34, %f50 + fxor %f52, %f36, %f52 + fxor %f54, %f38, %f54 + fxor %f56, %f40, %f56 + fxor %f58, %f42, %f58 + fxor %f60, %f44, %f60 + fxor %f62, %f46, %f62 + stda %f48, [%i1] %asi + membar #Sync|#StoreStore|#StoreLoad + wr %g1, %g0, %asi + wr %g0, 0, %fprs + ret + restore +ENDPROC(xor_vis_5) + + /* Niagara versions. */ +ENTRY(xor_niagara_2) /* %o0=bytes, %o1=dest, %o2=src */ + save %sp, -192, %sp + prefetch [%i1], #n_writes + prefetch [%i2], #one_read + rd %asi, %g7 + wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi + srlx %i0, 6, %g1 + mov %i1, %i0 + mov %i2, %i1 +1: ldda [%i1 + 0x00] %asi, %i2 /* %i2/%i3 = src + 0x00 */ + ldda [%i1 + 0x10] %asi, %i4 /* %i4/%i5 = src + 0x10 */ + ldda [%i1 + 0x20] %asi, %g2 /* %g2/%g3 = src + 0x20 */ + ldda [%i1 + 0x30] %asi, %l0 /* %l0/%l1 = src + 0x30 */ + prefetch [%i1 + 0x40], #one_read + ldda [%i0 + 0x00] %asi, %o0 /* %o0/%o1 = dest + 0x00 */ + ldda [%i0 + 0x10] %asi, %o2 /* %o2/%o3 = dest + 0x10 */ + ldda [%i0 + 0x20] %asi, %o4 /* %o4/%o5 = dest + 0x20 */ + ldda [%i0 + 0x30] %asi, %l2 /* %l2/%l3 = dest + 0x30 */ + prefetch [%i0 + 0x40], #n_writes + xor %o0, %i2, %o0 + xor %o1, %i3, %o1 + stxa %o0, [%i0 + 0x00] %asi + stxa %o1, [%i0 + 0x08] %asi + xor %o2, %i4, %o2 + xor %o3, %i5, %o3 + stxa %o2, [%i0 + 0x10] %asi + stxa %o3, [%i0 + 0x18] %asi + xor %o4, %g2, %o4 + xor %o5, %g3, %o5 + stxa %o4, [%i0 + 0x20] %asi + stxa %o5, [%i0 + 0x28] %asi + xor %l2, %l0, %l2 + xor %l3, %l1, %l3 + stxa %l2, [%i0 + 0x30] %asi + stxa %l3, [%i0 + 0x38] %asi + add %i0, 0x40, %i0 + subcc %g1, 1, %g1 + bne,pt %xcc, 1b + add %i1, 0x40, %i1 + membar #Sync + wr %g7, 0x0, %asi + ret + restore +ENDPROC(xor_niagara_2) + +ENTRY(xor_niagara_3) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2 */ + save %sp, -192, %sp + prefetch [%i1], #n_writes + prefetch [%i2], #one_read + prefetch [%i3], #one_read + rd %asi, %g7 + wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi + srlx %i0, 6, %g1 + mov %i1, %i0 + mov %i2, %i1 + mov %i3, %l7 +1: ldda [%i1 + 0x00] %asi, %i2 /* %i2/%i3 = src1 + 0x00 */ + ldda [%i1 + 0x10] %asi, %i4 /* %i4/%i5 = src1 + 0x10 */ + ldda [%l7 + 0x00] %asi, %g2 /* %g2/%g3 = src2 + 0x00 */ + ldda [%l7 + 0x10] %asi, %l0 /* %l0/%l1 = src2 + 0x10 */ + ldda [%i0 + 0x00] %asi, %o0 /* %o0/%o1 = dest + 0x00 */ + ldda [%i0 + 0x10] %asi, %o2 /* %o2/%o3 = dest + 0x10 */ + xor %g2, %i2, %g2 + xor %g3, %i3, %g3 + xor %o0, %g2, %o0 + xor %o1, %g3, %o1 + stxa %o0, [%i0 + 0x00] %asi + stxa %o1, [%i0 + 0x08] %asi + ldda [%i1 + 0x20] %asi, %i2 /* %i2/%i3 = src1 + 0x20 */ + ldda [%l7 + 0x20] %asi, %g2 /* %g2/%g3 = src2 + 0x20 */ + ldda [%i0 + 0x20] %asi, %o0 /* %o0/%o1 = dest + 0x20 */ + xor %l0, %i4, %l0 + xor %l1, %i5, %l1 + xor %o2, %l0, %o2 + xor %o3, %l1, %o3 + stxa %o2, [%i0 + 0x10] %asi + stxa %o3, [%i0 + 0x18] %asi + ldda [%i1 + 0x30] %asi, %i4 /* %i4/%i5 = src1 + 0x30 */ + ldda [%l7 + 0x30] %asi, %l0 /* %l0/%l1 = src2 + 0x30 */ + ldda [%i0 + 0x30] %asi, %o2 /* %o2/%o3 = dest + 0x30 */ + prefetch [%i1 + 0x40], #one_read + prefetch [%l7 + 0x40], #one_read + prefetch [%i0 + 0x40], #n_writes + xor %g2, %i2, %g2 + xor %g3, %i3, %g3 + xor %o0, %g2, %o0 + xor %o1, %g3, %o1 + stxa %o0, [%i0 + 0x20] %asi + stxa %o1, [%i0 + 0x28] %asi + xor %l0, %i4, %l0 + xor %l1, %i5, %l1 + xor %o2, %l0, %o2 + xor %o3, %l1, %o3 + stxa %o2, [%i0 + 0x30] %asi + stxa %o3, [%i0 + 0x38] %asi + add %i0, 0x40, %i0 + add %i1, 0x40, %i1 + subcc %g1, 1, %g1 + bne,pt %xcc, 1b + add %l7, 0x40, %l7 + membar #Sync + wr %g7, 0x0, %asi + ret + restore +ENDPROC(xor_niagara_3) + +ENTRY(xor_niagara_4) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3 */ + save %sp, -192, %sp + prefetch [%i1], #n_writes + prefetch [%i2], #one_read + prefetch [%i3], #one_read + prefetch [%i4], #one_read + rd %asi, %g7 + wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi + srlx %i0, 6, %g1 + mov %i1, %i0 + mov %i2, %i1 + mov %i3, %l7 + mov %i4, %l6 +1: ldda [%i1 + 0x00] %asi, %i2 /* %i2/%i3 = src1 + 0x00 */ + ldda [%l7 + 0x00] %asi, %i4 /* %i4/%i5 = src2 + 0x00 */ + ldda [%l6 + 0x00] %asi, %g2 /* %g2/%g3 = src3 + 0x00 */ + ldda [%i0 + 0x00] %asi, %l0 /* %l0/%l1 = dest + 0x00 */ + xor %i4, %i2, %i4 + xor %i5, %i3, %i5 + ldda [%i1 + 0x10] %asi, %i2 /* %i2/%i3 = src1 + 0x10 */ + xor %g2, %i4, %g2 + xor %g3, %i5, %g3 + ldda [%l7 + 0x10] %asi, %i4 /* %i4/%i5 = src2 + 0x10 */ + xor %l0, %g2, %l0 + xor %l1, %g3, %l1 + stxa %l0, [%i0 + 0x00] %asi + stxa %l1, [%i0 + 0x08] %asi + ldda [%l6 + 0x10] %asi, %g2 /* %g2/%g3 = src3 + 0x10 */ + ldda [%i0 + 0x10] %asi, %l0 /* %l0/%l1 = dest + 0x10 */ + + xor %i4, %i2, %i4 + xor %i5, %i3, %i5 + ldda [%i1 + 0x20] %asi, %i2 /* %i2/%i3 = src1 + 0x20 */ + xor %g2, %i4, %g2 + xor %g3, %i5, %g3 + ldda [%l7 + 0x20] %asi, %i4 /* %i4/%i5 = src2 + 0x20 */ + xor %l0, %g2, %l0 + xor %l1, %g3, %l1 + stxa %l0, [%i0 + 0x10] %asi + stxa %l1, [%i0 + 0x18] %asi + ldda [%l6 + 0x20] %asi, %g2 /* %g2/%g3 = src3 + 0x20 */ + ldda [%i0 + 0x20] %asi, %l0 /* %l0/%l1 = dest + 0x20 */ + + xor %i4, %i2, %i4 + xor %i5, %i3, %i5 + ldda [%i1 + 0x30] %asi, %i2 /* %i2/%i3 = src1 + 0x30 */ + xor %g2, %i4, %g2 + xor %g3, %i5, %g3 + ldda [%l7 + 0x30] %asi, %i4 /* %i4/%i5 = src2 + 0x30 */ + xor %l0, %g2, %l0 + xor %l1, %g3, %l1 + stxa %l0, [%i0 + 0x20] %asi + stxa %l1, [%i0 + 0x28] %asi + ldda [%l6 + 0x30] %asi, %g2 /* %g2/%g3 = src3 + 0x30 */ + ldda [%i0 + 0x30] %asi, %l0 /* %l0/%l1 = dest + 0x30 */ + + prefetch [%i1 + 0x40], #one_read + prefetch [%l7 + 0x40], #one_read + prefetch [%l6 + 0x40], #one_read + prefetch [%i0 + 0x40], #n_writes + + xor %i4, %i2, %i4 + xor %i5, %i3, %i5 + xor %g2, %i4, %g2 + xor %g3, %i5, %g3 + xor %l0, %g2, %l0 + xor %l1, %g3, %l1 + stxa %l0, [%i0 + 0x30] %asi + stxa %l1, [%i0 + 0x38] %asi + + add %i0, 0x40, %i0 + add %i1, 0x40, %i1 + add %l7, 0x40, %l7 + subcc %g1, 1, %g1 + bne,pt %xcc, 1b + add %l6, 0x40, %l6 + membar #Sync + wr %g7, 0x0, %asi + ret + restore +ENDPROC(xor_niagara_4) + +ENTRY(xor_niagara_5) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3, %o5=src4 */ + save %sp, -192, %sp + prefetch [%i1], #n_writes + prefetch [%i2], #one_read + prefetch [%i3], #one_read + prefetch [%i4], #one_read + prefetch [%i5], #one_read + rd %asi, %g7 + wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi + srlx %i0, 6, %g1 + mov %i1, %i0 + mov %i2, %i1 + mov %i3, %l7 + mov %i4, %l6 + mov %i5, %l5 +1: ldda [%i1 + 0x00] %asi, %i2 /* %i2/%i3 = src1 + 0x00 */ + ldda [%l7 + 0x00] %asi, %i4 /* %i4/%i5 = src2 + 0x00 */ + ldda [%l6 + 0x00] %asi, %g2 /* %g2/%g3 = src3 + 0x00 */ + ldda [%l5 + 0x00] %asi, %l0 /* %l0/%l1 = src4 + 0x00 */ + ldda [%i0 + 0x00] %asi, %l2 /* %l2/%l3 = dest + 0x00 */ + xor %i4, %i2, %i4 + xor %i5, %i3, %i5 + ldda [%i1 + 0x10] %asi, %i2 /* %i2/%i3 = src1 + 0x10 */ + xor %g2, %i4, %g2 + xor %g3, %i5, %g3 + ldda [%l7 + 0x10] %asi, %i4 /* %i4/%i5 = src2 + 0x10 */ + xor %l0, %g2, %l0 + xor %l1, %g3, %l1 + ldda [%l6 + 0x10] %asi, %g2 /* %g2/%g3 = src3 + 0x10 */ + xor %l2, %l0, %l2 + xor %l3, %l1, %l3 + stxa %l2, [%i0 + 0x00] %asi + stxa %l3, [%i0 + 0x08] %asi + ldda [%l5 + 0x10] %asi, %l0 /* %l0/%l1 = src4 + 0x10 */ + ldda [%i0 + 0x10] %asi, %l2 /* %l2/%l3 = dest + 0x10 */ + + xor %i4, %i2, %i4 + xor %i5, %i3, %i5 + ldda [%i1 + 0x20] %asi, %i2 /* %i2/%i3 = src1 + 0x20 */ + xor %g2, %i4, %g2 + xor %g3, %i5, %g3 + ldda [%l7 + 0x20] %asi, %i4 /* %i4/%i5 = src2 + 0x20 */ + xor %l0, %g2, %l0 + xor %l1, %g3, %l1 + ldda [%l6 + 0x20] %asi, %g2 /* %g2/%g3 = src3 + 0x20 */ + xor %l2, %l0, %l2 + xor %l3, %l1, %l3 + stxa %l2, [%i0 + 0x10] %asi + stxa %l3, [%i0 + 0x18] %asi + ldda [%l5 + 0x20] %asi, %l0 /* %l0/%l1 = src4 + 0x20 */ + ldda [%i0 + 0x20] %asi, %l2 /* %l2/%l3 = dest + 0x20 */ + + xor %i4, %i2, %i4 + xor %i5, %i3, %i5 + ldda [%i1 + 0x30] %asi, %i2 /* %i2/%i3 = src1 + 0x30 */ + xor %g2, %i4, %g2 + xor %g3, %i5, %g3 + ldda [%l7 + 0x30] %asi, %i4 /* %i4/%i5 = src2 + 0x30 */ + xor %l0, %g2, %l0 + xor %l1, %g3, %l1 + ldda [%l6 + 0x30] %asi, %g2 /* %g2/%g3 = src3 + 0x30 */ + xor %l2, %l0, %l2 + xor %l3, %l1, %l3 + stxa %l2, [%i0 + 0x20] %asi + stxa %l3, [%i0 + 0x28] %asi + ldda [%l5 + 0x30] %asi, %l0 /* %l0/%l1 = src4 + 0x30 */ + ldda [%i0 + 0x30] %asi, %l2 /* %l2/%l3 = dest + 0x30 */ + + prefetch [%i1 + 0x40], #one_read + prefetch [%l7 + 0x40], #one_read + prefetch [%l6 + 0x40], #one_read + prefetch [%l5 + 0x40], #one_read + prefetch [%i0 + 0x40], #n_writes + + xor %i4, %i2, %i4 + xor %i5, %i3, %i5 + xor %g2, %i4, %g2 + xor %g3, %i5, %g3 + xor %l0, %g2, %l0 + xor %l1, %g3, %l1 + xor %l2, %l0, %l2 + xor %l3, %l1, %l3 + stxa %l2, [%i0 + 0x30] %asi + stxa %l3, [%i0 + 0x38] %asi + + add %i0, 0x40, %i0 + add %i1, 0x40, %i1 + add %l7, 0x40, %l7 + add %l6, 0x40, %l6 + subcc %g1, 1, %g1 + bne,pt %xcc, 1b + add %l5, 0x40, %l5 + membar #Sync + wr %g7, 0x0, %asi + ret + restore +ENDPROC(xor_niagara_5) diff --git a/lib/raid/xor/sparc/xor_arch.h b/lib/raid/xor/sparc/xor_arch.h new file mode 100644 index 000000000000..af288abe4e91 --- /dev/null +++ b/lib/raid/xor/sparc/xor_arch.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz) + * Copyright (C) 2006 David S. Miller <davem@davemloft.net> + */ +#if defined(__sparc__) && defined(__arch64__) +#include <asm/spitfire.h> + +extern struct xor_block_template xor_block_VIS; +extern struct xor_block_template xor_block_niagara; + +static __always_inline void __init arch_xor_init(void) +{ + /* Force VIS for everything except Niagara. */ + if (tlb_type == hypervisor && + (sun4v_chip_type == SUN4V_CHIP_NIAGARA1 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA2 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA5)) + xor_force(&xor_block_niagara); + else + xor_force(&xor_block_VIS); +} +#else /* sparc64 */ + +extern struct xor_block_template xor_block_SPARC; + +static __always_inline void __init arch_xor_init(void) +{ + xor_register(&xor_block_8regs); + xor_register(&xor_block_32regs); + xor_register(&xor_block_SPARC); +} +#endif /* !sparc64 */ diff --git a/lib/raid/xor/tests/Makefile b/lib/raid/xor/tests/Makefile new file mode 100644 index 000000000000..1cce833cd7fd --- /dev/null +++ b/lib/raid/xor/tests/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only + +CONTEXT_ANALYSIS := y + +obj-$(CONFIG_XOR_KUNIT_TEST) += xor_kunit.o diff --git a/lib/raid/xor/tests/xor_kunit.c b/lib/raid/xor/tests/xor_kunit.c new file mode 100644 index 000000000000..1fb30b2c7c47 --- /dev/null +++ b/lib/raid/xor/tests/xor_kunit.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Unit test the XOR library functions. + * + * Copyright 2024 Google LLC + * Copyright 2026 Christoph Hellwig + * + * Based on the CRC tests by Eric Biggers <ebiggers@google.com>. + */ +#include <kunit/test.h> +#include <linux/prandom.h> +#include <linux/string_choices.h> +#include <linux/vmalloc.h> +#include <linux/raid/xor.h> + +#define XOR_KUNIT_SEED 42 +#define XOR_KUNIT_MAX_BYTES 16384 +#define XOR_KUNIT_MAX_BUFFERS 64 +#define XOR_KUNIT_NUM_TEST_ITERS 1000 + +static struct rnd_state rng; +static void *test_buffers[XOR_KUNIT_MAX_BUFFERS]; +static void *test_dest; +static void *test_ref; +static size_t test_buflen; + +static u32 rand32(void) +{ + return prandom_u32_state(&rng); +} + +/* Reference implementation using dumb byte-wise XOR */ +static void xor_ref(void *dest, void **srcs, unsigned int src_cnt, + unsigned int bytes) +{ + unsigned int off, idx; + u8 *d = dest; + + for (off = 0; off < bytes; off++) { + for (idx = 0; idx < src_cnt; idx++) { + u8 *src = srcs[idx]; + + d[off] ^= src[off]; + } + } +} + +/* Generate a random length that is a multiple of 512. */ +static unsigned int random_length(unsigned int max_length) +{ + return round_up((rand32() % max_length) + 1, 512); +} + +/* Generate a random alignment that is a multiple of 64. */ +static unsigned int random_alignment(unsigned int max_alignment) +{ + return ((rand32() % max_alignment) + 1) & ~63; +} + +static void xor_generate_random_data(void) +{ + int i; + + prandom_bytes_state(&rng, test_dest, test_buflen); + memcpy(test_ref, test_dest, test_buflen); + for (i = 0; i < XOR_KUNIT_MAX_BUFFERS; i++) + prandom_bytes_state(&rng, test_buffers[i], test_buflen); +} + +/* Test that xor_gen gives the same result as a reference implementation. */ +static void xor_test(struct kunit *test) +{ + void *aligned_buffers[XOR_KUNIT_MAX_BUFFERS]; + size_t i; + + for (i = 0; i < XOR_KUNIT_NUM_TEST_ITERS; i++) { + unsigned int nr_buffers = + (rand32() % XOR_KUNIT_MAX_BUFFERS) + 1; + unsigned int len = random_length(XOR_KUNIT_MAX_BYTES); + unsigned int max_alignment, align = 0; + void *buffers; + + if (rand32() % 8 == 0) + /* Refresh the data occasionally. */ + xor_generate_random_data(); + + /* + * If we're not using the entire buffer size, inject randomized + * alignment into the buffer. + */ + max_alignment = XOR_KUNIT_MAX_BYTES - len; + if (max_alignment == 0) { + buffers = test_buffers; + } else if (rand32() % 2 == 0) { + /* Use random alignments mod 64 */ + int j; + + for (j = 0; j < nr_buffers; j++) + aligned_buffers[j] = test_buffers[j] + + random_alignment(max_alignment); + buffers = aligned_buffers; + align = random_alignment(max_alignment); + } else { + /* Go up to the guard page, to catch buffer overreads */ + int j; + + align = test_buflen - len; + for (j = 0; j < nr_buffers; j++) + aligned_buffers[j] = test_buffers[j] + align; + buffers = aligned_buffers; + } + + /* + * Compute the XOR, and verify that it equals the XOR computed + * by a simple byte-at-a-time reference implementation. + */ + xor_ref(test_ref + align, buffers, nr_buffers, len); + xor_gen(test_dest + align, buffers, nr_buffers, len); + KUNIT_EXPECT_MEMEQ_MSG(test, test_ref + align, + test_dest + align, len, + "Wrong result with buffers=%u, len=%u, unaligned=%s, at_end=%s", + nr_buffers, len, + str_yes_no(max_alignment), + str_yes_no(align + len == test_buflen)); + } +} + +static void xor_benchmark(struct kunit *test) +{ + static const unsigned int nr_to_test[] = { + 4, 5, 6, 7, 8, 10, 12, 15, 16, 32, + }; + static const unsigned int len_to_test[] = { + SZ_4K, SZ_16K, + }; + unsigned int i, j, l; + u64 t; + + if (!IS_ENABLED(CONFIG_XOR_BENCHMARK)) + kunit_skip(test, "not enabled"); + + /* warm-up */ + for (i = 0; i < ARRAY_SIZE(nr_to_test); i++) { + for (j = 0; j < ARRAY_SIZE(len_to_test); j++) { + for (l = 0; l < 10; l++) { + xor_gen(test_dest, test_buffers, nr_to_test[i], + len_to_test[j]); + } + } + } + + /* + * Preferably this would be a loop over len_to_test, but the kunit + * logging always adds a newline to each logged format string. + */ + static_assert(ARRAY_SIZE(len_to_test) == 2); + kunit_info(test, " \t%5u bytes\t%5u bytes\n", + len_to_test[0], len_to_test[1]); + + for (i = 0; i < ARRAY_SIZE(nr_to_test); i++) { + unsigned int nr = nr_to_test[i]; + u64 speed[ARRAY_SIZE(len_to_test)]; + + KUNIT_ASSERT_LE(test, nr, XOR_KUNIT_MAX_BUFFERS); + + for (j = 0; j < ARRAY_SIZE(len_to_test); j++) { + unsigned int len = len_to_test[j]; + const unsigned long num_iters = 1000; + + KUNIT_ASSERT_GT(test, len, 0); + KUNIT_ASSERT_LE(test, len, XOR_KUNIT_MAX_BYTES); + + preempt_disable(); + t = ktime_get_ns(); + for (l = 0; l < num_iters; l++) + xor_gen(test_dest, test_buffers, nr, len); + t = max(ktime_get_ns() - t, 1); + preempt_enable(); + + speed[j] = div64_u64((u64)len * num_iters * nr, t); + } + + static_assert(ARRAY_SIZE(len_to_test) == 2); + kunit_info(test, "%3u disks:\t%5llu GB/s\t%5llu GB/s\n", + nr, speed[0], speed[1]); + } +} + +static struct kunit_case xor_test_cases[] = { + KUNIT_CASE(xor_test), + KUNIT_CASE(xor_benchmark), + {}, +}; + +static int xor_suite_init(struct kunit_suite *suite) +{ + int i; + + /* + * Allocate the test buffer using vmalloc() with a page-aligned length + * so that it is immediately followed by a guard page. This allows + * buffer overreads to be detected, even in assembly code. + */ + test_buflen = round_up(XOR_KUNIT_MAX_BYTES, PAGE_SIZE); + test_ref = vmalloc(test_buflen); + if (!test_ref) + return -ENOMEM; + test_dest = vmalloc(test_buflen); + if (!test_dest) + goto out_free_ref; + for (i = 0; i < XOR_KUNIT_MAX_BUFFERS; i++) { + test_buffers[i] = vmalloc(test_buflen); + if (!test_buffers[i]) + goto out_free_buffers; + } + + prandom_seed_state(&rng, XOR_KUNIT_SEED); + xor_generate_random_data(); + return 0; + +out_free_buffers: + while (--i >= 0) + vfree(test_buffers[i]); + vfree(test_dest); +out_free_ref: + vfree(test_ref); + return -ENOMEM; +} + +static void xor_suite_exit(struct kunit_suite *suite) +{ + int i; + + vfree(test_ref); + vfree(test_dest); + for (i = 0; i < XOR_KUNIT_MAX_BUFFERS; i++) + vfree(test_buffers[i]); +} + +static struct kunit_suite xor_test_suite = { + .name = "xor", + .test_cases = xor_test_cases, + .suite_init = xor_suite_init, + .suite_exit = xor_suite_exit, +}; +kunit_test_suite(xor_test_suite); + +MODULE_DESCRIPTION("Unit test for the XOR library functions"); +MODULE_LICENSE("GPL"); diff --git a/lib/raid/xor/um/xor_arch.h b/lib/raid/xor/um/xor_arch.h new file mode 100644 index 000000000000..a33e57a26c5e --- /dev/null +++ b/lib/raid/xor/um/xor_arch.h @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include <../x86/xor_arch.h> diff --git a/lib/raid/xor/x86/xor-avx.c b/lib/raid/xor/x86/xor-avx.c new file mode 100644 index 000000000000..f7777d7aa269 --- /dev/null +++ b/lib/raid/xor/x86/xor-avx.c @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Optimized XOR parity functions for AVX + * + * Copyright (C) 2012 Intel Corporation + * Author: Jim Kukunas <james.t.kukunas@linux.intel.com> + * + * Based on Ingo Molnar and Zach Brown's respective MMX and SSE routines + */ +#include <linux/compiler.h> +#include <asm/fpu/api.h> +#include "xor_impl.h" +#include "xor_arch.h" + +#define BLOCK4(i) \ + BLOCK(32 * i, 0) \ + BLOCK(32 * (i + 1), 1) \ + BLOCK(32 * (i + 2), 2) \ + BLOCK(32 * (i + 3), 3) + +#define BLOCK16() \ + BLOCK4(0) \ + BLOCK4(4) \ + BLOCK4(8) \ + BLOCK4(12) + +static void xor_avx_2(unsigned long bytes, unsigned long * __restrict p0, + const unsigned long * __restrict p1) +{ + unsigned long lines = bytes >> 9; + + while (lines--) { +#undef BLOCK +#define BLOCK(i, reg) \ +do { \ + asm volatile("vmovdqa %0, %%ymm" #reg : : "m" (p1[i / sizeof(*p1)])); \ + asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \ + "m" (p0[i / sizeof(*p0)])); \ + asm volatile("vmovdqa %%ymm" #reg ", %0" : \ + "=m" (p0[i / sizeof(*p0)])); \ +} while (0); + + BLOCK16() + + p0 = (unsigned long *)((uintptr_t)p0 + 512); + p1 = (unsigned long *)((uintptr_t)p1 + 512); + } +} + +static void xor_avx_3(unsigned long bytes, unsigned long * __restrict p0, + const unsigned long * __restrict p1, + const unsigned long * __restrict p2) +{ + unsigned long lines = bytes >> 9; + + while (lines--) { +#undef BLOCK +#define BLOCK(i, reg) \ +do { \ + asm volatile("vmovdqa %0, %%ymm" #reg : : "m" (p2[i / sizeof(*p2)])); \ + asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \ + "m" (p1[i / sizeof(*p1)])); \ + asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \ + "m" (p0[i / sizeof(*p0)])); \ + asm volatile("vmovdqa %%ymm" #reg ", %0" : \ + "=m" (p0[i / sizeof(*p0)])); \ +} while (0); + + BLOCK16() + + p0 = (unsigned long *)((uintptr_t)p0 + 512); + p1 = (unsigned long *)((uintptr_t)p1 + 512); + p2 = (unsigned long *)((uintptr_t)p2 + 512); + } +} + +static void xor_avx_4(unsigned long bytes, unsigned long * __restrict p0, + const unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) +{ + unsigned long lines = bytes >> 9; + + while (lines--) { +#undef BLOCK +#define BLOCK(i, reg) \ +do { \ + asm volatile("vmovdqa %0, %%ymm" #reg : : "m" (p3[i / sizeof(*p3)])); \ + asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \ + "m" (p2[i / sizeof(*p2)])); \ + asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \ + "m" (p1[i / sizeof(*p1)])); \ + asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \ + "m" (p0[i / sizeof(*p0)])); \ + asm volatile("vmovdqa %%ymm" #reg ", %0" : \ + "=m" (p0[i / sizeof(*p0)])); \ +} while (0); + + BLOCK16(); + + p0 = (unsigned long *)((uintptr_t)p0 + 512); + p1 = (unsigned long *)((uintptr_t)p1 + 512); + p2 = (unsigned long *)((uintptr_t)p2 + 512); + p3 = (unsigned long *)((uintptr_t)p3 + 512); + } +} + +static void xor_avx_5(unsigned long bytes, unsigned long * __restrict p0, + const unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) +{ + unsigned long lines = bytes >> 9; + + while (lines--) { +#undef BLOCK +#define BLOCK(i, reg) \ +do { \ + asm volatile("vmovdqa %0, %%ymm" #reg : : "m" (p4[i / sizeof(*p4)])); \ + asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \ + "m" (p3[i / sizeof(*p3)])); \ + asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \ + "m" (p2[i / sizeof(*p2)])); \ + asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \ + "m" (p1[i / sizeof(*p1)])); \ + asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \ + "m" (p0[i / sizeof(*p0)])); \ + asm volatile("vmovdqa %%ymm" #reg ", %0" : \ + "=m" (p0[i / sizeof(*p0)])); \ +} while (0); + + BLOCK16() + + p0 = (unsigned long *)((uintptr_t)p0 + 512); + p1 = (unsigned long *)((uintptr_t)p1 + 512); + p2 = (unsigned long *)((uintptr_t)p2 + 512); + p3 = (unsigned long *)((uintptr_t)p3 + 512); + p4 = (unsigned long *)((uintptr_t)p4 + 512); + } +} + +DO_XOR_BLOCKS(avx_inner, xor_avx_2, xor_avx_3, xor_avx_4, xor_avx_5); + +static void xor_gen_avx(void *dest, void **srcs, unsigned int src_cnt, + unsigned int bytes) +{ + kernel_fpu_begin(); + xor_gen_avx_inner(dest, srcs, src_cnt, bytes); + kernel_fpu_end(); +} + +struct xor_block_template xor_block_avx = { + .name = "avx", + .xor_gen = xor_gen_avx, +}; diff --git a/lib/raid/xor/x86/xor-mmx.c b/lib/raid/xor/x86/xor-mmx.c new file mode 100644 index 000000000000..63a8b0444fce --- /dev/null +++ b/lib/raid/xor/x86/xor-mmx.c @@ -0,0 +1,515 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Optimized XOR parity functions for MMX. + * + * Copyright (C) 1998 Ingo Molnar. + */ +#include <asm/fpu/api.h> +#include "xor_impl.h" +#include "xor_arch.h" + +#define LD(x, y) " movq 8*("#x")(%1), %%mm"#y" ;\n" +#define ST(x, y) " movq %%mm"#y", 8*("#x")(%1) ;\n" +#define XO1(x, y) " pxor 8*("#x")(%2), %%mm"#y" ;\n" +#define XO2(x, y) " pxor 8*("#x")(%3), %%mm"#y" ;\n" +#define XO3(x, y) " pxor 8*("#x")(%4), %%mm"#y" ;\n" +#define XO4(x, y) " pxor 8*("#x")(%5), %%mm"#y" ;\n" + +static void +xor_pII_mmx_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) +{ + unsigned long lines = bytes >> 7; + + asm volatile( +#undef BLOCK +#define BLOCK(i) \ + LD(i, 0) \ + LD(i + 1, 1) \ + LD(i + 2, 2) \ + LD(i + 3, 3) \ + XO1(i, 0) \ + ST(i, 0) \ + XO1(i+1, 1) \ + ST(i+1, 1) \ + XO1(i + 2, 2) \ + ST(i + 2, 2) \ + XO1(i + 3, 3) \ + ST(i + 3, 3) + + " .align 32 ;\n" + " 1: ;\n" + + BLOCK(0) + BLOCK(4) + BLOCK(8) + BLOCK(12) + + " addl $128, %1 ;\n" + " addl $128, %2 ;\n" + " decl %0 ;\n" + " jnz 1b ;\n" + : "+r" (lines), + "+r" (p1), "+r" (p2) + : + : "memory"); +} + +static void +xor_pII_mmx_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) +{ + unsigned long lines = bytes >> 7; + + asm volatile( +#undef BLOCK +#define BLOCK(i) \ + LD(i, 0) \ + LD(i + 1, 1) \ + LD(i + 2, 2) \ + LD(i + 3, 3) \ + XO1(i, 0) \ + XO1(i + 1, 1) \ + XO1(i + 2, 2) \ + XO1(i + 3, 3) \ + XO2(i, 0) \ + ST(i, 0) \ + XO2(i + 1, 1) \ + ST(i + 1, 1) \ + XO2(i + 2, 2) \ + ST(i + 2, 2) \ + XO2(i + 3, 3) \ + ST(i + 3, 3) + + " .align 32 ;\n" + " 1: ;\n" + + BLOCK(0) + BLOCK(4) + BLOCK(8) + BLOCK(12) + + " addl $128, %1 ;\n" + " addl $128, %2 ;\n" + " addl $128, %3 ;\n" + " decl %0 ;\n" + " jnz 1b ;\n" + : "+r" (lines), + "+r" (p1), "+r" (p2), "+r" (p3) + : + : "memory"); +} + +static void +xor_pII_mmx_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) +{ + unsigned long lines = bytes >> 7; + + asm volatile( +#undef BLOCK +#define BLOCK(i) \ + LD(i, 0) \ + LD(i + 1, 1) \ + LD(i + 2, 2) \ + LD(i + 3, 3) \ + XO1(i, 0) \ + XO1(i + 1, 1) \ + XO1(i + 2, 2) \ + XO1(i + 3, 3) \ + XO2(i, 0) \ + XO2(i + 1, 1) \ + XO2(i + 2, 2) \ + XO2(i + 3, 3) \ + XO3(i, 0) \ + ST(i, 0) \ + XO3(i + 1, 1) \ + ST(i + 1, 1) \ + XO3(i + 2, 2) \ + ST(i + 2, 2) \ + XO3(i + 3, 3) \ + ST(i + 3, 3) + + " .align 32 ;\n" + " 1: ;\n" + + BLOCK(0) + BLOCK(4) + BLOCK(8) + BLOCK(12) + + " addl $128, %1 ;\n" + " addl $128, %2 ;\n" + " addl $128, %3 ;\n" + " addl $128, %4 ;\n" + " decl %0 ;\n" + " jnz 1b ;\n" + : "+r" (lines), + "+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4) + : + : "memory"); +} + + +static void +xor_pII_mmx_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) +{ + unsigned long lines = bytes >> 7; + + /* Make sure GCC forgets anything it knows about p4 or p5, + such that it won't pass to the asm volatile below a + register that is shared with any other variable. That's + because we modify p4 and p5 there, but we can't mark them + as read/write, otherwise we'd overflow the 10-asm-operands + limit of GCC < 3.1. */ + asm("" : "+r" (p4), "+r" (p5)); + + asm volatile( +#undef BLOCK +#define BLOCK(i) \ + LD(i, 0) \ + LD(i + 1, 1) \ + LD(i + 2, 2) \ + LD(i + 3, 3) \ + XO1(i, 0) \ + XO1(i + 1, 1) \ + XO1(i + 2, 2) \ + XO1(i + 3, 3) \ + XO2(i, 0) \ + XO2(i + 1, 1) \ + XO2(i + 2, 2) \ + XO2(i + 3, 3) \ + XO3(i, 0) \ + XO3(i + 1, 1) \ + XO3(i + 2, 2) \ + XO3(i + 3, 3) \ + XO4(i, 0) \ + ST(i, 0) \ + XO4(i + 1, 1) \ + ST(i + 1, 1) \ + XO4(i + 2, 2) \ + ST(i + 2, 2) \ + XO4(i + 3, 3) \ + ST(i + 3, 3) + + " .align 32 ;\n" + " 1: ;\n" + + BLOCK(0) + BLOCK(4) + BLOCK(8) + BLOCK(12) + + " addl $128, %1 ;\n" + " addl $128, %2 ;\n" + " addl $128, %3 ;\n" + " addl $128, %4 ;\n" + " addl $128, %5 ;\n" + " decl %0 ;\n" + " jnz 1b ;\n" + : "+r" (lines), + "+r" (p1), "+r" (p2), "+r" (p3) + : "r" (p4), "r" (p5) + : "memory"); + + /* p4 and p5 were modified, and now the variables are dead. + Clobber them just to be sure nobody does something stupid + like assuming they have some legal value. */ + asm("" : "=r" (p4), "=r" (p5)); +} + +#undef LD +#undef XO1 +#undef XO2 +#undef XO3 +#undef XO4 +#undef ST +#undef BLOCK + +static void +xor_p5_mmx_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) +{ + unsigned long lines = bytes >> 6; + + asm volatile( + " .align 32 ;\n" + " 1: ;\n" + " movq (%1), %%mm0 ;\n" + " movq 8(%1), %%mm1 ;\n" + " pxor (%2), %%mm0 ;\n" + " movq 16(%1), %%mm2 ;\n" + " movq %%mm0, (%1) ;\n" + " pxor 8(%2), %%mm1 ;\n" + " movq 24(%1), %%mm3 ;\n" + " movq %%mm1, 8(%1) ;\n" + " pxor 16(%2), %%mm2 ;\n" + " movq 32(%1), %%mm4 ;\n" + " movq %%mm2, 16(%1) ;\n" + " pxor 24(%2), %%mm3 ;\n" + " movq 40(%1), %%mm5 ;\n" + " movq %%mm3, 24(%1) ;\n" + " pxor 32(%2), %%mm4 ;\n" + " movq 48(%1), %%mm6 ;\n" + " movq %%mm4, 32(%1) ;\n" + " pxor 40(%2), %%mm5 ;\n" + " movq 56(%1), %%mm7 ;\n" + " movq %%mm5, 40(%1) ;\n" + " pxor 48(%2), %%mm6 ;\n" + " pxor 56(%2), %%mm7 ;\n" + " movq %%mm6, 48(%1) ;\n" + " movq %%mm7, 56(%1) ;\n" + + " addl $64, %1 ;\n" + " addl $64, %2 ;\n" + " decl %0 ;\n" + " jnz 1b ;\n" + : "+r" (lines), + "+r" (p1), "+r" (p2) + : + : "memory"); +} + +static void +xor_p5_mmx_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) +{ + unsigned long lines = bytes >> 6; + + asm volatile( + " .align 32,0x90 ;\n" + " 1: ;\n" + " movq (%1), %%mm0 ;\n" + " movq 8(%1), %%mm1 ;\n" + " pxor (%2), %%mm0 ;\n" + " movq 16(%1), %%mm2 ;\n" + " pxor 8(%2), %%mm1 ;\n" + " pxor (%3), %%mm0 ;\n" + " pxor 16(%2), %%mm2 ;\n" + " movq %%mm0, (%1) ;\n" + " pxor 8(%3), %%mm1 ;\n" + " pxor 16(%3), %%mm2 ;\n" + " movq 24(%1), %%mm3 ;\n" + " movq %%mm1, 8(%1) ;\n" + " movq 32(%1), %%mm4 ;\n" + " movq 40(%1), %%mm5 ;\n" + " pxor 24(%2), %%mm3 ;\n" + " movq %%mm2, 16(%1) ;\n" + " pxor 32(%2), %%mm4 ;\n" + " pxor 24(%3), %%mm3 ;\n" + " pxor 40(%2), %%mm5 ;\n" + " movq %%mm3, 24(%1) ;\n" + " pxor 32(%3), %%mm4 ;\n" + " pxor 40(%3), %%mm5 ;\n" + " movq 48(%1), %%mm6 ;\n" + " movq %%mm4, 32(%1) ;\n" + " movq 56(%1), %%mm7 ;\n" + " pxor 48(%2), %%mm6 ;\n" + " movq %%mm5, 40(%1) ;\n" + " pxor 56(%2), %%mm7 ;\n" + " pxor 48(%3), %%mm6 ;\n" + " pxor 56(%3), %%mm7 ;\n" + " movq %%mm6, 48(%1) ;\n" + " movq %%mm7, 56(%1) ;\n" + + " addl $64, %1 ;\n" + " addl $64, %2 ;\n" + " addl $64, %3 ;\n" + " decl %0 ;\n" + " jnz 1b ;\n" + : "+r" (lines), + "+r" (p1), "+r" (p2), "+r" (p3) + : + : "memory" ); +} + +static void +xor_p5_mmx_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) +{ + unsigned long lines = bytes >> 6; + + asm volatile( + " .align 32,0x90 ;\n" + " 1: ;\n" + " movq (%1), %%mm0 ;\n" + " movq 8(%1), %%mm1 ;\n" + " pxor (%2), %%mm0 ;\n" + " movq 16(%1), %%mm2 ;\n" + " pxor 8(%2), %%mm1 ;\n" + " pxor (%3), %%mm0 ;\n" + " pxor 16(%2), %%mm2 ;\n" + " pxor 8(%3), %%mm1 ;\n" + " pxor (%4), %%mm0 ;\n" + " movq 24(%1), %%mm3 ;\n" + " pxor 16(%3), %%mm2 ;\n" + " pxor 8(%4), %%mm1 ;\n" + " movq %%mm0, (%1) ;\n" + " movq 32(%1), %%mm4 ;\n" + " pxor 24(%2), %%mm3 ;\n" + " pxor 16(%4), %%mm2 ;\n" + " movq %%mm1, 8(%1) ;\n" + " movq 40(%1), %%mm5 ;\n" + " pxor 32(%2), %%mm4 ;\n" + " pxor 24(%3), %%mm3 ;\n" + " movq %%mm2, 16(%1) ;\n" + " pxor 40(%2), %%mm5 ;\n" + " pxor 32(%3), %%mm4 ;\n" + " pxor 24(%4), %%mm3 ;\n" + " movq %%mm3, 24(%1) ;\n" + " movq 56(%1), %%mm7 ;\n" + " movq 48(%1), %%mm6 ;\n" + " pxor 40(%3), %%mm5 ;\n" + " pxor 32(%4), %%mm4 ;\n" + " pxor 48(%2), %%mm6 ;\n" + " movq %%mm4, 32(%1) ;\n" + " pxor 56(%2), %%mm7 ;\n" + " pxor 40(%4), %%mm5 ;\n" + " pxor 48(%3), %%mm6 ;\n" + " pxor 56(%3), %%mm7 ;\n" + " movq %%mm5, 40(%1) ;\n" + " pxor 48(%4), %%mm6 ;\n" + " pxor 56(%4), %%mm7 ;\n" + " movq %%mm6, 48(%1) ;\n" + " movq %%mm7, 56(%1) ;\n" + + " addl $64, %1 ;\n" + " addl $64, %2 ;\n" + " addl $64, %3 ;\n" + " addl $64, %4 ;\n" + " decl %0 ;\n" + " jnz 1b ;\n" + : "+r" (lines), + "+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4) + : + : "memory"); +} + +static void +xor_p5_mmx_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) +{ + unsigned long lines = bytes >> 6; + + /* Make sure GCC forgets anything it knows about p4 or p5, + such that it won't pass to the asm volatile below a + register that is shared with any other variable. That's + because we modify p4 and p5 there, but we can't mark them + as read/write, otherwise we'd overflow the 10-asm-operands + limit of GCC < 3.1. */ + asm("" : "+r" (p4), "+r" (p5)); + + asm volatile( + " .align 32,0x90 ;\n" + " 1: ;\n" + " movq (%1), %%mm0 ;\n" + " movq 8(%1), %%mm1 ;\n" + " pxor (%2), %%mm0 ;\n" + " pxor 8(%2), %%mm1 ;\n" + " movq 16(%1), %%mm2 ;\n" + " pxor (%3), %%mm0 ;\n" + " pxor 8(%3), %%mm1 ;\n" + " pxor 16(%2), %%mm2 ;\n" + " pxor (%4), %%mm0 ;\n" + " pxor 8(%4), %%mm1 ;\n" + " pxor 16(%3), %%mm2 ;\n" + " movq 24(%1), %%mm3 ;\n" + " pxor (%5), %%mm0 ;\n" + " pxor 8(%5), %%mm1 ;\n" + " movq %%mm0, (%1) ;\n" + " pxor 16(%4), %%mm2 ;\n" + " pxor 24(%2), %%mm3 ;\n" + " movq %%mm1, 8(%1) ;\n" + " pxor 16(%5), %%mm2 ;\n" + " pxor 24(%3), %%mm3 ;\n" + " movq 32(%1), %%mm4 ;\n" + " movq %%mm2, 16(%1) ;\n" + " pxor 24(%4), %%mm3 ;\n" + " pxor 32(%2), %%mm4 ;\n" + " movq 40(%1), %%mm5 ;\n" + " pxor 24(%5), %%mm3 ;\n" + " pxor 32(%3), %%mm4 ;\n" + " pxor 40(%2), %%mm5 ;\n" + " movq %%mm3, 24(%1) ;\n" + " pxor 32(%4), %%mm4 ;\n" + " pxor 40(%3), %%mm5 ;\n" + " movq 48(%1), %%mm6 ;\n" + " movq 56(%1), %%mm7 ;\n" + " pxor 32(%5), %%mm4 ;\n" + " pxor 40(%4), %%mm5 ;\n" + " pxor 48(%2), %%mm6 ;\n" + " pxor 56(%2), %%mm7 ;\n" + " movq %%mm4, 32(%1) ;\n" + " pxor 48(%3), %%mm6 ;\n" + " pxor 56(%3), %%mm7 ;\n" + " pxor 40(%5), %%mm5 ;\n" + " pxor 48(%4), %%mm6 ;\n" + " pxor 56(%4), %%mm7 ;\n" + " movq %%mm5, 40(%1) ;\n" + " pxor 48(%5), %%mm6 ;\n" + " pxor 56(%5), %%mm7 ;\n" + " movq %%mm6, 48(%1) ;\n" + " movq %%mm7, 56(%1) ;\n" + + " addl $64, %1 ;\n" + " addl $64, %2 ;\n" + " addl $64, %3 ;\n" + " addl $64, %4 ;\n" + " addl $64, %5 ;\n" + " decl %0 ;\n" + " jnz 1b ;\n" + : "+r" (lines), + "+r" (p1), "+r" (p2), "+r" (p3) + : "r" (p4), "r" (p5) + : "memory"); + + /* p4 and p5 were modified, and now the variables are dead. + Clobber them just to be sure nobody does something stupid + like assuming they have some legal value. */ + asm("" : "=r" (p4), "=r" (p5)); +} + +DO_XOR_BLOCKS(pII_mmx_inner, xor_pII_mmx_2, xor_pII_mmx_3, xor_pII_mmx_4, + xor_pII_mmx_5); + +static void xor_gen_pII_mmx(void *dest, void **srcs, unsigned int src_cnt, + unsigned int bytes) +{ + kernel_fpu_begin(); + xor_gen_pII_mmx_inner(dest, srcs, src_cnt, bytes); + kernel_fpu_end(); +} + +struct xor_block_template xor_block_pII_mmx = { + .name = "pII_mmx", + .xor_gen = xor_gen_pII_mmx, +}; + +DO_XOR_BLOCKS(p5_mmx_inner, xor_p5_mmx_2, xor_p5_mmx_3, xor_p5_mmx_4, + xor_p5_mmx_5); + +static void xor_gen_p5_mmx(void *dest, void **srcs, unsigned int src_cnt, + unsigned int bytes) +{ + kernel_fpu_begin(); + xor_gen_p5_mmx_inner(dest, srcs, src_cnt, bytes); + kernel_fpu_end(); +} + +struct xor_block_template xor_block_p5_mmx = { + .name = "p5_mmx", + .xor_gen = xor_gen_p5_mmx, +}; diff --git a/lib/raid/xor/x86/xor-sse.c b/lib/raid/xor/x86/xor-sse.c new file mode 100644 index 000000000000..c6626ecae6ba --- /dev/null +++ b/lib/raid/xor/x86/xor-sse.c @@ -0,0 +1,459 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Optimized XOR parity functions for SSE. + * + * Cache avoiding checksumming functions utilizing KNI instructions + * Copyright (C) 1999 Zach Brown (with obvious credit due Ingo) + * + * Based on + * High-speed RAID5 checksumming functions utilizing SSE instructions. + * Copyright (C) 1998 Ingo Molnar. + * + * x86-64 changes / gcc fixes from Andi Kleen. + * Copyright 2002 Andi Kleen, SuSE Labs. + */ +#include <asm/fpu/api.h> +#include "xor_impl.h" +#include "xor_arch.h" + +#ifdef CONFIG_X86_32 +/* reduce register pressure */ +# define XOR_CONSTANT_CONSTRAINT "i" +#else +# define XOR_CONSTANT_CONSTRAINT "re" +#endif + +#define OFFS(x) "16*("#x")" +#define PF_OFFS(x) "256+16*("#x")" +#define PF0(x) " prefetchnta "PF_OFFS(x)"(%[p1]) ;\n" +#define LD(x, y) " movaps "OFFS(x)"(%[p1]), %%xmm"#y" ;\n" +#define ST(x, y) " movaps %%xmm"#y", "OFFS(x)"(%[p1]) ;\n" +#define PF1(x) " prefetchnta "PF_OFFS(x)"(%[p2]) ;\n" +#define PF2(x) " prefetchnta "PF_OFFS(x)"(%[p3]) ;\n" +#define PF3(x) " prefetchnta "PF_OFFS(x)"(%[p4]) ;\n" +#define PF4(x) " prefetchnta "PF_OFFS(x)"(%[p5]) ;\n" +#define XO1(x, y) " xorps "OFFS(x)"(%[p2]), %%xmm"#y" ;\n" +#define XO2(x, y) " xorps "OFFS(x)"(%[p3]), %%xmm"#y" ;\n" +#define XO3(x, y) " xorps "OFFS(x)"(%[p4]), %%xmm"#y" ;\n" +#define XO4(x, y) " xorps "OFFS(x)"(%[p5]), %%xmm"#y" ;\n" +#define NOP(x) + +#define BLK64(pf, op, i) \ + pf(i) \ + op(i, 0) \ + op(i + 1, 1) \ + op(i + 2, 2) \ + op(i + 3, 3) + +static void +xor_sse_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) +{ + unsigned long lines = bytes >> 8; + + asm volatile( +#undef BLOCK +#define BLOCK(i) \ + LD(i, 0) \ + LD(i + 1, 1) \ + PF1(i) \ + PF1(i + 2) \ + LD(i + 2, 2) \ + LD(i + 3, 3) \ + PF0(i + 4) \ + PF0(i + 6) \ + XO1(i, 0) \ + XO1(i + 1, 1) \ + XO1(i + 2, 2) \ + XO1(i + 3, 3) \ + ST(i, 0) \ + ST(i + 1, 1) \ + ST(i + 2, 2) \ + ST(i + 3, 3) \ + + + PF0(0) + PF0(2) + + " .align 32 ;\n" + " 1: ;\n" + + BLOCK(0) + BLOCK(4) + BLOCK(8) + BLOCK(12) + + " add %[inc], %[p1] ;\n" + " add %[inc], %[p2] ;\n" + " dec %[cnt] ;\n" + " jnz 1b ;\n" + : [cnt] "+r" (lines), + [p1] "+r" (p1), [p2] "+r" (p2) + : [inc] XOR_CONSTANT_CONSTRAINT (256UL) + : "memory"); +} + +static void +xor_sse_2_pf64(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) +{ + unsigned long lines = bytes >> 8; + + asm volatile( +#undef BLOCK +#define BLOCK(i) \ + BLK64(PF0, LD, i) \ + BLK64(PF1, XO1, i) \ + BLK64(NOP, ST, i) \ + + " .align 32 ;\n" + " 1: ;\n" + + BLOCK(0) + BLOCK(4) + BLOCK(8) + BLOCK(12) + + " add %[inc], %[p1] ;\n" + " add %[inc], %[p2] ;\n" + " dec %[cnt] ;\n" + " jnz 1b ;\n" + : [cnt] "+r" (lines), + [p1] "+r" (p1), [p2] "+r" (p2) + : [inc] XOR_CONSTANT_CONSTRAINT (256UL) + : "memory"); +} + +static void +xor_sse_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) +{ + unsigned long lines = bytes >> 8; + + asm volatile( +#undef BLOCK +#define BLOCK(i) \ + PF1(i) \ + PF1(i + 2) \ + LD(i, 0) \ + LD(i + 1, 1) \ + LD(i + 2, 2) \ + LD(i + 3, 3) \ + PF2(i) \ + PF2(i + 2) \ + PF0(i + 4) \ + PF0(i + 6) \ + XO1(i, 0) \ + XO1(i + 1, 1) \ + XO1(i + 2, 2) \ + XO1(i + 3, 3) \ + XO2(i, 0) \ + XO2(i + 1, 1) \ + XO2(i + 2, 2) \ + XO2(i + 3, 3) \ + ST(i, 0) \ + ST(i + 1, 1) \ + ST(i + 2, 2) \ + ST(i + 3, 3) \ + + + PF0(0) + PF0(2) + + " .align 32 ;\n" + " 1: ;\n" + + BLOCK(0) + BLOCK(4) + BLOCK(8) + BLOCK(12) + + " add %[inc], %[p1] ;\n" + " add %[inc], %[p2] ;\n" + " add %[inc], %[p3] ;\n" + " dec %[cnt] ;\n" + " jnz 1b ;\n" + : [cnt] "+r" (lines), + [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3) + : [inc] XOR_CONSTANT_CONSTRAINT (256UL) + : "memory"); +} + +static void +xor_sse_3_pf64(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) +{ + unsigned long lines = bytes >> 8; + + asm volatile( +#undef BLOCK +#define BLOCK(i) \ + BLK64(PF0, LD, i) \ + BLK64(PF1, XO1, i) \ + BLK64(PF2, XO2, i) \ + BLK64(NOP, ST, i) \ + + " .align 32 ;\n" + " 1: ;\n" + + BLOCK(0) + BLOCK(4) + BLOCK(8) + BLOCK(12) + + " add %[inc], %[p1] ;\n" + " add %[inc], %[p2] ;\n" + " add %[inc], %[p3] ;\n" + " dec %[cnt] ;\n" + " jnz 1b ;\n" + : [cnt] "+r" (lines), + [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3) + : [inc] XOR_CONSTANT_CONSTRAINT (256UL) + : "memory"); +} + +static void +xor_sse_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) +{ + unsigned long lines = bytes >> 8; + + asm volatile( +#undef BLOCK +#define BLOCK(i) \ + PF1(i) \ + PF1(i + 2) \ + LD(i, 0) \ + LD(i + 1, 1) \ + LD(i + 2, 2) \ + LD(i + 3, 3) \ + PF2(i) \ + PF2(i + 2) \ + XO1(i, 0) \ + XO1(i + 1, 1) \ + XO1(i + 2, 2) \ + XO1(i + 3, 3) \ + PF3(i) \ + PF3(i + 2) \ + PF0(i + 4) \ + PF0(i + 6) \ + XO2(i, 0) \ + XO2(i + 1, 1) \ + XO2(i + 2, 2) \ + XO2(i + 3, 3) \ + XO3(i, 0) \ + XO3(i + 1, 1) \ + XO3(i + 2, 2) \ + XO3(i + 3, 3) \ + ST(i, 0) \ + ST(i + 1, 1) \ + ST(i + 2, 2) \ + ST(i + 3, 3) \ + + + PF0(0) + PF0(2) + + " .align 32 ;\n" + " 1: ;\n" + + BLOCK(0) + BLOCK(4) + BLOCK(8) + BLOCK(12) + + " add %[inc], %[p1] ;\n" + " add %[inc], %[p2] ;\n" + " add %[inc], %[p3] ;\n" + " add %[inc], %[p4] ;\n" + " dec %[cnt] ;\n" + " jnz 1b ;\n" + : [cnt] "+r" (lines), [p1] "+r" (p1), + [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4) + : [inc] XOR_CONSTANT_CONSTRAINT (256UL) + : "memory"); +} + +static void +xor_sse_4_pf64(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) +{ + unsigned long lines = bytes >> 8; + + asm volatile( +#undef BLOCK +#define BLOCK(i) \ + BLK64(PF0, LD, i) \ + BLK64(PF1, XO1, i) \ + BLK64(PF2, XO2, i) \ + BLK64(PF3, XO3, i) \ + BLK64(NOP, ST, i) \ + + " .align 32 ;\n" + " 1: ;\n" + + BLOCK(0) + BLOCK(4) + BLOCK(8) + BLOCK(12) + + " add %[inc], %[p1] ;\n" + " add %[inc], %[p2] ;\n" + " add %[inc], %[p3] ;\n" + " add %[inc], %[p4] ;\n" + " dec %[cnt] ;\n" + " jnz 1b ;\n" + : [cnt] "+r" (lines), [p1] "+r" (p1), + [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4) + : [inc] XOR_CONSTANT_CONSTRAINT (256UL) + : "memory"); +} + +static void +xor_sse_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) +{ + unsigned long lines = bytes >> 8; + + asm volatile( +#undef BLOCK +#define BLOCK(i) \ + PF1(i) \ + PF1(i + 2) \ + LD(i, 0) \ + LD(i + 1, 1) \ + LD(i + 2, 2) \ + LD(i + 3, 3) \ + PF2(i) \ + PF2(i + 2) \ + XO1(i, 0) \ + XO1(i + 1, 1) \ + XO1(i + 2, 2) \ + XO1(i + 3, 3) \ + PF3(i) \ + PF3(i + 2) \ + XO2(i, 0) \ + XO2(i + 1, 1) \ + XO2(i + 2, 2) \ + XO2(i + 3, 3) \ + PF4(i) \ + PF4(i + 2) \ + PF0(i + 4) \ + PF0(i + 6) \ + XO3(i, 0) \ + XO3(i + 1, 1) \ + XO3(i + 2, 2) \ + XO3(i + 3, 3) \ + XO4(i, 0) \ + XO4(i + 1, 1) \ + XO4(i + 2, 2) \ + XO4(i + 3, 3) \ + ST(i, 0) \ + ST(i + 1, 1) \ + ST(i + 2, 2) \ + ST(i + 3, 3) \ + + + PF0(0) + PF0(2) + + " .align 32 ;\n" + " 1: ;\n" + + BLOCK(0) + BLOCK(4) + BLOCK(8) + BLOCK(12) + + " add %[inc], %[p1] ;\n" + " add %[inc], %[p2] ;\n" + " add %[inc], %[p3] ;\n" + " add %[inc], %[p4] ;\n" + " add %[inc], %[p5] ;\n" + " dec %[cnt] ;\n" + " jnz 1b ;\n" + : [cnt] "+r" (lines), [p1] "+r" (p1), [p2] "+r" (p2), + [p3] "+r" (p3), [p4] "+r" (p4), [p5] "+r" (p5) + : [inc] XOR_CONSTANT_CONSTRAINT (256UL) + : "memory"); +} + +static void +xor_sse_5_pf64(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) +{ + unsigned long lines = bytes >> 8; + + asm volatile( +#undef BLOCK +#define BLOCK(i) \ + BLK64(PF0, LD, i) \ + BLK64(PF1, XO1, i) \ + BLK64(PF2, XO2, i) \ + BLK64(PF3, XO3, i) \ + BLK64(PF4, XO4, i) \ + BLK64(NOP, ST, i) \ + + " .align 32 ;\n" + " 1: ;\n" + + BLOCK(0) + BLOCK(4) + BLOCK(8) + BLOCK(12) + + " add %[inc], %[p1] ;\n" + " add %[inc], %[p2] ;\n" + " add %[inc], %[p3] ;\n" + " add %[inc], %[p4] ;\n" + " add %[inc], %[p5] ;\n" + " dec %[cnt] ;\n" + " jnz 1b ;\n" + : [cnt] "+r" (lines), [p1] "+r" (p1), [p2] "+r" (p2), + [p3] "+r" (p3), [p4] "+r" (p4), [p5] "+r" (p5) + : [inc] XOR_CONSTANT_CONSTRAINT (256UL) + : "memory"); +} + +DO_XOR_BLOCKS(sse_inner, xor_sse_2, xor_sse_3, xor_sse_4, xor_sse_5); + +static void xor_gen_sse(void *dest, void **srcs, unsigned int src_cnt, + unsigned int bytes) +{ + kernel_fpu_begin(); + xor_gen_sse_inner(dest, srcs, src_cnt, bytes); + kernel_fpu_end(); +} + +struct xor_block_template xor_block_sse = { + .name = "sse", + .xor_gen = xor_gen_sse, +}; + +DO_XOR_BLOCKS(sse_pf64_inner, xor_sse_2_pf64, xor_sse_3_pf64, xor_sse_4_pf64, + xor_sse_5_pf64); + +static void xor_gen_sse_pf64(void *dest, void **srcs, unsigned int src_cnt, + unsigned int bytes) +{ + kernel_fpu_begin(); + xor_gen_sse_pf64_inner(dest, srcs, src_cnt, bytes); + kernel_fpu_end(); +} + +struct xor_block_template xor_block_sse_pf64 = { + .name = "prefetch64-sse", + .xor_gen = xor_gen_sse_pf64, +}; diff --git a/lib/raid/xor/x86/xor_arch.h b/lib/raid/xor/x86/xor_arch.h new file mode 100644 index 000000000000..99fe85a213c6 --- /dev/null +++ b/lib/raid/xor/x86/xor_arch.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#include <asm/cpufeature.h> + +extern struct xor_block_template xor_block_pII_mmx; +extern struct xor_block_template xor_block_p5_mmx; +extern struct xor_block_template xor_block_sse; +extern struct xor_block_template xor_block_sse_pf64; +extern struct xor_block_template xor_block_avx; + +/* + * When SSE is available, use it as it can write around L2. We may also be able + * to load into the L1 only depending on how the cpu deals with a load to a line + * that is being prefetched. + * + * When AVX2 is available, force using it as it is better by all measures. + * + * 32-bit without MMX can fall back to the generic routines. + */ +static __always_inline void __init arch_xor_init(void) +{ + if (boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_OSXSAVE)) { + xor_force(&xor_block_avx); + } else if (IS_ENABLED(CONFIG_X86_64) || boot_cpu_has(X86_FEATURE_XMM)) { + xor_register(&xor_block_sse); + xor_register(&xor_block_sse_pf64); + } else if (boot_cpu_has(X86_FEATURE_MMX)) { + xor_register(&xor_block_pII_mmx); + xor_register(&xor_block_p5_mmx); + } else { + xor_register(&xor_block_8regs); + xor_register(&xor_block_8regs_p); + xor_register(&xor_block_32regs); + xor_register(&xor_block_32regs_p); + } +} diff --git a/lib/raid/xor/xor-32regs-prefetch.c b/lib/raid/xor/xor-32regs-prefetch.c new file mode 100644 index 000000000000..ade2a7d8cbe2 --- /dev/null +++ b/lib/raid/xor/xor-32regs-prefetch.c @@ -0,0 +1,267 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include <linux/prefetch.h> +#include "xor_impl.h" + +static void +xor_32regs_p_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) +{ + long lines = bytes / (sizeof (long)) / 8 - 1; + + prefetchw(p1); + prefetch(p2); + + do { + register long d0, d1, d2, d3, d4, d5, d6, d7; + + prefetchw(p1+8); + prefetch(p2+8); + once_more: + d0 = p1[0]; /* Pull the stuff into registers */ + d1 = p1[1]; /* ... in bursts, if possible. */ + d2 = p1[2]; + d3 = p1[3]; + d4 = p1[4]; + d5 = p1[5]; + d6 = p1[6]; + d7 = p1[7]; + d0 ^= p2[0]; + d1 ^= p2[1]; + d2 ^= p2[2]; + d3 ^= p2[3]; + d4 ^= p2[4]; + d5 ^= p2[5]; + d6 ^= p2[6]; + d7 ^= p2[7]; + p1[0] = d0; /* Store the result (in bursts) */ + p1[1] = d1; + p1[2] = d2; + p1[3] = d3; + p1[4] = d4; + p1[5] = d5; + p1[6] = d6; + p1[7] = d7; + p1 += 8; + p2 += 8; + } while (--lines > 0); + if (lines == 0) + goto once_more; +} + +static void +xor_32regs_p_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) +{ + long lines = bytes / (sizeof (long)) / 8 - 1; + + prefetchw(p1); + prefetch(p2); + prefetch(p3); + + do { + register long d0, d1, d2, d3, d4, d5, d6, d7; + + prefetchw(p1+8); + prefetch(p2+8); + prefetch(p3+8); + once_more: + d0 = p1[0]; /* Pull the stuff into registers */ + d1 = p1[1]; /* ... in bursts, if possible. */ + d2 = p1[2]; + d3 = p1[3]; + d4 = p1[4]; + d5 = p1[5]; + d6 = p1[6]; + d7 = p1[7]; + d0 ^= p2[0]; + d1 ^= p2[1]; + d2 ^= p2[2]; + d3 ^= p2[3]; + d4 ^= p2[4]; + d5 ^= p2[5]; + d6 ^= p2[6]; + d7 ^= p2[7]; + d0 ^= p3[0]; + d1 ^= p3[1]; + d2 ^= p3[2]; + d3 ^= p3[3]; + d4 ^= p3[4]; + d5 ^= p3[5]; + d6 ^= p3[6]; + d7 ^= p3[7]; + p1[0] = d0; /* Store the result (in bursts) */ + p1[1] = d1; + p1[2] = d2; + p1[3] = d3; + p1[4] = d4; + p1[5] = d5; + p1[6] = d6; + p1[7] = d7; + p1 += 8; + p2 += 8; + p3 += 8; + } while (--lines > 0); + if (lines == 0) + goto once_more; +} + +static void +xor_32regs_p_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) +{ + long lines = bytes / (sizeof (long)) / 8 - 1; + + prefetchw(p1); + prefetch(p2); + prefetch(p3); + prefetch(p4); + + do { + register long d0, d1, d2, d3, d4, d5, d6, d7; + + prefetchw(p1+8); + prefetch(p2+8); + prefetch(p3+8); + prefetch(p4+8); + once_more: + d0 = p1[0]; /* Pull the stuff into registers */ + d1 = p1[1]; /* ... in bursts, if possible. */ + d2 = p1[2]; + d3 = p1[3]; + d4 = p1[4]; + d5 = p1[5]; + d6 = p1[6]; + d7 = p1[7]; + d0 ^= p2[0]; + d1 ^= p2[1]; + d2 ^= p2[2]; + d3 ^= p2[3]; + d4 ^= p2[4]; + d5 ^= p2[5]; + d6 ^= p2[6]; + d7 ^= p2[7]; + d0 ^= p3[0]; + d1 ^= p3[1]; + d2 ^= p3[2]; + d3 ^= p3[3]; + d4 ^= p3[4]; + d5 ^= p3[5]; + d6 ^= p3[6]; + d7 ^= p3[7]; + d0 ^= p4[0]; + d1 ^= p4[1]; + d2 ^= p4[2]; + d3 ^= p4[3]; + d4 ^= p4[4]; + d5 ^= p4[5]; + d6 ^= p4[6]; + d7 ^= p4[7]; + p1[0] = d0; /* Store the result (in bursts) */ + p1[1] = d1; + p1[2] = d2; + p1[3] = d3; + p1[4] = d4; + p1[5] = d5; + p1[6] = d6; + p1[7] = d7; + p1 += 8; + p2 += 8; + p3 += 8; + p4 += 8; + } while (--lines > 0); + if (lines == 0) + goto once_more; +} + +static void +xor_32regs_p_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) +{ + long lines = bytes / (sizeof (long)) / 8 - 1; + + prefetchw(p1); + prefetch(p2); + prefetch(p3); + prefetch(p4); + prefetch(p5); + + do { + register long d0, d1, d2, d3, d4, d5, d6, d7; + + prefetchw(p1+8); + prefetch(p2+8); + prefetch(p3+8); + prefetch(p4+8); + prefetch(p5+8); + once_more: + d0 = p1[0]; /* Pull the stuff into registers */ + d1 = p1[1]; /* ... in bursts, if possible. */ + d2 = p1[2]; + d3 = p1[3]; + d4 = p1[4]; + d5 = p1[5]; + d6 = p1[6]; + d7 = p1[7]; + d0 ^= p2[0]; + d1 ^= p2[1]; + d2 ^= p2[2]; + d3 ^= p2[3]; + d4 ^= p2[4]; + d5 ^= p2[5]; + d6 ^= p2[6]; + d7 ^= p2[7]; + d0 ^= p3[0]; + d1 ^= p3[1]; + d2 ^= p3[2]; + d3 ^= p3[3]; + d4 ^= p3[4]; + d5 ^= p3[5]; + d6 ^= p3[6]; + d7 ^= p3[7]; + d0 ^= p4[0]; + d1 ^= p4[1]; + d2 ^= p4[2]; + d3 ^= p4[3]; + d4 ^= p4[4]; + d5 ^= p4[5]; + d6 ^= p4[6]; + d7 ^= p4[7]; + d0 ^= p5[0]; + d1 ^= p5[1]; + d2 ^= p5[2]; + d3 ^= p5[3]; + d4 ^= p5[4]; + d5 ^= p5[5]; + d6 ^= p5[6]; + d7 ^= p5[7]; + p1[0] = d0; /* Store the result (in bursts) */ + p1[1] = d1; + p1[2] = d2; + p1[3] = d3; + p1[4] = d4; + p1[5] = d5; + p1[6] = d6; + p1[7] = d7; + p1 += 8; + p2 += 8; + p3 += 8; + p4 += 8; + p5 += 8; + } while (--lines > 0); + if (lines == 0) + goto once_more; +} + +DO_XOR_BLOCKS(32regs_p, xor_32regs_p_2, xor_32regs_p_3, xor_32regs_p_4, + xor_32regs_p_5); + +struct xor_block_template xor_block_32regs_p = { + .name = "32regs_prefetch", + .xor_gen = xor_gen_32regs_p, +}; diff --git a/lib/raid/xor/xor-32regs.c b/lib/raid/xor/xor-32regs.c new file mode 100644 index 000000000000..acb4a10d1e95 --- /dev/null +++ b/lib/raid/xor/xor-32regs.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include "xor_impl.h" + +static void +xor_32regs_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) +{ + long lines = bytes / (sizeof (long)) / 8; + + do { + register long d0, d1, d2, d3, d4, d5, d6, d7; + d0 = p1[0]; /* Pull the stuff into registers */ + d1 = p1[1]; /* ... in bursts, if possible. */ + d2 = p1[2]; + d3 = p1[3]; + d4 = p1[4]; + d5 = p1[5]; + d6 = p1[6]; + d7 = p1[7]; + d0 ^= p2[0]; + d1 ^= p2[1]; + d2 ^= p2[2]; + d3 ^= p2[3]; + d4 ^= p2[4]; + d5 ^= p2[5]; + d6 ^= p2[6]; + d7 ^= p2[7]; + p1[0] = d0; /* Store the result (in bursts) */ + p1[1] = d1; + p1[2] = d2; + p1[3] = d3; + p1[4] = d4; + p1[5] = d5; + p1[6] = d6; + p1[7] = d7; + p1 += 8; + p2 += 8; + } while (--lines > 0); +} + +static void +xor_32regs_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) +{ + long lines = bytes / (sizeof (long)) / 8; + + do { + register long d0, d1, d2, d3, d4, d5, d6, d7; + d0 = p1[0]; /* Pull the stuff into registers */ + d1 = p1[1]; /* ... in bursts, if possible. */ + d2 = p1[2]; + d3 = p1[3]; + d4 = p1[4]; + d5 = p1[5]; + d6 = p1[6]; + d7 = p1[7]; + d0 ^= p2[0]; + d1 ^= p2[1]; + d2 ^= p2[2]; + d3 ^= p2[3]; + d4 ^= p2[4]; + d5 ^= p2[5]; + d6 ^= p2[6]; + d7 ^= p2[7]; + d0 ^= p3[0]; + d1 ^= p3[1]; + d2 ^= p3[2]; + d3 ^= p3[3]; + d4 ^= p3[4]; + d5 ^= p3[5]; + d6 ^= p3[6]; + d7 ^= p3[7]; + p1[0] = d0; /* Store the result (in bursts) */ + p1[1] = d1; + p1[2] = d2; + p1[3] = d3; + p1[4] = d4; + p1[5] = d5; + p1[6] = d6; + p1[7] = d7; + p1 += 8; + p2 += 8; + p3 += 8; + } while (--lines > 0); +} + +static void +xor_32regs_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) +{ + long lines = bytes / (sizeof (long)) / 8; + + do { + register long d0, d1, d2, d3, d4, d5, d6, d7; + d0 = p1[0]; /* Pull the stuff into registers */ + d1 = p1[1]; /* ... in bursts, if possible. */ + d2 = p1[2]; + d3 = p1[3]; + d4 = p1[4]; + d5 = p1[5]; + d6 = p1[6]; + d7 = p1[7]; + d0 ^= p2[0]; + d1 ^= p2[1]; + d2 ^= p2[2]; + d3 ^= p2[3]; + d4 ^= p2[4]; + d5 ^= p2[5]; + d6 ^= p2[6]; + d7 ^= p2[7]; + d0 ^= p3[0]; + d1 ^= p3[1]; + d2 ^= p3[2]; + d3 ^= p3[3]; + d4 ^= p3[4]; + d5 ^= p3[5]; + d6 ^= p3[6]; + d7 ^= p3[7]; + d0 ^= p4[0]; + d1 ^= p4[1]; + d2 ^= p4[2]; + d3 ^= p4[3]; + d4 ^= p4[4]; + d5 ^= p4[5]; + d6 ^= p4[6]; + d7 ^= p4[7]; + p1[0] = d0; /* Store the result (in bursts) */ + p1[1] = d1; + p1[2] = d2; + p1[3] = d3; + p1[4] = d4; + p1[5] = d5; + p1[6] = d6; + p1[7] = d7; + p1 += 8; + p2 += 8; + p3 += 8; + p4 += 8; + } while (--lines > 0); +} + +static void +xor_32regs_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) +{ + long lines = bytes / (sizeof (long)) / 8; + + do { + register long d0, d1, d2, d3, d4, d5, d6, d7; + d0 = p1[0]; /* Pull the stuff into registers */ + d1 = p1[1]; /* ... in bursts, if possible. */ + d2 = p1[2]; + d3 = p1[3]; + d4 = p1[4]; + d5 = p1[5]; + d6 = p1[6]; + d7 = p1[7]; + d0 ^= p2[0]; + d1 ^= p2[1]; + d2 ^= p2[2]; + d3 ^= p2[3]; + d4 ^= p2[4]; + d5 ^= p2[5]; + d6 ^= p2[6]; + d7 ^= p2[7]; + d0 ^= p3[0]; + d1 ^= p3[1]; + d2 ^= p3[2]; + d3 ^= p3[3]; + d4 ^= p3[4]; + d5 ^= p3[5]; + d6 ^= p3[6]; + d7 ^= p3[7]; + d0 ^= p4[0]; + d1 ^= p4[1]; + d2 ^= p4[2]; + d3 ^= p4[3]; + d4 ^= p4[4]; + d5 ^= p4[5]; + d6 ^= p4[6]; + d7 ^= p4[7]; + d0 ^= p5[0]; + d1 ^= p5[1]; + d2 ^= p5[2]; + d3 ^= p5[3]; + d4 ^= p5[4]; + d5 ^= p5[5]; + d6 ^= p5[6]; + d7 ^= p5[7]; + p1[0] = d0; /* Store the result (in bursts) */ + p1[1] = d1; + p1[2] = d2; + p1[3] = d3; + p1[4] = d4; + p1[5] = d5; + p1[6] = d6; + p1[7] = d7; + p1 += 8; + p2 += 8; + p3 += 8; + p4 += 8; + p5 += 8; + } while (--lines > 0); +} + +DO_XOR_BLOCKS(32regs, xor_32regs_2, xor_32regs_3, xor_32regs_4, xor_32regs_5); + +struct xor_block_template xor_block_32regs = { + .name = "32regs", + .xor_gen = xor_gen_32regs, +}; diff --git a/lib/raid/xor/xor-8regs-prefetch.c b/lib/raid/xor/xor-8regs-prefetch.c new file mode 100644 index 000000000000..451527a951b1 --- /dev/null +++ b/lib/raid/xor/xor-8regs-prefetch.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include <linux/prefetch.h> +#include "xor_impl.h" + +static void +xor_8regs_p_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) +{ + long lines = bytes / (sizeof (long)) / 8 - 1; + prefetchw(p1); + prefetch(p2); + + do { + prefetchw(p1+8); + prefetch(p2+8); + once_more: + p1[0] ^= p2[0]; + p1[1] ^= p2[1]; + p1[2] ^= p2[2]; + p1[3] ^= p2[3]; + p1[4] ^= p2[4]; + p1[5] ^= p2[5]; + p1[6] ^= p2[6]; + p1[7] ^= p2[7]; + p1 += 8; + p2 += 8; + } while (--lines > 0); + if (lines == 0) + goto once_more; +} + +static void +xor_8regs_p_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) +{ + long lines = bytes / (sizeof (long)) / 8 - 1; + prefetchw(p1); + prefetch(p2); + prefetch(p3); + + do { + prefetchw(p1+8); + prefetch(p2+8); + prefetch(p3+8); + once_more: + p1[0] ^= p2[0] ^ p3[0]; + p1[1] ^= p2[1] ^ p3[1]; + p1[2] ^= p2[2] ^ p3[2]; + p1[3] ^= p2[3] ^ p3[3]; + p1[4] ^= p2[4] ^ p3[4]; + p1[5] ^= p2[5] ^ p3[5]; + p1[6] ^= p2[6] ^ p3[6]; + p1[7] ^= p2[7] ^ p3[7]; + p1 += 8; + p2 += 8; + p3 += 8; + } while (--lines > 0); + if (lines == 0) + goto once_more; +} + +static void +xor_8regs_p_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) +{ + long lines = bytes / (sizeof (long)) / 8 - 1; + + prefetchw(p1); + prefetch(p2); + prefetch(p3); + prefetch(p4); + + do { + prefetchw(p1+8); + prefetch(p2+8); + prefetch(p3+8); + prefetch(p4+8); + once_more: + p1[0] ^= p2[0] ^ p3[0] ^ p4[0]; + p1[1] ^= p2[1] ^ p3[1] ^ p4[1]; + p1[2] ^= p2[2] ^ p3[2] ^ p4[2]; + p1[3] ^= p2[3] ^ p3[3] ^ p4[3]; + p1[4] ^= p2[4] ^ p3[4] ^ p4[4]; + p1[5] ^= p2[5] ^ p3[5] ^ p4[5]; + p1[6] ^= p2[6] ^ p3[6] ^ p4[6]; + p1[7] ^= p2[7] ^ p3[7] ^ p4[7]; + p1 += 8; + p2 += 8; + p3 += 8; + p4 += 8; + } while (--lines > 0); + if (lines == 0) + goto once_more; +} + +static void +xor_8regs_p_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) +{ + long lines = bytes / (sizeof (long)) / 8 - 1; + + prefetchw(p1); + prefetch(p2); + prefetch(p3); + prefetch(p4); + prefetch(p5); + + do { + prefetchw(p1+8); + prefetch(p2+8); + prefetch(p3+8); + prefetch(p4+8); + prefetch(p5+8); + once_more: + p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0]; + p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1]; + p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2]; + p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3]; + p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4]; + p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5]; + p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6]; + p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7]; + p1 += 8; + p2 += 8; + p3 += 8; + p4 += 8; + p5 += 8; + } while (--lines > 0); + if (lines == 0) + goto once_more; +} + + +DO_XOR_BLOCKS(8regs_p, xor_8regs_p_2, xor_8regs_p_3, xor_8regs_p_4, + xor_8regs_p_5); + +struct xor_block_template xor_block_8regs_p = { + .name = "8regs_prefetch", + .xor_gen = xor_gen_8regs_p, +}; diff --git a/lib/raid/xor/xor-8regs.c b/lib/raid/xor/xor-8regs.c new file mode 100644 index 000000000000..46b3c8bdc27f --- /dev/null +++ b/lib/raid/xor/xor-8regs.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include "xor_impl.h" + +static void +xor_8regs_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) +{ + long lines = bytes / (sizeof (long)) / 8; + + do { + p1[0] ^= p2[0]; + p1[1] ^= p2[1]; + p1[2] ^= p2[2]; + p1[3] ^= p2[3]; + p1[4] ^= p2[4]; + p1[5] ^= p2[5]; + p1[6] ^= p2[6]; + p1[7] ^= p2[7]; + p1 += 8; + p2 += 8; + } while (--lines > 0); +} + +static void +xor_8regs_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) +{ + long lines = bytes / (sizeof (long)) / 8; + + do { + p1[0] ^= p2[0] ^ p3[0]; + p1[1] ^= p2[1] ^ p3[1]; + p1[2] ^= p2[2] ^ p3[2]; + p1[3] ^= p2[3] ^ p3[3]; + p1[4] ^= p2[4] ^ p3[4]; + p1[5] ^= p2[5] ^ p3[5]; + p1[6] ^= p2[6] ^ p3[6]; + p1[7] ^= p2[7] ^ p3[7]; + p1 += 8; + p2 += 8; + p3 += 8; + } while (--lines > 0); +} + +static void +xor_8regs_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) +{ + long lines = bytes / (sizeof (long)) / 8; + + do { + p1[0] ^= p2[0] ^ p3[0] ^ p4[0]; + p1[1] ^= p2[1] ^ p3[1] ^ p4[1]; + p1[2] ^= p2[2] ^ p3[2] ^ p4[2]; + p1[3] ^= p2[3] ^ p3[3] ^ p4[3]; + p1[4] ^= p2[4] ^ p3[4] ^ p4[4]; + p1[5] ^= p2[5] ^ p3[5] ^ p4[5]; + p1[6] ^= p2[6] ^ p3[6] ^ p4[6]; + p1[7] ^= p2[7] ^ p3[7] ^ p4[7]; + p1 += 8; + p2 += 8; + p3 += 8; + p4 += 8; + } while (--lines > 0); +} + +static void +xor_8regs_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) +{ + long lines = bytes / (sizeof (long)) / 8; + + do { + p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0]; + p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1]; + p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2]; + p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3]; + p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4]; + p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5]; + p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6]; + p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7]; + p1 += 8; + p2 += 8; + p3 += 8; + p4 += 8; + p5 += 8; + } while (--lines > 0); +} + +DO_XOR_BLOCKS(8regs, xor_8regs_2, xor_8regs_3, xor_8regs_4, xor_8regs_5); + +struct xor_block_template xor_block_8regs = { + .name = "8regs", + .xor_gen = xor_gen_8regs, +}; diff --git a/lib/raid/xor/xor-core.c b/lib/raid/xor/xor-core.c new file mode 100644 index 000000000000..f9d61b096b11 --- /dev/null +++ b/lib/raid/xor/xor-core.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 1996, 1997, 1998, 1999, 2000, + * Ingo Molnar, Matti Aarnio, Jakub Jelinek, Richard Henderson. + * + * Dispatch optimized XOR parity functions. + */ + +#include <linux/module.h> +#include <linux/gfp.h> +#include <linux/slab.h> +#include <linux/raid/xor.h> +#include <linux/preempt.h> +#include <linux/static_call.h> +#include "xor_impl.h" + +DEFINE_STATIC_CALL_NULL(xor_gen_impl, *xor_block_8regs.xor_gen); + +/** + * xor_gen - generate RAID-style XOR information + * @dest: destination vector + * @srcs: source vectors + * @src_cnt: number of source vectors + * @bytes: length in bytes of each vector + * + * Performs bit-wise XOR operation into @dest for each of the @src_cnt vectors + * in @srcs for a length of @bytes bytes. @src_cnt must be non-zero, and the + * memory pointed to by @dest and each member of @srcs must be at least 64-byte + * aligned. @bytes must be non-zero and a multiple of 512. + * + * Note: for typical RAID uses, @dest either needs to be zeroed, or filled with + * the first disk, which then needs to be removed from @srcs. + */ +void xor_gen(void *dest, void **srcs, unsigned int src_cnt, unsigned int bytes) +{ + WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); + WARN_ON_ONCE(bytes == 0); + WARN_ON_ONCE(bytes & 511); + + static_call(xor_gen_impl)(dest, srcs, src_cnt, bytes); +} +EXPORT_SYMBOL(xor_gen); + +/* Set of all registered templates. */ +static struct xor_block_template *__initdata template_list; +static struct xor_block_template *forced_template; + +/** + * xor_register - register a XOR template + * @tmpl: template to register + * + * Register a XOR implementation with the core. Registered implementations + * will be measured by a trivial benchmark, and the fastest one is chosen + * unless an implementation is forced using xor_force(). + */ +void __init xor_register(struct xor_block_template *tmpl) +{ + tmpl->next = template_list; + template_list = tmpl; +} + +/** + * xor_force - force use of a XOR template + * @tmpl: template to register + * + * Register a XOR implementation with the core and force using it. Forcing + * an implementation will make the core ignore any template registered using + * xor_register(), or any previous implementation forced using xor_force(). + */ +void __init xor_force(struct xor_block_template *tmpl) +{ + forced_template = tmpl; +} + +#define BENCH_SIZE SZ_4K +#define NR_SRCS 4 +#define REPS 800U + +static void __init do_xor_speed(struct xor_block_template *tmpl, void *dest, + void *srcs[NR_SRCS]) +{ + u64 t; + int i; + + preempt_disable(); + t = ktime_get_ns(); + for (i = 0; i < REPS; i++) { + mb(); /* prevent loop optimization */ + tmpl->xor_gen(dest, srcs, NR_SRCS, BENCH_SIZE); + mb(); + } + t = max(ktime_get_ns() - t, 1); + preempt_enable(); + + /* bytes/ns == GB/s, multiply by 1000 to get MB/s [not MiB/s] */ + tmpl->speed = div64_u64((u64)BENCH_SIZE * REPS * NR_SRCS * 1000, t); + + pr_info(" %-16s: %5d MB/sec\n", tmpl->name, tmpl->speed); +} + +static int __init calibrate_xor_blocks(void) +{ + struct xor_block_template *f, *fastest; + void *srcs[NR_SRCS]; + void *buf, *dest; + int i; + + if (forced_template) + return 0; + + buf = kmalloc(BENCH_SIZE * (NR_SRCS + 1), GFP_KERNEL); + if (!buf) { + pr_warn("xor: Yikes! No memory available.\n"); + return -ENOMEM; + } + get_random_bytes(buf, BENCH_SIZE * (NR_SRCS + 1)); + dest = buf; + for (i = 0; i < NR_SRCS; i++) + srcs[i] = buf + (i + 1) * BENCH_SIZE; + + pr_info("xor: measuring software checksum speed\n"); + fastest = template_list; + for (f = template_list; f; f = f->next) { + do_xor_speed(f, dest, srcs); + if (f->speed > fastest->speed) + fastest = f; + } + static_call_update(xor_gen_impl, fastest->xor_gen); + pr_info("xor: using function: %s (%d MB/sec)\n", + fastest->name, fastest->speed); + + kfree(buf); + return 0; +} +#undef NR_SRCS + +#ifdef CONFIG_XOR_BLOCKS_ARCH +#include "xor_arch.h" /* $SRCARCH/xor_arch.h */ +#else +static void __init arch_xor_init(void) +{ + xor_register(&xor_block_8regs); + xor_register(&xor_block_8regs_p); + xor_register(&xor_block_32regs); + xor_register(&xor_block_32regs_p); +} +#endif /* CONFIG_XOR_BLOCKS_ARCH */ + +static int __init xor_init(void) +{ + arch_xor_init(); + + /* + * If this arch/cpu has a short-circuited selection, don't loop through + * all the possible functions, just use the best one. + */ + if (forced_template) { + pr_info("xor: automatically using best checksumming function %-10s\n", + forced_template->name); + static_call_update(xor_gen_impl, forced_template->xor_gen); + return 0; + } + +#ifdef MODULE + return calibrate_xor_blocks(); +#else + /* + * Pick the first template as the temporary default until calibration + * happens. + */ + static_call_update(xor_gen_impl, template_list->xor_gen); + return 0; +#endif +} + +static __exit void xor_exit(void) +{ +} + +MODULE_DESCRIPTION("RAID-5 checksumming functions"); +MODULE_LICENSE("GPL"); + +/* + * When built-in we must register the default template before md, but we don't + * want calibration to run that early as that would delay the boot process. + */ +#ifndef MODULE +__initcall(calibrate_xor_blocks); +#endif +core_initcall(xor_init); +module_exit(xor_exit); diff --git a/lib/raid/xor/xor-neon.c b/lib/raid/xor/xor-neon.c new file mode 100644 index 000000000000..c7c3cf634e23 --- /dev/null +++ b/lib/raid/xor/xor-neon.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Authors: Jackie Liu <liuyun01@kylinos.cn> + * Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd. + */ + +#include "xor_impl.h" +#include "xor-neon.h" + +#include <asm/neon-intrinsics.h> + +static void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) +{ + uint64_t *dp1 = (uint64_t *)p1; + uint64_t *dp2 = (uint64_t *)p2; + + register uint64x2_t v0, v1, v2, v3; + long lines = bytes / (sizeof(uint64x2_t) * 4); + + do { + /* p1 ^= p2 */ + v0 = veorq_u64(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0)); + v1 = veorq_u64(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2)); + v2 = veorq_u64(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4)); + v3 = veorq_u64(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6)); + + /* store */ + vst1q_u64(dp1 + 0, v0); + vst1q_u64(dp1 + 2, v1); + vst1q_u64(dp1 + 4, v2); + vst1q_u64(dp1 + 6, v3); + + dp1 += 8; + dp2 += 8; + } while (--lines > 0); +} + +static void __xor_neon_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) +{ + uint64_t *dp1 = (uint64_t *)p1; + uint64_t *dp2 = (uint64_t *)p2; + uint64_t *dp3 = (uint64_t *)p3; + + register uint64x2_t v0, v1, v2, v3; + long lines = bytes / (sizeof(uint64x2_t) * 4); + + do { + /* p1 ^= p2 */ + v0 = veorq_u64(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0)); + v1 = veorq_u64(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2)); + v2 = veorq_u64(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4)); + v3 = veorq_u64(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6)); + + /* p1 ^= p3 */ + v0 = veorq_u64(v0, vld1q_u64(dp3 + 0)); + v1 = veorq_u64(v1, vld1q_u64(dp3 + 2)); + v2 = veorq_u64(v2, vld1q_u64(dp3 + 4)); + v3 = veorq_u64(v3, vld1q_u64(dp3 + 6)); + + /* store */ + vst1q_u64(dp1 + 0, v0); + vst1q_u64(dp1 + 2, v1); + vst1q_u64(dp1 + 4, v2); + vst1q_u64(dp1 + 6, v3); + + dp1 += 8; + dp2 += 8; + dp3 += 8; + } while (--lines > 0); +} + +static void __xor_neon_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) +{ + uint64_t *dp1 = (uint64_t *)p1; + uint64_t *dp2 = (uint64_t *)p2; + uint64_t *dp3 = (uint64_t *)p3; + uint64_t *dp4 = (uint64_t *)p4; + + register uint64x2_t v0, v1, v2, v3; + long lines = bytes / (sizeof(uint64x2_t) * 4); + + do { + /* p1 ^= p2 */ + v0 = veorq_u64(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0)); + v1 = veorq_u64(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2)); + v2 = veorq_u64(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4)); + v3 = veorq_u64(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6)); + + /* p1 ^= p3 */ + v0 = veorq_u64(v0, vld1q_u64(dp3 + 0)); + v1 = veorq_u64(v1, vld1q_u64(dp3 + 2)); + v2 = veorq_u64(v2, vld1q_u64(dp3 + 4)); + v3 = veorq_u64(v3, vld1q_u64(dp3 + 6)); + + /* p1 ^= p4 */ + v0 = veorq_u64(v0, vld1q_u64(dp4 + 0)); + v1 = veorq_u64(v1, vld1q_u64(dp4 + 2)); + v2 = veorq_u64(v2, vld1q_u64(dp4 + 4)); + v3 = veorq_u64(v3, vld1q_u64(dp4 + 6)); + + /* store */ + vst1q_u64(dp1 + 0, v0); + vst1q_u64(dp1 + 2, v1); + vst1q_u64(dp1 + 4, v2); + vst1q_u64(dp1 + 6, v3); + + dp1 += 8; + dp2 += 8; + dp3 += 8; + dp4 += 8; + } while (--lines > 0); +} + +static void __xor_neon_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) +{ + uint64_t *dp1 = (uint64_t *)p1; + uint64_t *dp2 = (uint64_t *)p2; + uint64_t *dp3 = (uint64_t *)p3; + uint64_t *dp4 = (uint64_t *)p4; + uint64_t *dp5 = (uint64_t *)p5; + + register uint64x2_t v0, v1, v2, v3; + long lines = bytes / (sizeof(uint64x2_t) * 4); + + do { + /* p1 ^= p2 */ + v0 = veorq_u64(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0)); + v1 = veorq_u64(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2)); + v2 = veorq_u64(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4)); + v3 = veorq_u64(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6)); + + /* p1 ^= p3 */ + v0 = veorq_u64(v0, vld1q_u64(dp3 + 0)); + v1 = veorq_u64(v1, vld1q_u64(dp3 + 2)); + v2 = veorq_u64(v2, vld1q_u64(dp3 + 4)); + v3 = veorq_u64(v3, vld1q_u64(dp3 + 6)); + + /* p1 ^= p4 */ + v0 = veorq_u64(v0, vld1q_u64(dp4 + 0)); + v1 = veorq_u64(v1, vld1q_u64(dp4 + 2)); + v2 = veorq_u64(v2, vld1q_u64(dp4 + 4)); + v3 = veorq_u64(v3, vld1q_u64(dp4 + 6)); + + /* p1 ^= p5 */ + v0 = veorq_u64(v0, vld1q_u64(dp5 + 0)); + v1 = veorq_u64(v1, vld1q_u64(dp5 + 2)); + v2 = veorq_u64(v2, vld1q_u64(dp5 + 4)); + v3 = veorq_u64(v3, vld1q_u64(dp5 + 6)); + + /* store */ + vst1q_u64(dp1 + 0, v0); + vst1q_u64(dp1 + 2, v1); + vst1q_u64(dp1 + 4, v2); + vst1q_u64(dp1 + 6, v3); + + dp1 += 8; + dp2 += 8; + dp3 += 8; + dp4 += 8; + dp5 += 8; + } while (--lines > 0); +} + +__DO_XOR_BLOCKS(neon_inner, __xor_neon_2, __xor_neon_3, __xor_neon_4, + __xor_neon_5); + +#ifdef CONFIG_ARM64 +extern typeof(__xor_neon_2) __xor_eor3_2 __alias(__xor_neon_2); +#endif diff --git a/lib/raid/xor/xor_impl.h b/lib/raid/xor/xor_impl.h new file mode 100644 index 000000000000..09ae2916f71e --- /dev/null +++ b/lib/raid/xor/xor_impl.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _XOR_IMPL_H +#define _XOR_IMPL_H + +#include <linux/init.h> +#include <linux/minmax.h> + +struct xor_block_template { + struct xor_block_template *next; + const char *name; + int speed; + void (*xor_gen)(void *dest, void **srcs, unsigned int src_cnt, + unsigned int bytes); +}; + +#define __DO_XOR_BLOCKS(_name, _handle1, _handle2, _handle3, _handle4) \ +void \ +xor_gen_##_name(void *dest, void **srcs, unsigned int src_cnt, \ + unsigned int bytes) \ +{ \ + unsigned int src_off = 0; \ + \ + while (src_cnt > 0) { \ + unsigned int this_cnt = min(src_cnt, 4); \ + \ + if (this_cnt == 1) \ + _handle1(bytes, dest, srcs[src_off]); \ + else if (this_cnt == 2) \ + _handle2(bytes, dest, srcs[src_off], \ + srcs[src_off + 1]); \ + else if (this_cnt == 3) \ + _handle3(bytes, dest, srcs[src_off], \ + srcs[src_off + 1], srcs[src_off + 2]); \ + else \ + _handle4(bytes, dest, srcs[src_off], \ + srcs[src_off + 1], srcs[src_off + 2], \ + srcs[src_off + 3]); \ + \ + src_cnt -= this_cnt; \ + src_off += this_cnt; \ + } \ +} + +#define DO_XOR_BLOCKS(_name, _handle1, _handle2, _handle3, _handle4) \ + static __DO_XOR_BLOCKS(_name, _handle1, _handle2, _handle3, _handle4) + +/* generic implementations */ +extern struct xor_block_template xor_block_8regs; +extern struct xor_block_template xor_block_32regs; +extern struct xor_block_template xor_block_8regs_p; +extern struct xor_block_template xor_block_32regs_p; + +void __init xor_register(struct xor_block_template *tmpl); +void __init xor_force(struct xor_block_template *tmpl); + +#endif /* _XOR_IMPL_H */ diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile deleted file mode 100644 index 5be0a4e60ab1..000000000000 --- a/lib/raid6/Makefile +++ /dev/null @@ -1,83 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_RAID6_PQ) += raid6_pq.o - -raid6_pq-y += algos.o recov.o tables.o int1.o int2.o int4.o \ - int8.o - -raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o avx512.o recov_avx512.o -raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o \ - vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o -raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o -raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o -raid6_pq-$(CONFIG_LOONGARCH) += loongarch_simd.o recov_loongarch_simd.o -raid6_pq-$(CONFIG_RISCV_ISA_V) += rvv.o recov_rvv.o - -hostprogs += mktables - -ifeq ($(CONFIG_ALTIVEC),y) -altivec_flags := -maltivec $(call cc-option,-mabi=altivec) -# Enable <altivec.h> -altivec_flags += -isystem $(shell $(CC) -print-file-name=include) - -ifdef CONFIG_CC_IS_CLANG -# clang ppc port does not yet support -maltivec when -msoft-float is -# enabled. A future release of clang will resolve this -# https://llvm.org/pr31177 -CFLAGS_REMOVE_altivec1.o += -msoft-float -CFLAGS_REMOVE_altivec2.o += -msoft-float -CFLAGS_REMOVE_altivec4.o += -msoft-float -CFLAGS_REMOVE_altivec8.o += -msoft-float -CFLAGS_REMOVE_vpermxor1.o += -msoft-float -CFLAGS_REMOVE_vpermxor2.o += -msoft-float -CFLAGS_REMOVE_vpermxor4.o += -msoft-float -CFLAGS_REMOVE_vpermxor8.o += -msoft-float -endif -endif - -quiet_cmd_unroll = UNROLL $@ - cmd_unroll = $(AWK) -v N=$* -f $(src)/unroll.awk < $< > $@ - -targets += int1.c int2.c int4.c int8.c -$(obj)/int%.c: $(src)/int.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -CFLAGS_altivec1.o += $(altivec_flags) -CFLAGS_altivec2.o += $(altivec_flags) -CFLAGS_altivec4.o += $(altivec_flags) -CFLAGS_altivec8.o += $(altivec_flags) -targets += altivec1.c altivec2.c altivec4.c altivec8.c -$(obj)/altivec%.c: $(src)/altivec.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -CFLAGS_vpermxor1.o += $(altivec_flags) -CFLAGS_vpermxor2.o += $(altivec_flags) -CFLAGS_vpermxor4.o += $(altivec_flags) -CFLAGS_vpermxor8.o += $(altivec_flags) -targets += vpermxor1.c vpermxor2.c vpermxor4.c vpermxor8.c -$(obj)/vpermxor%.c: $(src)/vpermxor.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -CFLAGS_neon1.o += $(CC_FLAGS_FPU) -CFLAGS_neon2.o += $(CC_FLAGS_FPU) -CFLAGS_neon4.o += $(CC_FLAGS_FPU) -CFLAGS_neon8.o += $(CC_FLAGS_FPU) -CFLAGS_recov_neon_inner.o += $(CC_FLAGS_FPU) -CFLAGS_REMOVE_neon1.o += $(CC_FLAGS_NO_FPU) -CFLAGS_REMOVE_neon2.o += $(CC_FLAGS_NO_FPU) -CFLAGS_REMOVE_neon4.o += $(CC_FLAGS_NO_FPU) -CFLAGS_REMOVE_neon8.o += $(CC_FLAGS_NO_FPU) -CFLAGS_REMOVE_recov_neon_inner.o += $(CC_FLAGS_NO_FPU) -targets += neon1.c neon2.c neon4.c neon8.c -$(obj)/neon%.c: $(src)/neon.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -targets += s390vx8.c -$(obj)/s390vx%.c: $(src)/s390vx.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -quiet_cmd_mktable = TABLE $@ - cmd_mktable = $(obj)/mktables > $@ - -targets += tables.c -$(obj)/tables.c: $(obj)/mktables FORCE - $(call if_changed,mktable) diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c deleted file mode 100644 index 799e0e5eac26..000000000000 --- a/lib/raid6/algos.c +++ /dev/null @@ -1,291 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - -/* - * raid6/algos.c - * - * Algorithm list and algorithm selection for RAID-6 - */ - -#include <linux/raid/pq.h> -#ifndef __KERNEL__ -#include <sys/mman.h> -#include <stdio.h> -#else -#include <linux/module.h> -#include <linux/gfp.h> -#endif - -struct raid6_calls raid6_call; -EXPORT_SYMBOL_GPL(raid6_call); - -const struct raid6_calls * const raid6_algos[] = { -#if defined(__i386__) && !defined(__arch_um__) - &raid6_avx512x2, - &raid6_avx512x1, - &raid6_avx2x2, - &raid6_avx2x1, - &raid6_sse2x2, - &raid6_sse2x1, - &raid6_sse1x2, - &raid6_sse1x1, - &raid6_mmxx2, - &raid6_mmxx1, -#endif -#if defined(__x86_64__) && !defined(__arch_um__) - &raid6_avx512x4, - &raid6_avx512x2, - &raid6_avx512x1, - &raid6_avx2x4, - &raid6_avx2x2, - &raid6_avx2x1, - &raid6_sse2x4, - &raid6_sse2x2, - &raid6_sse2x1, -#endif -#ifdef CONFIG_ALTIVEC - &raid6_vpermxor8, - &raid6_vpermxor4, - &raid6_vpermxor2, - &raid6_vpermxor1, - &raid6_altivec8, - &raid6_altivec4, - &raid6_altivec2, - &raid6_altivec1, -#endif -#if defined(CONFIG_S390) - &raid6_s390vx8, -#endif -#ifdef CONFIG_KERNEL_MODE_NEON - &raid6_neonx8, - &raid6_neonx4, - &raid6_neonx2, - &raid6_neonx1, -#endif -#ifdef CONFIG_LOONGARCH -#ifdef CONFIG_CPU_HAS_LASX - &raid6_lasx, -#endif -#ifdef CONFIG_CPU_HAS_LSX - &raid6_lsx, -#endif -#endif -#ifdef CONFIG_RISCV_ISA_V - &raid6_rvvx1, - &raid6_rvvx2, - &raid6_rvvx4, - &raid6_rvvx8, -#endif - &raid6_intx8, - &raid6_intx4, - &raid6_intx2, - &raid6_intx1, - NULL -}; - -void (*raid6_2data_recov)(int, size_t, int, int, void **); -EXPORT_SYMBOL_GPL(raid6_2data_recov); - -void (*raid6_datap_recov)(int, size_t, int, void **); -EXPORT_SYMBOL_GPL(raid6_datap_recov); - -const struct raid6_recov_calls *const raid6_recov_algos[] = { -#ifdef CONFIG_X86 - &raid6_recov_avx512, - &raid6_recov_avx2, - &raid6_recov_ssse3, -#endif -#ifdef CONFIG_S390 - &raid6_recov_s390xc, -#endif -#if defined(CONFIG_KERNEL_MODE_NEON) - &raid6_recov_neon, -#endif -#ifdef CONFIG_LOONGARCH -#ifdef CONFIG_CPU_HAS_LASX - &raid6_recov_lasx, -#endif -#ifdef CONFIG_CPU_HAS_LSX - &raid6_recov_lsx, -#endif -#endif -#ifdef CONFIG_RISCV_ISA_V - &raid6_recov_rvv, -#endif - &raid6_recov_intx1, - NULL -}; - -#ifdef __KERNEL__ -#define RAID6_TIME_JIFFIES_LG2 4 -#else -/* Need more time to be stable in userspace */ -#define RAID6_TIME_JIFFIES_LG2 9 -#define time_before(x, y) ((x) < (y)) -#endif - -#define RAID6_TEST_DISKS 8 -#define RAID6_TEST_DISKS_ORDER 3 - -static inline const struct raid6_recov_calls *raid6_choose_recov(void) -{ - const struct raid6_recov_calls *const *algo; - const struct raid6_recov_calls *best; - - for (best = NULL, algo = raid6_recov_algos; *algo; algo++) - if (!best || (*algo)->priority > best->priority) - if (!(*algo)->valid || (*algo)->valid()) - best = *algo; - - if (best) { - raid6_2data_recov = best->data2; - raid6_datap_recov = best->datap; - - pr_info("raid6: using %s recovery algorithm\n", best->name); - } else - pr_err("raid6: Yikes! No recovery algorithm found!\n"); - - return best; -} - -static inline const struct raid6_calls *raid6_choose_gen( - void *(*const dptrs)[RAID6_TEST_DISKS], const int disks) -{ - unsigned long perf, bestgenperf, j0, j1; - int start = (disks>>1)-1, stop = disks-3; /* work on the second half of the disks */ - const struct raid6_calls *const *algo; - const struct raid6_calls *best; - - for (bestgenperf = 0, best = NULL, algo = raid6_algos; *algo; algo++) { - if (!best || (*algo)->priority >= best->priority) { - if ((*algo)->valid && !(*algo)->valid()) - continue; - - if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) { - best = *algo; - break; - } - - perf = 0; - - preempt_disable(); - j0 = jiffies; - while ((j1 = jiffies) == j0) - cpu_relax(); - while (time_before(jiffies, - j1 + (1<<RAID6_TIME_JIFFIES_LG2))) { - (*algo)->gen_syndrome(disks, PAGE_SIZE, *dptrs); - perf++; - } - preempt_enable(); - - if (perf > bestgenperf) { - bestgenperf = perf; - best = *algo; - } - pr_info("raid6: %-8s gen() %5ld MB/s\n", (*algo)->name, - (perf * HZ * (disks-2)) >> - (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); - } - } - - if (!best) { - pr_err("raid6: Yikes! No algorithm found!\n"); - goto out; - } - - raid6_call = *best; - - if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) { - pr_info("raid6: skipped pq benchmark and selected %s\n", - best->name); - goto out; - } - - pr_info("raid6: using algorithm %s gen() %ld MB/s\n", - best->name, - (bestgenperf * HZ * (disks - 2)) >> - (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); - - if (best->xor_syndrome) { - perf = 0; - - preempt_disable(); - j0 = jiffies; - while ((j1 = jiffies) == j0) - cpu_relax(); - while (time_before(jiffies, - j1 + (1 << RAID6_TIME_JIFFIES_LG2))) { - best->xor_syndrome(disks, start, stop, - PAGE_SIZE, *dptrs); - perf++; - } - preempt_enable(); - - pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n", - (perf * HZ * (disks - 2)) >> - (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1)); - } - -out: - return best; -} - - -/* Try to pick the best algorithm */ -/* This code uses the gfmul table as convenient data set to abuse */ - -int __init raid6_select_algo(void) -{ - const int disks = RAID6_TEST_DISKS; - - const struct raid6_calls *gen_best; - const struct raid6_recov_calls *rec_best; - char *disk_ptr, *p; - void *dptrs[RAID6_TEST_DISKS]; - int i, cycle; - - /* prepare the buffer and fill it circularly with gfmul table */ - disk_ptr = (char *)__get_free_pages(GFP_KERNEL, RAID6_TEST_DISKS_ORDER); - if (!disk_ptr) { - pr_err("raid6: Yikes! No memory available.\n"); - return -ENOMEM; - } - - p = disk_ptr; - for (i = 0; i < disks; i++) - dptrs[i] = p + PAGE_SIZE * i; - - cycle = ((disks - 2) * PAGE_SIZE) / 65536; - for (i = 0; i < cycle; i++) { - memcpy(p, raid6_gfmul, 65536); - p += 65536; - } - - if ((disks - 2) * PAGE_SIZE % 65536) - memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536); - - /* select raid gen_syndrome function */ - gen_best = raid6_choose_gen(&dptrs, disks); - - /* select raid recover functions */ - rec_best = raid6_choose_recov(); - - free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER); - - return gen_best && rec_best ? 0 : -EINVAL; -} - -static void raid6_exit(void) -{ - do { } while (0); -} - -subsys_initcall(raid6_select_algo); -module_exit(raid6_exit); -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("RAID6 Q-syndrome calculations"); diff --git a/lib/raid6/loongarch.h b/lib/raid6/loongarch.h deleted file mode 100644 index acfc33ce7056..000000000000 --- a/lib/raid6/loongarch.h +++ /dev/null @@ -1,38 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) 2023 WANG Xuerui <git@xen0n.name> - * - * raid6/loongarch.h - * - * Definitions common to LoongArch RAID-6 code only - */ - -#ifndef _LIB_RAID6_LOONGARCH_H -#define _LIB_RAID6_LOONGARCH_H - -#ifdef __KERNEL__ - -#include <asm/cpu-features.h> -#include <asm/fpu.h> - -#else /* for user-space testing */ - -#include <sys/auxv.h> - -/* have to supply these defines for glibc 2.37- and musl */ -#ifndef HWCAP_LOONGARCH_LSX -#define HWCAP_LOONGARCH_LSX (1 << 4) -#endif -#ifndef HWCAP_LOONGARCH_LASX -#define HWCAP_LOONGARCH_LASX (1 << 5) -#endif - -#define kernel_fpu_begin() -#define kernel_fpu_end() - -#define cpu_has_lsx (getauxval(AT_HWCAP) & HWCAP_LOONGARCH_LSX) -#define cpu_has_lasx (getauxval(AT_HWCAP) & HWCAP_LOONGARCH_LASX) - -#endif /* __KERNEL__ */ - -#endif /* _LIB_RAID6_LOONGARCH_H */ diff --git a/lib/raid6/test/.gitignore b/lib/raid6/test/.gitignore deleted file mode 100644 index 1b68a77f348f..000000000000 --- a/lib/raid6/test/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/int.uc -/neon.uc -/raid6test diff --git a/lib/raid6/test/Makefile b/lib/raid6/test/Makefile deleted file mode 100644 index 09bbe2b14cce..000000000000 --- a/lib/raid6/test/Makefile +++ /dev/null @@ -1,156 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# This is a simple Makefile to test some of the RAID-6 code -# from userspace. -# - -pound := \# - -# Adjust as desired -CC = gcc -OPTFLAGS = -O2 -CFLAGS = -I.. -I ../../../include -g $(OPTFLAGS) -LD = ld -AWK = awk -f -AR = ar -RANLIB = ranlib -OBJS = int1.o int2.o int4.o int8.o int16.o int32.o recov.o algos.o tables.o - -ARCH := $(shell uname -m 2>/dev/null | sed -e /s/i.86/i386/) -ifeq ($(ARCH),i386) - CFLAGS += -DCONFIG_X86_32 - IS_X86 = yes -endif -ifeq ($(ARCH),x86_64) - CFLAGS += -DCONFIG_X86_64 - IS_X86 = yes -endif - -ifeq ($(ARCH),arm) - CFLAGS += -I../../../arch/arm/include -mfpu=neon - HAS_NEON = yes -endif -ifeq ($(ARCH),aarch64) - CFLAGS += -I../../../arch/arm64/include - HAS_NEON = yes -endif - -ifeq ($(findstring riscv,$(ARCH)),riscv) - CFLAGS += -I../../../arch/riscv/include -DCONFIG_RISCV=1 - HAS_RVV = yes -endif - -ifeq ($(findstring ppc,$(ARCH)),ppc) - CFLAGS += -I../../../arch/powerpc/include - HAS_ALTIVEC := $(shell printf '$(pound)include <altivec.h>\nvector int a;\n' |\ - gcc -c -x c - >/dev/null && rm ./-.o && echo yes) -endif - -ifeq ($(ARCH),loongarch64) - CFLAGS += -I../../../arch/loongarch/include -DCONFIG_LOONGARCH=1 - CFLAGS += $(shell echo 'vld $$vr0, $$zero, 0' | \ - gcc -c -x assembler - >/dev/null 2>&1 && \ - rm ./-.o && echo -DCONFIG_CPU_HAS_LSX=1) - CFLAGS += $(shell echo 'xvld $$xr0, $$zero, 0' | \ - gcc -c -x assembler - >/dev/null 2>&1 && \ - rm ./-.o && echo -DCONFIG_CPU_HAS_LASX=1) -endif - -ifeq ($(IS_X86),yes) - OBJS += mmx.o sse1.o sse2.o avx2.o recov_ssse3.o recov_avx2.o avx512.o recov_avx512.o - CFLAGS += -DCONFIG_X86 -else ifeq ($(HAS_NEON),yes) - OBJS += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o - CFLAGS += -DCONFIG_KERNEL_MODE_NEON=1 -else ifeq ($(HAS_ALTIVEC),yes) - CFLAGS += -DCONFIG_ALTIVEC - OBJS += altivec1.o altivec2.o altivec4.o altivec8.o \ - vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o -else ifeq ($(ARCH),loongarch64) - OBJS += loongarch_simd.o recov_loongarch_simd.o -else ifeq ($(HAS_RVV),yes) - OBJS += rvv.o recov_rvv.o - CFLAGS += -DCONFIG_RISCV_ISA_V=1 -endif - -.c.o: - $(CC) $(CFLAGS) -c -o $@ $< - -%.c: ../%.c - cp -f $< $@ - -%.uc: ../%.uc - cp -f $< $@ - -all: raid6.a raid6test - -raid6.a: $(OBJS) - rm -f $@ - $(AR) cq $@ $^ - $(RANLIB) $@ - -raid6test: test.c raid6.a - $(CC) $(CFLAGS) -o raid6test $^ - -neon1.c: neon.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=1 < neon.uc > $@ - -neon2.c: neon.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=2 < neon.uc > $@ - -neon4.c: neon.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=4 < neon.uc > $@ - -neon8.c: neon.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=8 < neon.uc > $@ - -altivec1.c: altivec.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=1 < altivec.uc > $@ - -altivec2.c: altivec.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=2 < altivec.uc > $@ - -altivec4.c: altivec.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=4 < altivec.uc > $@ - -altivec8.c: altivec.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=8 < altivec.uc > $@ - -vpermxor1.c: vpermxor.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=1 < vpermxor.uc > $@ - -vpermxor2.c: vpermxor.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=2 < vpermxor.uc > $@ - -vpermxor4.c: vpermxor.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=4 < vpermxor.uc > $@ - -vpermxor8.c: vpermxor.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=8 < vpermxor.uc > $@ - -int1.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=1 < int.uc > $@ - -int2.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=2 < int.uc > $@ - -int4.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=4 < int.uc > $@ - -int8.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=8 < int.uc > $@ - -int16.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=16 < int.uc > $@ - -int32.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=32 < int.uc > $@ - -tables.c: mktables - ./mktables > tables.c - -clean: - rm -f *.o *.a mktables mktables.c *.uc int*.c altivec*.c vpermxor*.c neon*.c tables.c raid6test - -spotless: clean - rm -f *~ diff --git a/lib/raid6/test/test.c b/lib/raid6/test/test.c deleted file mode 100644 index 841a55242aba..000000000000 --- a/lib/raid6/test/test.c +++ /dev/null @@ -1,152 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002-2007 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - -/* - * raid6test.c - * - * Test RAID-6 recovery with various algorithms - */ - -#include <stdlib.h> -#include <stdio.h> -#include <string.h> -#include <linux/raid/pq.h> - -#define NDISKS 16 /* Including P and Q */ - -const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); - -char *dataptrs[NDISKS]; -char data[NDISKS][PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); -char recovi[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); -char recovj[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); - -static void makedata(int start, int stop) -{ - int i, j; - - for (i = start; i <= stop; i++) { - for (j = 0; j < PAGE_SIZE; j++) - data[i][j] = rand(); - - dataptrs[i] = data[i]; - } -} - -static char disk_type(int d) -{ - switch (d) { - case NDISKS-2: - return 'P'; - case NDISKS-1: - return 'Q'; - default: - return 'D'; - } -} - -static int test_disks(int i, int j) -{ - int erra, errb; - - memset(recovi, 0xf0, PAGE_SIZE); - memset(recovj, 0xba, PAGE_SIZE); - - dataptrs[i] = recovi; - dataptrs[j] = recovj; - - raid6_dual_recov(NDISKS, PAGE_SIZE, i, j, (void **)&dataptrs); - - erra = memcmp(data[i], recovi, PAGE_SIZE); - errb = memcmp(data[j], recovj, PAGE_SIZE); - - if (i < NDISKS-2 && j == NDISKS-1) { - /* We don't implement the DQ failure scenario, since it's - equivalent to a RAID-5 failure (XOR, then recompute Q) */ - erra = errb = 0; - } else { - printf("algo=%-8s faila=%3d(%c) failb=%3d(%c) %s\n", - raid6_call.name, - i, disk_type(i), - j, disk_type(j), - (!erra && !errb) ? "OK" : - !erra ? "ERRB" : - !errb ? "ERRA" : "ERRAB"); - } - - dataptrs[i] = data[i]; - dataptrs[j] = data[j]; - - return erra || errb; -} - -int main(int argc, char *argv[]) -{ - const struct raid6_calls *const *algo; - const struct raid6_recov_calls *const *ra; - int i, j, p1, p2; - int err = 0; - - makedata(0, NDISKS-1); - - for (ra = raid6_recov_algos; *ra; ra++) { - if ((*ra)->valid && !(*ra)->valid()) - continue; - - raid6_2data_recov = (*ra)->data2; - raid6_datap_recov = (*ra)->datap; - - printf("using recovery %s\n", (*ra)->name); - - for (algo = raid6_algos; *algo; algo++) { - if ((*algo)->valid && !(*algo)->valid()) - continue; - - raid6_call = **algo; - - /* Nuke syndromes */ - memset(data[NDISKS-2], 0xee, 2*PAGE_SIZE); - - /* Generate assumed good syndrome */ - raid6_call.gen_syndrome(NDISKS, PAGE_SIZE, - (void **)&dataptrs); - - for (i = 0; i < NDISKS-1; i++) - for (j = i+1; j < NDISKS; j++) - err += test_disks(i, j); - - if (!raid6_call.xor_syndrome) - continue; - - for (p1 = 0; p1 < NDISKS-2; p1++) - for (p2 = p1; p2 < NDISKS-2; p2++) { - - /* Simulate rmw run */ - raid6_call.xor_syndrome(NDISKS, p1, p2, PAGE_SIZE, - (void **)&dataptrs); - makedata(p1, p2); - raid6_call.xor_syndrome(NDISKS, p1, p2, PAGE_SIZE, - (void **)&dataptrs); - - for (i = 0; i < NDISKS-1; i++) - for (j = i+1; j < NDISKS; j++) - err += test_disks(i, j); - } - - } - printf("\n"); - } - - printf("\n"); - /* Pick the best algorithm test */ - raid6_select_algo(); - - if (err) - printf("\n*** ERRORS FOUND ***\n"); - - return err; -} diff --git a/lib/raid6/x86.h b/lib/raid6/x86.h deleted file mode 100644 index 9a6ff37115e7..000000000000 --- a/lib/raid6/x86.h +++ /dev/null @@ -1,75 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* ----------------------------------------------------------------------- * - * - * Copyright 2002-2004 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - -/* - * raid6/x86.h - * - * Definitions common to x86 and x86-64 RAID-6 code only - */ - -#ifndef LINUX_RAID_RAID6X86_H -#define LINUX_RAID_RAID6X86_H - -#if (defined(__i386__) || defined(__x86_64__)) && !defined(__arch_um__) - -#ifdef __KERNEL__ /* Real code */ - -#include <asm/fpu/api.h> - -#else /* Dummy code for user space testing */ - -static inline void kernel_fpu_begin(void) -{ -} - -static inline void kernel_fpu_end(void) -{ -} - -#define __aligned(x) __attribute__((aligned(x))) - -#define X86_FEATURE_MMX (0*32+23) /* Multimedia Extensions */ -#define X86_FEATURE_FXSR (0*32+24) /* FXSAVE and FXRSTOR instructions - * (fast save and restore) */ -#define X86_FEATURE_XMM (0*32+25) /* Streaming SIMD Extensions */ -#define X86_FEATURE_XMM2 (0*32+26) /* Streaming SIMD Extensions-2 */ -#define X86_FEATURE_XMM3 (4*32+ 0) /* "pni" SSE-3 */ -#define X86_FEATURE_SSSE3 (4*32+ 9) /* Supplemental SSE-3 */ -#define X86_FEATURE_AVX (4*32+28) /* Advanced Vector Extensions */ -#define X86_FEATURE_AVX2 (9*32+ 5) /* AVX2 instructions */ -#define X86_FEATURE_AVX512F (9*32+16) /* AVX-512 Foundation */ -#define X86_FEATURE_AVX512DQ (9*32+17) /* AVX-512 DQ (Double/Quad granular) - * Instructions - */ -#define X86_FEATURE_AVX512BW (9*32+30) /* AVX-512 BW (Byte/Word granular) - * Instructions - */ -#define X86_FEATURE_AVX512VL (9*32+31) /* AVX-512 VL (128/256 Vector Length) - * Extensions - */ -#define X86_FEATURE_MMXEXT (1*32+22) /* AMD MMX extensions */ - -/* Should work well enough on modern CPUs for testing */ -static inline int boot_cpu_has(int flag) -{ - u32 eax, ebx, ecx, edx; - - eax = (flag & 0x100) ? 7 : - (flag & 0x20) ? 0x80000001 : 1; - ecx = 0; - - asm volatile("cpuid" - : "+a" (eax), "=b" (ebx), "=d" (edx), "+c" (ecx)); - - return ((flag & 0x100 ? ebx : - (flag & 0x80) ? ecx : edx) >> (flag & 31)) & 1; -} - -#endif /* ndef __KERNEL__ */ - -#endif -#endif diff --git a/lib/random32.c b/lib/random32.c index 24e7acd9343f..dad90219c351 100644 --- a/lib/random32.c +++ b/lib/random32.c @@ -41,6 +41,7 @@ #include <linux/bitops.h> #include <linux/slab.h> #include <linux/unaligned.h> +#include <kunit/visibility.h> /** * prandom_u32_state - seeded pseudo-random number generator. @@ -92,7 +93,14 @@ void prandom_bytes_state(struct rnd_state *state, void *buf, size_t bytes) } EXPORT_SYMBOL(prandom_bytes_state); -static void prandom_warmup(struct rnd_state *state) +/* + * Only declared here so that it has a prototype when made + * non-static for KUnit testing (avoids -Wmissing-prototypes). + */ +#if IS_ENABLED(CONFIG_KUNIT) +void prandom_warmup(struct rnd_state *state); +#endif +VISIBLE_IF_KUNIT void prandom_warmup(struct rnd_state *state) { /* Calling RNG ten times to satisfy recurrence condition */ prandom_u32_state(state); @@ -106,6 +114,7 @@ static void prandom_warmup(struct rnd_state *state) prandom_u32_state(state); prandom_u32_state(state); } +EXPORT_SYMBOL_IF_KUNIT(prandom_warmup); void prandom_seed_full_state(struct rnd_state __percpu *pcpu_state) { @@ -125,176 +134,3 @@ void prandom_seed_full_state(struct rnd_state __percpu *pcpu_state) } } EXPORT_SYMBOL(prandom_seed_full_state); - -#ifdef CONFIG_RANDOM32_SELFTEST -static struct prandom_test1 { - u32 seed; - u32 result; -} test1[] = { - { 1U, 3484351685U }, - { 2U, 2623130059U }, - { 3U, 3125133893U }, - { 4U, 984847254U }, -}; - -static struct prandom_test2 { - u32 seed; - u32 iteration; - u32 result; -} test2[] = { - /* Test cases against taus113 from GSL library. */ - { 931557656U, 959U, 2975593782U }, - { 1339693295U, 876U, 3887776532U }, - { 1545556285U, 961U, 1615538833U }, - { 601730776U, 723U, 1776162651U }, - { 1027516047U, 687U, 511983079U }, - { 416526298U, 700U, 916156552U }, - { 1395522032U, 652U, 2222063676U }, - { 366221443U, 617U, 2992857763U }, - { 1539836965U, 714U, 3783265725U }, - { 556206671U, 994U, 799626459U }, - { 684907218U, 799U, 367789491U }, - { 2121230701U, 931U, 2115467001U }, - { 1668516451U, 644U, 3620590685U }, - { 768046066U, 883U, 2034077390U }, - { 1989159136U, 833U, 1195767305U }, - { 536585145U, 996U, 3577259204U }, - { 1008129373U, 642U, 1478080776U }, - { 1740775604U, 939U, 1264980372U }, - { 1967883163U, 508U, 10734624U }, - { 1923019697U, 730U, 3821419629U }, - { 442079932U, 560U, 3440032343U }, - { 1961302714U, 845U, 841962572U }, - { 2030205964U, 962U, 1325144227U }, - { 1160407529U, 507U, 240940858U }, - { 635482502U, 779U, 4200489746U }, - { 1252788931U, 699U, 867195434U }, - { 1961817131U, 719U, 668237657U }, - { 1071468216U, 983U, 917876630U }, - { 1281848367U, 932U, 1003100039U }, - { 582537119U, 780U, 1127273778U }, - { 1973672777U, 853U, 1071368872U }, - { 1896756996U, 762U, 1127851055U }, - { 847917054U, 500U, 1717499075U }, - { 1240520510U, 951U, 2849576657U }, - { 1685071682U, 567U, 1961810396U }, - { 1516232129U, 557U, 3173877U }, - { 1208118903U, 612U, 1613145022U }, - { 1817269927U, 693U, 4279122573U }, - { 1510091701U, 717U, 638191229U }, - { 365916850U, 807U, 600424314U }, - { 399324359U, 702U, 1803598116U }, - { 1318480274U, 779U, 2074237022U }, - { 697758115U, 840U, 1483639402U }, - { 1696507773U, 840U, 577415447U }, - { 2081979121U, 981U, 3041486449U }, - { 955646687U, 742U, 3846494357U }, - { 1250683506U, 749U, 836419859U }, - { 595003102U, 534U, 366794109U }, - { 47485338U, 558U, 3521120834U }, - { 619433479U, 610U, 3991783875U }, - { 704096520U, 518U, 4139493852U }, - { 1712224984U, 606U, 2393312003U }, - { 1318233152U, 922U, 3880361134U }, - { 855572992U, 761U, 1472974787U }, - { 64721421U, 703U, 683860550U }, - { 678931758U, 840U, 380616043U }, - { 692711973U, 778U, 1382361947U }, - { 677703619U, 530U, 2826914161U }, - { 92393223U, 586U, 1522128471U }, - { 1222592920U, 743U, 3466726667U }, - { 358288986U, 695U, 1091956998U }, - { 1935056945U, 958U, 514864477U }, - { 735675993U, 990U, 1294239989U }, - { 1560089402U, 897U, 2238551287U }, - { 70616361U, 829U, 22483098U }, - { 368234700U, 731U, 2913875084U }, - { 20221190U, 879U, 1564152970U }, - { 539444654U, 682U, 1835141259U }, - { 1314987297U, 840U, 1801114136U }, - { 2019295544U, 645U, 3286438930U }, - { 469023838U, 716U, 1637918202U }, - { 1843754496U, 653U, 2562092152U }, - { 400672036U, 809U, 4264212785U }, - { 404722249U, 965U, 2704116999U }, - { 600702209U, 758U, 584979986U }, - { 519953954U, 667U, 2574436237U }, - { 1658071126U, 694U, 2214569490U }, - { 420480037U, 749U, 3430010866U }, - { 690103647U, 969U, 3700758083U }, - { 1029424799U, 937U, 3787746841U }, - { 2012608669U, 506U, 3362628973U }, - { 1535432887U, 998U, 42610943U }, - { 1330635533U, 857U, 3040806504U }, - { 1223800550U, 539U, 3954229517U }, - { 1322411537U, 680U, 3223250324U }, - { 1877847898U, 945U, 2915147143U }, - { 1646356099U, 874U, 965988280U }, - { 805687536U, 744U, 4032277920U }, - { 1948093210U, 633U, 1346597684U }, - { 392609744U, 783U, 1636083295U }, - { 690241304U, 770U, 1201031298U }, - { 1360302965U, 696U, 1665394461U }, - { 1220090946U, 780U, 1316922812U }, - { 447092251U, 500U, 3438743375U }, - { 1613868791U, 592U, 828546883U }, - { 523430951U, 548U, 2552392304U }, - { 726692899U, 810U, 1656872867U }, - { 1364340021U, 836U, 3710513486U }, - { 1986257729U, 931U, 935013962U }, - { 407983964U, 921U, 728767059U }, -}; - -static void prandom_state_selftest_seed(struct rnd_state *state, u32 seed) -{ -#define LCG(x) ((x) * 69069U) /* super-duper LCG */ - state->s1 = __seed(LCG(seed), 2U); - state->s2 = __seed(LCG(state->s1), 8U); - state->s3 = __seed(LCG(state->s2), 16U); - state->s4 = __seed(LCG(state->s3), 128U); -} - -static int __init prandom_state_selftest(void) -{ - int i, j, errors = 0, runs = 0; - bool error = false; - - for (i = 0; i < ARRAY_SIZE(test1); i++) { - struct rnd_state state; - - prandom_state_selftest_seed(&state, test1[i].seed); - prandom_warmup(&state); - - if (test1[i].result != prandom_u32_state(&state)) - error = true; - } - - if (error) - pr_warn("prandom: seed boundary self test failed\n"); - else - pr_info("prandom: seed boundary self test passed\n"); - - for (i = 0; i < ARRAY_SIZE(test2); i++) { - struct rnd_state state; - - prandom_state_selftest_seed(&state, test2[i].seed); - prandom_warmup(&state); - - for (j = 0; j < test2[i].iteration - 1; j++) - prandom_u32_state(&state); - - if (test2[i].result != prandom_u32_state(&state)) - errors++; - - runs++; - cond_resched(); - } - - if (errors) - pr_warn("prandom: %d/%d self tests failed\n", errors, runs); - else - pr_info("prandom: %d self tests passed\n", runs); - return 0; -} -core_initcall(prandom_state_selftest); -#endif diff --git a/lib/rbtree.c b/lib/rbtree.c index 18d42bcf4ec9..5790d6ecba4e 100644 --- a/lib/rbtree.c +++ b/lib/rbtree.c @@ -446,6 +446,23 @@ void rb_erase(struct rb_node *node, struct rb_root *root) } EXPORT_SYMBOL(rb_erase); +bool rb_erase_linked(struct rb_node_linked *node, struct rb_root_linked *root) +{ + if (node->prev) + node->prev->next = node->next; + else + root->rb_leftmost = node->next; + + if (node->next) + node->next->prev = node->prev; + + rb_erase(&node->node, &root->rb_root); + RB_CLEAR_LINKED_NODE(node); + + return !!root->rb_leftmost; +} +EXPORT_SYMBOL_GPL(rb_erase_linked); + /* * Augmented rbtree manipulation functions. * diff --git a/lib/region_alloc_benchmark.c b/lib/region_alloc_benchmark.c new file mode 100644 index 000000000000..e88b4cf55c62 --- /dev/null +++ b/lib/region_alloc_benchmark.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Benchmark bitmap, IDA and Maple Tree allocation of variable-sized regions. */ + +#include <linux/bitmap.h> +#include <linux/idr.h> +#include <linux/kernel.h> +#include <linux/maple_tree.h> +#include <linux/module.h> +#include <linux/printk.h> +#include <linux/random.h> +#include <linux/slab.h> +#include <linux/xarray.h> + +#define REGION_MAX_SIZE 32 + +static unsigned long *bitmap __initdata; +/* One more request guarantees that even an all-ones trace reaches ENOSPC. */ +static u8 *reg_sz __initdata; +static unsigned long *reg_idx __initdata; +static unsigned long capacities[64] = { 1000000, 100000, 10000, 1000, 100, 10 }; +static unsigned int cap_cnt = 6; + +module_param_array(capacities, ulong, &cap_cnt, 0400); +MODULE_PARM_DESC(capacities, "Region capacities to benchmark"); + +static unsigned long __init benchmark_bitmap(unsigned long cap) +{ + unsigned long cnt, idx; + ktime_t alloc_time, free_time; + size_t sz; + + bitmap_zero(bitmap, cap); + alloc_time = ktime_get(); + for (cnt = 0; cnt <= cap; cnt++) { + idx = bitmap_find_next_zero_area(bitmap, cap, 0, reg_sz[cnt], 0); + if (idx >= cap) + break; + + reg_idx[cnt] = idx; + bitmap_set(bitmap, idx, reg_sz[cnt]); + } + alloc_time = ktime_get() - alloc_time; + + idx = cnt; + + free_time = ktime_get(); + while (idx--) + bitmap_clear(bitmap, reg_idx[idx], reg_sz[idx]); + free_time = ktime_get() - free_time; + + WARN_ON(!bitmap_empty(bitmap, cap)); + + sz = BITS_TO_LONGS(cap) * sizeof(unsigned long); + pr_err("Bitmap %12llu %12llu %8lu %8lu %10zu\n", + alloc_time, free_time, cnt, cap, sz); + + return cnt; +} + +static size_t __init ida_size(unsigned long nr_ids) +{ + unsigned long entries = DIV_ROUND_UP(nr_ids, IDA_BITMAP_BITS); + unsigned long bitmaps = nr_ids / IDA_BITMAP_BITS; + unsigned long nodes = 0; + + if (nr_ids % IDA_BITMAP_BITS > BITS_PER_XA_VALUE) + bitmaps++; + + while (entries > 1) { + entries = DIV_ROUND_UP(entries, XA_CHUNK_SIZE); + nodes += entries; + } + + return sizeof(struct ida) + + bitmaps * sizeof(struct ida_bitmap) + + nodes * sizeof(struct xa_node); +} + +static unsigned long __init benchmark_ida(unsigned long cap) +{ + struct ida ida = IDA_INIT(ida); + unsigned long cnt, idx, off, nr_ids = 0; + ktime_t alloc_time, free_time; + int id = -ENOSPC; + + alloc_time = ktime_get(); + for (cnt = 0; cnt <= cap; cnt++) { + for (off = 0; off < reg_sz[cnt]; off++) { + id = ida_alloc_max(&ida, cap - 1, GFP_KERNEL); + if (id < 0) + break; + + if (!off) + reg_idx[cnt] = id; + } + if (id < 0) { + while (off--) + ida_free(&ida, reg_idx[cnt] + off); + break; + } + WARN_ON(id != reg_idx[cnt] + reg_sz[cnt] - 1); + nr_ids += reg_sz[cnt]; + } + alloc_time = ktime_get() - alloc_time; + + WARN_ON(id != -ENOSPC); + + idx = cnt; + + free_time = ktime_get(); + while (idx--) { + for (off = 0; off < reg_sz[idx]; off++) + ida_free(&ida, reg_idx[idx] + off); + } + free_time = ktime_get() - free_time; + + WARN_ON(!ida_is_empty(&ida)); + + pr_err("IDA %12llu %12llu %8lu %8lu %10zu\n", + alloc_time, free_time, cnt, cap, ida_size(nr_ids)); + + ida_destroy(&ida); + return cnt; +} + +static unsigned long __init benchmark_maple_tree(unsigned long cap) +{ + struct maple_tree mt = MTREE_INIT(mt, MT_FLAGS_ALLOC_RANGE); + unsigned long cnt, idx; + ktime_t alloc_time, free_time; + size_t sz; + int ret; + + alloc_time = ktime_get(); + for (cnt = 0; cnt <= cap; cnt++) { + ret = mtree_alloc_range(&mt, &idx, xa_mk_value(cnt + 1), + reg_sz[cnt], 0, cap - 1, GFP_KERNEL); + if (ret) + break; + + reg_idx[cnt] = idx; + } + alloc_time = ktime_get() - alloc_time; + + WARN_ON(ret != -EBUSY); + + idx = cnt; + + free_time = ktime_get(); + while (idx--) + mtree_erase(&mt, reg_idx[idx]); + free_time = ktime_get() - free_time; + + WARN_ON(!mtree_empty(&mt)); + + /* Minimum storage assuming fully occupied allocation-range leaf nodes. */ + sz = sizeof(mt) + DIV_ROUND_UP(cnt, MAPLE_ARANGE64_SLOTS) * sizeof(struct maple_node); + pr_err("Maple %12llu %12llu %8lu %8lu %10zu\n", + alloc_time, free_time, cnt, cap, sz); + + mtree_destroy(&mt); + return cnt; +} + +static int __init region_alloc_benchmark(void) +{ + unsigned long bitmap_count, ida_count, maple_count; + unsigned long i, max_cap = 0; + int ret = -ENOMEM; + + for (i = 0; i < cap_cnt; i++) { + if (capacities[i] == 0) { + pr_err("capacity must be nonzero\n"); + return -EINVAL; + } + max_cap = max(max_cap, capacities[i]); + } + + bitmap = kvmalloc_array(BITS_TO_LONGS(max_cap), sizeof(*bitmap), GFP_KERNEL); + reg_sz = kvmalloc_array(max_cap + 1, sizeof(*reg_sz), GFP_KERNEL); + reg_idx = kvmalloc_array(max_cap, sizeof(*reg_idx), GFP_KERNEL); + if (!bitmap || !reg_sz || !reg_idx) + goto out; + + pr_err("\nStart testing bitmap vs IDA vs Maple Tree region allocation\n"); + pr_err("memory: bitmap is exact; IDA and Maple Tree are lower bounds\n"); + pr_err("Type alloc (ns) free (ns) regions capacity memory (B)\n"); + + for (i = 0; i < cap_cnt; i++) { + unsigned long idx, max_size; + + max_size = min(REGION_MAX_SIZE, capacities[i] / 10) ? : 1; + for (idx = 0; idx <= capacities[i]; idx++) + reg_sz[idx] = get_random_u32_below(max_size) + 1; + + bitmap_count = benchmark_bitmap(capacities[i]); + maple_count = benchmark_maple_tree(capacities[i]); + ida_count = benchmark_ida(capacities[i]); + + WARN_ON(bitmap_count != ida_count); + WARN_ON(bitmap_count != maple_count); + } + + /* Return an error so the benchmark can run repeatedly without rmmod. */ + pr_info("Region allocation benchmark complete\n"); + ret = -EAGAIN; +out: + kvfree(reg_idx); + kvfree(reg_sz); + kvfree(bitmap); + return ret; +} +module_init(region_alloc_benchmark); + +MODULE_AUTHOR("Yury Norov <ynorov@nvidia.com>"); +MODULE_DESCRIPTION("Benchmark bitmap, IDA and Maple Tree region allocation"); +MODULE_LICENSE("GPL"); diff --git a/lib/rhashtable.c b/lib/rhashtable.c index 6074ed5f66f3..6362896e4f09 100644 --- a/lib/rhashtable.c +++ b/lib/rhashtable.c @@ -14,6 +14,7 @@ #include <linux/atomic.h> #include <linux/kernel.h> #include <linux/init.h> +#include <linux/irq_work.h> #include <linux/log2.h> #include <linux/sched.h> #include <linux/rculist.h> @@ -25,6 +26,7 @@ #include <linux/rhashtable.h> #include <linux/err.h> #include <linux/export.h> +#include <linux/workqueue.h> #define HASH_DEFAULT_SIZE 64UL #define HASH_MIN_SIZE 4U @@ -114,6 +116,14 @@ static void bucket_table_free(const struct bucket_table *tbl) kvfree(tbl); } +static void bucket_table_free_atomic(const struct bucket_table *tbl) +{ + if (tbl->nest) + nested_bucket_table_free(tbl); + + kvfree_atomic(tbl); +} + static void bucket_table_free_rcu(struct rcu_head *head) { bucket_table_free(container_of(head, struct bucket_table, rcu)); @@ -441,10 +451,33 @@ static void rht_deferred_worker(struct work_struct *work) mutex_unlock(&ht->mutex); + /* + * Re-arm via @run_work, not @run_irq_work. + * rhashtable_free_and_destroy() drains async work as irq_work_sync() + * followed by cancel_work_sync(). If this site queued irq_work while + * cancel_work_sync() was waiting for us, irq_work_sync() would already + * have returned and the stale irq_work could fire post-teardown. + * cancel_work_sync() natively handles self-requeue on @run_work. + */ if (err) schedule_work(&ht->run_work); } +/* + * Insert-path callers can run under a raw spinlock (e.g. an insecure_elasticity + * user). Calling schedule_work() under that lock records caller_lock -> + * pool->lock -> pi_lock -> rq->__lock, closing a locking cycle if any of + * these is acquired in the reverse direction elsewhere. Bounce through + * irq_work so the schedule_work() runs with the caller's lock no longer held. + */ +static void rht_deferred_irq_work(struct irq_work *irq_work) +{ + struct rhashtable *ht = container_of(irq_work, struct rhashtable, + run_irq_work); + + schedule_work(&ht->run_work); +} + static int rhashtable_insert_rehash(struct rhashtable *ht, struct bucket_table *tbl) { @@ -473,11 +506,11 @@ static int rhashtable_insert_rehash(struct rhashtable *ht, err = rhashtable_rehash_attach(ht, tbl, new_tbl); if (err) { - bucket_table_free(new_tbl); + bucket_table_free_atomic(new_tbl); if (err == -EEXIST) err = 0; } else - schedule_work(&ht->run_work); + irq_work_queue(&ht->run_irq_work); return err; @@ -488,7 +521,7 @@ fail: /* Schedule async rehash to retry allocation in process context. */ if (err == -ENOMEM) - schedule_work(&ht->run_work); + irq_work_queue(&ht->run_irq_work); return err; } @@ -538,7 +571,7 @@ static void *rhashtable_lookup_one(struct rhashtable *ht, return NULL; } - if (elasticity <= 0) + if (elasticity <= 0 && !ht->p.insecure_elasticity) return ERR_PTR(-EAGAIN); return ERR_PTR(-ENOENT); @@ -568,7 +601,8 @@ static struct bucket_table *rhashtable_insert_one( if (unlikely(rht_grow_above_max(ht, tbl))) return ERR_PTR(-E2BIG); - if (unlikely(rht_grow_above_100(ht, tbl))) + if (unlikely(rht_grow_above_100(ht, tbl)) && + !ht->p.insecure_elasticity) return ERR_PTR(-EAGAIN); head = rht_ptr(bkt, tbl, hash); @@ -629,7 +663,7 @@ static void *rhashtable_try_insert(struct rhashtable *ht, const void *key, rht_unlock(tbl, bkt, flags); if (inserted && rht_grow_above_75(ht, tbl)) - schedule_work(&ht->run_work); + irq_work_queue(&ht->run_irq_work); } } while (!IS_ERR_OR_NULL(new_tbl)); @@ -655,6 +689,108 @@ void *rhashtable_insert_slow(struct rhashtable *ht, const void *key, } EXPORT_SYMBOL_GPL(rhashtable_insert_slow); +/* Scan one element forward from prev_key's position in @tbl. + * Returns first rhash_head whose bucket > prev_key's bucket, or the + * element immediately after prev_key inside prev_key's bucket. + * Returns the first element if prev_key is NULL, NULL when @tbl is + * exhausted, or ERR_PTR(-ENOENT) if prev_key is not found in @tbl. + */ +static struct rhash_head *__rhashtable_next_in_table( + struct rhashtable *ht, struct bucket_table *tbl, + const void *prev_key) +{ + struct rhashtable_compare_arg arg = { .ht = ht, .key = prev_key }; + const struct rhashtable_params params = ht->p; + struct rhash_head *he; + unsigned int b = 0; + bool found = false; + + if (prev_key) { + b = rht_key_hashfn(ht, tbl, prev_key, params); + rht_for_each_rcu(he, tbl, b) { + bool match = params.obj_cmpfn + ? !params.obj_cmpfn(&arg, rht_obj(ht, he)) + : !rhashtable_compare(&arg, rht_obj(ht, he)); + if (found) { + if (match) + continue; + return he; + } + if (match) + found = true; + } + if (!found) + return ERR_PTR(-ENOENT); + b++; + } + + for (; b < tbl->size; b++) + rht_for_each_rcu(he, tbl, b) + return he; + return NULL; +} + +/** + * rhashtable_next_key - return next element after a given key + * @ht: hash table + * @prev_key: pointer to previous key, or NULL for the first element + * + * WARNING: this walk is highly unstable. Unlike rhashtable_walk_*(), + * it cannot detect a concurrent resize or rehash, so a full iteration + * is NOT guaranteed to terminate under adversarial or sustained + * rehashing. Callers MUST tolerate skipped and duplicated elements and + * SHOULD bound their loop externally. + * + * Returns the next element in best-effort iteration order, walking the + * @tbl chain (including any future_tbl in flight). Caller must hold RCU. + * + * Pass @prev_key == NULL to obtain the first element. To iterate, set + * @prev_key to the key of the previously returned element on each call, + * and stop when NULL is returned. + * + * Best-effort semantics: + * - Across the tbl->future_tbl chain, an element being migrated may + * transiently appear in both tables and be observed twice. + * - Concurrent inserts may or may not be observed. + * - Termination of a full iteration loop is NOT guaranteed under + * adversarial continuous rehash; callers MUST tolerate skips and + * repeats and SHOULD bound their loop externally. + * - Behavior on tables that contain duplicate keys is undefined: + * duplicates may be skipped, repeated, or trap the walk in a + * cycle. Callers requiring duplicate-key iteration must use + * rhashtable_walk_*() instead. + * - rhltable instances are not supported and return + * ERR_PTR(-EOPNOTSUPP). + * - If prev_key was concurrently deleted and is not present in any + * in-flight table, returns ERR_PTR(-ENOENT). + * + * Returns entry of the next element, or NULL when iteration is exhausted, + * or ERR_PTR(-ENOENT) if prev_key is not found, or + * ERR_PTR(-EOPNOTSUPP) if @ht is an rhltable. + */ +void *rhashtable_next_key(struct rhashtable *ht, const void *prev_key) +{ + struct bucket_table *tbl; + struct rhash_head *he; + + if (unlikely(ht->rhlist)) + return ERR_PTR(-EOPNOTSUPP); + + tbl = rht_dereference_rcu(ht->tbl, ht); + do { + he = __rhashtable_next_in_table(ht, tbl, prev_key); + if (!IS_ERR_OR_NULL(he)) + return rht_obj(ht, he); + if (!he) + prev_key = NULL; + /* See any new future_tbl attached during a rehash. */ + smp_rmb(); + tbl = rht_dereference_rcu(tbl->future_tbl, ht); + } while (tbl); + return he; /* NULL or -ENOENT */ +} +EXPORT_SYMBOL_GPL(rhashtable_next_key); + /** * rhashtable_walk_enter - Initialise an iterator * @ht: Table to walk over @@ -744,6 +880,7 @@ int rhashtable_walk_start_check(struct rhashtable_iter *iter) iter->walker.tbl = rht_dereference_rcu(ht->tbl, ht); iter->slot = 0; iter->skip = 0; + iter->p = NULL; return -EAGAIN; } @@ -1025,8 +1162,9 @@ static u32 rhashtable_jhash2(const void *key, u32 length, u32 seed) * .obj_hashfn = my_hash_fn, * }; */ -int rhashtable_init_noprof(struct rhashtable *ht, - const struct rhashtable_params *params) +int __rhashtable_init_noprof(struct rhashtable *ht, + const struct rhashtable_params *params, + struct lock_class_key *key) { struct bucket_table *tbl; size_t size; @@ -1036,7 +1174,7 @@ int rhashtable_init_noprof(struct rhashtable *ht, return -EINVAL; memset(ht, 0, sizeof(*ht)); - mutex_init(&ht->mutex); + mutex_init_with_key(&ht->mutex, key); spin_lock_init(&ht->lock); memcpy(&ht->p, params, sizeof(*params)); @@ -1084,10 +1222,11 @@ int rhashtable_init_noprof(struct rhashtable *ht, RCU_INIT_POINTER(ht->tbl, tbl); INIT_WORK(&ht->run_work, rht_deferred_worker); + init_irq_work(&ht->run_irq_work, rht_deferred_irq_work); return 0; } -EXPORT_SYMBOL_GPL(rhashtable_init_noprof); +EXPORT_SYMBOL_GPL(__rhashtable_init_noprof); /** * rhltable_init - initialize a new hash list table @@ -1098,15 +1237,17 @@ EXPORT_SYMBOL_GPL(rhashtable_init_noprof); * * See documentation for rhashtable_init. */ -int rhltable_init_noprof(struct rhltable *hlt, const struct rhashtable_params *params) +int __rhltable_init_noprof(struct rhltable *hlt, + const struct rhashtable_params *params, + struct lock_class_key *key) { int err; - err = rhashtable_init_noprof(&hlt->ht, params); + err = __rhashtable_init_noprof(&hlt->ht, params, key); hlt->ht.rhlist = true; return err; } -EXPORT_SYMBOL_GPL(rhltable_init_noprof); +EXPORT_SYMBOL_GPL(__rhltable_init_noprof); static void rhashtable_free_one(struct rhashtable *ht, struct rhash_head *obj, void (*free_fn)(void *ptr, void *arg), @@ -1122,7 +1263,7 @@ static void rhashtable_free_one(struct rhashtable *ht, struct rhash_head *obj, list = container_of(obj, struct rhlist_head, rhead); do { obj = &list->rhead; - list = rht_dereference(list->next, ht); + list = rcu_dereference_raw(list->next); free_fn(rht_obj(ht, obj), arg); } while (list); } @@ -1141,6 +1282,11 @@ static void rhashtable_free_one(struct rhashtable *ht, struct rhash_head *obj, * This function will eventually sleep to wait for an async resize * to complete. The caller is responsible that no further write operations * occurs in parallel. + * + * After cancel_work_sync() has returned, the deferred rehash worker is + * quiesced and, per the contract above, no other concurrent access to the + * rhashtable is possible. The tables are therefore owned exclusively by + * this function and can be walked without ht->mutex held. */ void rhashtable_free_and_destroy(struct rhashtable *ht, void (*free_fn)(void *ptr, void *arg), @@ -1149,10 +1295,18 @@ void rhashtable_free_and_destroy(struct rhashtable *ht, struct bucket_table *tbl, *next_tbl; unsigned int i; + irq_work_sync(&ht->run_irq_work); cancel_work_sync(&ht->run_work); - mutex_lock(&ht->mutex); - tbl = rht_dereference(ht->tbl, ht); + /* + * Do NOT take ht->mutex here. The rehash worker establishes + * ht->mutex -> fs_reclaim via GFP_KERNEL bucket allocation under + * the mutex; callers on the reclaim path (e.g. simple_xattr_ht_free() + * from evict() under the dcache shrinker for shmem/kernfs/pidfs + * inodes) would otherwise close a circular dependency + * fs_reclaim -> ht->mutex. + */ + tbl = rcu_dereference_raw(ht->tbl); restart: if (free_fn) { for (i = 0; i < tbl->size; i++) { @@ -1161,22 +1315,21 @@ restart: cond_resched(); for (pos = rht_ptr_exclusive(rht_bucket(tbl, i)), next = !rht_is_a_nulls(pos) ? - rht_dereference(pos->next, ht) : NULL; + rcu_dereference_raw(pos->next) : NULL; !rht_is_a_nulls(pos); pos = next, next = !rht_is_a_nulls(pos) ? - rht_dereference(pos->next, ht) : NULL) + rcu_dereference_raw(pos->next) : NULL) rhashtable_free_one(ht, pos, free_fn, arg); } } - next_tbl = rht_dereference(tbl->future_tbl, ht); + next_tbl = rcu_dereference_raw(tbl->future_tbl); bucket_table_free(tbl); if (next_tbl) { tbl = next_tbl; goto restart; } - mutex_unlock(&ht->mutex); } EXPORT_SYMBOL_GPL(rhashtable_free_and_destroy); diff --git a/lib/scatterlist.c b/lib/scatterlist.c index d773720d11bf..6ea40d2e6247 100644 --- a/lib/scatterlist.c +++ b/lib/scatterlist.c @@ -1123,8 +1123,7 @@ static ssize_t extract_user_to_sg(struct iov_iter *iter, size_t len, off; /* We decant the page list into the tail of the scatterlist */ - pages = (void *)sgtable->sgl + - array_size(sg_max, sizeof(struct scatterlist)); + pages = (void *)sg + array_size(sg_max, sizeof(struct scatterlist)); pages -= sg_max; do { @@ -1247,7 +1246,7 @@ static ssize_t extract_kvec_to_sg(struct iov_iter *iter, else page = virt_to_page((void *)kaddr); - sg_set_page(sg, page, len, off); + sg_set_page(sg, page, seg, off); sgtable->nents++; sg++; sg_max--; @@ -1256,6 +1255,7 @@ static ssize_t extract_kvec_to_sg(struct iov_iter *iter, kaddr += PAGE_SIZE; off = 0; } while (len > 0 && sg_max > 0); + ret -= len; if (maxsize <= 0 || sg_max == 0) break; @@ -1366,6 +1366,7 @@ static ssize_t extract_xarray_to_sg(struct iov_iter *iter, sg_max--; maxsize -= len; + start += len; ret += len; if (maxsize <= 0 || sg_max == 0) break; @@ -1409,7 +1410,7 @@ ssize_t extract_iter_to_sg(struct iov_iter *iter, size_t maxsize, struct sg_table *sgtable, unsigned int sg_max, iov_iter_extraction_t extraction_flags) { - if (maxsize == 0) + if (maxsize == 0 || sg_max == 0) return 0; switch (iov_iter_type(iter)) { diff --git a/lib/seq_buf.c b/lib/seq_buf.c index f3f3436d60a9..a92093f346da 100644 --- a/lib/seq_buf.c +++ b/lib/seq_buf.c @@ -298,6 +298,7 @@ int seq_buf_putmem_hex(struct seq_buf *s, const void *mem, } return 0; } +EXPORT_SYMBOL_GPL(seq_buf_putmem_hex); /** * seq_buf_path - copy a path into the sequence buffer @@ -320,7 +321,7 @@ int seq_buf_path(struct seq_buf *s, const struct path *path, const char *esc) if (size) { char *p = d_path(path, buf, size); if (!IS_ERR(p)) { - char *end = mangle_path(buf, p, esc); + char *end = seq_mangle_path(buf, p, esc); if (end) res = end - buf; } diff --git a/lib/string.c b/lib/string.c index b632c71df1a5..a3778d5aab4a 100644 --- a/lib/string.c +++ b/lib/string.c @@ -21,6 +21,7 @@ #include <linux/errno.h> #include <linux/limits.h> #include <linux/linkage.h> +#include <linux/minmax.h> #include <linux/stddef.h> #include <linux/string.h> #include <linux/types.h> @@ -88,22 +89,6 @@ char *strcpy(char *dest, const char *src) EXPORT_SYMBOL(strcpy); #endif -#ifndef __HAVE_ARCH_STRNCPY -char *strncpy(char *dest, const char *src, size_t count) -{ - char *tmp = dest; - - while (count) { - if ((*tmp = *src) != 0) - src++; - tmp++; - count--; - } - return dest; -} -EXPORT_SYMBOL(strncpy); -#endif - #ifdef __BIG_ENDIAN # define ALLBUTLAST_BYTE_MASK (~255ul) #else @@ -125,11 +110,8 @@ ssize_t sized_strscpy(char *dest, const char *src, size_t count) * If src is unaligned, don't cross a page boundary, * since we don't know if the next page is mapped. */ - if ((long)src & (sizeof(long) - 1)) { - size_t limit = PAGE_SIZE - ((long)src & (PAGE_SIZE - 1)); - if (limit < max) - max = limit; - } + if ((long)src & (sizeof(long) - 1)) + max = min(PAGE_SIZE - ((long)src & (PAGE_SIZE - 1)), max); #else /* If src or dest is unaligned, don't do word-at-a-time. */ if (((long) dest | (long) src) & (sizeof(long) - 1)) @@ -839,7 +821,8 @@ void *memchr_inv(const void *start, int c, size_t bytes) { u8 value = c; u64 value64; - unsigned int words, prefix; + size_t words; + unsigned int prefix; if (bytes <= 16) return check_bytes8(start, value, bytes); diff --git a/lib/string_helpers.c b/lib/string_helpers.c index 169eaf583494..98d6ed0eaab7 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -752,7 +752,7 @@ EXPORT_SYMBOL_GPL(kstrdup_and_replace); * kasprintf_strarray - allocate and fill array of sequential strings * @gfp: flags for the slab allocator * @prefix: prefix to be used - * @n: amount of lines to be allocated and filled + * @n: number of strings to be allocated and filled * * Allocates and fills @n strings using pattern "%s-%zu", where prefix * is provided by caller. The caller is responsible to free them with @@ -765,7 +765,7 @@ char **kasprintf_strarray(gfp_t gfp, const char *prefix, size_t n) char **names; size_t i; - names = kcalloc(n + 1, sizeof(char *), gfp); + names = kcalloc(n, sizeof(char *), gfp); if (!names) return NULL; @@ -805,7 +805,7 @@ void kfree_strarray(char **array, size_t n) EXPORT_SYMBOL_GPL(kfree_strarray); struct strarray { - char **array; + char **array __counted_by_ptr(n); size_t n; }; @@ -824,13 +824,13 @@ char **devm_kasprintf_strarray(struct device *dev, const char *prefix, size_t n) if (!ptr) return ERR_PTR(-ENOMEM); + ptr->n = n; ptr->array = kasprintf_strarray(GFP_KERNEL, prefix, n); if (!ptr->array) { devres_free(ptr); return ERR_PTR(-ENOMEM); } - ptr->n = n; devres_add(dev, ptr); return ptr->array; diff --git a/lib/test-kstrtox.c b/lib/test-kstrtox.c index ee87fef66cb5..9255234752db 100644 --- a/lib/test-kstrtox.c +++ b/lib/test-kstrtox.c @@ -198,6 +198,7 @@ static void __init test_kstrtoull_fail(void) {"10000000000000000000000000000000000000000000000000000000000000000", 2}, {"2000000000000000000000", 8}, {"18446744073709551616", 10}, + {"569202370375329612767", 10}, {"10000000000000000", 16}, /* negative */ {"-0", 0}, @@ -275,9 +276,11 @@ static void __init test_kstrtoll_fail(void) {"9223372036854775809", 10}, {"18446744073709551614", 10}, {"18446744073709551615", 10}, + {"569202370375329612767", 10}, {"-9223372036854775809", 10}, {"-18446744073709551614", 10}, {"-18446744073709551615", 10}, + {"-569202370375329612767", 10}, /* sign is first character if any */ {"-+1", 0}, {"-+1", 8}, @@ -334,6 +337,7 @@ static void __init test_kstrtou64_fail(void) {"-1", 10}, {"18446744073709551616", 10}, {"18446744073709551617", 10}, + {"569202370375329612767", 10}, }; TEST_FAIL(kstrtou64, u64, "%llu", test_u64_fail); } @@ -386,6 +390,8 @@ static void __init test_kstrtos64_fail(void) {"18446744073709551615", 10}, {"18446744073709551616", 10}, {"18446744073709551617", 10}, + {"569202370375329612767", 10}, + {"-569202370375329612767", 10}, }; TEST_FAIL(kstrtos64, s64, "%lld", test_s64_fail); } @@ -703,6 +709,182 @@ static void __init test_kstrtos8_fail(void) TEST_FAIL(kstrtos8, s8, "%hhd", test_s8_fail); } +static void __init test_kstrtoudec64_ok(void) +{ + DECLARE_TEST_OK(u64, struct test_udec64); + static DEFINE_TEST_OK(struct test_udec64, test_udec64_ok) = { + /* basic: integer.fraction, exact digits */ + {"0.0", 1, 0}, + {"1.5", 1, 15}, + {"1.234", 3, 1234}, + {"42.0", 1, 420}, + /* zero */ + {"0.0", 1, 0}, + {"0.000", 3, 0}, + /* integer only */ + {"0", 1, 0}, + {"42", 3, 42000}, + {"123.", 2, 12300}, + {"1", 1, 10}, + /* fractional only (leading dot) */ + {".5", 1, 5}, + {".5", 0, 0}, + {".123", 3, 123}, + {".001", 3, 1}, + /* zero padding: fewer fractional digits than scale */ + {"1.2", 3, 1200}, + {"1.2", 6, 1200000}, + {"0.01", 3, 10}, + {"0.1", 9, 100000000ULL}, + {"0.01", 9, 10000000}, + /* truncation: more fractional digits than scale */ + {"1.23456", 3, 1234}, + {"3.1415926535", 6, 3141592}, + {"0.999999999", 3, 999}, + {"1.99", 1, 19}, + {"1.234", 0, 1}, + /* trailing newline */ + {"1.5\n", 1, 15}, + {"42\n", 3, 42000}, + /* plus sign */ + {"+1.5", 1, 15}, + {"+.5", 1, 5}, + /* scale progression */ + {"1.", 0, 1}, + {"1.0", 1, 10}, + {"1.00", 2, 100}, + {"1.000", 3, 1000}, + {"1.000000", 6, 1000000}, + {"1.000000000", 9, 1000000000ULL}, + /* max limit check */ + {"18446744073.709551615", 9, ULLONG_MAX}, + {"18446744073709.551615", 6, ULLONG_MAX}, + {"0.18446744073709551615", 20, ULLONG_MAX}, + /* scale > 19: representable when integer part is small */ + {"0.00000000000000000001", 20, 1}, + {"0.1", 20, 10000000000000000000ULL}, + {"0.00000000000000000000001", 23, 1}, + /* truncation with scale > 19 */ + {"0.0000000000000000000012345", 23, 123}, + /* truncation with many excess digits */ + {"0.00000000000000000000000000000000423", 34, 42}, + {"1.99999999999999999999999999999999999", 3, 1999}, + }; + TEST_OK(kstrtoudec64, u64, "%llu", test_udec64_ok); +} + +static void __init test_kstrtoudec64_fail(void) +{ + static DEFINE_TEST_FAIL(test_udec64_fail) = { + /* empty / whitespace */ + {"", 3}, + {"\n", 3}, + /* minus sign (unsigned) */ + {"-1.5", 1}, + {"-0.5", 1}, + /* only a decimal point */ + {".", 3}, + {".", 0}, + /* only a sign */ + {"+", 3}, + /* non-digit characters */ + {"abc", 3}, + {"1.2x", 3}, + /* leading/trailing space */ + {" 1.5", 1}, + {"1.5 ", 1}, + /* overflow */ + {"18446744073710.551615", 6}, + {"99999999999999999999", 1}, + /* overflow with scale > 19 */ + {"1.0", 21}, + {"0.2", 20}, + {"0.18446744073709551616", 20}, + {"1", 20}, + }; + TEST_FAIL(kstrtoudec64, u64, "%llu", test_udec64_fail); +} + +static void __init test_kstrtodec64_ok(void) +{ + DECLARE_TEST_OK(s64, struct test_dec64); + static DEFINE_TEST_OK(struct test_dec64, test_dec64_ok) = { + /* basic positive */ + {"0.0", 1, 0}, + {"1.5", 1, 15}, + {"1.234", 3, 1234}, + /* basic negative */ + {"-1.5", 1, -15}, + {"-1.234", 3, -1234}, + {"-0.5", 1, -5}, + {"-0.001", 3, -1}, + /* zero (signed) */ + {"-0", 1, 0}, + {"-0.0", 1, 0}, + {"0.000", 3, 0}, + /* integer only */ + {"42", 3, 42000}, + {"-42", 3, -42000}, + /* fractional only */ + {".5", 1, 5}, + {"-.5", 1, -5}, + /* zero padding */ + {"1.2", 3, 1200}, + {"-1.2", 3, -1200}, + {"0.01", 3, 10}, + {"-0.01", 3, -10}, + /* truncation */ + {"1.23456", 3, 1234}, + {"-1.23456", 3, -1234}, + {"0.999999999", 3, 999}, + {"-0.999999999", 3, -999}, + /* trailing newline */ + {"1.5\n", 1, 15}, + {"-1.5\n", 1, -15}, + /* plus sign */ + {"+1.5", 1, 15}, + /* limits */ + {"9223372036.854775807", 9, LLONG_MAX}, + {"-9223372036.854775808", 9, LLONG_MIN}, + /* scale > 19 */ + {"0.0", 20, 0}, + {"-0.0", 20, 0}, + {"0.00000000000000000001", 20, 1}, + {"-0.00000000000000000001", 20, -1}, + {"0.009223372036854775807", 21, LLONG_MAX}, + {"-0.009223372036854775808", 21, LLONG_MIN}, + }; + TEST_OK(kstrtodec64, s64, "%lld", test_dec64_ok); +} + +static void __init test_kstrtodec64_fail(void) +{ + static DEFINE_TEST_FAIL(test_dec64_fail) = { + /* empty / whitespace */ + {"", 3}, + {"\n", 3}, + /* no digits after dot */ + {".", 3}, + {"-.", 3}, + /* no digits at all */ + {"+", 3}, + {"-", 3}, + /* non-digit characters */ + {"abc", 3}, + {"-1.2x", 3}, + /* signed overflow */ + {"9223372036.854775808", 9}, + {"-9223372036.854775809", 9}, + {"99999999999999999999", 1}, + /* signed overflow with scale > 19 */ + {"0.1", 20}, + {"-0.1", 20}, + {"0.09223372036854775808", 20}, + {"-0.09223372036854775809", 20}, + }; + TEST_FAIL(kstrtodec64, s64, "%lld", test_dec64_fail); +} + static int __init test_kstrtox_init(void) { test_kstrtoull_ok(); @@ -729,6 +911,12 @@ static int __init test_kstrtox_init(void) test_kstrtou8_fail(); test_kstrtos8_ok(); test_kstrtos8_fail(); + + test_kstrtoudec64_ok(); + test_kstrtoudec64_fail(); + test_kstrtodec64_ok(); + test_kstrtodec64_fail(); + return -EINVAL; } module_init(test_kstrtox_init); diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c index c83829ef557f..56bd23059b26 100644 --- a/lib/test_bitmap.c +++ b/lib/test_bitmap.c @@ -234,6 +234,43 @@ static void __init test_find_nth_bit(void) } } +static void __init +test_bitmap_find_next_zero_area_off(void) +{ + DECLARE_BITMAP(bmap, 192); + + bitmap_set(bmap, 0, 192); + + bitmap_clear(bmap, 0, 8); + __clear_bit(50, bmap); + bitmap_clear(bmap, 60, 18); + __set_bit(69, bmap); + __clear_bit(80, bmap); + bitmap_clear(bmap, 100, 10); + __clear_bit(120, bmap); + bitmap_clear(bmap, 145, 8); + bitmap_clear(bmap, 160, 32); + + expect_eq_uint(0, + bitmap_find_next_zero_area_off(bmap, 192, 0, 8, 0, 0)); + expect_eq_uint(0, + bitmap_find_next_zero_area_off(bmap, 192, 0, 8, 3, 0)); + expect_eq_uint(163, + bitmap_find_next_zero_area_off(bmap, 192, 0, 8, 3, 1)); + expect_eq_uint(60, + bitmap_find_next_zero_area_off(bmap, 192, 1, 8, 0, 0)); + expect_eq_uint(160, + bitmap_find_next_zero_area_off(bmap, 192, 1, 8, 7, 0)); + expect_eq_uint(60, + bitmap_find_next_zero_area_off(bmap, 192, 1, 8, 7, 4)); + expect_eq_uint(100, + bitmap_find_next_zero_area_off(bmap, 192, 0, 10, 0, 0)); + expect_eq_uint(160, + bitmap_find_next_zero_area_off(bmap, 192, 0, 32, 0, 0)); + expect_eq_uint(1, + !!(bitmap_find_next_zero_area_off(bmap, 192, 0, 33, 0, 0) >= 192)); +} + static void __init test_fill_set(void) { DECLARE_BITMAP(bmap, 1024); @@ -354,18 +391,22 @@ static void __init test_replace(void) static const unsigned long sg_mask[] __initconst = { BITMAP_FROM_U64(0x000000000000035aULL), + BITMAP_FROM_U64(0x0000000000000000ULL), }; static const unsigned long sg_src[] __initconst = { BITMAP_FROM_U64(0x0000000000000667ULL), + BITMAP_FROM_U64(0x0000000000000000ULL), }; static const unsigned long sg_gather_exp[] __initconst = { BITMAP_FROM_U64(0x0000000000000029ULL), + BITMAP_FROM_U64(0x0000000000000000ULL), }; static const unsigned long sg_scatter_exp[] __initconst = { BITMAP_FROM_U64(0x000000000000021aULL), + BITMAP_FROM_U64(0x0000000000000000ULL), }; static void __init test_bitmap_sg(void) @@ -379,18 +420,19 @@ static void __init test_bitmap_sg(void) /* Simple gather call */ bitmap_zero(bmap_gather, 100); bitmap_gather(bmap_gather, sg_src, sg_mask, nbits); - expect_eq_bitmap(sg_gather_exp, bmap_gather, nbits); + expect_eq_bitmap(sg_gather_exp, bmap_gather, 100); /* Simple scatter call */ bitmap_zero(bmap_scatter, 100); bitmap_scatter(bmap_scatter, sg_src, sg_mask, nbits); - expect_eq_bitmap(sg_scatter_exp, bmap_scatter, nbits); + expect_eq_bitmap(sg_scatter_exp, bmap_scatter, 100); /* Scatter/gather relationship */ bitmap_zero(bmap_tmp, 100); + bitmap_zero(bmap_res, 100); bitmap_gather(bmap_tmp, bmap_scatter, sg_mask, nbits); bitmap_scatter(bmap_res, bmap_tmp, sg_mask, nbits); - expect_eq_bitmap(bmap_scatter, bmap_res, nbits); + expect_eq_bitmap(bmap_scatter, bmap_res, 100); } #define PARSE_TIME 0x1 @@ -520,8 +562,7 @@ static void __init test_bitmap_parselist(void) } if (ptest.flags & PARSE_TIME) - pr_info("parselist: %d: input is '%s' OK, Time: %llu\n", - i, ptest.in, time); + pr_info("parselist('%s'):\t%llu\n", ptest.in, time); #undef ptest } @@ -544,22 +585,22 @@ static void __init test_bitmap_printlist(void) goto out; time = ktime_get(); - ret = bitmap_print_to_pagebuf(true, buf, bmap, PAGE_SIZE * 8); + ret = scnprintf(buf, PAGE_SIZE, "%*pbl", (int)PAGE_SIZE * 8, bmap); time = ktime_get() - time; - if (ret != slen + 1) { - pr_err("bitmap_print_to_pagebuf: result is %d, expected %d\n", ret, slen); + if (ret != slen) { + pr_err("scnprintf(\"%%*pbl\"): result is %d, expected %d\n", ret, slen); failed_tests++; goto out; } if (strncmp(buf, expected, slen)) { - pr_err("bitmap_print_to_pagebuf: result is %s, expected %s\n", buf, expected); + pr_err("scnprintf(\"%%*pbl\"): result is %s, expected %s\n", buf, expected); failed_tests++; goto out; } - pr_info("bitmap_print_to_pagebuf: input is '%s', Time: %llu\n", buf, time); + pr_info("scnprintf(\"%%*pbl\", '%s'):\t%llu\n", buf, time); out: kfree(buf); kfree(bmap); @@ -650,7 +691,7 @@ static void __init test_bitmap_arr32(void) memset(arr, 0xa5, sizeof(arr)); - for (nbits = 0; nbits < EXP1_IN_BITS; ++nbits) { + for (nbits = 1; nbits < EXP1_IN_BITS; ++nbits) { bitmap_to_arr32(arr, exp1, nbits); bitmap_from_arr32(bmap2, arr, nbits); expect_eq_bitmap(bmap2, exp1, nbits); @@ -678,7 +719,7 @@ static void __init test_bitmap_arr64(void) memset(arr, 0xa5, sizeof(arr)); - for (nbits = 0; nbits < EXP1_IN_BITS; ++nbits) { + for (nbits = 1; nbits < EXP1_IN_BITS; ++nbits) { memset(bmap2, 0xff, sizeof(arr)); bitmap_to_arr64(arr, exp1, nbits); bitmap_from_arr64(bmap2, arr, nbits); @@ -711,7 +752,7 @@ static void noinline __init test_mem_optimisations(void) unsigned int start, nbits; for (start = 0; start < 1024; start += 8) { - for (nbits = 0; nbits < 1024 - start; nbits += 8) { + for (nbits = 1; nbits < 1024 - start; nbits += 8) { memset(bmap1, 0x5a, sizeof(bmap1)); memset(bmap2, 0x5a, sizeof(bmap2)); @@ -851,6 +892,50 @@ static void __init test_for_each_set_bit_from(void) } } +static void __init test_bitmap_weight(void) +{ + unsigned int bit, w1, w2, w; + DECLARE_BITMAP(b, 30); + DECLARE_BITMAP(b1, 128); + + bitmap_parselist("all:1/2", b, 30); + + /* Test inline implementation */ + w = bitmap_weight(b, 30); + w1 = bitmap_weight(b, 15); + w2 = bitmap_weight_from(b, 15, 30); + + expect_eq_uint(15, w); + expect_eq_uint(8, w1); + expect_eq_uint(7, w2); + + /* Test outline implementation */ + w = bitmap_weight(exp1, EXP1_IN_BITS); + for (bit = 1; bit < EXP1_IN_BITS; bit++) { + w1 = bitmap_weight(exp1, bit); + w2 = bitmap_weight_from(exp1, bit, EXP1_IN_BITS); + expect_eq_uint(w1 + w2, w); + } + + /* Test out-of-range */ + w = bitmap_weight_from(b, 31, 30); + expect_eq_uint(0, !!(w < 30)); + + /* + * Test bitmap_weight() for correctness in case of some bits set between + * nbits and end of the last word. + */ + bitmap_fill(b1, 128); + + /* Inline */ + expect_eq_uint(30, bitmap_weight(b1, 30)); + expect_eq_uint(100, bitmap_weight(b1, 100)); + + /* Outline */ + for (int i = 1; i < 128; i++) + expect_eq_uint(i, bitmap_weight(b1, i)); +} + static void __init test_for_each_clear_bit(void) { DECLARE_BITMAP(orig, 500); @@ -1395,7 +1480,7 @@ static void __init test_bitmap_read_perf(void) } } time = ktime_get() - time; - pr_info("Time spent in %s:\t%llu\n", __func__, time); + pr_info("%s:\t\t%llu\n", __func__, time); } static void __init test_bitmap_write_perf(void) @@ -1417,7 +1502,63 @@ static void __init test_bitmap_write_perf(void) } } time = ktime_get() - time; - pr_info("Time spent in %s:\t%llu\n", __func__, time); + pr_info("%s:\t\t%llu\n", __func__, time); +} + +/* + * nbits == 0 is most commonly not a valid case. Bitmap users should revisit + * the caller logic. Bitmap API doesn't provide any guarantees on returned + * value. The pointers are not dereferenced. The return value is intentionally + * ignored. + */ +static void __init test_zero_nbits(void) +{ + static volatile __always_used unsigned long ret __initdata; + + bitmap_clear(NULL, 0, 0); + bitmap_complement(NULL, NULL, 0); + bitmap_copy(NULL, NULL, 0); + bitmap_copy_clear_tail(NULL, NULL, 0); + bitmap_fill(NULL, 0); + bitmap_from_arr32(NULL, NULL, 0); + bitmap_from_arr64(NULL, NULL, 0); + bitmap_or(NULL, NULL, NULL, 0); + bitmap_set(NULL, 0, 0); + bitmap_shift_left(NULL, NULL, 0, 0); + bitmap_shift_right(NULL, NULL, 0, 0); + bitmap_to_arr32(NULL, NULL, 0); + bitmap_to_arr64(NULL, NULL, 0); + bitmap_write(NULL, 0, 0, 0); + bitmap_xor(NULL, NULL, NULL, 0); + bitmap_zero(NULL, 0); + + ret = bitmap_and(NULL, NULL, NULL, 0); + ret = bitmap_empty(NULL, 0); + ret = bitmap_equal(NULL, NULL, 0); + ret = bitmap_full(NULL, 0); + ret = bitmap_or_equal(NULL, NULL, NULL, 0); + ret = bitmap_read(NULL, 0, 0); + ret = bitmap_subset(NULL, NULL, 0); + ret = bitmap_weight(NULL, 0); + ret = bitmap_weight_and(NULL, NULL, 0); + ret = bitmap_weight_andnot(NULL, NULL, 0); + ret = bitmap_weight_from(NULL, 0, 0); + ret = bitmap_weighted_or(NULL, NULL, NULL, 0); + + ret = find_first_and_and_bit(NULL, NULL, NULL, 0); + ret = find_first_and_bit(NULL, NULL, 0); + ret = find_first_andnot_bit(NULL, NULL, 0); + ret = find_first_bit(NULL, 0); + ret = find_first_zero_bit(NULL, 0); + ret = find_last_bit(NULL, 0); + ret = find_next_and_bit(NULL, NULL, 0, 0); + ret = find_next_andnot_bit(NULL, NULL, 0, 0); + ret = find_next_bit(NULL, 0, 0); + ret = find_next_clump8(NULL, NULL, 0, 0); + ret = find_next_zero_bit(NULL, 0, 0); + ret = find_nth_and_bit(NULL, NULL, 0, 0); + ret = find_nth_bit(NULL, 0, 0); + ret = find_random_bit(NULL, 0); } #undef TEST_BIT_LEN @@ -1441,7 +1582,9 @@ static void __init selftest(void) test_bitmap_const_eval(); test_bitmap_read_write(); test_bitmap_read_perf(); + test_bitmap_weight(); test_bitmap_write_perf(); + test_zero_nbits(); test_find_nth_bit(); test_for_each_set_bit(); @@ -1454,6 +1597,7 @@ static void __init selftest(void) test_for_each_clear_bitrange_from(); test_for_each_set_clump8(); test_for_each_set_bit_wrap(); + test_bitmap_find_next_zero_area_off(); } KSTM_MODULE_LOADERS(test_bitmap); diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 5892c0f17ddc..af6f3340c034 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -560,8 +560,23 @@ static int bpf_fill_max_jmp_never_taken(struct bpf_test *self) } /* ALU result computation used in tests */ -static bool __bpf_alu_result(u64 *res, u64 v1, u64 v2, u8 op) +enum { F_ALU32 = 1, F_SIGNED = 2 }; + +static bool __bpf_alu_result(u64 *res, u64 v1, u64 v2, u8 op, u32 flags) { + bool is_signed = flags & F_SIGNED; + + /* Narrow operands for ALU32 */ + if (flags & F_ALU32) { + if (is_signed) { + v1 = (u64)(s32)v1; + v2 = (u64)(s32)v2; + } else { + v1 = (u32)v1; + v2 = (u32)v2; + } + } + *res = 0; switch (op) { case BPF_MOV: @@ -599,12 +614,28 @@ static bool __bpf_alu_result(u64 *res, u64 v1, u64 v2, u8 op) case BPF_DIV: if (v2 == 0) return false; - *res = div64_u64(v1, v2); + if (!is_signed) { + *res = div64_u64(v1, v2); + } else { + if ((s64)v2 == -1) /* Handled by verifier */ + return false; + *res = (u64)div64_s64(v1, v2); + } break; case BPF_MOD: if (v2 == 0) return false; - div64_u64_rem(v1, v2, res); + if (!is_signed) { + div64_u64_rem(v1, v2, res); + } else { + if ((s64)v2 == -1) + return false; + /* + * Avoid s64 % s64 which generates __moddi3 on + * 32-bit architectures. Use div64_s64 instead. + */ + *res = (u64)((s64)v1 - div64_s64(v1, v2) * (s64)v2); + } break; } return true; @@ -612,7 +643,7 @@ static bool __bpf_alu_result(u64 *res, u64 v1, u64 v2, u8 op) /* Test an ALU shift operation for all valid shift values */ static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op, - u8 mode, bool alu32) + u8 mode, u32 flags) { static const s64 regs[] = { 0x0123456789abcdefLL, /* dword > 0, word < 0 */ @@ -620,7 +651,7 @@ static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op, 0xfedcba0198765432LL, /* dword < 0, word < 0 */ 0x0123458967abcdefLL, /* dword > 0, word > 0 */ }; - int bits = alu32 ? 32 : 64; + int bits = (flags & F_ALU32) ? 32 : 64; int len = (2 + 7 * bits) * ARRAY_SIZE(regs) + 3; struct bpf_insn *insn; int imm, k; @@ -643,7 +674,7 @@ static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op, /* Perform operation */ insn[i++] = BPF_ALU64_REG(BPF_MOV, R1, R3); insn[i++] = BPF_ALU64_IMM(BPF_MOV, R2, imm); - if (alu32) { + if (flags & F_ALU32) { if (mode == BPF_K) insn[i++] = BPF_ALU32_IMM(op, R1, imm); else @@ -653,14 +684,14 @@ static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op, reg = (s32)reg; else reg = (u32)reg; - __bpf_alu_result(&val, reg, imm, op); + __bpf_alu_result(&val, reg, imm, op, 0); val = (u32)val; } else { if (mode == BPF_K) insn[i++] = BPF_ALU64_IMM(op, R1, imm); else insn[i++] = BPF_ALU64_REG(op, R1, R2); - __bpf_alu_result(&val, reg, imm, op); + __bpf_alu_result(&val, reg, imm, op, 0); } /* @@ -688,62 +719,62 @@ static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op, static int bpf_fill_alu64_lsh_imm(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, false); + return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, 0); } static int bpf_fill_alu64_rsh_imm(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, false); + return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, 0); } static int bpf_fill_alu64_arsh_imm(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, false); + return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, 0); } static int bpf_fill_alu64_lsh_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, false); + return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, 0); } static int bpf_fill_alu64_rsh_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, false); + return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, 0); } static int bpf_fill_alu64_arsh_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, false); + return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, 0); } static int bpf_fill_alu32_lsh_imm(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, true); + return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, F_ALU32); } static int bpf_fill_alu32_rsh_imm(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, true); + return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, F_ALU32); } static int bpf_fill_alu32_arsh_imm(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, true); + return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, F_ALU32); } static int bpf_fill_alu32_lsh_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, true); + return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, F_ALU32); } static int bpf_fill_alu32_rsh_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, true); + return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, F_ALU32); } static int bpf_fill_alu32_arsh_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, true); + return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, F_ALU32); } /* @@ -751,9 +782,9 @@ static int bpf_fill_alu32_arsh_reg(struct bpf_test *self) * for the case when the source and destination are the same. */ static int __bpf_fill_alu_shift_same_reg(struct bpf_test *self, u8 op, - bool alu32) + u32 flags) { - int bits = alu32 ? 32 : 64; + int bits = (flags & F_ALU32) ? 32 : 64; int len = 3 + 6 * bits; struct bpf_insn *insn; int i = 0; @@ -770,14 +801,14 @@ static int __bpf_fill_alu_shift_same_reg(struct bpf_test *self, u8 op, /* Perform operation */ insn[i++] = BPF_ALU64_IMM(BPF_MOV, R1, val); - if (alu32) + if (flags & F_ALU32) insn[i++] = BPF_ALU32_REG(op, R1, R1); else insn[i++] = BPF_ALU64_REG(op, R1, R1); /* Compute the reference result */ - __bpf_alu_result(&res, val, val, op); - if (alu32) + __bpf_alu_result(&res, val, val, op, 0); + if (flags & F_ALU32) res = (u32)res; i += __bpf_ld_imm64(&insn[i], R2, res); @@ -798,32 +829,32 @@ static int __bpf_fill_alu_shift_same_reg(struct bpf_test *self, u8 op, static int bpf_fill_alu64_lsh_same_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, false); + return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, 0); } static int bpf_fill_alu64_rsh_same_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, false); + return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, 0); } static int bpf_fill_alu64_arsh_same_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, false); + return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, 0); } static int bpf_fill_alu32_lsh_same_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, true); + return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, F_ALU32); } static int bpf_fill_alu32_rsh_same_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, true); + return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, F_ALU32); } static int bpf_fill_alu32_arsh_same_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, true); + return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, F_ALU32); } /* @@ -936,17 +967,20 @@ static int __bpf_fill_pattern(struct bpf_test *self, void *arg, static int __bpf_emit_alu64_imm(struct bpf_test *self, void *arg, struct bpf_insn *insns, s64 dst, s64 imm) { - int op = *(int *)arg; + int *a = arg; + int op = a[0]; + u32 flags = a[1]; + s16 off = (flags & F_SIGNED) ? 1 : 0; int i = 0; u64 res; if (!insns) return 7; - if (__bpf_alu_result(&res, dst, (s32)imm, op)) { + if (__bpf_alu_result(&res, dst, (s32)imm, op, flags)) { i += __bpf_ld_imm64(&insns[i], R1, dst); i += __bpf_ld_imm64(&insns[i], R3, res); - insns[i++] = BPF_ALU64_IMM(op, R1, imm); + insns[i++] = BPF_ALU64_IMM_OFF(op, R1, imm, off); insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); insns[i++] = BPF_EXIT_INSN(); } @@ -957,17 +991,20 @@ static int __bpf_emit_alu64_imm(struct bpf_test *self, void *arg, static int __bpf_emit_alu32_imm(struct bpf_test *self, void *arg, struct bpf_insn *insns, s64 dst, s64 imm) { - int op = *(int *)arg; + int *a = arg; + int op = a[0]; + u32 flags = a[1]; + s16 off = (flags & F_SIGNED) ? 1 : 0; int i = 0; u64 res; if (!insns) return 7; - if (__bpf_alu_result(&res, (u32)dst, (u32)imm, op)) { + if (__bpf_alu_result(&res, dst, (s32)imm, op, flags | F_ALU32)) { i += __bpf_ld_imm64(&insns[i], R1, dst); i += __bpf_ld_imm64(&insns[i], R3, (u32)res); - insns[i++] = BPF_ALU32_IMM(op, R1, imm); + insns[i++] = BPF_ALU32_IMM_OFF(op, R1, imm, off); insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); insns[i++] = BPF_EXIT_INSN(); } @@ -985,7 +1022,7 @@ static int __bpf_emit_alu64_reg(struct bpf_test *self, void *arg, if (!insns) return 9; - if (__bpf_alu_result(&res, dst, src, op)) { + if (__bpf_alu_result(&res, dst, src, op, 0)) { i += __bpf_ld_imm64(&insns[i], R1, dst); i += __bpf_ld_imm64(&insns[i], R2, src); i += __bpf_ld_imm64(&insns[i], R3, res); @@ -1007,7 +1044,7 @@ static int __bpf_emit_alu32_reg(struct bpf_test *self, void *arg, if (!insns) return 9; - if (__bpf_alu_result(&res, (u32)dst, (u32)src, op)) { + if (__bpf_alu_result(&res, (u32)dst, (u32)src, op, 0)) { i += __bpf_ld_imm64(&insns[i], R1, dst); i += __bpf_ld_imm64(&insns[i], R2, src); i += __bpf_ld_imm64(&insns[i], R3, (u32)res); @@ -1019,16 +1056,20 @@ static int __bpf_emit_alu32_reg(struct bpf_test *self, void *arg, return i; } -static int __bpf_fill_alu64_imm(struct bpf_test *self, int op) +static int __bpf_fill_alu64_imm(struct bpf_test *self, int op, u32 flags) { - return __bpf_fill_pattern(self, &op, 64, 32, + int arg[2] = {op, flags}; + + return __bpf_fill_pattern(self, &arg, 64, 32, PATTERN_BLOCK1, PATTERN_BLOCK2, &__bpf_emit_alu64_imm); } -static int __bpf_fill_alu32_imm(struct bpf_test *self, int op) +static int __bpf_fill_alu32_imm(struct bpf_test *self, int op, u32 flags) { - return __bpf_fill_pattern(self, &op, 64, 32, + int arg[2] = {op, flags}; + + return __bpf_fill_pattern(self, &arg, 64, 32, PATTERN_BLOCK1, PATTERN_BLOCK2, &__bpf_emit_alu32_imm); } @@ -1050,93 +1091,115 @@ static int __bpf_fill_alu32_reg(struct bpf_test *self, int op) /* ALU64 immediate operations */ static int bpf_fill_alu64_mov_imm(struct bpf_test *self) { - return __bpf_fill_alu64_imm(self, BPF_MOV); + return __bpf_fill_alu64_imm(self, BPF_MOV, 0); } static int bpf_fill_alu64_and_imm(struct bpf_test *self) { - return __bpf_fill_alu64_imm(self, BPF_AND); + return __bpf_fill_alu64_imm(self, BPF_AND, 0); } static int bpf_fill_alu64_or_imm(struct bpf_test *self) { - return __bpf_fill_alu64_imm(self, BPF_OR); + return __bpf_fill_alu64_imm(self, BPF_OR, 0); } static int bpf_fill_alu64_xor_imm(struct bpf_test *self) { - return __bpf_fill_alu64_imm(self, BPF_XOR); + return __bpf_fill_alu64_imm(self, BPF_XOR, 0); } static int bpf_fill_alu64_add_imm(struct bpf_test *self) { - return __bpf_fill_alu64_imm(self, BPF_ADD); + return __bpf_fill_alu64_imm(self, BPF_ADD, 0); } static int bpf_fill_alu64_sub_imm(struct bpf_test *self) { - return __bpf_fill_alu64_imm(self, BPF_SUB); + return __bpf_fill_alu64_imm(self, BPF_SUB, 0); } static int bpf_fill_alu64_mul_imm(struct bpf_test *self) { - return __bpf_fill_alu64_imm(self, BPF_MUL); + return __bpf_fill_alu64_imm(self, BPF_MUL, 0); } static int bpf_fill_alu64_div_imm(struct bpf_test *self) { - return __bpf_fill_alu64_imm(self, BPF_DIV); + return __bpf_fill_alu64_imm(self, BPF_DIV, 0); } static int bpf_fill_alu64_mod_imm(struct bpf_test *self) { - return __bpf_fill_alu64_imm(self, BPF_MOD); + return __bpf_fill_alu64_imm(self, BPF_MOD, 0); +} + +/* Signed ALU64 immediate operations */ +static int bpf_fill_alu64_sdiv_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_DIV, F_SIGNED); +} + +static int bpf_fill_alu64_smod_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_MOD, F_SIGNED); +} + +/* Signed ALU32 immediate operations */ +static int bpf_fill_alu32_sdiv_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_DIV, F_SIGNED); +} + +static int bpf_fill_alu32_smod_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_MOD, F_SIGNED); } /* ALU32 immediate operations */ static int bpf_fill_alu32_mov_imm(struct bpf_test *self) { - return __bpf_fill_alu32_imm(self, BPF_MOV); + return __bpf_fill_alu32_imm(self, BPF_MOV, 0); } static int bpf_fill_alu32_and_imm(struct bpf_test *self) { - return __bpf_fill_alu32_imm(self, BPF_AND); + return __bpf_fill_alu32_imm(self, BPF_AND, 0); } static int bpf_fill_alu32_or_imm(struct bpf_test *self) { - return __bpf_fill_alu32_imm(self, BPF_OR); + return __bpf_fill_alu32_imm(self, BPF_OR, 0); } static int bpf_fill_alu32_xor_imm(struct bpf_test *self) { - return __bpf_fill_alu32_imm(self, BPF_XOR); + return __bpf_fill_alu32_imm(self, BPF_XOR, 0); } static int bpf_fill_alu32_add_imm(struct bpf_test *self) { - return __bpf_fill_alu32_imm(self, BPF_ADD); + return __bpf_fill_alu32_imm(self, BPF_ADD, 0); } static int bpf_fill_alu32_sub_imm(struct bpf_test *self) { - return __bpf_fill_alu32_imm(self, BPF_SUB); + return __bpf_fill_alu32_imm(self, BPF_SUB, 0); } static int bpf_fill_alu32_mul_imm(struct bpf_test *self) { - return __bpf_fill_alu32_imm(self, BPF_MUL); + return __bpf_fill_alu32_imm(self, BPF_MUL, 0); } static int bpf_fill_alu32_div_imm(struct bpf_test *self) { - return __bpf_fill_alu32_imm(self, BPF_DIV); + return __bpf_fill_alu32_imm(self, BPF_DIV, 0); } static int bpf_fill_alu32_mod_imm(struct bpf_test *self) { - return __bpf_fill_alu32_imm(self, BPF_MOD); + return __bpf_fill_alu32_imm(self, BPF_MOD, 0); } /* ALU64 register operations */ @@ -1235,7 +1298,8 @@ static int bpf_fill_alu32_mod_reg(struct bpf_test *self) * Test JITs that implement complex ALU operations as function * calls, and must re-arrange operands for argument passing. */ -static int __bpf_fill_alu_imm_regs(struct bpf_test *self, u8 op, bool alu32) +static int __bpf_fill_alu_imm_regs(struct bpf_test *self, u8 op, + u32 flags) { int len = 2 + 10 * 10; struct bpf_insn *insns; @@ -1249,28 +1313,37 @@ static int __bpf_fill_alu_imm_regs(struct bpf_test *self, u8 op, bool alu32) return -ENOMEM; /* Operand and result values according to operation */ - if (alu32) - dst = 0x76543210U; - else - dst = 0x7edcba9876543210ULL; + if (flags & F_SIGNED) { + if (flags & F_ALU32) + dst = -76543210; + else + dst = -7654321076543210LL; + } else { + if (flags & F_ALU32) + dst = 0x76543210U; + else + dst = 0x7edcba9876543210ULL; + } imm = 0x01234567U; if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) imm &= 31; - __bpf_alu_result(&res, dst, imm, op); + __bpf_alu_result(&res, dst, imm, op, flags); - if (alu32) + if (flags & F_ALU32) res = (u32)res; /* Check all operand registers */ for (rd = R0; rd <= R9; rd++) { i += __bpf_ld_imm64(&insns[i], rd, dst); - if (alu32) - insns[i++] = BPF_ALU32_IMM(op, rd, imm); + s16 off = (flags & F_SIGNED) ? 1 : 0; + + if (flags & F_ALU32) + insns[i++] = BPF_ALU32_IMM_OFF(op, rd, imm, off); else - insns[i++] = BPF_ALU64_IMM(op, rd, imm); + insns[i++] = BPF_ALU64_IMM_OFF(op, rd, imm, off); insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, res, 2); insns[i++] = BPF_MOV64_IMM(R0, __LINE__); @@ -1295,123 +1368,145 @@ static int __bpf_fill_alu_imm_regs(struct bpf_test *self, u8 op, bool alu32) /* ALU64 K registers */ static int bpf_fill_alu64_mov_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_MOV, false); + return __bpf_fill_alu_imm_regs(self, BPF_MOV, 0); } static int bpf_fill_alu64_and_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_AND, false); + return __bpf_fill_alu_imm_regs(self, BPF_AND, 0); } static int bpf_fill_alu64_or_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_OR, false); + return __bpf_fill_alu_imm_regs(self, BPF_OR, 0); } static int bpf_fill_alu64_xor_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_XOR, false); + return __bpf_fill_alu_imm_regs(self, BPF_XOR, 0); } static int bpf_fill_alu64_lsh_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_LSH, false); + return __bpf_fill_alu_imm_regs(self, BPF_LSH, 0); } static int bpf_fill_alu64_rsh_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_RSH, false); + return __bpf_fill_alu_imm_regs(self, BPF_RSH, 0); } static int bpf_fill_alu64_arsh_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_ARSH, false); + return __bpf_fill_alu_imm_regs(self, BPF_ARSH, 0); } static int bpf_fill_alu64_add_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_ADD, false); + return __bpf_fill_alu_imm_regs(self, BPF_ADD, 0); } static int bpf_fill_alu64_sub_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_SUB, false); + return __bpf_fill_alu_imm_regs(self, BPF_SUB, 0); } static int bpf_fill_alu64_mul_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_MUL, false); + return __bpf_fill_alu_imm_regs(self, BPF_MUL, 0); } static int bpf_fill_alu64_div_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_DIV, false); + return __bpf_fill_alu_imm_regs(self, BPF_DIV, 0); } static int bpf_fill_alu64_mod_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_MOD, false); + return __bpf_fill_alu_imm_regs(self, BPF_MOD, 0); +} + +/* Signed ALU64 K registers */ +static int bpf_fill_alu64_sdiv_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_DIV, F_SIGNED); +} + +static int bpf_fill_alu64_smod_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MOD, F_SIGNED); } /* ALU32 K registers */ static int bpf_fill_alu32_mov_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_MOV, true); + return __bpf_fill_alu_imm_regs(self, BPF_MOV, F_ALU32); } static int bpf_fill_alu32_and_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_AND, true); + return __bpf_fill_alu_imm_regs(self, BPF_AND, F_ALU32); } static int bpf_fill_alu32_or_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_OR, true); + return __bpf_fill_alu_imm_regs(self, BPF_OR, F_ALU32); } static int bpf_fill_alu32_xor_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_XOR, true); + return __bpf_fill_alu_imm_regs(self, BPF_XOR, F_ALU32); } static int bpf_fill_alu32_lsh_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_LSH, true); + return __bpf_fill_alu_imm_regs(self, BPF_LSH, F_ALU32); } static int bpf_fill_alu32_rsh_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_RSH, true); + return __bpf_fill_alu_imm_regs(self, BPF_RSH, F_ALU32); } static int bpf_fill_alu32_arsh_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_ARSH, true); + return __bpf_fill_alu_imm_regs(self, BPF_ARSH, F_ALU32); } static int bpf_fill_alu32_add_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_ADD, true); + return __bpf_fill_alu_imm_regs(self, BPF_ADD, F_ALU32); } static int bpf_fill_alu32_sub_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_SUB, true); + return __bpf_fill_alu_imm_regs(self, BPF_SUB, F_ALU32); } static int bpf_fill_alu32_mul_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_MUL, true); + return __bpf_fill_alu_imm_regs(self, BPF_MUL, F_ALU32); } static int bpf_fill_alu32_div_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_DIV, true); + return __bpf_fill_alu_imm_regs(self, BPF_DIV, F_ALU32); } static int bpf_fill_alu32_mod_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_MOD, true); + return __bpf_fill_alu_imm_regs(self, BPF_MOD, F_ALU32); +} + +/* Signed ALU32 K registers */ +static int bpf_fill_alu32_sdiv_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_DIV, F_ALU32 | F_SIGNED); +} + +static int bpf_fill_alu32_smod_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MOD, F_ALU32 | F_SIGNED); } /* @@ -1442,8 +1537,8 @@ static int __bpf_fill_alu_reg_pairs(struct bpf_test *self, u8 op, bool alu32) if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) src &= 31; - __bpf_alu_result(&res, dst, src, op); - __bpf_alu_result(&same, src, src, op); + __bpf_alu_result(&res, dst, src, op, 0); + __bpf_alu_result(&same, src, src, op, 0); if (alu32) { res = (u32)res; @@ -1626,7 +1721,7 @@ static int __bpf_emit_atomic64(struct bpf_test *self, void *arg, res = src; break; default: - __bpf_alu_result(&res, dst, src, BPF_OP(op)); + __bpf_alu_result(&res, dst, src, BPF_OP(op), 0); } keep = 0x0123456789abcdefULL; @@ -1673,7 +1768,7 @@ static int __bpf_emit_atomic32(struct bpf_test *self, void *arg, res = src; break; default: - __bpf_alu_result(&res, (u32)dst, (u32)src, BPF_OP(op)); + __bpf_alu_result(&res, (u32)dst, (u32)src, BPF_OP(op), 0); } keep = 0x0123456789abcdefULL; @@ -1939,7 +2034,7 @@ static int __bpf_fill_atomic_reg_pairs(struct bpf_test *self, u8 width, u8 op) res = mem; break; default: - __bpf_alu_result(&res, mem, upd, BPF_OP(op)); + __bpf_alu_result(&res, mem, upd, BPF_OP(op), 0); } /* Test all operand registers */ @@ -12354,6 +12449,22 @@ static struct bpf_test tests[] = { { { 0, 1 } }, .fill_helper = bpf_fill_alu64_mod_imm_regs, }, + { + "ALU64_SDIV_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_sdiv_imm_regs, + }, + { + "ALU64_SMOD_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_smod_imm_regs, + }, /* ALU32 K registers */ { "ALU32_MOV_K: registers", @@ -12451,6 +12562,22 @@ static struct bpf_test tests[] = { { { 0, 1 } }, .fill_helper = bpf_fill_alu32_mod_imm_regs, }, + { + "ALU32_SDIV_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_sdiv_imm_regs, + }, + { + "ALU32_SMOD_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_smod_imm_regs, + }, /* ALU64 X register combinations */ { "ALU64_MOV_X: register combinations", @@ -12881,6 +13008,24 @@ static struct bpf_test tests[] = { .fill_helper = bpf_fill_alu64_mod_imm, .nr_testruns = NR_PATTERN_RUNS, }, + { + "ALU64_SDIV_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_sdiv_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_SMOD_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_smod_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, /* ALU32 immediate magnitudes */ { "ALU32_MOV_K: all immediate value magnitudes", @@ -12963,6 +13108,24 @@ static struct bpf_test tests[] = { .fill_helper = bpf_fill_alu32_mod_imm, .nr_testruns = NR_PATTERN_RUNS, }, + { + "ALU32_SDIV_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_sdiv_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_SMOD_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_smod_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, /* ALU64 register magnitudes */ { "ALU64_MOV_X: all register value magnitudes", diff --git a/lib/test_context-analysis.c b/lib/test_context-analysis.c index 140efa8a9763..316f4dfcda65 100644 --- a/lib/test_context-analysis.c +++ b/lib/test_context-analysis.c @@ -159,6 +159,10 @@ TEST_SPINLOCK_COMMON(read_lock, struct test_mutex_data { struct mutex mtx; int counter __guarded_by(&mtx); + + struct mutex mtx2; + int anyread __guarded_by(&mtx, &mtx2); + int *anyptr __pt_guarded_by(&mtx, &mtx2); }; static void __used test_mutex_init(struct test_mutex_data *d) @@ -219,6 +223,26 @@ static void __used test_mutex_cond_guard(struct test_mutex_data *d) } } +static void __used test_mutex_multiguard(struct test_mutex_data *d) +{ + mutex_lock(&d->mtx); + (void)d->anyread; + (void)*d->anyptr; + mutex_unlock(&d->mtx); + + mutex_lock(&d->mtx2); + (void)d->anyread; + (void)*d->anyptr; + mutex_unlock(&d->mtx2); + + mutex_lock(&d->mtx); + mutex_lock(&d->mtx2); + d->anyread++; + (*d->anyptr)++; + mutex_unlock(&d->mtx2); + mutex_unlock(&d->mtx); +} + struct test_seqlock_data { seqlock_t sl; int counter __guarded_by(&sl); @@ -596,3 +620,14 @@ static void __used test_ww_mutex_lock_ctx(struct test_ww_mutex_data *d) ww_mutex_destroy(&d->mtx); } + +static DEFINE_PER_CPU(raw_spinlock_t, test_per_cpu_lock); + +static void __used test_per_cpu(int cpu) +{ + raw_spin_lock(&per_cpu(test_per_cpu_lock, cpu)); + raw_spin_unlock(&per_cpu(test_per_cpu_lock, cpu)); + + raw_spin_lock(per_cpu_ptr(&test_per_cpu_lock, cpu)); + raw_spin_unlock(per_cpu_ptr(&test_per_cpu_lock, cpu)); +} diff --git a/lib/test_firmware.c b/lib/test_firmware.c index b471d720879a..7459bba65444 100644 --- a/lib/test_firmware.c +++ b/lib/test_firmware.c @@ -867,7 +867,7 @@ static int test_fw_run_batch_request(void *data) if (test_fw_config->into_buf) { void *test_buf; - test_buf = kzalloc(TEST_FIRMWARE_BUF_SIZE, GFP_KERNEL); + test_buf = kzalloc(test_fw_config->buf_size, GFP_KERNEL); if (!test_buf) return -ENOMEM; diff --git a/lib/test_fortify/Makefile b/lib/test_fortify/Makefile index 399cae880e1d..44cd5df41a81 100644 --- a/lib/test_fortify/Makefile +++ b/lib/test_fortify/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 ccflags-y := $(call cc-disable-warning,fortify-source) +ccflags-y += $(call cc-disable-warning,stringop-overread) quiet_cmd_test_fortify = TEST $@ cmd_test_fortify = $(CONFIG_SHELL) $(src)/test_fortify.sh \ diff --git a/lib/test_fortify/write_overflow-strncpy-src.c b/lib/test_fortify/write_overflow-strncpy-src.c deleted file mode 100644 index 8dcfb8c788dd..000000000000 --- a/lib/test_fortify/write_overflow-strncpy-src.c +++ /dev/null @@ -1,5 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -#define TEST \ - strncpy(small, large_src, sizeof(small) + 1) - -#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow-strncpy.c b/lib/test_fortify/write_overflow-strncpy.c deleted file mode 100644 index b85f079c815d..000000000000 --- a/lib/test_fortify/write_overflow-strncpy.c +++ /dev/null @@ -1,5 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -#define TEST \ - strncpy(instance.buf, large_src, sizeof(instance.buf) + 1) - -#include "test_fortify.h" diff --git a/lib/test_hmm.c b/lib/test_hmm.c index 0964d53365e6..6911daa9f854 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -185,11 +185,73 @@ static int dmirror_fops_open(struct inode *inode, struct file *filp) return 0; } +static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk) +{ + unsigned long start_pfn = chunk->pagemap.range.start >> PAGE_SHIFT; + unsigned long end_pfn = chunk->pagemap.range.end >> PAGE_SHIFT; + unsigned long npages = end_pfn - start_pfn + 1; + unsigned long i; + unsigned long *src_pfns; + unsigned long *dst_pfns; + unsigned int order = 0; + + src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); + dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); + + migrate_device_range(src_pfns, start_pfn, npages); + for (i = 0; i < npages; i++) { + struct page *dpage, *spage; + + spage = migrate_pfn_to_page(src_pfns[i]); + if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) + continue; + + if (WARN_ON(!is_device_private_page(spage) && + !is_device_coherent_page(spage))) + continue; + + order = folio_order(page_folio(spage)); + spage = BACKING_PAGE(spage); + if (src_pfns[i] & MIGRATE_PFN_COMPOUND) { + dpage = folio_page(folio_alloc(GFP_HIGHUSER_MOVABLE, + order), 0); + } else { + dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL); + order = 0; + } + + /* TODO Support splitting here */ + lock_page(dpage); + dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)); + if (src_pfns[i] & MIGRATE_PFN_WRITE) + dst_pfns[i] |= MIGRATE_PFN_WRITE; + if (order) + dst_pfns[i] |= MIGRATE_PFN_COMPOUND; + folio_copy(page_folio(dpage), page_folio(spage)); + } + migrate_device_pages(src_pfns, dst_pfns, npages); + migrate_device_finalize(src_pfns, dst_pfns, npages); + kvfree(src_pfns); + kvfree(dst_pfns); +} + static int dmirror_fops_release(struct inode *inode, struct file *filp) { struct dmirror *dmirror = filp->private_data; + struct dmirror_device *mdevice = dmirror->mdevice; + int i; mmu_interval_notifier_remove(&dmirror->notifier); + + if (mdevice->devmem_chunks) { + for (i = 0; i < mdevice->devmem_count; i++) { + struct dmirror_chunk *devmem = + mdevice->devmem_chunks[i]; + + dmirror_device_evict_chunk(devmem); + } + } + xa_destroy(&dmirror->pt); kfree(dmirror); return 0; @@ -327,6 +389,67 @@ out: return ret; } +static int dmirror_range_fault_unlocked(struct dmirror *dmirror, + struct hmm_range *range, + unsigned long timeout) +{ + int ret; + + while (true) { + ret = hmm_range_fault_unlocked_timeout(range, timeout); + if (ret) + goto out; + + mutex_lock(&dmirror->mutex); + if (mmu_interval_read_retry(range->notifier, + range->notifier_seq)) { + mutex_unlock(&dmirror->mutex); + continue; + } + break; + } + + ret = dmirror_do_fault(dmirror, range); + + mutex_unlock(&dmirror->mutex); +out: + return ret; +} + +static int dmirror_fault_unlocked(struct dmirror *dmirror, + unsigned long start, + unsigned long end, bool write, + unsigned long timeout) +{ + struct mm_struct *mm = dmirror->notifier.mm; + unsigned long addr; + unsigned long pfns[32]; + struct hmm_range range = { + .notifier = &dmirror->notifier, + .hmm_pfns = pfns, + .pfn_flags_mask = 0, + .default_flags = + HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), + .dev_private_owner = dmirror->mdevice, + }; + int ret = 0; + + if (!mmget_not_zero(mm)) + return -EFAULT; + + for (addr = start; addr < end; addr = range.end) { + range.start = addr; + range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); + + ret = dmirror_range_fault_unlocked(dmirror, &range, timeout); + if (ret) + break; + } + + mmput(mm); + return ret; +} + static int dmirror_fault(struct dmirror *dmirror, unsigned long start, unsigned long end, bool write) { @@ -345,7 +468,7 @@ static int dmirror_fault(struct dmirror *dmirror, unsigned long start, /* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) - return 0; + return -EFAULT; for (addr = start; addr < end; addr = range.end) { range.start = addr; @@ -426,6 +549,48 @@ static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) return ret; } +static int dmirror_read_unlocked(struct dmirror *dmirror, + struct hmm_dmirror_cmd *cmd, + unsigned long timeout) +{ + struct dmirror_bounce bounce; + unsigned long start, end; + unsigned long size = cmd->npages << PAGE_SHIFT; + int ret; + + start = cmd->addr; + end = start + size; + if (end < start) + return -EINVAL; + + ret = dmirror_bounce_init(&bounce, start, size); + if (ret) + return ret; + + while (1) { + mutex_lock(&dmirror->mutex); + ret = dmirror_do_read(dmirror, start, end, &bounce); + mutex_unlock(&dmirror->mutex); + if (ret != -ENOENT) + break; + + start = cmd->addr + (bounce.cpages << PAGE_SHIFT); + ret = dmirror_fault_unlocked(dmirror, start, end, false, timeout); + if (ret) + break; + cmd->faults++; + } + + if (ret == 0) { + if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, + bounce.size)) + ret = -EFAULT; + } + cmd->cpages = bounce.cpages; + dmirror_bounce_fini(&bounce); + return ret; +} + static int dmirror_do_write(struct dmirror *dmirror, unsigned long start, unsigned long end, struct dmirror_bounce *bounce) { @@ -519,7 +684,7 @@ static int dmirror_allocate_chunk(struct dmirror_device *mdevice, devmem->pagemap.type = MEMORY_DEVICE_PRIVATE; break; case HMM_DMIRROR_MEMORY_DEVICE_COHERENT: - devmem->pagemap.range.start = (MINOR(mdevice->cdevice.dev) - 2) ? + devmem->pagemap.range.start = (MINOR(mdevice->device.devt) - 2) ? spm_addr_dev0 : spm_addr_dev1; devmem->pagemap.range.end = devmem->pagemap.range.start + @@ -986,10 +1151,9 @@ static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args, if (!dpage && !order) return VM_FAULT_OOM; - pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n", - page_to_pfn(spage), page_to_pfn(dpage)); - if (dpage) { + pr_debug("migrating from dev to sys pfn src: 0x%lx pfn dst: 0x%lx\n", + page_to_pfn(spage), page_to_pfn(dpage)); lock_page(dpage); *dst |= migrate_pfn(page_to_pfn(dpage)); } @@ -1001,6 +1165,25 @@ static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args, /* Try with smaller pages if large allocation fails */ if (!dpage && order) { dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr); + if (!dpage) { + /* Unlock and free pages already allocated. */ + while (i > 0) { + struct page *fpage; + + fpage = migrate_pfn_to_page(dst[--i]); + unlock_page(fpage); + __free_page(fpage); + } + /* Clear remaining dst entries to avoid + * migrate_vma_pages/finalize() using + * uninitialized values. + */ + while (i < (1 << order)) { + dst[i] = 0; + i++; + } + return VM_FAULT_OOM; + } lock_page(dpage); dst[i] = migrate_pfn(page_to_pfn(dpage)); dst_page = pfn_to_page(page_to_pfn(dpage)); @@ -1049,9 +1232,6 @@ static int dmirror_migrate_to_system(struct dmirror *dmirror, unsigned long *src_pfns; unsigned long *dst_pfns; - src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); - dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); - start = cmd->addr; end = start + size; if (end < start) @@ -1061,6 +1241,9 @@ static int dmirror_migrate_to_system(struct dmirror *dmirror, if (!mmget_not_zero(mm)) return -EINVAL; + src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); + dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); + cmd->cpages = 0; mmap_read_lock(mm); for (addr = start; addr < end; addr = next) { @@ -1086,7 +1269,11 @@ static int dmirror_migrate_to_system(struct dmirror *dmirror, goto out; pr_debug("Migrating from device mem to sys mem\n"); - dmirror_devmem_fault_alloc_and_copy(&args, dmirror); + if (dmirror_devmem_fault_alloc_and_copy(&args, dmirror)) { + migrate_vma_finalize(&args); + ret = -ENOMEM; + goto out; + } migrate_vma_pages(&args); cmd->cpages += dmirror_successful_migrated_pages(&args); @@ -1124,16 +1311,10 @@ static int dmirror_migrate_to_device(struct dmirror *dmirror, if (!mmget_not_zero(mm)) return -EINVAL; - ret = -ENOMEM; src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); - if (!src_pfns) - goto free_mem; - dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); - if (!dst_pfns) - goto free_mem; ret = 0; mmap_read_lock(mm); @@ -1191,8 +1372,8 @@ out: mmap_read_unlock(mm); mmput(mm); free_mem: - kfree(src_pfns); - kfree(dst_pfns); + kvfree(src_pfns); + kvfree(dst_pfns); return ret; } @@ -1377,56 +1558,6 @@ static int dmirror_snapshot(struct dmirror *dmirror, return ret; } -static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk) -{ - unsigned long start_pfn = chunk->pagemap.range.start >> PAGE_SHIFT; - unsigned long end_pfn = chunk->pagemap.range.end >> PAGE_SHIFT; - unsigned long npages = end_pfn - start_pfn + 1; - unsigned long i; - unsigned long *src_pfns; - unsigned long *dst_pfns; - unsigned int order = 0; - - src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); - dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); - - migrate_device_range(src_pfns, start_pfn, npages); - for (i = 0; i < npages; i++) { - struct page *dpage, *spage; - - spage = migrate_pfn_to_page(src_pfns[i]); - if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) - continue; - - if (WARN_ON(!is_device_private_page(spage) && - !is_device_coherent_page(spage))) - continue; - - order = folio_order(page_folio(spage)); - spage = BACKING_PAGE(spage); - if (src_pfns[i] & MIGRATE_PFN_COMPOUND) { - dpage = folio_page(folio_alloc(GFP_HIGHUSER_MOVABLE, - order), 0); - } else { - dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL); - order = 0; - } - - /* TODO Support splitting here */ - lock_page(dpage); - dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)); - if (src_pfns[i] & MIGRATE_PFN_WRITE) - dst_pfns[i] |= MIGRATE_PFN_WRITE; - if (order) - dst_pfns[i] |= MIGRATE_PFN_COMPOUND; - folio_copy(page_folio(dpage), page_folio(spage)); - } - migrate_device_pages(src_pfns, dst_pfns, npages); - migrate_device_finalize(src_pfns, dst_pfns, npages); - kvfree(src_pfns); - kvfree(dst_pfns); -} - /* Removes free pages from the free list so they can't be re-allocated */ static void dmirror_remove_free_pages(struct dmirror_chunk *devmem) { @@ -1537,7 +1668,9 @@ static long dmirror_fops_unlocked_ioctl(struct file *filp, dmirror->flags = cmd.npages; ret = 0; break; - + case HMM_DMIRROR_READ_UNLOCKED: + ret = dmirror_read_unlocked(dmirror, &cmd, 0); + break; default: return -EINVAL; } @@ -1667,12 +1800,20 @@ static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf) if (order) args.flags |= MIGRATE_VMA_SELECT_COMPOUND; - if (migrate_vma_setup(&args)) - return VM_FAULT_SIGBUS; + /* + * In practice migrate_vma_setup() should never fail unless the + * test is wrong as it just tests some static VMA properties. + */ + if (migrate_vma_setup(&args)) { + ret = VM_FAULT_SIGBUS; + goto err; + } ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror); - if (ret) + if (ret) { + migrate_vma_finalize(&args); goto err; + } migrate_vma_pages(&args); /* * No device finalize step is needed since @@ -1726,6 +1867,13 @@ static const struct dev_pagemap_ops dmirror_devmem_ops = { .folio_split = dmirror_devmem_folio_split, }; +static void dmirror_device_release(struct device *dev) +{ + struct dmirror_device *mdevice = container_of(dev, struct dmirror_device, device); + + dmirror_device_remove_chunks(mdevice); +} + static int dmirror_device_init(struct dmirror_device *mdevice, int id) { dev_t dev; @@ -1737,6 +1885,8 @@ static int dmirror_device_init(struct dmirror_device *mdevice, int id) cdev_init(&mdevice->cdevice, &dmirror_fops); mdevice->cdevice.owner = THIS_MODULE; + mdevice->device.release = dmirror_device_release; + device_initialize(&mdevice->device); mdevice->device.devt = dev; @@ -1744,12 +1894,16 @@ static int dmirror_device_init(struct dmirror_device *mdevice, int id) if (ret) goto put_device; + /* Build a list of free ZONE_DEVICE struct pages */ + ret = dmirror_allocate_chunk(mdevice, NULL, false); + if (ret) + goto put_device; + ret = cdev_device_add(&mdevice->cdevice, &mdevice->device); if (ret) goto put_device; - /* Build a list of free ZONE_DEVICE struct pages */ - return dmirror_allocate_chunk(mdevice, NULL, false); + return 0; put_device: put_device(&mdevice->device); @@ -1758,7 +1912,6 @@ put_device: static void dmirror_device_remove(struct dmirror_device *mdevice) { - dmirror_device_remove_chunks(mdevice); cdev_device_del(&mdevice->cdevice, &mdevice->device); put_device(&mdevice->device); } diff --git a/lib/test_hmm_uapi.h b/lib/test_hmm_uapi.h index f94c6d457338..ea9b0ec404fb 100644 --- a/lib/test_hmm_uapi.h +++ b/lib/test_hmm_uapi.h @@ -38,6 +38,7 @@ struct hmm_dmirror_cmd { #define HMM_DMIRROR_CHECK_EXCLUSIVE _IOWR('H', 0x06, struct hmm_dmirror_cmd) #define HMM_DMIRROR_RELEASE _IOWR('H', 0x07, struct hmm_dmirror_cmd) #define HMM_DMIRROR_FLAGS _IOWR('H', 0x08, struct hmm_dmirror_cmd) +#define HMM_DMIRROR_READ_UNLOCKED _IOWR('H', 0x09, struct hmm_dmirror_cmd) #define HMM_DMIRROR_FLAG_FAIL_ALLOC (1ULL << 0) diff --git a/lib/test_kho.c b/lib/test_kho.c index 7ef9e4061869..aa6a0956bb8b 100644 --- a/lib/test_kho.c +++ b/lib/test_kho.c @@ -143,7 +143,8 @@ static int kho_test_preserve(struct kho_test_state *state) if (err) goto err_unpreserve_data; - err = kho_add_subtree(KHO_TEST_FDT, folio_address(state->fdt)); + err = kho_add_subtree(KHO_TEST_FDT, folio_address(state->fdt), + fdt_totalsize(folio_address(state->fdt))); if (err) goto err_unpreserve_data; @@ -318,7 +319,7 @@ static int __init kho_test_init(void) if (!kho_is_enabled()) return 0; - err = kho_retrieve_subtree(KHO_TEST_FDT, &fdt_phys); + err = kho_retrieve_subtree(KHO_TEST_FDT, &fdt_phys, NULL); if (!err) { err = kho_test_restore(fdt_phys); if (err) diff --git a/lib/test_maple_tree.c b/lib/test_maple_tree.c index a182e48b5f5e..b9367c61e8b5 100644 --- a/lib/test_maple_tree.c +++ b/lib/test_maple_tree.c @@ -2,7 +2,7 @@ /* * test_maple_tree.c: Test the maple tree API * Copyright (c) 2018-2022 Oracle Corporation - * Author: Liam R. Howlett <Liam.Howlett@Oracle.com> + * Author: Liam R. Howlett <liam@infradead.org> * * Any tests that only require the interface of the tree. */ @@ -1024,6 +1024,7 @@ static noinline void __init check_ranges(struct maple_tree *mt) mt_set_non_kernel(10); check_store_range(mt, r[10], r[11], xa_mk_value(r[10]), 0); MT_BUG_ON(mt, !mt_height(mt)); + mt_validate(mt); mtree_destroy(mt); /* Create tree of 1-200 */ @@ -1031,11 +1032,13 @@ static noinline void __init check_ranges(struct maple_tree *mt) /* Store 45-168 */ check_store_range(mt, r[10], r[11], xa_mk_value(r[10]), 0); MT_BUG_ON(mt, !mt_height(mt)); + mt_validate(mt); mtree_destroy(mt); check_seq(mt, 30, false); check_store_range(mt, 6, 18, xa_mk_value(6), 0); MT_BUG_ON(mt, !mt_height(mt)); + mt_validate(mt); mtree_destroy(mt); /* Overwrite across multiple levels. */ @@ -1061,6 +1064,7 @@ static noinline void __init check_ranges(struct maple_tree *mt) check_load(mt, r[13] + 1, xa_mk_value(r[13] + 1)); check_load(mt, 135, NULL); check_load(mt, 140, NULL); + mt_validate(mt); mt_set_non_kernel(0); MT_BUG_ON(mt, !mt_height(mt)); mtree_destroy(mt); @@ -1285,14 +1289,20 @@ static noinline void __init check_ranges(struct maple_tree *mt) MT_BUG_ON(mt, mt_height(mt) >= 4); } /* Cause a 3 child split all the way up the tree. */ - for (i = 5; i < 215; i += 10) + for (i = 5; i < 215; i += 10) { check_store_range(mt, 11450 + i, 11450 + i + 1, NULL, 0); - for (i = 5; i < 65; i += 10) + mt_validate(mt); + } + for (i = 5; i < 65; i += 10) { check_store_range(mt, 11770 + i, 11770 + i + 1, NULL, 0); + mt_validate(mt); + } MT_BUG_ON(mt, mt_height(mt) >= 4); - for (i = 5; i < 45; i += 10) + for (i = 5; i < 45; i += 10) { check_store_range(mt, 11700 + i, 11700 + i + 1, NULL, 0); + mt_validate(mt); + } if (!MAPLE_32BIT) MT_BUG_ON(mt, mt_height(mt) < 4); mtree_destroy(mt); @@ -1304,17 +1314,42 @@ static noinline void __init check_ranges(struct maple_tree *mt) val2 = (i+1)*10; check_store_range(mt, val, val2, xa_mk_value(val), 0); MT_BUG_ON(mt, mt_height(mt) >= 4); + mt_validate(mt); + } + /* Fill parents and leaves before split. */ + val = 7660; + for (i = 5; i < 490; i += 5) { + val += 5; + check_store_range(mt, val, val + 1, NULL, 0); + mt_validate(mt); + MT_BUG_ON(mt, mt_height(mt) >= 4); } + + val = 9460; /* Fill parents and leaves before split. */ - for (i = 5; i < 455; i += 10) - check_store_range(mt, 7800 + i, 7800 + i + 1, NULL, 0); + for (i = 1; i < 10; i++) { + val++; + check_store_range(mt, val, val + 1, xa_mk_value(val), 0); + mt_validate(mt); + } - for (i = 1; i < 16; i++) - check_store_range(mt, 8185 + i, 8185 + i + 1, - xa_mk_value(8185+i), 0); - MT_BUG_ON(mt, mt_height(mt) >= 4); + val = 8000; + for (i = 1; i < 14; i++) { + val++; + check_store_range(mt, val, val + 1, xa_mk_value(val), 0); + mt_validate(mt); + } + + + check_store_range(mt, 8051, 8051, xa_mk_value(8081), 0); + check_store_range(mt, 8052, 8052, xa_mk_value(8082), 0); + check_store_range(mt, 8083, 8083, xa_mk_value(8083), 0); + check_store_range(mt, 8084, 8084, xa_mk_value(8084), 0); + check_store_range(mt, 8085, 8085, xa_mk_value(8085), 0); /* triple split across multiple levels. */ - check_store_range(mt, 8184, 8184, xa_mk_value(8184), 0); + check_store_range(mt, 8099, 8100, xa_mk_value(1), 0); + + mt_validate(mt); if (!MAPLE_32BIT) MT_BUG_ON(mt, mt_height(mt) != 4); } @@ -3986,6 +4021,6 @@ static void __exit maple_tree_harvest(void) module_init(maple_tree_seed); module_exit(maple_tree_harvest); -MODULE_AUTHOR("Liam R. Howlett <Liam.Howlett@Oracle.com>"); +MODULE_AUTHOR("Liam R. Howlett <liam@infradead.org>"); MODULE_DESCRIPTION("maple tree API test module"); MODULE_LICENSE("GPL"); diff --git a/lib/test_meminit.c b/lib/test_meminit.c index 6298f66c964b..68c3b9da090e 100644 --- a/lib/test_meminit.c +++ b/lib/test_meminit.c @@ -229,16 +229,14 @@ static int __init do_kmem_cache_size(size_t size, bool want_ctor, for (iter = 0; iter < 10; iter++) { /* Do a test of bulk allocations */ if (!want_rcu && !want_ctor) { - int ret; - - ret = kmem_cache_alloc_bulk(c, alloc_mask, BULK_SIZE, bulk_array); - if (!ret) { + if (!kmem_cache_alloc_bulk(c, alloc_mask, BULK_SIZE, + bulk_array)) { fail = true; } else { int i; - for (i = 0; i < ret; i++) + for (i = 0; i < BULK_SIZE; i++) fail |= check_buf(bulk_array[i], size, want_ctor, want_rcu, want_zero); - kmem_cache_free_bulk(c, ret, bulk_array); + kmem_cache_free_bulk(c, BULK_SIZE, bulk_array); } } @@ -348,23 +346,24 @@ static int __init do_kmem_cache_size_bulk(int size, int *total_failures) { struct kmem_cache *c; int i, iter, maxiter = 1024; - int num, bytes; + int bytes; bool fail = false; void *objects[10]; c = kmem_cache_create("test_cache", size, size, 0, NULL); for (iter = 0; (iter < maxiter) && !fail; iter++) { - num = kmem_cache_alloc_bulk(c, GFP_KERNEL, ARRAY_SIZE(objects), - objects); - for (i = 0; i < num; i++) { + if (!kmem_cache_alloc_bulk(c, GFP_KERNEL, ARRAY_SIZE(objects), + objects)) + continue; + + for (i = 0; i < ARRAY_SIZE(objects); i++) { bytes = count_nonzero_bytes(objects[i], size); if (bytes) fail = true; fill_with_garbage(objects[i], size); } - if (num) - kmem_cache_free_bulk(c, num, objects); + kmem_cache_free_bulk(c, ARRAY_SIZE(objects), objects); } kmem_cache_destroy(c); *total_failures += fail; @@ -387,7 +386,7 @@ static int __init test_kmemcache(int *total_failures) ctor = flags & 1; rcu = flags & 2; zero = flags & 4; - if (ctor & zero) + if (ctor && zero) continue; num_tests += do_kmem_cache_size(size, ctor, rcu, zero, &failures); diff --git a/lib/test_rhashtable.c b/lib/test_rhashtable.c index 0b33559a910b..2f922b63d545 100644 --- a/lib/test_rhashtable.c +++ b/lib/test_rhashtable.c @@ -679,6 +679,78 @@ out: return err; } +static int __init test_rhashtable_next_key(void) +{ + struct rhashtable_params params = test_rht_params; + struct test_obj_val key_missing = { .id = 99999, .tid = 0 }; + struct test_obj_val *prev_key = NULL; + struct rhashtable ht; + struct test_obj *objs, *cur; + int i, count = 0, err; + int visited_keys[8] = { 0 }; + const int n = ARRAY_SIZE(visited_keys); + + params.nelem_hint = n; + + err = rhashtable_init(&ht, ¶ms); + if (err) + return err; + + objs = kzalloc_objs(*objs, n); + if (!objs) { + rhashtable_destroy(&ht); + return -ENOMEM; + } + + for (i = 0; i < n; i++) { + objs[i].value.id = i; + err = rhashtable_insert_fast(&ht, &objs[i].node, params); + if (err) + goto out; + } + + rcu_read_lock(); + + /* NULL prev_key: walk from the beginning, expect all n elements. */ + while ((cur = rhashtable_next_key(&ht, prev_key))) { + if (IS_ERR(cur)) { + err = -EINVAL; + goto unlock; + } + count++; + prev_key = &cur->value; + visited_keys[cur->value.id] = 1; + if (count > n) + break; + } + + if (count != n) { + err = -EINVAL; + goto unlock; + } + + for (i = 0; i < n; i++) { + if (!visited_keys[i]) { + err = -EINVAL; + goto unlock; + } + } + + /* Non-existing prev_key: must return ERR_PTR(-ENOENT). */ + cur = rhashtable_next_key(&ht, &key_missing); + if (!IS_ERR(cur) || PTR_ERR(cur) != -ENOENT) + err = -EINVAL; + +unlock: + rcu_read_unlock(); +out: + for (i = 0; i < n; i++) + rhashtable_remove_fast(&ht, &objs[i].node, params); + kfree(objs); + rhashtable_destroy(&ht); + return err; +} + static int __init test_rht_init(void) { unsigned int entries; @@ -738,6 +810,9 @@ static int __init test_rht_init(void) test_insert_duplicates_run(); + pr_info("Testing rhashtable_next_key: %s\n", + test_rhashtable_next_key() == 0 ? "pass" : "FAIL"); + if (!tcount) return 0; diff --git a/lib/test_sysctl.c b/lib/test_sysctl.c index c02aa9c868f2..909cfcf76dbf 100644 --- a/lib/test_sysctl.c +++ b/lib/test_sysctl.c @@ -7,7 +7,7 @@ /* * This module provides an interface to the proc sysctl interfaces. This - * driver requires CONFIG_PROC_SYSCTL. It will not normally be loaded by the + * driver requires CONFIG_SYSCTL. It will not normally be loaded by the * system unless explicitly requested by name. You can also build this driver * into your kernel. */ diff --git a/lib/test_vmalloc.c b/lib/test_vmalloc.c index 876c72c18a0c..b23f85e8f8ca 100644 --- a/lib/test_vmalloc.c +++ b/lib/test_vmalloc.c @@ -55,6 +55,7 @@ __param(int, run_test_mask, 7, "\t\tid: 512, name: kvfree_rcu_2_arg_vmalloc_test\n" "\t\tid: 1024, name: vm_map_ram_test\n" "\t\tid: 2048, name: no_block_alloc_test\n" + "\t\tid: 4096, name: vrealloc_test\n" /* Add a new test case description here. */ ); @@ -421,6 +422,66 @@ cleanup: return nr_allocated != map_nr_pages; } +static int vrealloc_test(void) +{ + void *ptr, *tmp; + int i; + + for (i = 0; i < test_loop_count; i++) { + int err = -1; + + ptr = vrealloc(NULL, PAGE_SIZE, GFP_KERNEL); + if (!ptr) + return -1; + + *((__u8 *)ptr) = 'a'; + + /* Grow: beyond allocated pages, triggers full realloc. */ + tmp = vrealloc(ptr, 4 * PAGE_SIZE, GFP_KERNEL); + if (!tmp) + goto error; + ptr = tmp; + + if (*((__u8 *)ptr) != 'a') + goto error; + + /* Shrink: crosses page boundary, frees tail pages. */ + tmp = vrealloc(ptr, PAGE_SIZE, GFP_KERNEL); + if (!tmp) + goto error; + ptr = tmp; + + if (*((__u8 *)ptr) != 'a') + goto error; + + /* Shrink: within same page, no page freeing. */ + tmp = vrealloc(ptr, PAGE_SIZE / 2, GFP_KERNEL); + if (!tmp) + goto error; + ptr = tmp; + + if (*((__u8 *)ptr) != 'a') + goto error; + + /* Grow: within allocated page, in-place, no realloc. */ + tmp = vrealloc(ptr, PAGE_SIZE, GFP_KERNEL); + if (!tmp) + goto error; + ptr = tmp; + + if (*((__u8 *)ptr) != 'a') + goto error; + + err = 0; +error: + vfree(ptr); + if (err) + return err; + } + + return 0; +} + struct test_case_desc { const char *test_name; int (*test_func)(void); @@ -440,6 +501,7 @@ static struct test_case_desc test_case_array[] = { { "kvfree_rcu_2_arg_vmalloc_test", kvfree_rcu_2_arg_vmalloc_test, }, { "vm_map_ram_test", vm_map_ram_test, }, { "no_block_alloc_test", no_block_alloc_test, true }, + { "vrealloc_test", vrealloc_test, }, /* Add a new test case here. */ }; diff --git a/lib/test_workqueue.c b/lib/test_workqueue.c new file mode 100644 index 000000000000..2bdfcbbcabb4 --- /dev/null +++ b/lib/test_workqueue.c @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Test module for stress and performance analysis of workqueue. + * + * Benchmarks queue_work() throughput on an unbound workqueue to measure + * pool->lock contention under different affinity scope configurations + * (e.g., cache vs cache_shard). + * + * The affinity scope is changed between runs via the workqueue's sysfs + * affinity_scope attribute (WQ_SYSFS). + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates + * Copyright (c) 2026 Breno Leitao <leitao@debian.org> + * + */ +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/workqueue.h> +#include <linux/kthread.h> +#include <linux/moduleparam.h> +#include <linux/completion.h> +#include <linux/atomic.h> +#include <linux/slab.h> +#include <linux/ktime.h> +#include <linux/cpumask.h> +#include <linux/sched.h> +#include <linux/sort.h> +#include <linux/fs.h> + +#define WQ_NAME "bench_wq" +#define SCOPE_PATH "/sys/bus/workqueue/devices/" WQ_NAME "/affinity_scope" + +static int nr_threads; +module_param(nr_threads, int, 0444); +MODULE_PARM_DESC(nr_threads, + "Number of threads to spawn (default: 0 = num_online_cpus())"); + +static int wq_items = 50000; +module_param(wq_items, int, 0444); +MODULE_PARM_DESC(wq_items, + "Number of work items each thread queues (default: 50000)"); + +static struct workqueue_struct *bench_wq; +static atomic_t threads_done; +static DECLARE_COMPLETION(start_comp); +static DECLARE_COMPLETION(all_done_comp); + +struct thread_ctx { + struct completion work_done; + struct work_struct work; + u64 *latencies; + int cpu; + int items; +}; + +static void bench_work_fn(struct work_struct *work) +{ + struct thread_ctx *ctx = container_of(work, struct thread_ctx, work); + + complete(&ctx->work_done); +} + +static int bench_kthread_fn(void *data) +{ + struct thread_ctx *ctx = data; + ktime_t t_start, t_end; + int i; + + /* Wait for all threads to be ready */ + wait_for_completion(&start_comp); + + if (kthread_should_stop()) + return 0; + + for (i = 0; i < ctx->items; i++) { + reinit_completion(&ctx->work_done); + INIT_WORK(&ctx->work, bench_work_fn); + + t_start = ktime_get(); + queue_work(bench_wq, &ctx->work); + t_end = ktime_get(); + + ctx->latencies[i] = ktime_to_ns(ktime_sub(t_end, t_start)); + wait_for_completion(&ctx->work_done); + } + + if (atomic_dec_and_test(&threads_done)) + complete(&all_done_comp); + + /* + * Wait for kthread_stop() so the module text isn't freed + * while we're still executing. + */ + while (!kthread_should_stop()) + schedule(); + + return 0; +} + +static int cmp_u64(const void *a, const void *b) +{ + u64 va = *(const u64 *)a; + u64 vb = *(const u64 *)b; + + if (va < vb) + return -1; + if (va > vb) + return 1; + return 0; +} + +static int __init set_affn_scope(const char *scope) +{ + struct file *f; + loff_t pos = 0; + ssize_t ret; + + f = filp_open(SCOPE_PATH, O_WRONLY, 0); + if (IS_ERR(f)) { + pr_err("test_workqueue: open %s failed: %ld\n", + SCOPE_PATH, PTR_ERR(f)); + return PTR_ERR(f); + } + + ret = kernel_write(f, scope, strlen(scope), &pos); + filp_close(f, NULL); + + if (ret < 0) { + pr_err("test_workqueue: write '%s' failed: %zd\n", scope, ret); + return ret; + } + + return 0; +} + +static int __init run_bench(int n_threads, const char *scope, const char *label) +{ + struct task_struct **tasks; + unsigned long total_items; + struct thread_ctx *ctxs; + u64 *all_latencies; + ktime_t start, end; + int cpu, i, j, ret; + s64 elapsed_us; + + ret = set_affn_scope(scope); + if (ret) + return ret; + + ctxs = kzalloc_objs(*ctxs, n_threads); + if (!ctxs) + return -ENOMEM; + + tasks = kzalloc_objs(*tasks, n_threads); + if (!tasks) { + kfree(ctxs); + return -ENOMEM; + } + + total_items = (unsigned long)n_threads * wq_items; + all_latencies = kvmalloc_array(total_items, sizeof(u64), GFP_KERNEL); + if (!all_latencies) { + kfree(tasks); + kfree(ctxs); + return -ENOMEM; + } + + /* Allocate per-thread latency arrays */ + for (i = 0; i < n_threads; i++) { + ctxs[i].latencies = kvmalloc_array(wq_items, sizeof(u64), + GFP_KERNEL); + if (!ctxs[i].latencies) { + while (--i >= 0) + kvfree(ctxs[i].latencies); + kvfree(all_latencies); + kfree(tasks); + kfree(ctxs); + return -ENOMEM; + } + } + + atomic_set(&threads_done, n_threads); + reinit_completion(&all_done_comp); + reinit_completion(&start_comp); + + /* Create kthreads, each bound to a different online CPU */ + i = 0; + for_each_online_cpu(cpu) { + if (i >= n_threads) + break; + + ctxs[i].cpu = cpu; + ctxs[i].items = wq_items; + init_completion(&ctxs[i].work_done); + + tasks[i] = kthread_create(bench_kthread_fn, &ctxs[i], + "wq_bench/%d", cpu); + if (IS_ERR(tasks[i])) { + ret = PTR_ERR(tasks[i]); + pr_err("test_workqueue: failed to create kthread %d: %d\n", + i, ret); + /* Unblock threads waiting on start_comp before stopping them */ + complete_all(&start_comp); + while (--i >= 0) + kthread_stop(tasks[i]); + goto out_free; + } + + kthread_bind(tasks[i], cpu); + wake_up_process(tasks[i]); + i++; + } + + /* Start timing and release all threads */ + start = ktime_get(); + complete_all(&start_comp); + + /* Wait for all threads to finish the benchmark */ + wait_for_completion(&all_done_comp); + + /* Drain any remaining work */ + flush_workqueue(bench_wq); + + /* Ensure all kthreads have fully exited before module memory is freed */ + for (i = 0; i < n_threads; i++) + kthread_stop(tasks[i]); + + end = ktime_get(); + elapsed_us = ktime_us_delta(end, start); + + /* Merge all per-thread latencies and sort for percentile calculation */ + j = 0; + for (i = 0; i < n_threads; i++) { + memcpy(&all_latencies[j], ctxs[i].latencies, + wq_items * sizeof(u64)); + j += wq_items; + } + + sort(all_latencies, total_items, sizeof(u64), cmp_u64, NULL); + + pr_info("test_workqueue: %-16s %llu items/sec\tp50=%llu\tp90=%llu\tp95=%llu ns\n", + label, + elapsed_us ? div_u64(total_items * 1000000ULL, elapsed_us) : 0, + all_latencies[total_items * 50 / 100], + all_latencies[total_items * 90 / 100], + all_latencies[total_items * 95 / 100]); + + ret = 0; +out_free: + for (i = 0; i < n_threads; i++) + kvfree(ctxs[i].latencies); + kvfree(all_latencies); + kfree(tasks); + kfree(ctxs); + + return ret; +} + +static const char * const bench_scopes[] = { + "cpu", "smt", "cache_shard", "cache", "numa", "system", +}; + +static int __init test_workqueue_init(void) +{ + int n_threads = min(nr_threads ?: num_online_cpus(), num_online_cpus()); + int i; + + if (wq_items <= 0) { + pr_err("test_workqueue: wq_items must be > 0\n"); + return -EINVAL; + } + + bench_wq = alloc_workqueue(WQ_NAME, WQ_UNBOUND | WQ_SYSFS, 0); + if (!bench_wq) + return -ENOMEM; + + pr_info("test_workqueue: running %d threads, %d items/thread\n", + n_threads, wq_items); + + for (i = 0; i < ARRAY_SIZE(bench_scopes); i++) + run_bench(n_threads, bench_scopes[i], bench_scopes[i]); + + destroy_workqueue(bench_wq); + + /* Return -EAGAIN so the module doesn't stay loaded after the benchmark */ + return -EAGAIN; +} + +module_init(test_workqueue_init); +MODULE_AUTHOR("Breno Leitao <leitao@debian.org>"); +MODULE_DESCRIPTION("Stress/performance benchmark for workqueue subsystem"); +MODULE_LICENSE("GPL"); diff --git a/lib/tests/Makefile b/lib/tests/Makefile index 05f74edbc62b..3cac3b63a752 100644 --- a/lib/tests/Makefile +++ b/lib/tests/Makefile @@ -8,6 +8,7 @@ obj-$(CONFIG_BASE64_KUNIT) += base64_kunit.o obj-$(CONFIG_BITOPS_KUNIT) += bitops_kunit.o obj-$(CONFIG_BITFIELD_KUNIT) += bitfield_kunit.o obj-$(CONFIG_BITS_TEST) += test_bits.o +obj-$(CONFIG_SHDI3_KUNIT_TEST) += shdi3_kunit.o obj-$(CONFIG_BLACKHOLE_DEV_KUNIT_TEST) += blackhole_dev_kunit.o obj-$(CONFIG_CHECKSUM_KUNIT) += checksum_kunit.o obj-$(CONFIG_CMDLINE_KUNIT_TEST) += cmdline_kunit.o @@ -40,6 +41,7 @@ obj-$(CONFIG_MEMCPY_KUNIT_TEST) += memcpy_kunit.o obj-$(CONFIG_MIN_HEAP_KUNIT_TEST) += min_heap_kunit.o CFLAGS_overflow_kunit.o = $(call cc-disable-warning, tautological-constant-out-of-range-compare) obj-$(CONFIG_OVERFLOW_KUNIT_TEST) += overflow_kunit.o +obj-$(CONFIG_PRANDOM_KUNIT_TEST) += random32_kunit.o obj-$(CONFIG_PRINTF_KUNIT_TEST) += printf_kunit.o obj-$(CONFIG_RANDSTRUCT_KUNIT_TEST) += randstruct_kunit.o obj-$(CONFIG_SCANF_KUNIT_TEST) += scanf_kunit.o diff --git a/lib/tests/blackhole_dev_kunit.c b/lib/tests/blackhole_dev_kunit.c index 06834ab35f43..fa3e0533038d 100644 --- a/lib/tests/blackhole_dev_kunit.c +++ b/lib/tests/blackhole_dev_kunit.c @@ -46,7 +46,7 @@ static void test_blackholedev(struct kunit *test) uh = (struct udphdr *)skb_push(skb, sizeof(struct udphdr)); skb_set_transport_header(skb, 0); uh->source = uh->dest = htons(UDP_PORT); - uh->len = htons(data_len); + udp_set_len_short(uh, data_len); uh->check = 0; /* (Network) IPv6 */ ip6h = (struct ipv6hdr *)skb_push(skb, sizeof(struct ipv6hdr)); diff --git a/lib/tests/cmdline_kunit.c b/lib/tests/cmdline_kunit.c index c1602f797637..3f61ff8d3178 100644 --- a/lib/tests/cmdline_kunit.c +++ b/lib/tests/cmdline_kunit.c @@ -6,6 +6,7 @@ #include <kunit/test.h> #include <linux/kernel.h> #include <linux/random.h> +#include <linux/sizes.h> #include <linux/string.h> static const char *cmdline_test_strings[] = { @@ -139,11 +140,128 @@ static void cmdline_test_range(struct kunit *test) } while (++i < ARRAY_SIZE(cmdline_test_range_strings)); } +static void cmdline_test_next_arg_quoted_value(struct kunit *test) +{ + char in[] = "foo=\"bar baz\" qux=1"; + char *next, *param, *val; + + next = next_arg(in, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "foo"); + KUNIT_ASSERT_NOT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, val, "bar baz"); + KUNIT_EXPECT_STREQ(test, next, "qux=1"); + + next = next_arg(next, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "qux"); + KUNIT_ASSERT_NOT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, val, "1"); + KUNIT_EXPECT_STREQ(test, next, ""); +} + +static void cmdline_test_next_arg_bare_quote_regression(struct kunit *test) +{ + char in[] = "foo=bar \""; + char *next, *param, *val; + + next = next_arg(in, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "foo"); + KUNIT_ASSERT_NOT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, val, "bar"); + KUNIT_EXPECT_STREQ(test, next, "\""); + + /* This hits the i == 0 quoted-token case fixed by 9847f21225c4. */ + next = next_arg(next, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, ""); + KUNIT_EXPECT_PTR_EQ(test, val, NULL); + KUNIT_EXPECT_STREQ(test, next, ""); +} + +static void cmdline_test_next_arg_mixed_tokens(struct kunit *test) +{ + char in[] = "bbb= jjj kkk=\"a=b\""; + char *next, *param, *val; + + next = next_arg(in, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "bbb"); + KUNIT_ASSERT_NOT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, val, ""); + KUNIT_EXPECT_STREQ(test, next, "jjj kkk=\"a=b\""); + + next = next_arg(next, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "jjj"); + KUNIT_EXPECT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, next, "kkk=\"a=b\""); + + next = next_arg(next, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "kkk"); + KUNIT_ASSERT_NOT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, val, "a=b"); + KUNIT_EXPECT_STREQ(test, next, ""); +} + +struct cmdline_test_memparse_entry { + const char *input; + const char *unrecognized; + unsigned long long result; +}; + +static const struct cmdline_test_memparse_entry testdata[] = { + { "0", "", 0ULL }, + { "1", "", 1ULL }, + { "a", "a", 0ULL }, + { "k", "k", 0ULL }, + { "E", "E", 0ULL }, + { "0xb", "", 11ULL }, + { "0xz", "x", 0ULL }, + { "1234", "", 1234ULL }, + { "04567", "", 2423ULL }, + { "0x9876", "", 39030LL }, + { "05678", "8", 375ULL }, + { "0xabcdefz", "z", 11259375ULL }, + { "0cdba", "c", 0ULL }, + { "4K", "", SZ_4K }, + { "0x10k@0xaaaabbbb", "@", SZ_16K }, + { "32M", "", SZ_32M }, + { "067m:foo", ":", 55 * SZ_1M }, + { "2G;bar=baz", ";", SZ_2G }, + { "07gz", "z", 7ULL * SZ_1G }, + { "3T+data", "+", 3 * SZ_1T }, + { "04t,ro", ",", SZ_4T }, + { "012p", "", 11258999068426240ULL }, + { "7P,sync", ",", 7881299347898368ULL }, + { "0x2e", "", 46ULL }, + { "2E and more", " ", 2305843009213693952ULL }, + { "18446744073709551615", "", ULLONG_MAX }, + { "0xffffffffffffffff0", "", ULLONG_MAX }, + { "1111111111111111111T", "", ULLONG_MAX }, + { "222222222222222222222G", "", ULLONG_MAX }, + { "3333333333333333333333M", "", ULLONG_MAX }, +}; + +static void cmdline_test_memparse(struct kunit *test) +{ + const struct cmdline_test_memparse_entry *e; + unsigned long long ret; + char *retptr; + + for (e = testdata; e < testdata + ARRAY_SIZE(testdata); e++) { + ret = memparse(e->input, &retptr); + KUNIT_EXPECT_EQ_MSG(test, ret, e->result, + " when parsing '%s'", e->input); + KUNIT_EXPECT_EQ_MSG(test, *retptr, *e->unrecognized, + " when parsing '%s'", e->input); + } +} + static struct kunit_case cmdline_test_cases[] = { KUNIT_CASE(cmdline_test_noint), KUNIT_CASE(cmdline_test_lead_int), KUNIT_CASE(cmdline_test_tail_int), KUNIT_CASE(cmdline_test_range), + KUNIT_CASE(cmdline_test_next_arg_quoted_value), + KUNIT_CASE(cmdline_test_next_arg_bare_quote_regression), + KUNIT_CASE(cmdline_test_next_arg_mixed_tokens), + KUNIT_CASE(cmdline_test_memparse), {} }; diff --git a/lib/tests/fortify_kunit.c b/lib/tests/fortify_kunit.c index fc9c76f026d6..413cdbf3dc0d 100644 --- a/lib/tests/fortify_kunit.c +++ b/lib/tests/fortify_kunit.c @@ -458,7 +458,7 @@ static void fortify_test_strnlen(struct kunit *test) /* Make string unterminated, and recount. */ pad.buf[end] = 'A'; end = sizeof(pad.buf); - /* Reading beyond with strncpy() will fail. */ + /* Reading beyond will fail. */ KUNIT_EXPECT_EQ(test, strnlen(pad.buf, end + 1), end); KUNIT_EXPECT_EQ(test, fortify_read_overflows, 1); KUNIT_EXPECT_EQ(test, strnlen(pad.buf, end + 2), end); @@ -531,64 +531,6 @@ static void fortify_test_strcpy(struct kunit *test) KUNIT_EXPECT_EQ(test, pad.bytes_after, 0); } -static void fortify_test_strncpy(struct kunit *test) -{ - struct fortify_padding pad = { }; - char src[] = "Copy me fully into a small buffer and I will overflow!"; - size_t sizeof_buf = sizeof(pad.buf); - - OPTIMIZER_HIDE_VAR(sizeof_buf); - - /* Destination is %NUL-filled to start with. */ - KUNIT_EXPECT_EQ(test, pad.bytes_before, 0); - KUNIT_EXPECT_EQ(test, pad.buf[sizeof_buf - 1], '\0'); - KUNIT_EXPECT_EQ(test, pad.buf[sizeof_buf - 2], '\0'); - KUNIT_EXPECT_EQ(test, pad.buf[sizeof_buf - 3], '\0'); - KUNIT_EXPECT_EQ(test, pad.bytes_after, 0); - - /* Legitimate strncpy() 1 less than of max size. */ - KUNIT_ASSERT_TRUE(test, strncpy(pad.buf, src, sizeof_buf - 1) - == pad.buf); - KUNIT_EXPECT_EQ(test, fortify_write_overflows, 0); - /* Only last byte should be %NUL */ - KUNIT_EXPECT_EQ(test, pad.buf[sizeof_buf - 1], '\0'); - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 2], '\0'); - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 3], '\0'); - - /* Legitimate (though unterminated) max-size strncpy. */ - KUNIT_ASSERT_TRUE(test, strncpy(pad.buf, src, sizeof_buf) - == pad.buf); - KUNIT_EXPECT_EQ(test, fortify_write_overflows, 0); - /* No trailing %NUL -- thanks strncpy API. */ - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 1], '\0'); - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 2], '\0'); - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 2], '\0'); - /* But we will not have gone beyond. */ - KUNIT_EXPECT_EQ(test, pad.bytes_after, 0); - - /* Now verify that FORTIFY is working... */ - KUNIT_ASSERT_TRUE(test, strncpy(pad.buf, src, sizeof_buf + 1) - == pad.buf); - /* Should catch the overflow. */ - KUNIT_EXPECT_EQ(test, fortify_write_overflows, 1); - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 1], '\0'); - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 2], '\0'); - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 2], '\0'); - /* And we will not have gone beyond. */ - KUNIT_EXPECT_EQ(test, pad.bytes_after, 0); - - /* And further... */ - KUNIT_ASSERT_TRUE(test, strncpy(pad.buf, src, sizeof_buf + 2) - == pad.buf); - /* Should catch the overflow. */ - KUNIT_EXPECT_EQ(test, fortify_write_overflows, 2); - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 1], '\0'); - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 2], '\0'); - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 2], '\0'); - /* And we will not have gone beyond. */ - KUNIT_EXPECT_EQ(test, pad.bytes_after, 0); -} - static void fortify_test_strscpy(struct kunit *test) { struct fortify_padding pad = { }; @@ -1100,7 +1042,6 @@ static struct kunit_case fortify_test_cases[] = { KUNIT_CASE(fortify_test_strlen), KUNIT_CASE(fortify_test_strnlen), KUNIT_CASE(fortify_test_strcpy), - KUNIT_CASE(fortify_test_strncpy), KUNIT_CASE(fortify_test_strscpy), KUNIT_CASE(fortify_test_strcat), KUNIT_CASE(fortify_test_strncat), diff --git a/lib/tests/kunit_iov_iter.c b/lib/tests/kunit_iov_iter.c index bb847e5010eb..32e42d8c7ca1 100644 --- a/lib/tests/kunit_iov_iter.c +++ b/lib/tests/kunit_iov_iter.c @@ -13,6 +13,9 @@ #include <linux/uio.h> #include <linux/bvec.h> #include <linux/folio_queue.h> +#include <linux/scatterlist.h> +#include <linux/minmax.h> +#include <linux/mman.h> #include <kunit/test.h> MODULE_DESCRIPTION("iov_iter testing"); @@ -37,12 +40,12 @@ static const struct kvec_test_range kvec_test_ranges[] = { static inline u8 pattern(unsigned long x) { - return x & 0xff; + return (u8)x + (u8)(x >> 8) + (u8)(x >> 16); } static void iov_kunit_unmap(void *data) { - vunmap(data); + vfree(data); } static void *__init iov_kunit_create_buffer(struct kunit *test, @@ -50,20 +53,37 @@ static void *__init iov_kunit_create_buffer(struct kunit *test, size_t npages) { struct page **pages; - unsigned long got; + unsigned long got, last; void *buffer; + unsigned int i; - pages = kunit_kcalloc(test, npages, sizeof(struct page *), GFP_KERNEL); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pages); + pages = kzalloc_objs(struct page *, npages); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pages); *ppages = pages; - got = alloc_pages_bulk(GFP_KERNEL, npages, pages); + got = 0; + while (true) { + last = got; + got = alloc_pages_bulk(GFP_KERNEL, npages, pages); + + if (last == got || got == npages) + break; + } + if (got != npages) { release_pages(pages, got); + kvfree(pages); KUNIT_ASSERT_EQ(test, got, npages); } + /* Make sure that we don't get a physically contiguous buffer. */ + for (i = 0; i < npages / 4; ++i) + swap(pages[i], pages[i + npages / 2]); buffer = vmap(pages, npages, VM_MAP | VM_MAP_PUT_PAGES, PAGE_KERNEL); + if (buffer == NULL) { + release_pages(pages, got); + kvfree(pages); + } KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buffer); kunit_add_action_or_reset(test, iov_kunit_unmap, buffer); @@ -263,7 +283,7 @@ static void __init iov_kunit_copy_to_bvec(struct kunit *test) struct page **spages, **bpages; u8 *scratch, *buffer; size_t bufsize, npages, size, copied; - int i, b, patt; + int i, patt; bufsize = 0x100000; npages = bufsize / PAGE_SIZE; @@ -286,10 +306,9 @@ static void __init iov_kunit_copy_to_bvec(struct kunit *test) KUNIT_EXPECT_EQ(test, iter.nr_segs, 0); /* Build the expected image in the scratch buffer. */ - b = 0; patt = 0; memset(scratch, 0, bufsize); - for (pr = bvec_test_ranges; pr->from >= 0; pr++, b++) { + for (pr = bvec_test_ranges; pr->from >= 0; pr++) { u8 *p = scratch + pr->page * PAGE_SIZE; for (i = pr->from; i < pr->to; i++) @@ -369,9 +388,6 @@ static void iov_kunit_destroy_folioq(void *data) for (folioq = data; folioq; folioq = next) { next = folioq->next; - for (int i = 0; i < folioq_nr_slots(folioq); i++) - if (folioq_folio(folioq, i)) - folio_put(folioq_folio(folioq, i)); kfree(folioq); } } @@ -1009,6 +1025,202 @@ stop: KUNIT_SUCCEED(test); } +struct iov_kunit_iter_to_sg_data { + struct sg_table *sgt; + u8 *buffer, *scratch; + u8 __user *ubuf; + struct page **pages; + size_t npages; +}; + +static void __init +iov_kunit_iter_unpin_sgt(void *data) +{ + struct sg_table *sgt = data; + + for (unsigned int i = 0; i < sgt->nents; ++i) + unpin_user_page(sg_page(&sgt->sgl[i])); +} + +static void __init +iov_kunit_iter_to_sg_init(struct kunit *test, size_t bufsize, bool user, + struct iov_kunit_iter_to_sg_data *data) +{ + struct page **spages; + struct scatterlist *sg; + unsigned long uaddr; + size_t i; + + data->npages = bufsize / PAGE_SIZE; + sg = kunit_kmalloc_array(test, data->npages, sizeof(*sg), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sg); + sg_init_table(sg, data->npages); + data->sgt = kunit_kzalloc(test, sizeof(*data->sgt), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, data->sgt); + data->sgt->orig_nents = 0; + data->sgt->sgl = sg; + + data->buffer = NULL; + data->ubuf = NULL; + if (user) { + uaddr = kunit_vm_mmap(test, NULL, 0, bufsize, + PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, 0); + KUNIT_ASSERT_NE(test, uaddr, 0); + data->ubuf = (u8 __user *)uaddr; + for (i = 0; i < bufsize; ++i) + put_user(pattern(i), data->ubuf + i); + } else { + data->buffer = iov_kunit_create_buffer(test, &data->pages, + data->npages); + for (i = 0; i < bufsize; ++i) + data->buffer[i] = pattern(i); + } + data->scratch = iov_kunit_create_buffer(test, &spages, data->npages); + memset(data->scratch, 0, bufsize); +} + +static void __init +iov_kunit_iter_to_sg_check(struct kunit *test, struct iov_iter *iter, + size_t bufsize, + struct iov_kunit_iter_to_sg_data *data) +{ + static const size_t tail = 16 * PAGE_SIZE; + size_t i; + + KUNIT_ASSERT_LT(test, tail, bufsize); + + if (iov_iter_extract_will_pin(iter)) + kunit_add_action_or_reset(test, iov_kunit_iter_unpin_sgt, + data->sgt); + + i = extract_iter_to_sg(iter, bufsize, data->sgt, 0, 0); + KUNIT_ASSERT_EQ(test, i, 0); + KUNIT_ASSERT_EQ(test, data->sgt->nents, 0); + + i = extract_iter_to_sg(iter, bufsize - tail, data->sgt, 1, 0); + KUNIT_ASSERT_LE(test, i, bufsize - tail); + KUNIT_ASSERT_EQ(test, data->sgt->nents, 1); + + i += extract_iter_to_sg(iter, bufsize - tail - i, data->sgt, + data->npages - data->sgt->nents, 0); + KUNIT_ASSERT_EQ(test, i, bufsize - tail); + KUNIT_ASSERT_LE(test, data->sgt->nents, data->npages); + + i += extract_iter_to_sg(iter, tail, data->sgt, + data->npages - data->sgt->nents, 0); + KUNIT_ASSERT_EQ(test, i, bufsize); + KUNIT_ASSERT_LE(test, data->sgt->nents, data->npages); + + sg_mark_end(&data->sgt->sgl[data->sgt->nents - 1]); + + i = sg_copy_to_buffer(data->sgt->sgl, data->sgt->nents, + data->scratch, bufsize); + KUNIT_ASSERT_EQ(test, i, bufsize); + + for (i = 0; i < bufsize; ++i) { + KUNIT_EXPECT_EQ_MSG(test, data->scratch[i], pattern(i), + "at i=%zx", i); + if (data->scratch[i] != pattern(i)) + break; + } + + KUNIT_EXPECT_EQ(test, i, bufsize); +} + +static void __init iov_kunit_iter_to_sg_kvec(struct kunit *test) +{ + struct iov_kunit_iter_to_sg_data data; + struct iov_iter iter; + struct kvec kvec; + size_t bufsize; + + bufsize = 0x200000; + iov_kunit_iter_to_sg_init(test, bufsize, false, &data); + + kvec.iov_base = data.buffer; + kvec.iov_len = bufsize; + iov_iter_kvec(&iter, READ, &kvec, 1, bufsize); + + iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data); +} + +static void __init iov_kunit_iter_to_sg_bvec(struct kunit *test) +{ + struct iov_kunit_iter_to_sg_data data; + struct page *p, *can_merge = NULL; + size_t i, k, bufsize; + struct bio_vec *bvec; + struct iov_iter iter; + + bufsize = 0x200000; + iov_kunit_iter_to_sg_init(test, bufsize, false, &data); + + bvec = kunit_kmalloc_array(test, data.npages, sizeof(*bvec), + GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bvec); + k = 0; + for (i = 0; i < data.npages; ++i) { + p = data.pages[i]; + if (p == can_merge) + bvec[k-1].bv_len += PAGE_SIZE; + else + bvec_set_page(&bvec[k++], p, PAGE_SIZE, 0); + can_merge = p + 1; + } + iov_iter_bvec(&iter, READ, bvec, k, bufsize); + + iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data); +} + +static void __init iov_kunit_iter_to_sg_folioq(struct kunit *test) +{ + struct iov_kunit_iter_to_sg_data data; + struct folio_queue *folioq; + struct iov_iter iter; + size_t bufsize; + + bufsize = 0x200000; + iov_kunit_iter_to_sg_init(test, bufsize, false, &data); + + folioq = iov_kunit_create_folioq(test); + iov_kunit_load_folioq(test, &iter, READ, folioq, data.pages, + data.npages); + + iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data); +} + +static void __init iov_kunit_iter_to_sg_xarray(struct kunit *test) +{ + struct iov_kunit_iter_to_sg_data data; + struct xarray *xarray; + struct iov_iter iter; + size_t bufsize; + + bufsize = 0x200000; + iov_kunit_iter_to_sg_init(test, bufsize, false, &data); + + xarray = iov_kunit_create_xarray(test); + iov_kunit_load_xarray(test, &iter, READ, xarray, data.pages, + data.npages); + + iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data); +} + +static void __init iov_kunit_iter_to_sg_ubuf(struct kunit *test) +{ + struct iov_kunit_iter_to_sg_data data; + struct iov_iter iter; + size_t bufsize; + + bufsize = 0x200000; + iov_kunit_iter_to_sg_init(test, bufsize, true, &data); + + iov_iter_ubuf(&iter, READ, data.ubuf, bufsize); + + iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data); +} + static struct kunit_case __refdata iov_kunit_cases[] = { KUNIT_CASE(iov_kunit_copy_to_kvec), KUNIT_CASE(iov_kunit_copy_from_kvec), @@ -1022,6 +1234,11 @@ static struct kunit_case __refdata iov_kunit_cases[] = { KUNIT_CASE(iov_kunit_extract_pages_bvec), KUNIT_CASE(iov_kunit_extract_pages_folioq), KUNIT_CASE(iov_kunit_extract_pages_xarray), + KUNIT_CASE(iov_kunit_iter_to_sg_kvec), + KUNIT_CASE(iov_kunit_iter_to_sg_bvec), + KUNIT_CASE(iov_kunit_iter_to_sg_folioq), + KUNIT_CASE(iov_kunit_iter_to_sg_xarray), + KUNIT_CASE(iov_kunit_iter_to_sg_ubuf), {} }; diff --git a/lib/tests/liveupdate.c b/lib/tests/liveupdate.c index 496d6ef91a30..4c08a7c6fb78 100644 --- a/lib/tests/liveupdate.c +++ b/lib/tests/liveupdate.c @@ -105,6 +105,9 @@ static void liveupdate_test_init(void) pr_err("liveupdate_flb_get_incoming for %s failed: %pe\n", flb->compatible, ERR_PTR(err)); } + + if (!err) + liveupdate_flb_put_incoming(flb); } initialized = true; } @@ -135,24 +138,6 @@ void liveupdate_test_register(struct liveupdate_file_handler *fh) TEST_NFLBS, fh->compatible); } -void liveupdate_test_unregister(struct liveupdate_file_handler *fh) -{ - int err, i; - - for (i = 0; i < TEST_NFLBS; i++) { - struct liveupdate_flb *flb = &test_flbs[i]; - - err = liveupdate_unregister_flb(fh, flb); - if (err) { - pr_err("Failed to unregister %s %pe\n", - flb->compatible, ERR_PTR(err)); - } - } - - pr_info("Unregistered %d FLBs from file handler: [%s]\n", - TEST_NFLBS, fh->compatible); -} - MODULE_LICENSE("GPL"); MODULE_AUTHOR("Pasha Tatashin <pasha.tatashin@soleen.com>"); MODULE_DESCRIPTION("In-kernel test for LUO mechanism"); diff --git a/lib/tests/printf_kunit.c b/lib/tests/printf_kunit.c index f6f21b445ece..eccf041ebd56 100644 --- a/lib/tests/printf_kunit.c +++ b/lib/tests/printf_kunit.c @@ -17,6 +17,7 @@ #include <linux/dcache.h> #include <linux/socket.h> #include <linux/in.h> +#include <linux/in6.h> #include <linux/gfp.h> #include <linux/mm.h> @@ -318,7 +319,27 @@ symbol_ptr(struct kunit *kunittest) static void kernel_ptr(struct kunit *kunittest) { - /* We can't test this without access to kptr_restrict. */ + switch (kptr_restrict) { + case 0: + if (no_hash_pointers) { + test(PTR_STR, "%pK", PTR); + } else { + char buf[PLAIN_BUF_SIZE]; + + plain_hash_to_buffer(kunittest, PTR, buf, PLAIN_BUF_SIZE); + /* %pK behaves the same as hashing */ + test(buf, "%pK", PTR); + } + break; + case 1: + /* The KUnit kthread has all capabilities, including CAP_SYSLOG */ + test(PTR_STR, "%pK", PTR); + break; + case 2: + default: + test(ZEROS "00000000", "%pK", PTR); + break; + } } static void @@ -414,8 +435,10 @@ mac(struct kunit *kunittest) test("2d:48:d6:fc:7a:05", "%pM", addr); test("05:7a:fc:d6:48:2d", "%pMR", addr); + test("05:7A:FC:D6:48:2D", "%pMRU", addr); test("2d-48-d6-fc-7a-05", "%pMF", addr); test("2d48d6fc7a05", "%pm", addr); + test("2D48D6FC7A05", "%pmU", addr); test("057afcd6482d", "%pmR", addr); } @@ -437,6 +460,27 @@ ip4(struct kunit *kunittest) static void ip6(struct kunit *kunittest) { + const struct in6_addr addr = { + .s6_addr = { 0x00, 0x01, 0x00, 0x02, 0x00, 0x03, 0x00, 0x04, + 0x00, 0x05, 0x00, 0x06, 0x00, 0x07, 0x00, 0x08 } + }; + const struct in6_addr single_zero = { + .s6_addr = { 0x00, 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x04, + 0x00, 0x05, 0x00, 0x06, 0x00, 0x07, 0x00, 0x08 } + }; + struct sockaddr_in6 sa = { + .sin6_family = AF_INET6, + .sin6_port = cpu_to_be16(12345), + .sin6_addr = addr, + }; + + test("00010002000300040005000600070008|0001:0002:0003:0004:0005:0006:0007:0008", + "%pi6|%pI6", &addr, &addr); + test("00010002000300040005000600070008|0001:0002:0003:0004:0005:0006:0007:0008", + "%piS|%pIS", &sa, &sa); + test("1:2:3:4:5:6:7:8", "%pI6c", &addr); + test("1:0:3:4:5:6:7:8", "%pI6c", &single_zero); + test("[1:2:3:4:5:6:7:8]:12345", "%pISpc", &sa); } static void diff --git a/lib/tests/random32_kunit.c b/lib/tests/random32_kunit.c new file mode 100644 index 000000000000..0b4af2b09c01 --- /dev/null +++ b/lib/tests/random32_kunit.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Test cases for random32 functions. + */ + +#include <linux/prandom.h> +#include <kunit/test.h> + +/* prandom_warmup() is static in lib/random32.c; exposed for testing only. */ +void prandom_warmup(struct rnd_state *state); + +static const struct prandom_test1 { + u32 seed; + u32 result; +} test1[] = { + { 1U, 3484351685U }, + { 2U, 2623130059U }, + { 3U, 3125133893U }, + { 4U, 984847254U }, +}; + +static const struct prandom_test2 { + u32 seed; + u32 iteration; + u32 result; +} test2[] = { + /* Test cases against taus113 from GSL library. */ + { 931557656U, 959U, 2975593782U }, + { 1339693295U, 876U, 3887776532U }, + { 1545556285U, 961U, 1615538833U }, + { 601730776U, 723U, 1776162651U }, + { 1027516047U, 687U, 511983079U }, + { 416526298U, 700U, 916156552U }, + { 1395522032U, 652U, 2222063676U }, + { 366221443U, 617U, 2992857763U }, + { 1539836965U, 714U, 3783265725U }, + { 556206671U, 994U, 799626459U }, + { 684907218U, 799U, 367789491U }, + { 2121230701U, 931U, 2115467001U }, + { 1668516451U, 644U, 3620590685U }, + { 768046066U, 883U, 2034077390U }, + { 1989159136U, 833U, 1195767305U }, + { 536585145U, 996U, 3577259204U }, + { 1008129373U, 642U, 1478080776U }, + { 1740775604U, 939U, 1264980372U }, + { 1967883163U, 508U, 10734624U }, + { 1923019697U, 730U, 3821419629U }, + { 442079932U, 560U, 3440032343U }, + { 1961302714U, 845U, 841962572U }, + { 2030205964U, 962U, 1325144227U }, + { 1160407529U, 507U, 240940858U }, + { 635482502U, 779U, 4200489746U }, + { 1252788931U, 699U, 867195434U }, + { 1961817131U, 719U, 668237657U }, + { 1071468216U, 983U, 917876630U }, + { 1281848367U, 932U, 1003100039U }, + { 582537119U, 780U, 1127273778U }, + { 1973672777U, 853U, 1071368872U }, + { 1896756996U, 762U, 1127851055U }, + { 847917054U, 500U, 1717499075U }, + { 1240520510U, 951U, 2849576657U }, + { 1685071682U, 567U, 1961810396U }, + { 1516232129U, 557U, 3173877U }, + { 1208118903U, 612U, 1613145022U }, + { 1817269927U, 693U, 4279122573U }, + { 1510091701U, 717U, 638191229U }, + { 365916850U, 807U, 600424314U }, + { 399324359U, 702U, 1803598116U }, + { 1318480274U, 779U, 2074237022U }, + { 697758115U, 840U, 1483639402U }, + { 1696507773U, 840U, 577415447U }, + { 2081979121U, 981U, 3041486449U }, + { 955646687U, 742U, 3846494357U }, + { 1250683506U, 749U, 836419859U }, + { 595003102U, 534U, 366794109U }, + { 47485338U, 558U, 3521120834U }, + { 619433479U, 610U, 3991783875U }, + { 704096520U, 518U, 4139493852U }, + { 1712224984U, 606U, 2393312003U }, + { 1318233152U, 922U, 3880361134U }, + { 855572992U, 761U, 1472974787U }, + { 64721421U, 703U, 683860550U }, + { 678931758U, 840U, 380616043U }, + { 692711973U, 778U, 1382361947U }, + { 677703619U, 530U, 2826914161U }, + { 92393223U, 586U, 1522128471U }, + { 1222592920U, 743U, 3466726667U }, + { 358288986U, 695U, 1091956998U }, + { 1935056945U, 958U, 514864477U }, + { 735675993U, 990U, 1294239989U }, + { 1560089402U, 897U, 2238551287U }, + { 70616361U, 829U, 22483098U }, + { 368234700U, 731U, 2913875084U }, + { 20221190U, 879U, 1564152970U }, + { 539444654U, 682U, 1835141259U }, + { 1314987297U, 840U, 1801114136U }, + { 2019295544U, 645U, 3286438930U }, + { 469023838U, 716U, 1637918202U }, + { 1843754496U, 653U, 2562092152U }, + { 400672036U, 809U, 4264212785U }, + { 404722249U, 965U, 2704116999U }, + { 600702209U, 758U, 584979986U }, + { 519953954U, 667U, 2574436237U }, + { 1658071126U, 694U, 2214569490U }, + { 420480037U, 749U, 3430010866U }, + { 690103647U, 969U, 3700758083U }, + { 1029424799U, 937U, 3787746841U }, + { 2012608669U, 506U, 3362628973U }, + { 1535432887U, 998U, 42610943U }, + { 1330635533U, 857U, 3040806504U }, + { 1223800550U, 539U, 3954229517U }, + { 1322411537U, 680U, 3223250324U }, + { 1877847898U, 945U, 2915147143U }, + { 1646356099U, 874U, 965988280U }, + { 805687536U, 744U, 4032277920U }, + { 1948093210U, 633U, 1346597684U }, + { 392609744U, 783U, 1636083295U }, + { 690241304U, 770U, 1201031298U }, + { 1360302965U, 696U, 1665394461U }, + { 1220090946U, 780U, 1316922812U }, + { 447092251U, 500U, 3438743375U }, + { 1613868791U, 592U, 828546883U }, + { 523430951U, 548U, 2552392304U }, + { 726692899U, 810U, 1656872867U }, + { 1364340021U, 836U, 3710513486U }, + { 1986257729U, 931U, 935013962U }, + { 407983964U, 921U, 728767059U }, +}; + +static void prandom_state_test_seed(struct rnd_state *state, u32 seed) +{ +#define LCG(x) ((x) * 69069U) /* super-duper LCG */ + state->s1 = __seed(LCG(seed), 2U); + state->s2 = __seed(LCG(state->s1), 8U); + state->s3 = __seed(LCG(state->s2), 16U); + state->s4 = __seed(LCG(state->s3), 128U); +} + +static void test_prandom_seed_boundary(struct kunit *test) +{ + int i; + struct rnd_state state; + + for (i = 0; i < ARRAY_SIZE(test1); i++) { + prandom_state_test_seed(&state, test1[i].seed); + prandom_warmup(&state); + KUNIT_EXPECT_EQ(test, test1[i].result, prandom_u32_state(&state)); + } +} + +static void test_prandom_taus113(struct kunit *test) +{ + int i, j; + struct rnd_state state; + + for (i = 0; i < ARRAY_SIZE(test2); i++) { + prandom_state_test_seed(&state, test2[i].seed); + prandom_warmup(&state); + + for (j = 0; j < test2[i].iteration - 1; j++) + prandom_u32_state(&state); + + KUNIT_EXPECT_EQ(test, test2[i].result, prandom_u32_state(&state)); + } +} + +static struct kunit_case prandom_test_cases[] = { + KUNIT_CASE(test_prandom_seed_boundary), + KUNIT_CASE(test_prandom_taus113), + {} +}; + +static struct kunit_suite prandom_test_suite = { + .name = "prandom", + .test_cases = prandom_test_cases, +}; + +kunit_test_suite(prandom_test_suite); + +MODULE_DESCRIPTION("KUnit test for prandom"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/lib/tests/seq_buf_kunit.c b/lib/tests/seq_buf_kunit.c index 8a01579a978e..eb466386bbef 100644 --- a/lib/tests/seq_buf_kunit.c +++ b/lib/tests/seq_buf_kunit.c @@ -184,6 +184,38 @@ static void seq_buf_get_buf_commit_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, seq_buf_has_overflowed(&s)); } +static void seq_buf_putmem_hex_test(struct kunit *test) +{ + DECLARE_SEQ_BUF(s, 24); + const u8 data[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; +#ifdef __BIG_ENDIAN + const char *expected = "0001020304050607 0809 "; +#else + const char *expected = "0706050403020100 0908 "; +#endif + + KUNIT_EXPECT_EQ(test, seq_buf_putmem_hex(&s, data, sizeof(data)), 0); + KUNIT_EXPECT_FALSE(test, seq_buf_has_overflowed(&s)); + KUNIT_EXPECT_EQ(test, seq_buf_used(&s), strlen(expected)); + KUNIT_EXPECT_STREQ(test, seq_buf_str(&s), expected); +} + +static void seq_buf_putmem_hex_overflow_test(struct kunit *test) +{ + DECLARE_SEQ_BUF(s, 20); + const u8 data[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; +#ifdef __BIG_ENDIAN + const char *expected = "0001020304050607 "; +#else + const char *expected = "0706050403020100 "; +#endif + + KUNIT_EXPECT_EQ(test, seq_buf_putmem_hex(&s, data, sizeof(data)), -1); + KUNIT_EXPECT_TRUE(test, seq_buf_has_overflowed(&s)); + KUNIT_EXPECT_EQ(test, seq_buf_used(&s), 20); + KUNIT_EXPECT_STREQ(test, seq_buf_str(&s), expected); +} + static struct kunit_case seq_buf_test_cases[] = { KUNIT_CASE(seq_buf_init_test), KUNIT_CASE(seq_buf_declare_test), @@ -194,6 +226,8 @@ static struct kunit_case seq_buf_test_cases[] = { KUNIT_CASE(seq_buf_printf_test), KUNIT_CASE(seq_buf_printf_overflow_test), KUNIT_CASE(seq_buf_get_buf_commit_test), + KUNIT_CASE(seq_buf_putmem_hex_test), + KUNIT_CASE(seq_buf_putmem_hex_overflow_test), {} }; diff --git a/lib/tests/shdi3_kunit.c b/lib/tests/shdi3_kunit.c new file mode 100644 index 000000000000..44f65e66512b --- /dev/null +++ b/lib/tests/shdi3_kunit.c @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: GPL-2.0-or-later OR Apache-2.0 +/* + * Test cases for __ashldi3(), __ashrdi3(), and __lshrdi3(). + */ + +#include <linux/array_size.h> +#include <linux/module.h> +#include <linux/libgcc.h> +#include <kunit/test.h> + +struct shdi3_test_entry { + long long input; + int shift; + long long result; +}; + +static const struct shdi3_test_entry ashldi3_testdata[] = { + /* https://github.com/llvm/llvm-project/compiler-rt/test/builtins/Unit/ashldi3_test.c */ + { 0x0123456789ABCDEFLL, 0, 0x123456789ABCDEFLL }, + { 0x0123456789ABCDEFLL, 1, 0x2468ACF13579BDELL }, + { 0x0123456789ABCDEFLL, 2, 0x48D159E26AF37BCLL }, + { 0x0123456789ABCDEFLL, 3, 0x91A2B3C4D5E6F78LL }, + { 0x0123456789ABCDEFLL, 4, 0x123456789ABCDEF0LL }, + { 0x0123456789ABCDEFLL, 28, 0x789ABCDEF0000000LL }, + { 0x0123456789ABCDEFLL, 29, 0xF13579BDE0000000LL }, + { 0x0123456789ABCDEFLL, 30, 0xE26AF37BC0000000LL }, + { 0x0123456789ABCDEFLL, 31, 0xC4D5E6F780000000LL }, + { 0x0123456789ABCDEFLL, 32, 0x89ABCDEF00000000LL }, + { 0x0123456789ABCDEFLL, 33, 0x13579BDE00000000LL }, + { 0x0123456789ABCDEFLL, 34, 0x26AF37BC00000000LL }, + { 0x0123456789ABCDEFLL, 35, 0x4D5E6F7800000000LL }, + { 0x0123456789ABCDEFLL, 36, 0x9ABCDEF000000000LL }, + { 0x0123456789ABCDEFLL, 60, 0xF000000000000000LL }, + { 0x0123456789ABCDEFLL, 61, 0xE000000000000000LL }, + { 0x0123456789ABCDEFLL, 62, 0xC000000000000000LL }, + { 0x0123456789ABCDEFLL, 63, 0x8000000000000000LL }, +}; + +static void shdi3_test_ashldi3(struct kunit *test) +{ + const struct shdi3_test_entry *e; + long long ret; + + for (e = ashldi3_testdata; + e < ashldi3_testdata + ARRAY_SIZE(ashldi3_testdata); e++) { + ret = __ashldi3(e->input, e->shift); + KUNIT_EXPECT_EQ_MSG(test, ret, e->result, + " when evaluating __ashldi3(%lld, %d)", + e->input, e->shift); + } +} + +static const struct shdi3_test_entry ashrdi3_testdata[] = { + /* https://github.com/llvm/llvm-project/compiler-rt/test/builtins/Unit/ashrdi3_test.c */ + { 0x0123456789ABCDEFLL, 0, 0x123456789ABCDEFLL }, + { 0x0123456789ABCDEFLL, 1, 0x91A2B3C4D5E6F7LL }, + { 0x0123456789ABCDEFLL, 2, 0x48D159E26AF37BLL }, + { 0x0123456789ABCDEFLL, 3, 0x2468ACF13579BDLL }, + { 0x0123456789ABCDEFLL, 4, 0x123456789ABCDELL }, + { 0x0123456789ABCDEFLL, 28, 0x12345678LL }, + { 0x0123456789ABCDEFLL, 29, 0x91A2B3CLL }, + { 0x0123456789ABCDEFLL, 30, 0x48D159ELL }, + { 0x0123456789ABCDEFLL, 31, 0x2468ACFLL }, + { 0x0123456789ABCDEFLL, 32, 0x1234567LL }, + { 0x0123456789ABCDEFLL, 33, 0x91A2B3LL }, + { 0x0123456789ABCDEFLL, 34, 0x48D159LL }, + { 0x0123456789ABCDEFLL, 35, 0x2468ACLL }, + { 0x0123456789ABCDEFLL, 36, 0x123456LL }, + { 0x0123456789ABCDEFLL, 60, 0 }, + { 0x0123456789ABCDEFLL, 61, 0 }, + { 0x0123456789ABCDEFLL, 62, 0 }, + { 0x0123456789ABCDEFLL, 63, 0 }, + { 0xFEDCBA9876543210LL, 0, 0xFEDCBA9876543210LL }, + { 0xFEDCBA9876543210LL, 1, 0xFF6E5D4C3B2A1908LL }, + { 0xFEDCBA9876543210LL, 2, 0xFFB72EA61D950C84LL }, + { 0xFEDCBA9876543210LL, 3, 0xFFDB97530ECA8642LL }, + { 0xFEDCBA9876543210LL, 4, 0xFFEDCBA987654321LL }, + { 0xFEDCBA9876543210LL, 28, 0xFFFFFFFFEDCBA987LL }, + { 0xFEDCBA9876543210LL, 29, 0xFFFFFFFFF6E5D4C3LL }, + { 0xFEDCBA9876543210LL, 30, 0xFFFFFFFFFB72EA61LL }, + { 0xFEDCBA9876543210LL, 31, 0xFFFFFFFFFDB97530LL }, + { 0xFEDCBA9876543210LL, 32, 0xFFFFFFFFFEDCBA98LL }, + { 0xFEDCBA9876543210LL, 33, 0xFFFFFFFFFF6E5D4CLL }, + { 0xFEDCBA9876543210LL, 34, 0xFFFFFFFFFFB72EA6LL }, + { 0xFEDCBA9876543210LL, 35, 0xFFFFFFFFFFDB9753LL }, + { 0xFEDCBA9876543210LL, 36, 0xFFFFFFFFFFEDCBA9LL }, + { 0xAEDCBA9876543210LL, 60, 0xFFFFFFFFFFFFFFFALL }, + { 0xAEDCBA9876543210LL, 61, 0xFFFFFFFFFFFFFFFDLL }, + { 0xAEDCBA9876543210LL, 62, 0xFFFFFFFFFFFFFFFELL }, + { 0xAEDCBA9876543210LL, 63, 0xFFFFFFFFFFFFFFFFLL }, +}; + +static void shdi3_test_ashrdi3(struct kunit *test) +{ + const struct shdi3_test_entry *e; + long long ret; + + for (e = ashrdi3_testdata; + e < ashrdi3_testdata + ARRAY_SIZE(ashrdi3_testdata); e++) { + ret = __ashrdi3(e->input, e->shift); + KUNIT_EXPECT_EQ_MSG(test, ret, e->result, + " when evaluating __ashrdi3(%lld, %d)", + e->input, e->shift); + } +} + +static const struct shdi3_test_entry lshrdi3_testdata[] = { + /* https://github.com/llvm/llvm-project/compiler-rt/test/builtins/Unit/lshrdi3_test.c */ + { 0x0123456789ABCDEFLL, 0, 0x123456789ABCDEFLL }, + { 0x0123456789ABCDEFLL, 1, 0x91A2B3C4D5E6F7LL }, + { 0x0123456789ABCDEFLL, 2, 0x48D159E26AF37BLL }, + { 0x0123456789ABCDEFLL, 3, 0x2468ACF13579BDLL }, + { 0x0123456789ABCDEFLL, 4, 0x123456789ABCDELL }, + { 0x0123456789ABCDEFLL, 28, 0x12345678LL }, + { 0x0123456789ABCDEFLL, 29, 0x91A2B3CLL }, + { 0x0123456789ABCDEFLL, 30, 0x48D159ELL }, + { 0x0123456789ABCDEFLL, 31, 0x2468ACFLL }, + { 0x0123456789ABCDEFLL, 32, 0x1234567LL }, + { 0x0123456789ABCDEFLL, 33, 0x91A2B3LL }, + { 0x0123456789ABCDEFLL, 34, 0x48D159LL }, + { 0x0123456789ABCDEFLL, 35, 0x2468ACLL }, + { 0x0123456789ABCDEFLL, 36, 0x123456LL }, + { 0x0123456789ABCDEFLL, 60, 0 }, + { 0x0123456789ABCDEFLL, 61, 0 }, + { 0x0123456789ABCDEFLL, 62, 0 }, + { 0x0123456789ABCDEFLL, 63, 0 }, + { 0xFEDCBA9876543210LL, 0, 0xFEDCBA9876543210LL }, + { 0xFEDCBA9876543210LL, 1, 0x7F6E5D4C3B2A1908LL }, + { 0xFEDCBA9876543210LL, 2, 0x3FB72EA61D950C84LL }, + { 0xFEDCBA9876543210LL, 3, 0x1FDB97530ECA8642LL }, + { 0xFEDCBA9876543210LL, 4, 0xFEDCBA987654321LL }, + { 0xFEDCBA9876543210LL, 28, 0xFEDCBA987LL }, + { 0xFEDCBA9876543210LL, 29, 0x7F6E5D4C3LL }, + { 0xFEDCBA9876543210LL, 30, 0x3FB72EA61LL }, + { 0xFEDCBA9876543210LL, 31, 0x1FDB97530LL }, + { 0xFEDCBA9876543210LL, 32, 0xFEDCBA98LL }, + { 0xFEDCBA9876543210LL, 33, 0x7F6E5D4CLL }, + { 0xFEDCBA9876543210LL, 34, 0x3FB72EA6LL }, + { 0xFEDCBA9876543210LL, 35, 0x1FDB9753LL }, + { 0xFEDCBA9876543210LL, 36, 0xFEDCBA9LL }, + { 0xAEDCBA9876543210LL, 60, 0xALL }, + { 0xAEDCBA9876543210LL, 61, 0x5LL }, + { 0xAEDCBA9876543210LL, 62, 0x2LL }, + { 0xAEDCBA9876543210LL, 63, 0x1LL }, +}; + +static void shdi3_test_lshrdi3(struct kunit *test) +{ + const struct shdi3_test_entry *e; + long long ret; + + for (e = lshrdi3_testdata; + e < lshrdi3_testdata + ARRAY_SIZE(lshrdi3_testdata); e++) { + ret = __lshrdi3(e->input, e->shift); + KUNIT_EXPECT_EQ_MSG(test, ret, e->result, + " when evaluating __lshrdi3(%lld, %d)", + e->input, e->shift); + } +} + +static struct kunit_case shdi3_test_cases[] = { + KUNIT_CASE(shdi3_test_ashldi3), + KUNIT_CASE(shdi3_test_ashrdi3), + KUNIT_CASE(shdi3_test_lshrdi3), + {} +}; + +static struct kunit_suite shdi3_test_suite = { + .name = "shdi3", + .test_cases = shdi3_test_cases, +}; +kunit_test_suite(shdi3_test_suite); + +MODULE_DESCRIPTION("Test cases for __ashldi3(), __ashrdi3(), and __lshrdi3()"); +MODULE_LICENSE("GPL"); diff --git a/lib/tests/slub_kunit.c b/lib/tests/slub_kunit.c index 848b682a2d70..e3b63f0338d5 100644 --- a/lib/tests/slub_kunit.c +++ b/lib/tests/slub_kunit.c @@ -7,6 +7,8 @@ #include <linux/kernel.h> #include <linux/rcupdate.h> #include <linux/delay.h> +#include <linux/perf_event.h> +#include <linux/kprobes.h> #include "../mm/slab.h" static struct kunit_resource resource; @@ -160,7 +162,10 @@ static void test_kmalloc_redzone_access(struct kunit *test) } struct test_kfree_rcu_struct { - struct rcu_head rcu; + union { + struct rcu_head rcu; + struct kvfree_rcu_head kvrcu; + }; }; static void test_kfree_rcu(struct kunit *test) @@ -291,6 +296,175 @@ static void test_krealloc_redzone_zeroing(struct kunit *test) kmem_cache_destroy(s); } +#if defined(CONFIG_PERF_EVENTS) || (defined(CONFIG_KPROBES) && defined(CONFIG_SMP)) +#define NR_ITERATIONS 1000 +#define NR_OBJECTS 1000 +static struct test_kfree_rcu_struct *objects[NR_OBJECTS]; + +struct test_nolock_context { + struct kunit *test; + int callback_count; + int alloc_ok; + int alloc_fail; +#ifdef CONFIG_PERF_EVENTS + struct perf_event *event; +#endif +#if defined(CONFIG_KPROBES) && defined(CONFIG_SMP) + struct kprobe kprobe; +#endif +}; + +static void test_kmalloc_and_friends(void) +{ + int i, j; + bool can_use_kfree_rcu = !IS_BUILTIN(CONFIG_SLUB_KUNIT_TEST); + + for (i = 0; i < NR_ITERATIONS; i++) { + for (j = 0; j < NR_OBJECTS; j++) { + gfp_t gfp = (i & 1) ? GFP_KERNEL : GFP_KERNEL_ACCOUNT; + + objects[j] = kmalloc_obj(*objects[j], gfp); + if (!objects[j]) { + j--; + while (j >= 0) + kfree(objects[j--]); + return; + } + } + + for (j = 0; j < NR_OBJECTS; j++) { + if (can_use_kfree_rcu && (i & 2)) + kfree_rcu(objects[j], rcu); + else + kfree(objects[j]); + } + } +} + +static void test_nolock(struct test_nolock_context *ctx) +{ + struct test_kfree_rcu_struct *objp; + gfp_t gfp; + bool can_use_kfree_rcu = !IS_BUILTIN(CONFIG_SLUB_KUNIT_TEST); + + /* __GFP_ACCOUNT to test kmalloc_nolock() in alloc_slab_obj_exts() */ + gfp = (ctx->callback_count & 1) ? 0 : __GFP_ACCOUNT; + objp = kmalloc_nolock(sizeof(*objp), gfp, NUMA_NO_NODE); + + if (objp) + ctx->alloc_ok++; + else + ctx->alloc_fail++; + + if (can_use_kfree_rcu && (ctx->callback_count & 2)) + kfree_rcu_nolock(objp, kvrcu); + else + kfree_nolock(objp); + + ctx->callback_count++; +} +#endif + +#ifdef CONFIG_PERF_EVENTS +static struct perf_event_attr hw_attr = { + .type = PERF_TYPE_HARDWARE, + .config = PERF_COUNT_HW_CPU_CYCLES, + .size = sizeof(struct perf_event_attr), + .pinned = 1, + .disabled = 1, + .freq = 1, + .sample_freq = 100000, +}; + +static void overflow_handler_test_nolock(struct perf_event *event, + struct perf_sample_data *data, + struct pt_regs *regs) +{ + struct test_nolock_context *ctx = event->overflow_handler_context; + + test_nolock(ctx); +} + +static bool enable_perf_events(struct test_nolock_context *ctx) +{ + struct perf_event *event; + + event = perf_event_create_kernel_counter(&hw_attr, -1, current, + overflow_handler_test_nolock, + ctx); + + if (IS_ERR(event)) + return false; + + ctx->event = event; + perf_event_enable(ctx->event); + return true; +} + +static void disable_perf_events(struct test_nolock_context *ctx) +{ + kunit_info(ctx->test, "HW perf events: callback_count: %d, alloc_ok: %d, alloc_fail: %d\n", + ctx->callback_count, ctx->alloc_ok, ctx->alloc_fail); + + perf_event_disable(ctx->event); + perf_event_release_kernel(ctx->event); +} + +static void test_kmalloc_nolock_and_friends_perf(struct kunit *test) +{ + struct test_nolock_context ctx = { .test = test }; + + if (!enable_perf_events(&ctx)) + kunit_skip(test, "Failed to enable perf event, skipping"); + + test_kmalloc_and_friends(); + + disable_perf_events(&ctx); + KUNIT_EXPECT_EQ(test, 0, slab_errors); +} +#endif + +#if defined(CONFIG_KPROBES) && defined(CONFIG_SMP) +static int slab_kprobe_pre_handler(struct kprobe *p, struct pt_regs *regs) +{ + struct test_nolock_context *ctx; + + ctx = container_of(p, struct test_nolock_context, kprobe); + test_nolock(ctx); + return 0; +} + +static bool register_slab_kprobes(struct test_nolock_context *ctx) +{ + ctx->kprobe.symbol_name = "slab_attach_kprobe_locked"; + ctx->kprobe.pre_handler = slab_kprobe_pre_handler; + + if (register_kprobe(&ctx->kprobe)) + return false; + return true; +} + +static void unregister_slab_kprobes(struct test_nolock_context *ctx) +{ + kunit_info(ctx->test, "kprobes: callback_count: %d, alloc_ok: %d, alloc_fail: %d\n", + ctx->callback_count, ctx->alloc_ok, ctx->alloc_fail); + unregister_kprobe(&ctx->kprobe); +} + +static void test_kmalloc_nolock_and_friends_kprobe(struct kunit *test) +{ + struct test_nolock_context ctx = { .test = test }; + + if (!register_slab_kprobes(&ctx)) + kunit_skip(test, "Failed to register kprobe, skipping"); + + test_kmalloc_and_friends(); + + unregister_slab_kprobes(&ctx); + KUNIT_EXPECT_EQ(test, 0, slab_errors); +} +#endif + static int test_init(struct kunit *test) { slab_errors = 0; @@ -315,6 +489,12 @@ static struct kunit_case test_cases[] = { KUNIT_CASE(test_kfree_rcu_wq_destroy), KUNIT_CASE(test_leak_destroy), KUNIT_CASE(test_krealloc_redzone_zeroing), +#ifdef CONFIG_PERF_EVENTS + KUNIT_CASE_SLOW(test_kmalloc_nolock_and_friends_perf), +#endif +#if defined(CONFIG_KPROBES) && defined(CONFIG_SMP) + KUNIT_CASE_SLOW(test_kmalloc_nolock_and_friends_kprobe), +#endif {} }; diff --git a/lib/tests/string_helpers_kunit.c b/lib/tests/string_helpers_kunit.c index c853046183d2..9fbe91079c7e 100644 --- a/lib/tests/string_helpers_kunit.c +++ b/lib/tests/string_helpers_kunit.c @@ -5,11 +5,16 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <kunit/test.h> + #include <linux/array_size.h> -#include <linux/kernel.h> +#include <linux/bug.h> +#include <linux/limits.h> +#include <linux/module.h> #include <linux/random.h> -#include <linux/string.h> +#include <linux/slab.h> +#include <linux/sprintf.h> #include <linux/string_helpers.h> +#include <linux/types.h> static void test_string_check_buf(struct kunit *test, const char *name, unsigned int flags, @@ -601,6 +606,11 @@ static void test_unescape(struct kunit *test) test_string_unescape(test, "unescape", i, false); test_string_unescape(test, "unescape inplace", get_random_u32_below(UNESCAPE_ALL_MASK + 1), true); +} + +static void test_escape(struct kunit *test) +{ + unsigned int i; /* Without dictionary */ for (i = 0; i < ESCAPE_ALL_MASK + 1; i++) @@ -615,6 +625,7 @@ static struct kunit_case string_helpers_test_cases[] = { KUNIT_CASE(test_get_size), KUNIT_CASE(test_upper_lower), KUNIT_CASE(test_unescape), + KUNIT_CASE(test_escape), {} }; diff --git a/lib/tests/string_kunit.c b/lib/tests/string_kunit.c index f9a8e557ba77..0819ace5b027 100644 --- a/lib/tests/string_kunit.c +++ b/lib/tests/string_kunit.c @@ -6,10 +6,18 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <kunit/test.h> +#include <linux/ktime.h> +#include <linux/math64.h> +#include <linux/minmax.h> +#include <linux/mm.h> #include <linux/module.h> +#include <linux/prandom.h> #include <linux/printk.h> #include <linux/slab.h> #include <linux/string.h> +#include <linux/time64.h> +#include <linux/units.h> +#include <linux/vmalloc.h> #define STRCMP_LARGE_BUF_LEN 2048 #define STRCMP_CHANGE_POINT 1337 @@ -17,6 +25,12 @@ #define STRCMP_TEST_EXPECT_LOWER(test, fn, ...) KUNIT_EXPECT_LT(test, fn(__VA_ARGS__), 0) #define STRCMP_TEST_EXPECT_GREATER(test, fn, ...) KUNIT_EXPECT_GT(test, fn(__VA_ARGS__), 0) +#define STRING_TEST_MAX_LEN 128 +#define STRING_TEST_MAX_OFFSET 16 + +#define STRING_BENCH_SEED 888 +#define STRING_BENCH_WORKLOAD (1 * MEGA) + static void string_test_memset16(struct kunit *test) { unsigned i, j, k; @@ -104,6 +118,64 @@ static void string_test_memset64(struct kunit *test) } } +static void string_test_strlen(struct kunit *test) +{ + size_t buf_size; + char *buf, *s; + + buf_size = PAGE_ALIGN(STRING_TEST_MAX_LEN + STRING_TEST_MAX_OFFSET + 1); + buf = vmalloc(buf_size); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buf); + + memset(buf, 'A', buf_size); + + for (size_t offset = 0; offset < STRING_TEST_MAX_OFFSET; offset++) { + for (size_t len = 0; len <= STRING_TEST_MAX_LEN; len++) { + s = buf + buf_size - 1 - offset - len; + s[len] = '\0'; + KUNIT_EXPECT_EQ_MSG(test, strlen(s), len, + "offset:%zu len:%zu", offset, len); + s[len] = 'A'; + } + } + + vfree(buf); +} + +static void string_test_strnlen(struct kunit *test) +{ + size_t buf_size; + char *buf, *s; + + buf_size = PAGE_ALIGN(STRING_TEST_MAX_LEN + STRING_TEST_MAX_OFFSET + 1); + buf = vmalloc(buf_size); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buf); + + memset(buf, 'A', buf_size); + + for (size_t offset = 0; offset < STRING_TEST_MAX_OFFSET; offset++) { + for (size_t len = 0; len <= STRING_TEST_MAX_LEN; len++) { + s = buf + buf_size - 1 - offset - len; + s[len] = '\0'; + + if (len > 0) + KUNIT_EXPECT_EQ(test, strnlen(s, len - 1), len - 1); + if (len > 1) + KUNIT_EXPECT_EQ(test, strnlen(s, len - 2), len - 2); + + KUNIT_EXPECT_EQ(test, strnlen(s, len), len); + + KUNIT_EXPECT_EQ(test, strnlen(s, len + 1), len); + KUNIT_EXPECT_EQ(test, strnlen(s, len + 2), len); + KUNIT_EXPECT_EQ(test, strnlen(s, len + 10), len); + + s[len] = 'A'; + } + } + + vfree(buf); +} + static void string_test_strchr(struct kunit *test) { const char *test_string = "abcdefghijkl"; @@ -127,6 +199,36 @@ static void string_test_strchr(struct kunit *test) KUNIT_ASSERT_NULL(test, result); } +static void string_test_strrchr(struct kunit *test) +{ + size_t buf_size; + char *buf, *s; + + buf_size = PAGE_ALIGN(STRING_TEST_MAX_LEN + STRING_TEST_MAX_OFFSET + 1); + buf = vmalloc(buf_size); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buf); + + memset(buf, 'A', buf_size); + + for (size_t offset = 0; offset < STRING_TEST_MAX_OFFSET; offset++) { + for (size_t len = 0; len <= STRING_TEST_MAX_LEN; len++) { + s = buf + buf_size - 1 - offset - len; + s[len] = '\0'; + + KUNIT_EXPECT_PTR_EQ(test, strrchr(s, 'Z'), NULL); + + if (len > 0) + KUNIT_EXPECT_PTR_EQ(test, strrchr(s, 'A'), s + len - 1); + else + KUNIT_EXPECT_PTR_EQ(test, strrchr(s, 'A'), NULL); + + s[len] = 'A'; + } + } + + vfree(buf); +} + static void string_test_strnchr(struct kunit *test) { const char *test_string = "abcdefghijkl"; @@ -614,12 +716,180 @@ static void string_test_strends(struct kunit *test) KUNIT_EXPECT_TRUE(test, strends("", "")); } +#if IS_ENABLED(CONFIG_STRING_KUNIT_BENCH) +/* Target string lengths for benchmarking */ +static const size_t bench_lens[] = { + 0, 1, 7, 8, 16, 31, 64, 127, 512, 1024, 3173, 4096, +}; + +/** + * alloc_max_bench_buffer() - Allocate buffer for the max test case. + * @test: KUnit context for managed allocation. + * @lens: Array of lengths used in the benchmark cases. + * @count: Number of elements in the @lens array. + * @buf_len: [out] Pointer to store the actually allocated buffer + * size (including NUL character). + * + * Return: Pointer to the allocated memory, or NULL on failure. + */ +static void *alloc_max_bench_buffer(struct kunit *test, const size_t *lens, + size_t count, size_t *buf_len) +{ + size_t max_len = 0; + void *buf; + + for (size_t i = 0; i < count; i++) + max_len = max(lens[i], max_len); + + /* Add space for NUL character */ + max_len += 1; + + buf = kunit_kzalloc(test, max_len, GFP_KERNEL); + if (!buf) + return NULL; + + if (buf_len) + *buf_len = max_len; + + return buf; +} + +/** + * fill_random_string() - Populate a buffer with a random NUL-terminated string. + * @buf: Buffer to fill. + * @len: Length of the buffer in bytes. + * + * Fills the buffer with random non-NUL bytes and ensures the string is + * properly NUL-terminated. + */ +static void fill_random_string(char *buf, size_t len) +{ + struct rnd_state state; + + if (!buf || !len) + return; + + /* Use a fixed seed to ensure deterministic benchmark results */ + prandom_seed_state(&state, STRING_BENCH_SEED); + prandom_bytes_state(&state, buf, len); + + /* Replace NUL characters to avoid early string termination */ + for (size_t i = 0; i < len; i++) { + if (buf[i] == '\0') + buf[i] = 0x01; + } + + buf[len - 1] = '\0'; +} + +/** + * STRING_BENCH() - Benchmark string functions. + * @iters: Number of iterations to run. + * @func: Function to benchmark. + * @...: Variable arguments passed to @func. + * + * Disables preemption and measures the total time in nanoseconds to execute + * @func(@__VA_ARGS__) for @iters times, including a small warm-up phase. + * + * Context: Disables preemption during measurement. + * Return: Total execution time in nanoseconds (u64). + */ +#define STRING_BENCH(iters, func, ...) \ +({ \ + /* Volatile function pointer prevents dead code elimination */ \ + typeof(func) (* volatile __func) = (func); \ + size_t __bn_iters = (iters); \ + size_t __bn_warm_iters; \ + u64 __bn_t; \ + \ + /* Use 10% of the given iterations (maximum 50) to warm up */ \ + __bn_warm_iters = max(__bn_iters / 10, 50U); \ + \ + for (size_t __bn_i = 0; __bn_i < __bn_warm_iters; __bn_i++) \ + (void)__func(__VA_ARGS__); \ + \ + preempt_disable(); \ + __bn_t = ktime_get_ns(); \ + for (size_t __bn_i = 0; __bn_i < __bn_iters; __bn_i++) \ + (void)__func(__VA_ARGS__); \ + __bn_t = ktime_get_ns() - __bn_t; \ + preempt_enable(); \ + __bn_t; \ +}) + +/** + * STRING_BENCH_BUF() - Benchmark harness for single-buffer functions. + * @test: KUnit context. + * @buf_name: Local char * variable name to be defined. + * @buf_size: Local size_t variable name to be defined. + * @func: Function to benchmark. + * @...: Extra arguments for @func. + * + * Prepares a randomized, NUL-terminated buffer and iterates through lengths + * in bench_lens, defining @buf_name and @buf_size in each loop. + */ +#define STRING_BENCH_BUF(test, buf_name, buf_size, func, ...) \ +do { \ + size_t _bn_i, _bn_iters, _bn_size = 0; \ + u64 _bn_t, _bn_mbps = 0, _bn_lat = 0; \ + char *_bn_buf; \ + \ + _bn_buf = alloc_max_bench_buffer(test, bench_lens, \ + ARRAY_SIZE(bench_lens), &_bn_size); \ + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, _bn_buf); \ + \ + fill_random_string(_bn_buf, _bn_size); \ + \ + for (_bn_i = 0; _bn_i < ARRAY_SIZE(bench_lens); _bn_i++) { \ + size_t buf_size = bench_lens[_bn_i]; \ + char *buf_name = _bn_buf + _bn_size - buf_size - 1; \ + _bn_iters = STRING_BENCH_WORKLOAD / max(buf_size, 1U); \ + \ + _bn_t = STRING_BENCH(_bn_iters, func, ##__VA_ARGS__); \ + if (_bn_t > 0) { \ + _bn_mbps = (u64)(buf_size) * _bn_iters * \ + (NSEC_PER_SEC / MEGA); \ + _bn_mbps = div64_u64(_bn_mbps, _bn_t); \ + _bn_lat = div64_u64(_bn_t, _bn_iters); \ + } \ + kunit_info(test, "len=%zu: %llu MB/s (%llu ns/call)\n", \ + buf_size, _bn_mbps, _bn_lat); \ + } \ +} while (0) +#else +#define STRING_BENCH_BUF(test, buf_name, buf_size, func, ...) \ + kunit_skip(test, "not enabled") +#endif /* IS_ENABLED(CONFIG_STRING_KUNIT_BENCH) */ + +static void string_bench_strlen(struct kunit *test) +{ + STRING_BENCH_BUF(test, buf, len, strlen, buf); +} + +static void string_bench_strnlen(struct kunit *test) +{ + STRING_BENCH_BUF(test, buf, len, strnlen, buf, len); +} + +static void string_bench_strchr(struct kunit *test) +{ + STRING_BENCH_BUF(test, buf, len, strchr, buf, '\0'); +} + +static void string_bench_strrchr(struct kunit *test) +{ + STRING_BENCH_BUF(test, buf, len, strrchr, buf, '\0'); +} + static struct kunit_case string_test_cases[] = { KUNIT_CASE(string_test_memset16), KUNIT_CASE(string_test_memset32), KUNIT_CASE(string_test_memset64), + KUNIT_CASE(string_test_strlen), + KUNIT_CASE(string_test_strnlen), KUNIT_CASE(string_test_strchr), KUNIT_CASE(string_test_strnchr), + KUNIT_CASE(string_test_strrchr), KUNIT_CASE(string_test_strspn), KUNIT_CASE(string_test_strcmp), KUNIT_CASE(string_test_strcmp_long_strings), @@ -636,6 +906,10 @@ static struct kunit_case string_test_cases[] = { KUNIT_CASE(string_test_strtomem), KUNIT_CASE(string_test_memtostr), KUNIT_CASE(string_test_strends), + KUNIT_CASE(string_bench_strlen), + KUNIT_CASE(string_bench_strnlen), + KUNIT_CASE(string_bench_strchr), + KUNIT_CASE(string_bench_strrchr), {} }; diff --git a/lib/tests/test_kprobes.c b/lib/tests/test_kprobes.c index b7582010125c..06e729e4de05 100644 --- a/lib/tests/test_kprobes.c +++ b/lib/tests/test_kprobes.c @@ -12,6 +12,12 @@ #define div_factor 3 +#define KP_CLEAR(_kp) \ +do { \ + (_kp).addr = NULL; \ + (_kp).flags = 0; \ +} while (0) + static u32 rand1, preh_val, posth_val; static u32 (*target)(u32 value); static u32 (*recursed_target)(u32 value); @@ -125,10 +131,6 @@ static void test_kprobes(struct kunit *test) current_test = test; - /* addr and flags should be cleard for reusing kprobe. */ - kp.addr = NULL; - kp.flags = 0; - KUNIT_EXPECT_EQ(test, 0, register_kprobes(kps, 2)); preh_val = 0; posth_val = 0; @@ -226,9 +228,6 @@ static void test_kretprobes(struct kunit *test) struct kretprobe *rps[2] = {&rp, &rp2}; current_test = test; - /* addr and flags should be cleard for reusing kprobe. */ - rp.kp.addr = NULL; - rp.kp.flags = 0; KUNIT_EXPECT_EQ(test, 0, register_kretprobes(rps, 2)); krph_val = 0; @@ -290,8 +289,6 @@ static void test_stacktrace_on_kretprobe(struct kunit *test) unsigned long myretaddr = (unsigned long)__builtin_return_address(0); current_test = test; - rp3.kp.addr = NULL; - rp3.kp.flags = 0; /* * Run the stacktrace_driver() to record correct return address in @@ -352,8 +349,6 @@ static void test_stacktrace_on_nested_kretprobe(struct kunit *test) struct kretprobe *rps[2] = {&rp3, &rp4}; current_test = test; - rp3.kp.addr = NULL; - rp3.kp.flags = 0; //KUNIT_ASSERT_NE(test, myretaddr, stacktrace_driver()); @@ -367,6 +362,18 @@ static void test_stacktrace_on_nested_kretprobe(struct kunit *test) static int kprobes_test_init(struct kunit *test) { + KP_CLEAR(kp); + KP_CLEAR(kp2); + KP_CLEAR(kp_missed); +#ifdef CONFIG_KRETPROBES + KP_CLEAR(rp.kp); + KP_CLEAR(rp2.kp); +#ifdef CONFIG_ARCH_CORRECT_STACKTRACE_ON_KRETPROBE + KP_CLEAR(rp3.kp); + KP_CLEAR(rp4.kp); +#endif +#endif + target = kprobe_target; target2 = kprobe_target2; recursed_target = kprobe_recursed_target; diff --git a/lib/tests/test_ratelimit.c b/lib/tests/test_ratelimit.c index 33cea5f3d28b..e244f8cd47d7 100644 --- a/lib/tests/test_ratelimit.c +++ b/lib/tests/test_ratelimit.c @@ -68,7 +68,6 @@ static void test_ratelimit_smoke(struct kunit *test) static struct ratelimit_state stressrl = RATELIMIT_STATE_INIT_FLAGS("stressrl", HZ / 10, 3, RATELIMIT_MSG_ON_RELEASE); -static int doneflag; static const int stress_duration = 2 * HZ; struct stress_kthread { @@ -84,9 +83,8 @@ static int test_ratelimit_stress_child(void *arg) struct stress_kthread *sktp = arg; set_user_nice(current, MAX_NICE); - WARN_ON_ONCE(!sktp->tp); - while (!READ_ONCE(doneflag)) { + while (!kthread_should_stop()) { sktp->nattempts++; if (___ratelimit(&stressrl, __func__)) sktp->nunlimited++; @@ -105,26 +103,37 @@ static void test_ratelimit_stress(struct kunit *test) const int n_stress_kthread = cpumask_weight(cpu_online_mask); struct stress_kthread skt = { 0 }; struct stress_kthread *sktp = kzalloc_objs(*sktp, n_stress_kthread); + int n_started = 0; - KUNIT_EXPECT_NOT_NULL_MSG(test, sktp, "Memory allocation failure"); + KUNIT_ASSERT_NOT_NULL_MSG(test, sktp, "Memory allocation failure"); for (i = 0; i < n_stress_kthread; i++) { sktp[i].tp = kthread_run(test_ratelimit_stress_child, &sktp[i], "%s/%i", "test_ratelimit_stress_child", i); - KUNIT_EXPECT_NOT_NULL_MSG(test, sktp, "kthread creation failure"); + if (IS_ERR(sktp[i].tp)) { + KUNIT_FAIL(test, "kthread_run failed: %ld", PTR_ERR(sktp[i].tp)); + goto out_stop; + } + n_started++; pr_alert("Spawned test_ratelimit_stress_child %d\n", i); } schedule_timeout_idle(stress_duration); - WRITE_ONCE(doneflag, 1); - for (i = 0; i < n_stress_kthread; i++) { + +out_stop: + for (i = 0; i < n_started; i++) { kthread_stop(sktp[i].tp); skt.nattempts += sktp[i].nattempts; skt.nunlimited += sktp[i].nunlimited; skt.nlimited += sktp[i].nlimited; skt.nmissed += sktp[i].nmissed; } - KUNIT_ASSERT_EQ_MSG(test, skt.nunlimited + skt.nlimited, skt.nattempts, - "Outcomes not equal to attempts"); - KUNIT_ASSERT_EQ_MSG(test, skt.nlimited, skt.nmissed, "Misses not equal to limits"); + if (n_started == n_stress_kthread) { + KUNIT_ASSERT_EQ_MSG(test, skt.nunlimited + skt.nlimited, skt.nattempts, + "Outcomes not equal to attempts"); + KUNIT_ASSERT_EQ_MSG(test, skt.nlimited, skt.nmissed, + "Misses not equal to limits"); + } + + kfree(sktp); } static struct kunit_case ratelimit_test_cases[] = { diff --git a/lib/tests/uuid_kunit.c b/lib/tests/uuid_kunit.c index de71b2649dac..2ef64fbe67d6 100644 --- a/lib/tests/uuid_kunit.c +++ b/lib/tests/uuid_kunit.c @@ -86,11 +86,67 @@ static void uuid_test_uuid_invalid(struct kunit *test) } } +/* + * RFC 4122 section 4.4 says random UUIDs/GUIDs (version 4) must have: + * - version 4 in the high nibble of the version byte, + * - variant DCE 1.1 (binary 10x) in the high bits of byte 8. + * + * The version byte is byte 6 in the "wire" uuid_t layout and byte 7 in + * the byte-swapped guid_t layout. + */ +static void uuid_test_uuid_gen(struct kunit *test) +{ + uuid_t u; + + for (unsigned int i = 0; i < 8; i++) { + uuid_gen(&u); + KUNIT_EXPECT_EQ(test, u.b[6] & 0xf0, 0x40); + KUNIT_EXPECT_EQ(test, u.b[8] & 0xc0, 0x80); + } +} + +static void uuid_test_guid_gen(struct kunit *test) +{ + guid_t g; + + for (unsigned int i = 0; i < 8; i++) { + guid_gen(&g); + KUNIT_EXPECT_EQ(test, g.b[7] & 0xf0, 0x40); + KUNIT_EXPECT_EQ(test, g.b[8] & 0xc0, 0x80); + } +} + +static void uuid_test_generate_random_uuid(struct kunit *test) +{ + unsigned char buf[16]; + + for (unsigned int i = 0; i < 8; i++) { + generate_random_uuid(buf); + KUNIT_EXPECT_EQ(test, buf[6] & 0xf0, 0x40); + KUNIT_EXPECT_EQ(test, buf[8] & 0xc0, 0x80); + } +} + +static void uuid_test_generate_random_guid(struct kunit *test) +{ + unsigned char buf[16]; + + for (unsigned int i = 0; i < 8; i++) { + generate_random_guid(buf); + KUNIT_EXPECT_EQ(test, buf[7] & 0xf0, 0x40); + KUNIT_EXPECT_EQ(test, buf[8] & 0xc0, 0x80); + } +} + static struct kunit_case uuid_test_cases[] = { KUNIT_CASE(uuid_test_guid_valid), KUNIT_CASE(uuid_test_uuid_valid), KUNIT_CASE(uuid_test_guid_invalid), KUNIT_CASE(uuid_test_uuid_invalid), + KUNIT_CASE(uuid_test_uuid_gen), + KUNIT_CASE(uuid_test_guid_gen), + KUNIT_CASE(uuid_test_generate_random_uuid), + KUNIT_CASE(uuid_test_generate_random_guid), {}, }; diff --git a/lib/timerqueue.c b/lib/timerqueue.c index cdb9c7658478..e2a1e08cb4bd 100644 --- a/lib/timerqueue.c +++ b/lib/timerqueue.c @@ -82,3 +82,17 @@ struct timerqueue_node *timerqueue_iterate_next(struct timerqueue_node *node) return container_of(next, struct timerqueue_node, node); } EXPORT_SYMBOL_GPL(timerqueue_iterate_next); + +#define __node_2_tq_linked(_n) \ + container_of(rb_entry((_n), struct rb_node_linked, node), struct timerqueue_linked_node, node) + +static __always_inline bool __tq_linked_less(struct rb_node *a, const struct rb_node *b) +{ + return __node_2_tq_linked(a)->expires < __node_2_tq_linked(b)->expires; +} + +bool timerqueue_linked_add(struct timerqueue_linked_head *head, struct timerqueue_linked_node *node) +{ + return rb_add_linked(&node->node, &head->rb_root, __tq_linked_less); +} +EXPORT_SYMBOL_GPL(timerqueue_linked_add); diff --git a/lib/ts_bm.c b/lib/ts_bm.c index eed5967238c5..676105e84005 100644 --- a/lib/ts_bm.c +++ b/lib/ts_bm.c @@ -163,8 +163,22 @@ static struct ts_config *bm_init(const void *pattern, unsigned int len, struct ts_config *conf; struct ts_bm *bm; int i; - unsigned int prefix_tbl_len = len * sizeof(unsigned int); - size_t priv_size = sizeof(*bm) + len + prefix_tbl_len; + unsigned int prefix_tbl_len; + size_t priv_size; + + /* Zero-length patterns would underflow bm_find()'s initial shift. */ + if (unlikely(!len)) + return ERR_PTR(-EINVAL); + + /* + * bm->pattern is stored immediately after the good_shift[] table. + * Reject lengths that would wrap while sizing either region. + */ + if (unlikely(check_mul_overflow(len, sizeof(*bm->good_shift), + &prefix_tbl_len) || + check_add_overflow(sizeof(*bm), (size_t)len, &priv_size) || + check_add_overflow(priv_size, prefix_tbl_len, &priv_size))) + return ERR_PTR(-EINVAL); conf = alloc_ts_config(priv_size, gfp_mask); if (IS_ERR(conf)) diff --git a/lib/ts_kmp.c b/lib/ts_kmp.c index 5520dc28255a..29466c1803c9 100644 --- a/lib/ts_kmp.c +++ b/lib/ts_kmp.c @@ -94,8 +94,22 @@ static struct ts_config *kmp_init(const void *pattern, unsigned int len, struct ts_config *conf; struct ts_kmp *kmp; int i; - unsigned int prefix_tbl_len = len * sizeof(unsigned int); - size_t priv_size = sizeof(*kmp) + len + prefix_tbl_len; + unsigned int prefix_tbl_len; + size_t priv_size; + + /* Zero-length patterns would make kmp_find() read beyond kmp->pattern. */ + if (unlikely(!len)) + return ERR_PTR(-EINVAL); + + /* + * kmp->pattern is stored immediately after the prefix_tbl[] table. + * Reject lengths that would wrap while sizing either region. + */ + if (unlikely(check_mul_overflow(len, sizeof(*kmp->prefix_tbl), + &prefix_tbl_len) || + check_add_overflow(sizeof(*kmp), (size_t)len, &priv_size) || + check_add_overflow(priv_size, prefix_tbl_len, &priv_size))) + return ERR_PTR(-EINVAL); conf = alloc_ts_config(priv_size, gfp_mask); if (IS_ERR(conf)) diff --git a/lib/ucs2_string.c b/lib/ucs2_string.c index dfb4f2358cab..1f7dd4eb640a 100644 --- a/lib/ucs2_string.c +++ b/lib/ucs2_string.c @@ -6,18 +6,18 @@ unsigned long ucs2_strnlen(const ucs2_char_t *s, size_t maxlength) { - unsigned long length = 0; + unsigned long length = 0; - while (*s++ != 0 && length < maxlength) - length++; - return length; + while (length < maxlength && *s++ != 0) + length++; + return length; } EXPORT_SYMBOL(ucs2_strnlen); unsigned long ucs2_strlen(const ucs2_char_t *s) { - return ucs2_strnlen(s, ~0UL); + return ucs2_strnlen(s, ~0UL); } EXPORT_SYMBOL(ucs2_strlen); @@ -28,7 +28,7 @@ EXPORT_SYMBOL(ucs2_strlen); unsigned long ucs2_strsize(const ucs2_char_t *data, unsigned long maxlength) { - return ucs2_strnlen(data, maxlength/sizeof(ucs2_char_t)) * sizeof(ucs2_char_t); + return ucs2_strnlen(data, maxlength/sizeof(ucs2_char_t)) * sizeof(ucs2_char_t); } EXPORT_SYMBOL(ucs2_strsize); @@ -87,19 +87,19 @@ EXPORT_SYMBOL(ucs2_strscpy); int ucs2_strncmp(const ucs2_char_t *a, const ucs2_char_t *b, size_t len) { - while (1) { - if (len == 0) - return 0; - if (*a < *b) - return -1; - if (*a > *b) - return 1; - if (*a == 0) /* implies *b == 0 */ - return 0; - a++; - b++; - len--; - } + while (1) { + if (len == 0) + return 0; + if (*a < *b) + return -1; + if (*a > *b) + return 1; + if (*a == 0) /* implies *b == 0 */ + return 0; + a++; + b++; + len--; + } } EXPORT_SYMBOL(ucs2_strncmp); diff --git a/lib/usercopy.c b/lib/usercopy.c index b00a3a957de6..e2f0bf104a59 100644 --- a/lib/usercopy.c +++ b/lib/usercopy.c @@ -12,15 +12,13 @@ /* out-of-line parts */ -#if !defined(INLINE_COPY_FROM_USER) +#if !defined(INLINE_COPY_USER) unsigned long _copy_from_user(void *to, const void __user *from, unsigned long n) { return _inline_copy_from_user(to, from, n); } EXPORT_SYMBOL(_copy_from_user); -#endif -#if !defined(INLINE_COPY_TO_USER) unsigned long _copy_to_user(void __user *to, const void *from, unsigned long n) { return _inline_copy_to_user(to, from, n); diff --git a/lib/uuid.c b/lib/uuid.c index e8543c668dc7..128a51f1879b 100644 --- a/lib/uuid.c +++ b/lib/uuid.c @@ -54,7 +54,7 @@ EXPORT_SYMBOL(generate_random_guid); static void __uuid_gen_common(__u8 b[16]) { get_random_bytes(b, 16); - /* reversion 0b10 */ + /* revision 0b10 */ b[8] = (b[8] & 0x3F) | 0x80; } diff --git a/lib/vdso/Kconfig b/lib/vdso/Kconfig index db87ba34ef19..597f5f0f9681 100644 --- a/lib/vdso/Kconfig +++ b/lib/vdso/Kconfig @@ -1,12 +1,11 @@ # SPDX-License-Identifier: GPL-2.0 -config HAVE_GENERIC_VDSO +config VDSO_DATASTORE bool -if HAVE_GENERIC_VDSO - config GENERIC_GETTIMEOFDAY bool + select VDSO_DATASTORE help This is a generic implementation of gettimeofday vdso. Each architecture that enables this feature has to @@ -21,7 +20,6 @@ config GENERIC_VDSO_OVERFLOW_PROTECT config VDSO_GETRANDOM bool + select VDSO_DATASTORE help Selected by architectures that support vDSO getrandom(). - -endif diff --git a/lib/vdso/Makefile b/lib/vdso/Makefile index 405f743253d7..ac304def42d6 100644 --- a/lib/vdso/Makefile +++ b/lib/vdso/Makefile @@ -1,3 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_HAVE_GENERIC_VDSO) += datastore.o +obj-$(CONFIG_VDSO_DATASTORE) += datastore.o diff --git a/lib/vdso/datastore.c b/lib/vdso/datastore.c index a565c30c71a0..1426bf4e0c12 100644 --- a/lib/vdso/datastore.c +++ b/lib/vdso/datastore.c @@ -1,65 +1,95 @@ // SPDX-License-Identifier: GPL-2.0-only -#include <linux/linkage.h> -#include <linux/mmap_lock.h> +#include <linux/gfp.h> +#include <linux/init.h> #include <linux/mm.h> #include <linux/time_namespace.h> #include <linux/types.h> #include <linux/vdso_datastore.h> #include <vdso/datapage.h> -/* - * The vDSO data page. - */ +static u8 vdso_initdata[VDSO_NR_PAGES * PAGE_SIZE] __aligned(PAGE_SIZE) __initdata = {}; + #ifdef CONFIG_GENERIC_GETTIMEOFDAY -static union { - struct vdso_time_data data; - u8 page[PAGE_SIZE]; -} vdso_time_data_store __page_aligned_data; -struct vdso_time_data *vdso_k_time_data = &vdso_time_data_store.data; -static_assert(sizeof(vdso_time_data_store) == PAGE_SIZE); +struct vdso_time_data *vdso_k_time_data __ro_after_init = + (void *)&vdso_initdata[VDSO_TIME_PAGE_OFFSET * PAGE_SIZE]; + +static_assert(sizeof(struct vdso_time_data) <= PAGE_SIZE); #endif /* CONFIG_GENERIC_GETTIMEOFDAY */ #ifdef CONFIG_VDSO_GETRANDOM -static union { - struct vdso_rng_data data; - u8 page[PAGE_SIZE]; -} vdso_rng_data_store __page_aligned_data; -struct vdso_rng_data *vdso_k_rng_data = &vdso_rng_data_store.data; -static_assert(sizeof(vdso_rng_data_store) == PAGE_SIZE); +struct vdso_rng_data *vdso_k_rng_data __ro_after_init = + (void *)&vdso_initdata[VDSO_RNG_PAGE_OFFSET * PAGE_SIZE]; + +static_assert(sizeof(struct vdso_rng_data) <= PAGE_SIZE); #endif /* CONFIG_VDSO_GETRANDOM */ #ifdef CONFIG_ARCH_HAS_VDSO_ARCH_DATA -static union { - struct vdso_arch_data data; - u8 page[VDSO_ARCH_DATA_SIZE]; -} vdso_arch_data_store __page_aligned_data; -struct vdso_arch_data *vdso_k_arch_data = &vdso_arch_data_store.data; +struct vdso_arch_data *vdso_k_arch_data __ro_after_init = + (void *)&vdso_initdata[VDSO_ARCH_PAGES_START * PAGE_SIZE]; #endif /* CONFIG_ARCH_HAS_VDSO_ARCH_DATA */ +static struct page *vdso_data_pages __ro_after_init; + +void __init vdso_setup_data_pages(void) +{ + unsigned int order = get_order(VDSO_NR_PAGES * PAGE_SIZE); + + /* + * Allocate the data pages dynamically. SPARC does not support mapping + * static pages to be mapped into userspace. + * It is also a requirement for mlockall() support. + * + * Do not use folios. In time namespaces the pages are mapped in a different order + * to userspace, which is not handled by the folio optimizations in finish_fault(). + */ + vdso_data_pages = alloc_pages(GFP_KERNEL, order); + if (!vdso_data_pages) + panic("Unable to allocate VDSO storage pages"); + + /* The pages are mapped one-by-one into userspace and each one needs to be refcounted. */ + split_page(vdso_data_pages, order); + + /* Move the data already written by other subsystems to the new pages */ + memcpy(page_address(vdso_data_pages), vdso_initdata, VDSO_NR_PAGES * PAGE_SIZE); + + if (IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY)) + vdso_k_time_data = page_address(vdso_data_pages + VDSO_TIME_PAGE_OFFSET); + + if (IS_ENABLED(CONFIG_VDSO_GETRANDOM)) + vdso_k_rng_data = page_address(vdso_data_pages + VDSO_RNG_PAGE_OFFSET); + + if (IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA)) + vdso_k_arch_data = page_address(vdso_data_pages + VDSO_ARCH_PAGES_START); +} + static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, struct vm_area_struct *vma, struct vm_fault *vmf) { - struct page *timens_page = find_timens_vvar_page(vma); - unsigned long addr, pfn; - vm_fault_t err; + struct page *page, *timens_page; + + if (unlikely(vmf->flags & FAULT_FLAG_REMOTE)) + return VM_FAULT_SIGBUS; + + page = vdso_data_pages + vmf->pgoff; + timens_page = find_timens_vvar_page(vma); switch (vmf->pgoff) { case VDSO_TIME_PAGE_OFFSET: - if (!IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY)) - return VM_FAULT_SIGBUS; - pfn = __phys_to_pfn(__pa_symbol(vdso_k_time_data)); - if (timens_page) { - /* - * Fault in VVAR page too, since it will be accessed - * to get clock data anyway. - */ - addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE; - err = vmf_insert_pfn(vma, addr, pfn); - if (unlikely(err & VM_FAULT_ERROR)) - return err; - pfn = page_to_pfn(timens_page); - } + if (!IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY) || !timens_page) + break; + /* + * Fault in VVAR page too, since it will be accessed + * to get clock data anyway. + */ + unsigned long addr; + vm_fault_t err; + + addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE; + err = vmf_insert_page(vma, addr, page); + if (unlikely(err & VM_FAULT_ERROR)) + return err; + page = timens_page; break; case VDSO_TIMENS_PAGE_OFFSET: /* @@ -70,25 +100,19 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, * See also the comment near timens_setup_vdso_data(). */ if (!IS_ENABLED(CONFIG_TIME_NS) || !timens_page) - return VM_FAULT_SIGBUS; - pfn = __phys_to_pfn(__pa_symbol(vdso_k_time_data)); + break; + page = vdso_data_pages + VDSO_TIME_PAGE_OFFSET; break; case VDSO_RNG_PAGE_OFFSET: - if (!IS_ENABLED(CONFIG_VDSO_GETRANDOM)) - return VM_FAULT_SIGBUS; - pfn = __phys_to_pfn(__pa_symbol(vdso_k_rng_data)); - break; case VDSO_ARCH_PAGES_START ... VDSO_ARCH_PAGES_END: - if (!IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA)) - return VM_FAULT_SIGBUS; - pfn = __phys_to_pfn(__pa_symbol(vdso_k_arch_data)) + - vmf->pgoff - VDSO_ARCH_PAGES_START; break; default: return VM_FAULT_SIGBUS; } - return vmf_insert_pfn(vma, vmf->address, pfn); + get_page(page); + vmf->page = page; + return 0; } const struct vm_special_mapping vdso_vvar_mapping = { @@ -99,32 +123,7 @@ const struct vm_special_mapping vdso_vvar_mapping = { struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr) { return _install_special_mapping(mm, addr, VDSO_NR_PAGES * PAGE_SIZE, - VM_READ | VM_MAYREAD | VM_IO | VM_DONTDUMP | - VM_PFNMAP | VM_SEALED_SYSMAP, + VM_READ | VM_MAYREAD | VM_DONTDUMP | + VM_MIXEDMAP | VM_SEALED_SYSMAP, &vdso_vvar_mapping); } - -#ifdef CONFIG_TIME_NS -/* - * The vvar page layout depends on whether a task belongs to the root or - * non-root time namespace. Whenever a task changes its namespace, the VVAR - * page tables are cleared and then they will be re-faulted with a - * corresponding layout. - * See also the comment near timens_setup_vdso_clock_data() for details. - */ -int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) -{ - struct mm_struct *mm = task->mm; - struct vm_area_struct *vma; - VMA_ITERATOR(vmi, mm, 0); - - mmap_read_lock(mm); - for_each_vma(vmi, vma) { - if (vma_is_special_mapping(vma, &vdso_vvar_mapping)) - zap_vma_pages(vma); - } - mmap_read_unlock(mm); - - return 0; -} -#endif diff --git a/lib/vdso/getrandom.c b/lib/vdso/getrandom.c index 440f8a6203a6..2851afa9154f 100644 --- a/lib/vdso/getrandom.c +++ b/lib/vdso/getrandom.c @@ -7,19 +7,18 @@ #include <linux/minmax.h> #include <vdso/datapage.h> #include <vdso/getrandom.h> +#include <vdso/limits.h> +#include <vdso/page.h> #include <vdso/unaligned.h> +#include <asm/barrier.h> #include <asm/vdso/getrandom.h> +#include <uapi/linux/errno.h> #include <uapi/linux/mman.h> #include <uapi/linux/random.h> /* Bring in default accessors */ #include <vdso/vsyscall.h> -#undef PAGE_SIZE -#undef PAGE_MASK -#define PAGE_SIZE (1UL << CONFIG_PAGE_SHIFT) -#define PAGE_MASK (~(PAGE_SIZE - 1)) - #define MEMCPY_AND_ZERO_SRC(type, dst, src, len) do { \ while (len >= sizeof(type)) { \ __put_unaligned_t(type, __get_unaligned_t(type, src), dst); \ diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index 4399e143d43a..f7a591aba59f 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -3,8 +3,29 @@ * Generic userspace implementations of gettimeofday() and similar. */ #include <vdso/auxclock.h> +#include <vdso/clocksource.h> #include <vdso/datapage.h> #include <vdso/helpers.h> +#include <vdso/ktime.h> +#include <vdso/limits.h> +#include <vdso/math64.h> +#include <vdso/time32.h> +#include <vdso/time64.h> + +#include <uapi/linux/unistd.h> + +/* + * The generic vDSO implementation requires that gettimeofday.h + * provides: + * - __arch_get_hw_counter(): to get the hw counter based on the + * clock_mode. + * - gettimeofday_fallback(): fallback for gettimeofday. + * - clock_gettime_fallback(): fallback for clock_gettime. + * - clock_getres_fallback(): fallback for clock_getres. + */ +#include <asm/vdso/gettimeofday.h> + +#include <linux/build_bug.h> /* Bring in default accessors */ #include <vdso/vsyscall.h> @@ -109,7 +130,7 @@ bool vdso_get_timestamp(const struct vdso_time_data *vd, const struct vdso_clock } static __always_inline -const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd) +const struct vdso_time_data *vdso_timens_data(const struct vdso_time_data *vd) { return (void *)vd + PAGE_SIZE; } @@ -118,7 +139,7 @@ static __always_inline bool do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, clockid_t clk, struct __kernel_timespec *ts) { - const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns); + const struct vdso_time_data *vd = vdso_timens_data(vdns); const struct timens_offset *offs = &vcns->offset[clk]; const struct vdso_clock *vc = vd->clock_data; u32 seq; @@ -135,7 +156,7 @@ bool do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock * if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns)) return false; - } while (unlikely(vdso_read_retry(vc, seq))); + } while (vdso_read_retry(vc, seq)); /* Add the namespace offset */ sec += offs->sec; @@ -158,28 +179,12 @@ bool do_hres(const struct vdso_time_data *vd, const struct vdso_clock *vc, return false; do { - /* - * Open coded function vdso_read_begin() to handle - * VDSO_CLOCKMODE_TIMENS. Time namespace enabled tasks have a - * special VVAR page installed which has vc->seq set to 1 and - * vc->clock_mode set to VDSO_CLOCKMODE_TIMENS. For non time - * namespace affected tasks this does not affect performance - * because if vc->seq is odd, i.e. a concurrent update is in - * progress the extra check for vc->clock_mode is just a few - * extra instructions while spin waiting for vc->seq to become - * even again. - */ - while (unlikely((seq = READ_ONCE(vc->seq)) & 1)) { - if (IS_ENABLED(CONFIG_TIME_NS) && - vc->clock_mode == VDSO_CLOCKMODE_TIMENS) - return do_hres_timens(vd, vc, clk, ts); - cpu_relax(); - } - smp_rmb(); + if (vdso_read_begin_timens(vc, &seq)) + return do_hres_timens(vd, vc, clk, ts); if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns)) return false; - } while (unlikely(vdso_read_retry(vc, seq))); + } while (vdso_read_retry(vc, seq)); vdso_set_timespec(ts, sec, ns); @@ -190,7 +195,7 @@ static __always_inline bool do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, clockid_t clk, struct __kernel_timespec *ts) { - const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns); + const struct vdso_time_data *vd = vdso_timens_data(vdns); const struct timens_offset *offs = &vcns->offset[clk]; const struct vdso_clock *vc = vd->clock_data; const struct vdso_timestamp *vdso_ts; @@ -204,7 +209,7 @@ bool do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock seq = vdso_read_begin(vc); sec = vdso_ts->sec; nsec = vdso_ts->nsec; - } while (unlikely(vdso_read_retry(vc, seq))); + } while (vdso_read_retry(vc, seq)); /* Add the namespace offset */ sec += offs->sec; @@ -223,21 +228,12 @@ bool do_coarse(const struct vdso_time_data *vd, const struct vdso_clock *vc, u32 seq; do { - /* - * Open coded function vdso_read_begin() to handle - * VDSO_CLOCK_TIMENS. See comment in do_hres(). - */ - while ((seq = READ_ONCE(vc->seq)) & 1) { - if (IS_ENABLED(CONFIG_TIME_NS) && - vc->clock_mode == VDSO_CLOCKMODE_TIMENS) - return do_coarse_timens(vd, vc, clk, ts); - cpu_relax(); - } - smp_rmb(); + if (vdso_read_begin_timens(vc, &seq)) + return do_coarse_timens(vd, vc, clk, ts); ts->tv_sec = vdso_ts->sec; ts->tv_nsec = vdso_ts->nsec; - } while (unlikely(vdso_read_retry(vc, seq))); + } while (vdso_read_retry(vc, seq)); return true; } @@ -256,20 +252,11 @@ bool do_aux(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_ti vc = &vd->aux_clock_data[idx]; do { - /* - * Open coded function vdso_read_begin() to handle - * VDSO_CLOCK_TIMENS. See comment in do_hres(). - */ - while ((seq = READ_ONCE(vc->seq)) & 1) { - if (IS_ENABLED(CONFIG_TIME_NS) && vc->clock_mode == VDSO_CLOCKMODE_TIMENS) { - vd = __arch_get_vdso_u_timens_data(vd); - vc = &vd->aux_clock_data[idx]; - /* Re-read from the real time data page */ - continue; - } - cpu_relax(); + while (vdso_read_begin_timens(vc, &seq)) { + /* Re-read from the real time data page, reload seq by looping */ + vd = vdso_timens_data(vd); + vc = &vd->aux_clock_data[idx]; } - smp_rmb(); /* Auxclock disabled? */ if (vc->clock_mode == VDSO_CLOCKMODE_NONE) @@ -277,7 +264,7 @@ bool do_aux(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_ti if (!vdso_get_timestamp(vd, vc, VDSO_BASE_AUX, &sec, &ns)) return false; - } while (unlikely(vdso_read_retry(vc, seq))); + } while (vdso_read_retry(vc, seq)); vdso_set_timespec(ts, sec, ns); @@ -313,7 +300,7 @@ __cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock, return do_hres(vd, vc, clock, ts); } -static __maybe_unused int +static int __cvdso_clock_gettime_data(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_timespec *ts) { @@ -333,13 +320,15 @@ __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) } #ifdef BUILD_VDSO32 -static __maybe_unused int +static int __cvdso_clock_gettime32_data(const struct vdso_time_data *vd, clockid_t clock, struct old_timespec32 *res) { struct __kernel_timespec ts; bool ok; + BUILD_BUG_ON(!IS_ENABLED(CONFIG_COMPAT_32BIT_TIME)); + ok = __cvdso_clock_gettime_common(vd, clock, &ts); if (unlikely(!ok)) @@ -359,12 +348,18 @@ __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) } #endif /* BUILD_VDSO32 */ -static __maybe_unused int +static int __cvdso_gettimeofday_data(const struct vdso_time_data *vd, struct __kernel_old_timeval *tv, struct timezone *tz) { const struct vdso_clock *vc = vd->clock_data; +#ifndef __NR_gettimeofday + BUILD_BUG(); +#endif + + BUILD_BUG_ON(sizeof(tv->tv_sec) != 8 && !IS_ENABLED(CONFIG_COMPAT_32BIT_TIME)); + if (likely(tv != NULL)) { struct __kernel_timespec ts; @@ -376,9 +371,8 @@ __cvdso_gettimeofday_data(const struct vdso_time_data *vd, } if (unlikely(tz != NULL)) { - if (IS_ENABLED(CONFIG_TIME_NS) && - vc->clock_mode == VDSO_CLOCKMODE_TIMENS) - vd = __arch_get_vdso_u_timens_data(vd); + if (vdso_is_timens_clock(vc)) + vd = vdso_timens_data(vd); tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; @@ -394,15 +388,20 @@ __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) } #ifdef VDSO_HAS_TIME -static __maybe_unused __kernel_old_time_t +static __kernel_old_time_t __cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time) { const struct vdso_clock *vc = vd->clock_data; __kernel_old_time_t t; - if (IS_ENABLED(CONFIG_TIME_NS) && - vc->clock_mode == VDSO_CLOCKMODE_TIMENS) { - vd = __arch_get_vdso_u_timens_data(vd); +#ifndef __NR_time + BUILD_BUG(); +#endif + + BUILD_BUG_ON(sizeof(*time) != 8 && !IS_ENABLED(CONFIG_COMPAT_32BIT_TIME)); + + if (vdso_is_timens_clock(vc)) { + vd = vdso_timens_data(vd); vc = vd->clock_data; } @@ -432,9 +431,8 @@ bool __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t cloc if (!vdso_clockid_valid(clock)) return false; - if (IS_ENABLED(CONFIG_TIME_NS) && - vc->clock_mode == VDSO_CLOCKMODE_TIMENS) - vd = __arch_get_vdso_u_timens_data(vd); + if (vdso_is_timens_clock(vc)) + vd = vdso_timens_data(vd); /* * Convert the clockid to a bitmask and use it to check which @@ -464,7 +462,7 @@ bool __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t cloc return true; } -static __maybe_unused +static int __cvdso_clock_getres_data(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_timespec *res) { @@ -484,13 +482,15 @@ int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) } #ifdef BUILD_VDSO32 -static __maybe_unused int +static int __cvdso_clock_getres_time32_data(const struct vdso_time_data *vd, clockid_t clock, struct old_timespec32 *res) { struct __kernel_timespec ts; bool ok; + BUILD_BUG_ON(!IS_ENABLED(CONFIG_COMPAT_32BIT_TIME)); + ok = __cvdso_clock_getres_common(vd, clock, &ts); if (unlikely(!ok)) diff --git a/lib/vsprintf.c b/lib/vsprintf.c index 800b8ac49f53..1d95ab9314bf 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -59,7 +59,7 @@ /* Disable pointer hashing if requested */ bool no_hash_pointers __ro_after_init; -EXPORT_SYMBOL_GPL(no_hash_pointers); +EXPORT_SYMBOL_FOR_MODULES(no_hash_pointers, "printf_kunit"); /* * Hashed pointers policy selected by "hash_pointers=..." boot param @@ -86,7 +86,7 @@ static unsigned long long simple_strntoull(const char *startp, char **endp, unsi cp = _parse_integer_fixup_radix(startp, &base); prefix_chars = cp - startp; if (prefix_chars < max_chars) { - rv = _parse_integer_limit(cp, base, &result, max_chars - prefix_chars); + rv = _parse_integer(cp, base, &result, max_chars - prefix_chars); /* FIXME */ cp += (rv & ~KSTRTOX_OVERFLOW); } else { @@ -129,13 +129,6 @@ unsigned long simple_strtoul(const char *cp, char **endp, unsigned int base) } EXPORT_SYMBOL(simple_strtoul); -unsigned long simple_strntoul(const char *cp, char **endp, unsigned int base, - size_t max_chars) -{ - return simple_strntoull(cp, endp, base, max_chars); -} -EXPORT_SYMBOL(simple_strntoul); - /** * simple_strtol - convert a string to a signed long * @cp: The start of the string @@ -857,6 +850,7 @@ static char *default_pointer(char *buf, char *end, const void *ptr, } int kptr_restrict __read_mostly; +EXPORT_SYMBOL_FOR_MODULES(kptr_restrict, "printf_kunit"); static noinline_for_stack char *restricted_pointer(char *buf, char *end, const void *ptr, @@ -1107,7 +1101,7 @@ char *resource_string(char *buf, char *end, struct resource *res, 2*RSRC_BUF_SIZE + FLAG_BUF_SIZE + RAW_BUF_SIZE)]; char *p = sym, *pend = sym + sizeof(sym); - int decode = (fmt[0] == 'R') ? 1 : 0; + bool decode = fmt[0] == 'R'; const struct printf_spec *specp; if (check_pointer(&buf, end, res, spec)) @@ -1132,7 +1126,7 @@ char *resource_string(char *buf, char *end, struct resource *res, } else { p = string_nocheck(p, pend, "??? ", str_spec); specp = &mem_spec; - decode = 0; + decode = false; } if (decode && res->flags & IORESOURCE_UNSET) { p = string_nocheck(p, pend, "size ", str_spec); @@ -1208,7 +1202,7 @@ char *hex_string(char *buf, char *end, u8 *addr, struct printf_spec spec, } if (spec.field_width > 0) - len = min_t(int, spec.field_width, 64); + len = min(spec.field_width, 64); for (i = 0; i < len; ++i) { if (buf < end) @@ -1233,7 +1227,7 @@ char *bitmap_string(char *buf, char *end, const unsigned long *bitmap, struct printf_spec spec, const char *fmt) { const int CHUNKSZ = 32; - int nr_bits = max_t(int, spec.field_width, 0); + int nr_bits = max(spec.field_width, 0); int i, chunksz; bool first = true; @@ -1276,7 +1270,7 @@ static noinline_for_stack char *bitmap_list_string(char *buf, char *end, const unsigned long *bitmap, struct printf_spec spec, const char *fmt) { - int nr_bits = max_t(int, spec.field_width, 0); + int nr_bits = max(spec.field_width, 0); bool first = true; int rbot, rtop; @@ -1309,31 +1303,39 @@ char *mac_address_string(char *buf, char *end, u8 *addr, char mac_addr[sizeof("xx:xx:xx:xx:xx:xx")]; char *p = mac_addr; int i; - char separator; + char separator = ':'; bool reversed = false; + bool uc = false; if (check_pointer(&buf, end, addr, spec)) return buf; switch (fmt[1]) { case 'F': + uc = fmt[2] == 'U'; separator = '-'; break; case 'R': + uc = fmt[2] == 'U'; reversed = true; - fallthrough; + break; + + case 'U': + uc = true; + break; default: - separator = ':'; break; } for (i = 0; i < 6; i++) { - if (reversed) - p = hex_byte_pack(p, addr[5 - i]); + u8 byte = reversed ? addr[5 - i] : addr[i]; + + if (uc) + p = hex_byte_pack_upper(p, byte); else - p = hex_byte_pack(p, addr[i]); + p = hex_byte_pack(p, byte); if (fmt[0] == 'M' && i != 5) *p++ = separator; @@ -2358,16 +2360,20 @@ void __init hash_pointers_finalize(bool slub_debug) static int __init hash_pointers_mode_parse(char *str) { + /* Avoid stale no_hash_pointers state when hash_pointers overrides it */ + no_hash_pointers = false; + if (!str) { pr_warn("Hash pointers mode empty; falling back to auto.\n"); hash_pointers_mode = HASH_PTR_AUTO; - } else if (strncmp(str, "auto", 4) == 0) { + } else if (strcmp(str, "auto") == 0) { pr_info("Hash pointers mode set to auto.\n"); hash_pointers_mode = HASH_PTR_AUTO; - } else if (strncmp(str, "never", 5) == 0) { + } else if (strcmp(str, "never") == 0) { pr_info("Hash pointers mode set to never.\n"); hash_pointers_mode = HASH_PTR_NEVER; - } else if (strncmp(str, "always", 6) == 0) { + no_hash_pointers = true; + } else if (strcmp(str, "always") == 0) { pr_info("Hash pointers mode set to always.\n"); hash_pointers_mode = HASH_PTR_ALWAYS; } else { @@ -2416,6 +2422,7 @@ early_param("no_hash_pointers", no_hash_pointers_enable); * - 'MF' For a 6-byte MAC FDDI address, it prints the address * with a dash-separated hex notation * - '[mM]R' For a 6-byte MAC address, Reverse order (Bluetooth) + * - '[mM][FR][U]' One of the above in the upper case * - 'I' [46] for IPv4/IPv6 addresses printed in the usual way * IPv4 uses dot-separated decimal without leading 0's (1.2.3.4) * IPv6 uses colon separated network-order 16 bit hex with leading 0's @@ -2550,6 +2557,7 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr, case 'm': /* Contiguous: 000102030405 */ /* [mM]F (FDDI) */ /* [mM]R (Reverse order; Bluetooth) */ + /* [mM][FR][U] (One of the above in the upper case) */ return mac_address_string(buf, end, ptr, spec, fmt); case 'I': /* Formatted IP supported * 4: 1.2.3.4 @@ -2640,6 +2648,18 @@ static unsigned char spec_flag(unsigned char c) return (c < sizeof(spec_flag_array)) ? spec_flag_array[c] : 0; } +static void set_field_width(struct printf_spec *spec, int width) +{ + spec->field_width = clamp(width, -FIELD_WIDTH_MAX, FIELD_WIDTH_MAX); + WARN_ONCE(spec->field_width != width, "field width %d out of range", width); +} + +static void set_precision(struct printf_spec *spec, int prec) +{ + spec->precision = clamp(prec, 0, PRECISION_MAX); + WARN_ONCE(spec->precision < prec, "precision %d too large", prec); +} + /* * Helper function to decode printf style format. * Each call decode a token from the format and return the @@ -2710,7 +2730,7 @@ struct fmt format_decode(struct fmt fmt, struct printf_spec *spec) spec->field_width = -1; if (isdigit(*fmt.str)) - spec->field_width = skip_atoi(&fmt.str); + set_field_width(spec, skip_atoi(&fmt.str)); else if (unlikely(*fmt.str == '*')) { /* it's the next argument */ fmt.state = FORMAT_STATE_WIDTH; @@ -2724,9 +2744,7 @@ precision: if (unlikely(*fmt.str == '.')) { fmt.str++; if (isdigit(*fmt.str)) { - spec->precision = skip_atoi(&fmt.str); - if (spec->precision < 0) - spec->precision = 0; + set_precision(spec, skip_atoi(&fmt.str)); } else if (*fmt.str == '*') { /* it's the next argument */ fmt.state = FORMAT_STATE_PRECISION; @@ -2799,24 +2817,6 @@ qualifier: return fmt; } -static void -set_field_width(struct printf_spec *spec, int width) -{ - spec->field_width = width; - if (WARN_ONCE(spec->field_width != width, "field width %d too large", width)) { - spec->field_width = clamp(width, -FIELD_WIDTH_MAX, FIELD_WIDTH_MAX); - } -} - -static void -set_precision(struct printf_spec *spec, int prec) -{ - spec->precision = prec; - if (WARN_ONCE(spec->precision != prec, "precision %d too large", prec)) { - spec->precision = clamp(prec, 0, PRECISION_MAX); - } -} - /* * Turn a 1/2/4-byte value into a 64-bit one for printing: truncate * as necessary and deal with signedness. @@ -2864,6 +2864,7 @@ static unsigned long long convert_num_spec(unsigned int val, int size, struct pr int vsnprintf(char *buf, size_t size, const char *fmt_str, va_list args) { char *str, *end; + size_t ret_size; struct printf_spec spec = {0}; struct fmt fmt = { .str = fmt_str, @@ -2983,8 +2984,12 @@ out: } /* the trailing null byte doesn't count towards the total */ - return str-buf; + ret_size = str - buf; + /* Make sure the return value is within the positive integer range */ + if (WARN_ON_ONCE(ret_size > INT_MAX)) + ret_size = INT_MAX; + return ret_size; } EXPORT_SYMBOL(vsnprintf); @@ -3288,6 +3293,7 @@ int bstr_printf(char *buf, size_t size, const char *fmt_str, const u32 *bin_buf) struct printf_spec spec = {0}; char *str, *end; const char *args = (const char *)bin_buf; + size_t ret_size; if (WARN_ON_ONCE(size > INT_MAX)) return 0; @@ -3436,7 +3442,12 @@ out: #undef get_arg /* the trailing null byte doesn't count towards the total */ - return str - buf; + ret_size = str - buf; + + /* Make sure the return value is within the positive integer range */ + if (WARN_ON_ONCE(ret_size > INT_MAX)) + ret_size = INT_MAX; + return ret_size; } EXPORT_SYMBOL_GPL(bstr_printf); diff --git a/lib/xarray.c b/lib/xarray.c index 9a8b4916540c..bfe7bef80f34 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -1053,6 +1053,9 @@ void xas_split_alloc(struct xa_state *xas, void *entry, unsigned int order, if (xas->xa_shift + XA_CHUNK_SHIFT > order) return; + if (xas->xa->xa_flags & XA_FLAGS_ACCOUNT) + gfp |= __GFP_ACCOUNT; + do { struct xa_node *node; diff --git a/lib/xz/xz_dec_bcj.c b/lib/xz/xz_dec_bcj.c index cc49a300a5b2..88922323f96e 100644 --- a/lib/xz/xz_dec_bcj.c +++ b/lib/xz/xz_dec_bcj.c @@ -466,7 +466,7 @@ static void bcj_flush(struct xz_dec_bcj *s, struct xz_buf *b) { size_t copy_size; - copy_size = min_t(size_t, s->temp.filtered, b->out_size - b->out_pos); + copy_size = min(s->temp.filtered, b->out_size - b->out_pos); memcpy(b->out + b->out_pos, s->temp.buf, copy_size); b->out_pos += copy_size; diff --git a/lib/xz/xz_dec_lzma2.c b/lib/xz/xz_dec_lzma2.c index 4b783ac94e71..46c2df6ad6f5 100644 --- a/lib/xz/xz_dec_lzma2.c +++ b/lib/xz/xz_dec_lzma2.c @@ -135,14 +135,16 @@ struct lzma_dec { uint32_t rep2; uint32_t rep3; - /* Types of the most recently seen LZMA symbols */ - enum lzma_state state; - /* * Length of a match. This is updated so that dict_repeat can - * be called again to finish repeating the whole match. + * be called again to finish repeating the whole match. This is + * size_t because a pointer to this is passed to dict_repeat, + * and there it's nicer to have size_t instead of uint32_t. */ - uint32_t len; + size_t len; + + /* Types of the most recently seen LZMA symbols */ + enum lzma_state state; /* * LZMA properties or related bit masks (number of literal @@ -228,13 +230,13 @@ struct lzma2_dec { enum lzma2_seq next_sequence; /* Uncompressed size of LZMA chunk (2 MiB at maximum) */ - uint32_t uncompressed; + size_t uncompressed; /* * Compressed size of LZMA chunk or compressed/uncompressed * size of uncompressed chunk (64 KiB at maximum) */ - uint32_t compressed; + size_t compressed; /* * True if dictionary reset is needed. This is false before @@ -269,11 +271,11 @@ struct xz_dec_lzma2 { struct lzma_dec lzma; /* - * Temporary buffer which holds small number of input bytes between + * Temporary buffer which holds a small number of input bytes between * decoder calls. See lzma2_lzma() for details. */ struct { - uint32_t size; + size_t size; uint8_t buf[3 * LZMA_IN_REQUIRED]; } temp; }; @@ -320,7 +322,7 @@ static inline bool dict_has_space(const struct dictionary *dict) * still empty. This special case is needed for single-call decoding to * avoid writing a '\0' to the end of the destination buffer. */ -static inline uint32_t dict_get(const struct dictionary *dict, uint32_t dist) +static inline uint32_t dict_get(const struct dictionary *dict, size_t dist) { size_t offset = dict->pos - dist - 1; @@ -346,15 +348,15 @@ static inline void dict_put(struct dictionary *dict, uint8_t byte) * invalid, false is returned. On success, true is returned and *len is * updated to indicate how many bytes were left to be repeated. */ -static bool dict_repeat(struct dictionary *dict, uint32_t *len, uint32_t dist) +static bool dict_repeat(struct dictionary *dict, size_t *len, size_t dist) { size_t back; - uint32_t left; + size_t left; if (dist >= dict->full || dist >= dict->size) return false; - left = min_t(size_t, dict->limit - dict->pos, *len); + left = min(dict->limit - dict->pos, *len); *len -= left; back = dict->pos - dist - 1; @@ -375,7 +377,7 @@ static bool dict_repeat(struct dictionary *dict, uint32_t *len, uint32_t dist) /* Copy uncompressed data as is from input to dictionary and output buffers. */ static void dict_uncompressed(struct dictionary *dict, struct xz_buf *b, - uint32_t *left) + size_t *left) { size_t copy_size; @@ -433,7 +435,7 @@ static void dict_uncompressed(struct dictionary *dict, struct xz_buf *b, * enough space in b->out. This is guaranteed because caller uses dict_limit() * before decoding data into the dictionary. */ -static uint32_t dict_flush(struct dictionary *dict, struct xz_buf *b) +static size_t dict_flush(struct dictionary *dict, struct xz_buf *b) { size_t copy_size = dict->pos - dict->start; @@ -755,8 +757,8 @@ static bool lzma_main(struct xz_dec_lzma2 *s) uint32_t pos_state; /* - * If the dictionary was reached during the previous call, try to - * finish the possibly pending repeat in the dictionary. + * If the dictionary write limit was reached during the previous call, + * try to finish the possibly pending repeat in the dictionary. */ if (dict_has_space(&s->dict) && s->lzma.len > 0) dict_repeat(&s->dict, &s->lzma.len, s->lzma.rep0); @@ -878,7 +880,7 @@ static bool lzma_props(struct xz_dec_lzma2 *s, uint8_t props) static bool lzma2_lzma(struct xz_dec_lzma2 *s, struct xz_buf *b) { size_t in_avail; - uint32_t tmp; + size_t tmp; in_avail = b->in_size - b->in_pos; if (s->temp.size > 0 || s->lzma2.compressed == 0) { @@ -976,7 +978,7 @@ enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, struct xz_buf *b) * an uncompressed chunk * 0x02 Uncompressed chunk (no dictionary reset) * - * Highest three bits (s->control & 0xE0): + * Highest three bits (tmp & 0xE0): * 0xE0 Dictionary reset, new properties and state * reset, followed by LZMA compressed chunk * 0xC0 New properties and state reset, followed @@ -988,7 +990,7 @@ enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, struct xz_buf *b) * 0x80 LZMA chunk (no dictionary or state reset) * * For LZMA compressed chunks, the lowest five bits - * (s->control & 1F) are the highest bits of the + * (tmp & 1F) are the highest bits of the * uncompressed size (bits 16-20). * * A new LZMA2 stream must begin with a dictionary @@ -1046,25 +1048,23 @@ enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, struct xz_buf *b) case SEQ_UNCOMPRESSED_1: s->lzma2.uncompressed - += (uint32_t)b->in[b->in_pos++] << 8; + += (size_t)b->in[b->in_pos++] << 8; s->lzma2.sequence = SEQ_UNCOMPRESSED_2; break; case SEQ_UNCOMPRESSED_2: s->lzma2.uncompressed - += (uint32_t)b->in[b->in_pos++] + 1; + += (size_t)b->in[b->in_pos++] + 1; s->lzma2.sequence = SEQ_COMPRESSED_0; break; case SEQ_COMPRESSED_0: - s->lzma2.compressed - = (uint32_t)b->in[b->in_pos++] << 8; + s->lzma2.compressed = (size_t)b->in[b->in_pos++] << 8; s->lzma2.sequence = SEQ_COMPRESSED_1; break; case SEQ_COMPRESSED_1: - s->lzma2.compressed - += (uint32_t)b->in[b->in_pos++] + 1; + s->lzma2.compressed += (size_t)b->in[b->in_pos++] + 1; s->lzma2.sequence = s->lzma2.next_sequence; break; @@ -1091,16 +1091,15 @@ enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, struct xz_buf *b) case SEQ_LZMA_RUN: /* * Set dictionary limit to indicate how much we want - * to be encoded at maximum. Decode new data into the + * to be decoded at maximum. Decode new data into the * dictionary. Flush the new data from dictionary to * b->out. Check if we finished decoding this chunk. * In case the dictionary got full but we didn't fill * the output buffer yet, we may run this loop * multiple times without changing s->lzma2.sequence. */ - dict_limit(&s->dict, min_t(size_t, - b->out_size - b->out_pos, - s->lzma2.uncompressed)); + dict_limit(&s->dict, min(b->out_size - b->out_pos, + s->lzma2.uncompressed)); if (!lzma2_lzma(s, b)) return XZ_DATA_ERROR; @@ -1260,8 +1259,8 @@ enum xz_ret xz_dec_microlzma_run(struct xz_dec_microlzma *s_ptr, s->dict.end = b->out_size - b->out_pos; while (true) { - dict_limit(&s->dict, min_t(size_t, b->out_size - b->out_pos, - s->lzma2.uncompressed)); + dict_limit(&s->dict, min(b->out_size - b->out_pos, + s->lzma2.uncompressed)); if (!lzma2_lzma(s, b)) return XZ_DATA_ERROR; diff --git a/lib/xz/xz_dec_stream.c b/lib/xz/xz_dec_stream.c index 59bfd54ffee7..0bed6daefac2 100644 --- a/lib/xz/xz_dec_stream.c +++ b/lib/xz/xz_dec_stream.c @@ -155,8 +155,8 @@ static const uint8_t check_sizes[16] = { */ static bool fill_temp(struct xz_dec *s, struct xz_buf *b) { - size_t copy_size = min_t(size_t, - b->in_size - b->in_pos, s->temp.size - s->temp.pos); + size_t copy_size = min(b->in_size - b->in_pos, + s->temp.size - s->temp.pos); memcpy(s->temp.buf + s->temp.pos, b->in + b->in_pos, copy_size); b->in_pos += copy_size; diff --git a/lib/xz/xz_lzma2.h b/lib/xz/xz_lzma2.h index d2632b7dfb9c..a612ce4fd450 100644 --- a/lib/xz/xz_lzma2.h +++ b/lib/xz/xz_lzma2.h @@ -143,7 +143,7 @@ static inline bool lzma_state_is_literal(enum lzma_state state) * Get the index of the appropriate probability array for decoding * the distance slot. */ -static inline uint32_t lzma_get_dist_state(uint32_t len) +static inline size_t lzma_get_dist_state(size_t len) { return len < DIST_STATES + MATCH_LEN_MIN ? len - MATCH_LEN_MIN : DIST_STATES - 1; |