From c0087d807ae86cc82cc356e366d2dccf0e3bb225 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 24 Nov 2025 16:04:29 +0100 Subject: s390/vdso: Rename vdso64 to vdso Since compat is gone there is only a 64 bit vdso left. Remove the superfluous "64" suffix everywhere. Reviewed-by: Jens Remus Signed-off-by: Heiko Carstens --- arch/s390/Makefile | 4 +- arch/s390/include/asm/vdso-symbols.h | 4 +- arch/s390/kernel/Makefile | 2 +- arch/s390/kernel/signal.c | 6 +- arch/s390/kernel/vdso.c | 16 ++- arch/s390/kernel/vdso/.gitignore | 2 + arch/s390/kernel/vdso/Makefile | 80 ++++++++++++ arch/s390/kernel/vdso/gen_vdso_offsets.sh | 15 +++ arch/s390/kernel/vdso/getcpu.c | 21 ++++ arch/s390/kernel/vdso/note.S | 13 ++ arch/s390/kernel/vdso/vdso.h | 15 +++ arch/s390/kernel/vdso/vdso.lds.S | 113 +++++++++++++++++ arch/s390/kernel/vdso/vdso_generic.c | 19 +++ arch/s390/kernel/vdso/vdso_user_wrapper.S | 52 ++++++++ arch/s390/kernel/vdso/vdso_wrapper.S | 15 +++ arch/s390/kernel/vdso/vgetrandom-chacha.S | 181 ++++++++++++++++++++++++++++ arch/s390/kernel/vdso/vgetrandom.c | 14 +++ arch/s390/kernel/vdso64/.gitignore | 2 - arch/s390/kernel/vdso64/Makefile | 80 ------------ arch/s390/kernel/vdso64/gen_vdso_offsets.sh | 15 --- arch/s390/kernel/vdso64/getcpu.c | 21 ---- arch/s390/kernel/vdso64/note.S | 13 -- arch/s390/kernel/vdso64/vdso.h | 15 --- arch/s390/kernel/vdso64/vdso64.lds.S | 113 ----------------- arch/s390/kernel/vdso64/vdso64_generic.c | 19 --- arch/s390/kernel/vdso64/vdso64_wrapper.S | 15 --- arch/s390/kernel/vdso64/vdso_user_wrapper.S | 52 -------- arch/s390/kernel/vdso64/vgetrandom-chacha.S | 181 ---------------------------- arch/s390/kernel/vdso64/vgetrandom.c | 14 --- 29 files changed, 555 insertions(+), 557 deletions(-) create mode 100644 arch/s390/kernel/vdso/.gitignore create mode 100644 arch/s390/kernel/vdso/Makefile create mode 100755 arch/s390/kernel/vdso/gen_vdso_offsets.sh create mode 100644 arch/s390/kernel/vdso/getcpu.c create mode 100644 arch/s390/kernel/vdso/note.S create mode 100644 arch/s390/kernel/vdso/vdso.h create mode 100644 arch/s390/kernel/vdso/vdso.lds.S create mode 100644 arch/s390/kernel/vdso/vdso_generic.c create mode 100644 arch/s390/kernel/vdso/vdso_user_wrapper.S create mode 100644 arch/s390/kernel/vdso/vdso_wrapper.S create mode 100644 arch/s390/kernel/vdso/vgetrandom-chacha.S create mode 100644 arch/s390/kernel/vdso/vgetrandom.c delete mode 100644 arch/s390/kernel/vdso64/.gitignore delete mode 100644 arch/s390/kernel/vdso64/Makefile delete mode 100755 arch/s390/kernel/vdso64/gen_vdso_offsets.sh delete mode 100644 arch/s390/kernel/vdso64/getcpu.c delete mode 100644 arch/s390/kernel/vdso64/note.S delete mode 100644 arch/s390/kernel/vdso64/vdso.h delete mode 100644 arch/s390/kernel/vdso64/vdso64.lds.S delete mode 100644 arch/s390/kernel/vdso64/vdso64_generic.c delete mode 100644 arch/s390/kernel/vdso64/vdso64_wrapper.S delete mode 100644 arch/s390/kernel/vdso64/vdso_user_wrapper.S delete mode 100644 arch/s390/kernel/vdso64/vgetrandom-chacha.S delete mode 100644 arch/s390/kernel/vdso64/vgetrandom.c diff --git a/arch/s390/Makefile b/arch/s390/Makefile index f0a670dcce71..3b8772628fde 100644 --- a/arch/s390/Makefile +++ b/arch/s390/Makefile @@ -151,9 +151,9 @@ ifeq ($(KBUILD_EXTMOD),) # this hack. prepare: vdso_prepare vdso_prepare: prepare0 - $(Q)$(MAKE) $(build)=arch/s390/kernel/vdso64 include/generated/vdso64-offsets.h + $(Q)$(MAKE) $(build)=arch/s390/kernel/vdso include/generated/vdso-offsets.h -vdso-install-y += arch/s390/kernel/vdso64/vdso64.so.dbg +vdso-install-y += arch/s390/kernel/vdso/vdso.so.dbg endif diff --git a/arch/s390/include/asm/vdso-symbols.h b/arch/s390/include/asm/vdso-symbols.h index 205da2c565c2..e3561e67c4e3 100644 --- a/arch/s390/include/asm/vdso-symbols.h +++ b/arch/s390/include/asm/vdso-symbols.h @@ -2,8 +2,8 @@ #ifndef __S390_VDSO_SYMBOLS_H__ #define __S390_VDSO_SYMBOLS_H__ -#include +#include -#define VDSO64_SYMBOL(tsk, name) ((tsk)->mm->context.vdso_base + (vdso64_offset_##name)) +#define VDSO_SYMBOL(tsk, name) ((tsk)->mm->context.vdso_base + (vdso_offset_##name)) #endif /* __S390_VDSO_SYMBOLS_H__ */ diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 7df060b8eb4e..42c83d60d6fa 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -81,4 +81,4 @@ obj-$(CONFIG_PERF_EVENTS) += perf_pai.o obj-$(CONFIG_TRACEPOINTS) += trace.o # vdso -obj-y += vdso64/ +obj-y += vdso/ diff --git a/arch/s390/kernel/signal.c b/arch/s390/kernel/signal.c index e7775d121fa1..4874de5edea0 100644 --- a/arch/s390/kernel/signal.c +++ b/arch/s390/kernel/signal.c @@ -326,7 +326,7 @@ static int setup_frame(int sig, struct k_sigaction *ka, if (ka->sa.sa_flags & SA_RESTORER) restorer = (unsigned long) ka->sa.sa_restorer; else - restorer = VDSO64_SYMBOL(current, sigreturn); + restorer = VDSO_SYMBOL(current, sigreturn); /* Set up registers for signal handler */ regs->gprs[14] = restorer; @@ -378,7 +378,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, if (ksig->ka.sa.sa_flags & SA_RESTORER) restorer = (unsigned long) ksig->ka.sa.sa_restorer; else - restorer = VDSO64_SYMBOL(current, rt_sigreturn); + restorer = VDSO_SYMBOL(current, rt_sigreturn); /* Create siginfo on the signal stack */ if (copy_siginfo_to_user(&frame->info, &ksig->info)) @@ -490,7 +490,7 @@ void arch_do_signal_or_restart(struct pt_regs *regs) /* Restart with sys_restart_syscall */ regs->gprs[2] = regs->orig_gpr2; current->restart_block.arch_data = regs->psw.addr; - regs->psw.addr = VDSO64_SYMBOL(current, restart_syscall); + regs->psw.addr = VDSO_SYMBOL(current, restart_syscall); if (test_thread_flag(TIF_SINGLE_STEP)) clear_thread_flag(TIF_PER_TRAP); break; diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c index 83cc67cf21c8..a27a90a199be 100644 --- a/arch/s390/kernel/vdso.c +++ b/arch/s390/kernel/vdso.c @@ -22,7 +22,7 @@ #include #include -extern char vdso64_start[], vdso64_end[]; +extern char vdso_start[], vdso_end[]; static int vdso_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *vma) @@ -31,7 +31,7 @@ static int vdso_mremap(const struct vm_special_mapping *sm, return 0; } -static struct vm_special_mapping vdso64_mapping = { +static struct vm_special_mapping vdso_mapping = { .name = "[vdso]", .mremap = vdso_mremap, }; @@ -46,7 +46,6 @@ early_initcall(vdso_getcpu_init); /* Must be called before SMP init */ static int map_vdso(unsigned long addr, unsigned long vdso_mapping_len) { unsigned long vvar_start, vdso_text_start, vdso_text_len; - struct vm_special_mapping *vdso_mapping; struct mm_struct *mm = current->mm; struct vm_area_struct *vma; int rc; @@ -55,8 +54,7 @@ static int map_vdso(unsigned long addr, unsigned long vdso_mapping_len) if (mmap_write_lock_killable(mm)) return -EINTR; - vdso_text_len = vdso64_end - vdso64_start; - vdso_mapping = &vdso64_mapping; + vdso_text_len = vdso_end - vdso_start; vvar_start = get_unmapped_area(NULL, addr, vdso_mapping_len, 0, 0); rc = vvar_start; if (IS_ERR_VALUE(vvar_start)) @@ -70,7 +68,7 @@ static int map_vdso(unsigned long addr, unsigned long vdso_mapping_len) vma = _install_special_mapping(mm, vdso_text_start, vdso_text_len, VM_READ|VM_EXEC|VM_SEALED_SYSMAP| VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, - vdso_mapping); + &vdso_mapping); if (IS_ERR(vma)) { do_munmap(mm, vvar_start, PAGE_SIZE, NULL); rc = PTR_ERR(vma); @@ -110,7 +108,7 @@ static unsigned long vdso_addr(unsigned long start, unsigned long len) unsigned long vdso_text_size(void) { - return PAGE_ALIGN(vdso64_end - vdso64_start); + return PAGE_ALIGN(vdso_end - vdso_start); } unsigned long vdso_size(void) @@ -148,7 +146,7 @@ static void vdso_apply_alternatives(void) struct alt_instr *start, *end; const struct elf64_hdr *hdr; - hdr = (struct elf64_hdr *)vdso64_start; + hdr = (struct elf64_hdr *)vdso_start; shdr = (void *)hdr + hdr->e_shoff; alt = find_section(hdr, shdr, ".altinstructions"); if (!alt) @@ -161,7 +159,7 @@ static void vdso_apply_alternatives(void) static int __init vdso_init(void) { vdso_apply_alternatives(); - vdso64_mapping.pages = vdso_setup_pages(vdso64_start, vdso64_end); + vdso_mapping.pages = vdso_setup_pages(vdso_start, vdso_end); return 0; } arch_initcall(vdso_init); diff --git a/arch/s390/kernel/vdso/.gitignore b/arch/s390/kernel/vdso/.gitignore new file mode 100644 index 000000000000..652e31d82582 --- /dev/null +++ b/arch/s390/kernel/vdso/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +vdso.lds diff --git a/arch/s390/kernel/vdso/Makefile b/arch/s390/kernel/vdso/Makefile new file mode 100644 index 000000000000..924be0a6a2df --- /dev/null +++ b/arch/s390/kernel/vdso/Makefile @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: GPL-2.0 +# List of files in the vdso + +# Include the generic Makefile to check the built vdso. +include $(srctree)/lib/vdso/Makefile.include +obj-vdso = vdso_user_wrapper.o note.o vgetrandom-chacha.o +obj-cvdso = vdso_generic.o getcpu.o vgetrandom.o +VDSO_CFLAGS_REMOVE := -pg $(CC_FLAGS_FTRACE) $(CC_FLAGS_EXPOLINE) +CFLAGS_REMOVE_getcpu.o = $(VDSO_CFLAGS_REMOVE) +CFLAGS_REMOVE_vgetrandom.o = $(VDSO_CFLAGS_REMOVE) +CFLAGS_REMOVE_vdso_generic.o = $(VDSO_CFLAGS_REMOVE) + +ifneq ($(c-getrandom-y),) + CFLAGS_vgetrandom.o += -include $(c-getrandom-y) +endif + +# Build rules + +targets := $(obj-vdso) $(obj-cvdso) vdso.so vdso.so.dbg +obj-vdso := $(addprefix $(obj)/, $(obj-vdso)) +obj-cvdso := $(addprefix $(obj)/, $(obj-cvdso)) + +KBUILD_AFLAGS += -DBUILD_VDSO +KBUILD_CFLAGS += -DBUILD_VDSO -DDISABLE_BRANCH_PROFILING + +KBUILD_AFLAGS_VDSO := $(filter-out -m64,$(KBUILD_AFLAGS)) +KBUILD_AFLAGS_VDSO += -m64 + +KBUILD_CFLAGS_VDSO := $(filter-out -m64,$(KBUILD_CFLAGS)) +KBUILD_CFLAGS_VDSO := $(filter-out -mpacked-stack,$(KBUILD_CFLAGS_VDSO)) +KBUILD_CFLAGS_VDSO := $(filter-out -mno-pic-data-is-text-relative,$(KBUILD_CFLAGS_VDSO)) +KBUILD_CFLAGS_VDSO := $(filter-out -munaligned-symbols,$(KBUILD_CFLAGS_VDSO)) +KBUILD_CFLAGS_VDSO := $(filter-out -fno-asynchronous-unwind-tables,$(KBUILD_CFLAGS_VDSO)) +KBUILD_CFLAGS_VDSO += -m64 -fPIC -fno-common -fno-builtin -fasynchronous-unwind-tables +KBUILD_CFLAGS_VDSO += -fno-stack-protector +ldflags-y := -shared -soname=linux-vdso.so.1 \ + --hash-style=both --build-id=sha1 -T + +$(targets:%=$(obj)/%.dbg): KBUILD_CFLAGS = $(KBUILD_CFLAGS_VDSO) +$(targets:%=$(obj)/%.dbg): KBUILD_AFLAGS = $(KBUILD_AFLAGS_VDSO) + +obj-y += vdso_wrapper.o +targets += vdso.lds +CPPFLAGS_vdso.lds += -P -C -U$(ARCH) + +# Force dependency (incbin is bad) +$(obj)/vdso_wrapper.o : $(obj)/vdso.so + +quiet_cmd_vdso_and_check = VDSO $@ + cmd_vdso_and_check = $(cmd_ld); $(cmd_vdso_check) + +# link rule for the .so file, .lds has to be first +$(obj)/vdso.so.dbg: $(obj)/vdso.lds $(obj-vdso) $(obj-cvdso) FORCE + $(call if_changed,vdso_and_check) + +# strip rule for the .so file +$(obj)/%.so: OBJCOPYFLAGS := -S +$(obj)/%.so: $(obj)/%.so.dbg FORCE + $(call if_changed,objcopy) + +# assembly rules for the .S files +$(obj-vdso): %.o: %.S FORCE + $(call if_changed_dep,vdsoas) + +$(obj-cvdso): %.o: %.c FORCE + $(call if_changed_dep,vdsocc) + +# actual build commands +quiet_cmd_vdsoas = VDSOA $@ + cmd_vdsoas = $(CC) $(a_flags) -c -o $@ $< +quiet_cmd_vdsocc = VDSOC $@ + cmd_vdsocc = $(CC) $(c_flags) -c -o $@ $< + +# Generate VDSO offsets using helper script +gen-vdsosym := $(src)/gen_vdso_offsets.sh +quiet_cmd_vdsosym = VDSOSYM $@ + cmd_vdsosym = $(NM) $< | $(gen-vdsosym) | LC_ALL=C sort > $@ + +include/generated/vdso-offsets.h: $(obj)/vdso.so.dbg FORCE + $(call if_changed,vdsosym) diff --git a/arch/s390/kernel/vdso/gen_vdso_offsets.sh b/arch/s390/kernel/vdso/gen_vdso_offsets.sh new file mode 100755 index 000000000000..359982fb002d --- /dev/null +++ b/arch/s390/kernel/vdso/gen_vdso_offsets.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +# +# Match symbols in the DSO that look like VDSO_*; produce a header file +# of constant offsets into the shared object. +# +# Doing this inside the Makefile will break the $(filter-out) function, +# causing Kbuild to rebuild the vdso-offsets header file every time. +# +# Inspired by arm64 version. +# + +LC_ALL=C +sed -n 's/\([0-9a-f]*\) . __kernel_\(.*\)/\#define vdso_offset_\2\t0x\1/p' diff --git a/arch/s390/kernel/vdso/getcpu.c b/arch/s390/kernel/vdso/getcpu.c new file mode 100644 index 000000000000..5c5d4a848b76 --- /dev/null +++ b/arch/s390/kernel/vdso/getcpu.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright IBM Corp. 2020 */ + +#include +#include +#include +#include "vdso.h" + +int __s390_vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused) +{ + union tod_clock clk; + + /* CPU number is stored in the programmable field of the TOD clock */ + store_tod_clock_ext(&clk); + if (cpu) + *cpu = clk.pf; + /* NUMA node is always zero */ + if (node) + *node = 0; + return 0; +} diff --git a/arch/s390/kernel/vdso/note.S b/arch/s390/kernel/vdso/note.S new file mode 100644 index 000000000000..db19d0680a0a --- /dev/null +++ b/arch/s390/kernel/vdso/note.S @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This supplies .note.* sections to go into the PT_NOTE inside the vDSO text. + * Here we can supply some information useful to userland. + */ + +#include +#include +#include + +ELFNOTE_START(Linux, 0, "a") + .long LINUX_VERSION_CODE +ELFNOTE_END diff --git a/arch/s390/kernel/vdso/vdso.h b/arch/s390/kernel/vdso/vdso.h new file mode 100644 index 000000000000..8cff033dd854 --- /dev/null +++ b/arch/s390/kernel/vdso/vdso.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ARCH_S390_KERNEL_VDSO_VDSO_H +#define __ARCH_S390_KERNEL_VDSO_VDSO_H + +#include + +struct getcpu_cache; + +int __s390_vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused); +int __s390_vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz); +int __s390_vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts); +int __s390_vdso_clock_getres(clockid_t clock, struct __kernel_timespec *ts); +ssize_t __kernel_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state, size_t opaque_len); + +#endif /* __ARCH_S390_KERNEL_VDSO_VDSO_H */ diff --git a/arch/s390/kernel/vdso/vdso.lds.S b/arch/s390/kernel/vdso/vdso.lds.S new file mode 100644 index 000000000000..7bec4de0e8e0 --- /dev/null +++ b/arch/s390/kernel/vdso/vdso.lds.S @@ -0,0 +1,113 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This is the infamous ld script for the 64 bits vdso + * library + */ + +#include +#include +#include +#include +#include + +OUTPUT_FORMAT("elf64-s390", "elf64-s390", "elf64-s390") +OUTPUT_ARCH(s390:64-bit) + +SECTIONS +{ + VDSO_VVAR_SYMS + + . = SIZEOF_HEADERS; + + .hash : { *(.hash) } :text + .gnu.hash : { *(.gnu.hash) } + .dynsym : { *(.dynsym) } + .dynstr : { *(.dynstr) } + .gnu.version : { *(.gnu.version) } + .gnu.version_d : { *(.gnu.version_d) } + .gnu.version_r : { *(.gnu.version_r) } + + .note : { *(.note.*) } :text :note + + . = ALIGN(16); + .text : { + *(.text .stub .text.* .gnu.linkonce.t.*) + } :text + PROVIDE(__etext = .); + PROVIDE(_etext = .); + PROVIDE(etext = .); + + /* + * Other stuff is appended to the text segment: + */ + .rodata : { *(.rodata .rodata.* .gnu.linkonce.r.*) } + .rodata1 : { *(.rodata1) } + + . = ALIGN(8); + .altinstructions : { *(.altinstructions) } + .altinstr_replacement : { *(.altinstr_replacement) } + + .dynamic : { *(.dynamic) } :text :dynamic + + .eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr + .eh_frame : { KEEP (*(.eh_frame)) } :text + .gcc_except_table : { *(.gcc_except_table .gcc_except_table.*) } + + .rela.dyn ALIGN(8) : { *(.rela.dyn) } + .got ALIGN(8) : { *(.got .toc) } + .got.plt ALIGN(8) : { *(.got.plt) } + + _end = .; + PROVIDE(end = .); + + STABS_DEBUG + DWARF_DEBUG + .comment 0 : { *(.comment) } + .gnu.attributes 0 : { KEEP (*(.gnu.attributes)) } + + /DISCARD/ : { + *(.note.GNU-stack) + *(.branch_lt) + *(.data .data.* .gnu.linkonce.d.* .sdata*) + *(.bss .sbss .dynbss .dynsbss) + } +} + +/* + * Very old versions of ld do not recognize this name token; use the constant. + */ +#define PT_GNU_EH_FRAME 0x6474e550 + +/* + * We must supply the ELF program headers explicitly to get just one + * PT_LOAD segment, and set the flags explicitly to make segments read-only. + */ +PHDRS +{ + text PT_LOAD FILEHDR PHDRS FLAGS(5); /* PF_R|PF_X */ + dynamic PT_DYNAMIC FLAGS(4); /* PF_R */ + note PT_NOTE FLAGS(4); /* PF_R */ + eh_frame_hdr PT_GNU_EH_FRAME; +} + +/* + * This controls what symbols we export from the DSO. + */ +VERSION +{ + VDSO_VERSION_STRING { + global: + /* + * Has to be there for the kernel to find + */ + __kernel_gettimeofday; + __kernel_clock_gettime; + __kernel_clock_getres; + __kernel_getcpu; + __kernel_restart_syscall; + __kernel_rt_sigreturn; + __kernel_sigreturn; + __kernel_getrandom; + local: *; + }; +} diff --git a/arch/s390/kernel/vdso/vdso_generic.c b/arch/s390/kernel/vdso/vdso_generic.c new file mode 100644 index 000000000000..a9aa75643c08 --- /dev/null +++ b/arch/s390/kernel/vdso/vdso_generic.c @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "../../../../lib/vdso/gettimeofday.c" +#include "vdso.h" + +int __s390_vdso_gettimeofday(struct __kernel_old_timeval *tv, + struct timezone *tz) +{ + return __cvdso_gettimeofday(tv, tz); +} + +int __s390_vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) +{ + return __cvdso_clock_gettime(clock, ts); +} + +int __s390_vdso_clock_getres(clockid_t clock, struct __kernel_timespec *ts) +{ + return __cvdso_clock_getres(clock, ts); +} diff --git a/arch/s390/kernel/vdso/vdso_user_wrapper.S b/arch/s390/kernel/vdso/vdso_user_wrapper.S new file mode 100644 index 000000000000..aa06c85bcbd3 --- /dev/null +++ b/arch/s390/kernel/vdso/vdso_user_wrapper.S @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include +#include +#include +#include + +/* + * Older glibc version called vdso without allocating a stackframe. This wrapper + * is just used to allocate a stackframe. See + * https://sourceware.org/git/?p=glibc.git;a=commit;h=478593e6374f3818da39332260dc453cb19cfa1e + * for details. + */ +.macro vdso_func func +SYM_FUNC_START(__kernel_\func) + CFI_STARTPROC + aghi %r15,-STACK_FRAME_VDSO_OVERHEAD + CFI_DEF_CFA_OFFSET (STACK_FRAME_USER_OVERHEAD + STACK_FRAME_VDSO_OVERHEAD) + CFI_VAL_OFFSET 15,-STACK_FRAME_USER_OVERHEAD + stg %r14,__SFVDSO_RETURN_ADDRESS(%r15) + CFI_REL_OFFSET 14,__SFVDSO_RETURN_ADDRESS + xc __SFUSER_BACKCHAIN(8,%r15),__SFUSER_BACKCHAIN(%r15) + brasl %r14,__s390_vdso_\func + lg %r14,__SFVDSO_RETURN_ADDRESS(%r15) + CFI_RESTORE 14 + aghi %r15,STACK_FRAME_VDSO_OVERHEAD + CFI_DEF_CFA_OFFSET STACK_FRAME_USER_OVERHEAD + CFI_RESTORE 15 + br %r14 + CFI_ENDPROC +SYM_FUNC_END(__kernel_\func) +.endm + +vdso_func gettimeofday +vdso_func clock_getres +vdso_func clock_gettime +vdso_func getcpu + +.macro vdso_syscall func,syscall +SYM_FUNC_START(__kernel_\func) + CFI_STARTPROC + svc \syscall + /* Make sure we notice when a syscall returns, which shouldn't happen */ + .word 0 + CFI_ENDPROC +SYM_FUNC_END(__kernel_\func) +.endm + +vdso_syscall restart_syscall,__NR_restart_syscall +vdso_syscall sigreturn,__NR_sigreturn +vdso_syscall rt_sigreturn,__NR_rt_sigreturn diff --git a/arch/s390/kernel/vdso/vdso_wrapper.S b/arch/s390/kernel/vdso/vdso_wrapper.S new file mode 100644 index 000000000000..f69e62a14978 --- /dev/null +++ b/arch/s390/kernel/vdso/vdso_wrapper.S @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include + + __PAGE_ALIGNED_DATA + + .globl vdso_start, vdso_end + .balign PAGE_SIZE +vdso_start: + .incbin "arch/s390/kernel/vdso/vdso.so" + .balign PAGE_SIZE +vdso_end: + + .previous diff --git a/arch/s390/kernel/vdso/vgetrandom-chacha.S b/arch/s390/kernel/vdso/vgetrandom-chacha.S new file mode 100644 index 000000000000..09c034c2f853 --- /dev/null +++ b/arch/s390/kernel/vdso/vgetrandom-chacha.S @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include +#include +#include +#include +#include + +#define STATE0 %v0 +#define STATE1 %v1 +#define STATE2 %v2 +#define STATE3 %v3 +#define COPY0 %v4 +#define COPY1 %v5 +#define COPY2 %v6 +#define COPY3 %v7 +#define BEPERM %v19 +#define TMP0 %v20 +#define TMP1 %v21 +#define TMP2 %v22 +#define TMP3 %v23 + + .section .rodata + + .balign 32 +SYM_DATA_START_LOCAL(chacha20_constants) + .long 0x61707865,0x3320646e,0x79622d32,0x6b206574 # endian-neutral + .long 0x03020100,0x07060504,0x0b0a0908,0x0f0e0d0c # byte swap +SYM_DATA_END(chacha20_constants) + + .text +/* + * s390 ChaCha20 implementation meant for vDSO. Produces a given positive + * number of blocks of output with nonce 0, taking an input key and 8-bytes + * counter. Does not spill to the stack. + * + * void __arch_chacha20_blocks_nostack(uint8_t *dst_bytes, + * const uint8_t *key, + * uint32_t *counter, + * size_t nblocks) + */ +SYM_FUNC_START(__arch_chacha20_blocks_nostack) + CFI_STARTPROC + larl %r1,chacha20_constants + + /* COPY0 = "expand 32-byte k" */ + VL COPY0,0,,%r1 + + /* BEPERM = byte selectors for VPERM */ + ALTERNATIVE __stringify(VL BEPERM,16,,%r1), "brcl 0,0", ALT_FACILITY(148) + + /* COPY1,COPY2 = key */ + VLM COPY1,COPY2,0,%r3 + + /* COPY3 = counter || zero nonce */ + lg %r3,0(%r4) + VZERO COPY3 + VLVGG COPY3,%r3,0 + + lghi %r1,0 +.Lblock: + VLR STATE0,COPY0 + VLR STATE1,COPY1 + VLR STATE2,COPY2 + VLR STATE3,COPY3 + + lghi %r0,10 +.Ldoubleround: + /* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 16) */ + VAF STATE0,STATE0,STATE1 + VX STATE3,STATE3,STATE0 + VERLLF STATE3,STATE3,16 + + /* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 12) */ + VAF STATE2,STATE2,STATE3 + VX STATE1,STATE1,STATE2 + VERLLF STATE1,STATE1,12 + + /* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 8) */ + VAF STATE0,STATE0,STATE1 + VX STATE3,STATE3,STATE0 + VERLLF STATE3,STATE3,8 + + /* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 7) */ + VAF STATE2,STATE2,STATE3 + VX STATE1,STATE1,STATE2 + VERLLF STATE1,STATE1,7 + + /* STATE1[0,1,2,3] = STATE1[1,2,3,0] */ + VSLDB STATE1,STATE1,STATE1,4 + /* STATE2[0,1,2,3] = STATE2[2,3,0,1] */ + VSLDB STATE2,STATE2,STATE2,8 + /* STATE3[0,1,2,3] = STATE3[3,0,1,2] */ + VSLDB STATE3,STATE3,STATE3,12 + + /* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 16) */ + VAF STATE0,STATE0,STATE1 + VX STATE3,STATE3,STATE0 + VERLLF STATE3,STATE3,16 + + /* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 12) */ + VAF STATE2,STATE2,STATE3 + VX STATE1,STATE1,STATE2 + VERLLF STATE1,STATE1,12 + + /* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 8) */ + VAF STATE0,STATE0,STATE1 + VX STATE3,STATE3,STATE0 + VERLLF STATE3,STATE3,8 + + /* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 7) */ + VAF STATE2,STATE2,STATE3 + VX STATE1,STATE1,STATE2 + VERLLF STATE1,STATE1,7 + + /* STATE1[0,1,2,3] = STATE1[3,0,1,2] */ + VSLDB STATE1,STATE1,STATE1,12 + /* STATE2[0,1,2,3] = STATE2[2,3,0,1] */ + VSLDB STATE2,STATE2,STATE2,8 + /* STATE3[0,1,2,3] = STATE3[1,2,3,0] */ + VSLDB STATE3,STATE3,STATE3,4 + brctg %r0,.Ldoubleround + + /* OUTPUT0 = STATE0 + COPY0 */ + VAF STATE0,STATE0,COPY0 + /* OUTPUT1 = STATE1 + COPY1 */ + VAF STATE1,STATE1,COPY1 + /* OUTPUT2 = STATE2 + COPY2 */ + VAF STATE2,STATE2,COPY2 + /* OUTPUT3 = STATE3 + COPY3 */ + VAF STATE3,STATE3,COPY3 + + ALTERNATIVE \ + __stringify( \ + /* Convert STATE to little endian and store to OUTPUT */\ + VPERM TMP0,STATE0,STATE0,BEPERM; \ + VPERM TMP1,STATE1,STATE1,BEPERM; \ + VPERM TMP2,STATE2,STATE2,BEPERM; \ + VPERM TMP3,STATE3,STATE3,BEPERM; \ + VSTM TMP0,TMP3,0,%r2), \ + __stringify( \ + /* 32 bit wise little endian store to OUTPUT */ \ + VSTBRF STATE0,0,,%r2; \ + VSTBRF STATE1,16,,%r2; \ + VSTBRF STATE2,32,,%r2; \ + VSTBRF STATE3,48,,%r2; \ + brcl 0,0), \ + ALT_FACILITY(148) + + /* ++COPY3.COUNTER */ + /* alsih %r3,1 */ + .insn rilu,0xcc0a00000000,%r3,1 + alcr %r3,%r1 + VLVGG COPY3,%r3,0 + + /* OUTPUT += 64, --NBLOCKS */ + aghi %r2,64 + brctg %r5,.Lblock + + /* COUNTER = COPY3.COUNTER */ + stg %r3,0(%r4) + + /* Zero out potentially sensitive regs */ + VZERO STATE0 + VZERO STATE1 + VZERO STATE2 + VZERO STATE3 + VZERO COPY1 + VZERO COPY2 + + /* Early exit if TMP0-TMP3 have not been used */ + ALTERNATIVE "nopr", "br %r14", ALT_FACILITY(148) + + VZERO TMP0 + VZERO TMP1 + VZERO TMP2 + VZERO TMP3 + + br %r14 + CFI_ENDPROC +SYM_FUNC_END(__arch_chacha20_blocks_nostack) diff --git a/arch/s390/kernel/vdso/vgetrandom.c b/arch/s390/kernel/vdso/vgetrandom.c new file mode 100644 index 000000000000..b5268b507fb5 --- /dev/null +++ b/arch/s390/kernel/vdso/vgetrandom.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "vdso.h" + +ssize_t __kernel_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state, size_t opaque_len) +{ + if (test_facility(129)) + return __cvdso_getrandom(buffer, len, flags, opaque_state, opaque_len); + if (unlikely(opaque_len == ~0UL && !buffer && !len && !flags)) + return -ENOSYS; + return getrandom_syscall(buffer, len, flags); +} diff --git a/arch/s390/kernel/vdso64/.gitignore b/arch/s390/kernel/vdso64/.gitignore deleted file mode 100644 index 4ec80685fecc..000000000000 --- a/arch/s390/kernel/vdso64/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -vdso64.lds diff --git a/arch/s390/kernel/vdso64/Makefile b/arch/s390/kernel/vdso64/Makefile deleted file mode 100644 index 49ad8dfc7c79..000000000000 --- a/arch/s390/kernel/vdso64/Makefile +++ /dev/null @@ -1,80 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# List of files in the vdso - -# Include the generic Makefile to check the built vdso. -include $(srctree)/lib/vdso/Makefile.include -obj-vdso64 = vdso_user_wrapper.o note.o vgetrandom-chacha.o -obj-cvdso64 = vdso64_generic.o getcpu.o vgetrandom.o -VDSO_CFLAGS_REMOVE := -pg $(CC_FLAGS_FTRACE) $(CC_FLAGS_EXPOLINE) -CFLAGS_REMOVE_getcpu.o = $(VDSO_CFLAGS_REMOVE) -CFLAGS_REMOVE_vgetrandom.o = $(VDSO_CFLAGS_REMOVE) -CFLAGS_REMOVE_vdso64_generic.o = $(VDSO_CFLAGS_REMOVE) - -ifneq ($(c-getrandom-y),) - CFLAGS_vgetrandom.o += -include $(c-getrandom-y) -endif - -# Build rules - -targets := $(obj-vdso64) $(obj-cvdso64) vdso64.so vdso64.so.dbg -obj-vdso64 := $(addprefix $(obj)/, $(obj-vdso64)) -obj-cvdso64 := $(addprefix $(obj)/, $(obj-cvdso64)) - -KBUILD_AFLAGS += -DBUILD_VDSO -KBUILD_CFLAGS += -DBUILD_VDSO -DDISABLE_BRANCH_PROFILING - -KBUILD_AFLAGS_64 := $(filter-out -m64,$(KBUILD_AFLAGS)) -KBUILD_AFLAGS_64 += -m64 - -KBUILD_CFLAGS_64 := $(filter-out -m64,$(KBUILD_CFLAGS)) -KBUILD_CFLAGS_64 := $(filter-out -mpacked-stack,$(KBUILD_CFLAGS_64)) -KBUILD_CFLAGS_64 := $(filter-out -mno-pic-data-is-text-relative,$(KBUILD_CFLAGS_64)) -KBUILD_CFLAGS_64 := $(filter-out -munaligned-symbols,$(KBUILD_CFLAGS_64)) -KBUILD_CFLAGS_64 := $(filter-out -fno-asynchronous-unwind-tables,$(KBUILD_CFLAGS_64)) -KBUILD_CFLAGS_64 += -m64 -fPIC -fno-common -fno-builtin -fasynchronous-unwind-tables -KBUILD_CFLAGS_64 += -fno-stack-protector -ldflags-y := -shared -soname=linux-vdso64.so.1 \ - --hash-style=both --build-id=sha1 -T - -$(targets:%=$(obj)/%.dbg): KBUILD_CFLAGS = $(KBUILD_CFLAGS_64) -$(targets:%=$(obj)/%.dbg): KBUILD_AFLAGS = $(KBUILD_AFLAGS_64) - -obj-y += vdso64_wrapper.o -targets += vdso64.lds -CPPFLAGS_vdso64.lds += -P -C -U$(ARCH) - -# Force dependency (incbin is bad) -$(obj)/vdso64_wrapper.o : $(obj)/vdso64.so - -quiet_cmd_vdso_and_check = VDSO $@ - cmd_vdso_and_check = $(cmd_ld); $(cmd_vdso_check) - -# link rule for the .so file, .lds has to be first -$(obj)/vdso64.so.dbg: $(obj)/vdso64.lds $(obj-vdso64) $(obj-cvdso64) FORCE - $(call if_changed,vdso_and_check) - -# strip rule for the .so file -$(obj)/%.so: OBJCOPYFLAGS := -S -$(obj)/%.so: $(obj)/%.so.dbg FORCE - $(call if_changed,objcopy) - -# assembly rules for the .S files -$(obj-vdso64): %.o: %.S FORCE - $(call if_changed_dep,vdso64as) - -$(obj-cvdso64): %.o: %.c FORCE - $(call if_changed_dep,vdso64cc) - -# actual build commands -quiet_cmd_vdso64as = VDSO64A $@ - cmd_vdso64as = $(CC) $(a_flags) -c -o $@ $< -quiet_cmd_vdso64cc = VDSO64C $@ - cmd_vdso64cc = $(CC) $(c_flags) -c -o $@ $< - -# Generate VDSO offsets using helper script -gen-vdsosym := $(src)/gen_vdso_offsets.sh -quiet_cmd_vdsosym = VDSOSYM $@ - cmd_vdsosym = $(NM) $< | $(gen-vdsosym) | LC_ALL=C sort > $@ - -include/generated/vdso64-offsets.h: $(obj)/vdso64.so.dbg FORCE - $(call if_changed,vdsosym) diff --git a/arch/s390/kernel/vdso64/gen_vdso_offsets.sh b/arch/s390/kernel/vdso64/gen_vdso_offsets.sh deleted file mode 100755 index 37f05cb38dad..000000000000 --- a/arch/s390/kernel/vdso64/gen_vdso_offsets.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: GPL-2.0 - -# -# Match symbols in the DSO that look like VDSO_*; produce a header file -# of constant offsets into the shared object. -# -# Doing this inside the Makefile will break the $(filter-out) function, -# causing Kbuild to rebuild the vdso-offsets header file every time. -# -# Inspired by arm64 version. -# - -LC_ALL=C -sed -n 's/\([0-9a-f]*\) . __kernel_\(.*\)/\#define vdso64_offset_\2\t0x\1/p' diff --git a/arch/s390/kernel/vdso64/getcpu.c b/arch/s390/kernel/vdso64/getcpu.c deleted file mode 100644 index 5c5d4a848b76..000000000000 --- a/arch/s390/kernel/vdso64/getcpu.c +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright IBM Corp. 2020 */ - -#include -#include -#include -#include "vdso.h" - -int __s390_vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused) -{ - union tod_clock clk; - - /* CPU number is stored in the programmable field of the TOD clock */ - store_tod_clock_ext(&clk); - if (cpu) - *cpu = clk.pf; - /* NUMA node is always zero */ - if (node) - *node = 0; - return 0; -} diff --git a/arch/s390/kernel/vdso64/note.S b/arch/s390/kernel/vdso64/note.S deleted file mode 100644 index db19d0680a0a..000000000000 --- a/arch/s390/kernel/vdso64/note.S +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * This supplies .note.* sections to go into the PT_NOTE inside the vDSO text. - * Here we can supply some information useful to userland. - */ - -#include -#include -#include - -ELFNOTE_START(Linux, 0, "a") - .long LINUX_VERSION_CODE -ELFNOTE_END diff --git a/arch/s390/kernel/vdso64/vdso.h b/arch/s390/kernel/vdso64/vdso.h deleted file mode 100644 index 9e5397e7b590..000000000000 --- a/arch/s390/kernel/vdso64/vdso.h +++ /dev/null @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __ARCH_S390_KERNEL_VDSO64_VDSO_H -#define __ARCH_S390_KERNEL_VDSO64_VDSO_H - -#include - -struct getcpu_cache; - -int __s390_vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused); -int __s390_vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz); -int __s390_vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts); -int __s390_vdso_clock_getres(clockid_t clock, struct __kernel_timespec *ts); -ssize_t __kernel_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state, size_t opaque_len); - -#endif /* __ARCH_S390_KERNEL_VDSO64_VDSO_H */ diff --git a/arch/s390/kernel/vdso64/vdso64.lds.S b/arch/s390/kernel/vdso64/vdso64.lds.S deleted file mode 100644 index 7bec4de0e8e0..000000000000 --- a/arch/s390/kernel/vdso64/vdso64.lds.S +++ /dev/null @@ -1,113 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * This is the infamous ld script for the 64 bits vdso - * library - */ - -#include -#include -#include -#include -#include - -OUTPUT_FORMAT("elf64-s390", "elf64-s390", "elf64-s390") -OUTPUT_ARCH(s390:64-bit) - -SECTIONS -{ - VDSO_VVAR_SYMS - - . = SIZEOF_HEADERS; - - .hash : { *(.hash) } :text - .gnu.hash : { *(.gnu.hash) } - .dynsym : { *(.dynsym) } - .dynstr : { *(.dynstr) } - .gnu.version : { *(.gnu.version) } - .gnu.version_d : { *(.gnu.version_d) } - .gnu.version_r : { *(.gnu.version_r) } - - .note : { *(.note.*) } :text :note - - . = ALIGN(16); - .text : { - *(.text .stub .text.* .gnu.linkonce.t.*) - } :text - PROVIDE(__etext = .); - PROVIDE(_etext = .); - PROVIDE(etext = .); - - /* - * Other stuff is appended to the text segment: - */ - .rodata : { *(.rodata .rodata.* .gnu.linkonce.r.*) } - .rodata1 : { *(.rodata1) } - - . = ALIGN(8); - .altinstructions : { *(.altinstructions) } - .altinstr_replacement : { *(.altinstr_replacement) } - - .dynamic : { *(.dynamic) } :text :dynamic - - .eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr - .eh_frame : { KEEP (*(.eh_frame)) } :text - .gcc_except_table : { *(.gcc_except_table .gcc_except_table.*) } - - .rela.dyn ALIGN(8) : { *(.rela.dyn) } - .got ALIGN(8) : { *(.got .toc) } - .got.plt ALIGN(8) : { *(.got.plt) } - - _end = .; - PROVIDE(end = .); - - STABS_DEBUG - DWARF_DEBUG - .comment 0 : { *(.comment) } - .gnu.attributes 0 : { KEEP (*(.gnu.attributes)) } - - /DISCARD/ : { - *(.note.GNU-stack) - *(.branch_lt) - *(.data .data.* .gnu.linkonce.d.* .sdata*) - *(.bss .sbss .dynbss .dynsbss) - } -} - -/* - * Very old versions of ld do not recognize this name token; use the constant. - */ -#define PT_GNU_EH_FRAME 0x6474e550 - -/* - * We must supply the ELF program headers explicitly to get just one - * PT_LOAD segment, and set the flags explicitly to make segments read-only. - */ -PHDRS -{ - text PT_LOAD FILEHDR PHDRS FLAGS(5); /* PF_R|PF_X */ - dynamic PT_DYNAMIC FLAGS(4); /* PF_R */ - note PT_NOTE FLAGS(4); /* PF_R */ - eh_frame_hdr PT_GNU_EH_FRAME; -} - -/* - * This controls what symbols we export from the DSO. - */ -VERSION -{ - VDSO_VERSION_STRING { - global: - /* - * Has to be there for the kernel to find - */ - __kernel_gettimeofday; - __kernel_clock_gettime; - __kernel_clock_getres; - __kernel_getcpu; - __kernel_restart_syscall; - __kernel_rt_sigreturn; - __kernel_sigreturn; - __kernel_getrandom; - local: *; - }; -} diff --git a/arch/s390/kernel/vdso64/vdso64_generic.c b/arch/s390/kernel/vdso64/vdso64_generic.c deleted file mode 100644 index a9aa75643c08..000000000000 --- a/arch/s390/kernel/vdso64/vdso64_generic.c +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include "../../../../lib/vdso/gettimeofday.c" -#include "vdso.h" - -int __s390_vdso_gettimeofday(struct __kernel_old_timeval *tv, - struct timezone *tz) -{ - return __cvdso_gettimeofday(tv, tz); -} - -int __s390_vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) -{ - return __cvdso_clock_gettime(clock, ts); -} - -int __s390_vdso_clock_getres(clockid_t clock, struct __kernel_timespec *ts) -{ - return __cvdso_clock_getres(clock, ts); -} diff --git a/arch/s390/kernel/vdso64/vdso64_wrapper.S b/arch/s390/kernel/vdso64/vdso64_wrapper.S deleted file mode 100644 index 672184998623..000000000000 --- a/arch/s390/kernel/vdso64/vdso64_wrapper.S +++ /dev/null @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#include -#include -#include - - __PAGE_ALIGNED_DATA - - .globl vdso64_start, vdso64_end - .balign PAGE_SIZE -vdso64_start: - .incbin "arch/s390/kernel/vdso64/vdso64.so" - .balign PAGE_SIZE -vdso64_end: - - .previous diff --git a/arch/s390/kernel/vdso64/vdso_user_wrapper.S b/arch/s390/kernel/vdso64/vdso_user_wrapper.S deleted file mode 100644 index aa06c85bcbd3..000000000000 --- a/arch/s390/kernel/vdso64/vdso_user_wrapper.S +++ /dev/null @@ -1,52 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#include -#include -#include -#include -#include -#include - -/* - * Older glibc version called vdso without allocating a stackframe. This wrapper - * is just used to allocate a stackframe. See - * https://sourceware.org/git/?p=glibc.git;a=commit;h=478593e6374f3818da39332260dc453cb19cfa1e - * for details. - */ -.macro vdso_func func -SYM_FUNC_START(__kernel_\func) - CFI_STARTPROC - aghi %r15,-STACK_FRAME_VDSO_OVERHEAD - CFI_DEF_CFA_OFFSET (STACK_FRAME_USER_OVERHEAD + STACK_FRAME_VDSO_OVERHEAD) - CFI_VAL_OFFSET 15,-STACK_FRAME_USER_OVERHEAD - stg %r14,__SFVDSO_RETURN_ADDRESS(%r15) - CFI_REL_OFFSET 14,__SFVDSO_RETURN_ADDRESS - xc __SFUSER_BACKCHAIN(8,%r15),__SFUSER_BACKCHAIN(%r15) - brasl %r14,__s390_vdso_\func - lg %r14,__SFVDSO_RETURN_ADDRESS(%r15) - CFI_RESTORE 14 - aghi %r15,STACK_FRAME_VDSO_OVERHEAD - CFI_DEF_CFA_OFFSET STACK_FRAME_USER_OVERHEAD - CFI_RESTORE 15 - br %r14 - CFI_ENDPROC -SYM_FUNC_END(__kernel_\func) -.endm - -vdso_func gettimeofday -vdso_func clock_getres -vdso_func clock_gettime -vdso_func getcpu - -.macro vdso_syscall func,syscall -SYM_FUNC_START(__kernel_\func) - CFI_STARTPROC - svc \syscall - /* Make sure we notice when a syscall returns, which shouldn't happen */ - .word 0 - CFI_ENDPROC -SYM_FUNC_END(__kernel_\func) -.endm - -vdso_syscall restart_syscall,__NR_restart_syscall -vdso_syscall sigreturn,__NR_sigreturn -vdso_syscall rt_sigreturn,__NR_rt_sigreturn diff --git a/arch/s390/kernel/vdso64/vgetrandom-chacha.S b/arch/s390/kernel/vdso64/vgetrandom-chacha.S deleted file mode 100644 index 09c034c2f853..000000000000 --- a/arch/s390/kernel/vdso64/vgetrandom-chacha.S +++ /dev/null @@ -1,181 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -#include -#include -#include -#include -#include - -#define STATE0 %v0 -#define STATE1 %v1 -#define STATE2 %v2 -#define STATE3 %v3 -#define COPY0 %v4 -#define COPY1 %v5 -#define COPY2 %v6 -#define COPY3 %v7 -#define BEPERM %v19 -#define TMP0 %v20 -#define TMP1 %v21 -#define TMP2 %v22 -#define TMP3 %v23 - - .section .rodata - - .balign 32 -SYM_DATA_START_LOCAL(chacha20_constants) - .long 0x61707865,0x3320646e,0x79622d32,0x6b206574 # endian-neutral - .long 0x03020100,0x07060504,0x0b0a0908,0x0f0e0d0c # byte swap -SYM_DATA_END(chacha20_constants) - - .text -/* - * s390 ChaCha20 implementation meant for vDSO. Produces a given positive - * number of blocks of output with nonce 0, taking an input key and 8-bytes - * counter. Does not spill to the stack. - * - * void __arch_chacha20_blocks_nostack(uint8_t *dst_bytes, - * const uint8_t *key, - * uint32_t *counter, - * size_t nblocks) - */ -SYM_FUNC_START(__arch_chacha20_blocks_nostack) - CFI_STARTPROC - larl %r1,chacha20_constants - - /* COPY0 = "expand 32-byte k" */ - VL COPY0,0,,%r1 - - /* BEPERM = byte selectors for VPERM */ - ALTERNATIVE __stringify(VL BEPERM,16,,%r1), "brcl 0,0", ALT_FACILITY(148) - - /* COPY1,COPY2 = key */ - VLM COPY1,COPY2,0,%r3 - - /* COPY3 = counter || zero nonce */ - lg %r3,0(%r4) - VZERO COPY3 - VLVGG COPY3,%r3,0 - - lghi %r1,0 -.Lblock: - VLR STATE0,COPY0 - VLR STATE1,COPY1 - VLR STATE2,COPY2 - VLR STATE3,COPY3 - - lghi %r0,10 -.Ldoubleround: - /* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 16) */ - VAF STATE0,STATE0,STATE1 - VX STATE3,STATE3,STATE0 - VERLLF STATE3,STATE3,16 - - /* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 12) */ - VAF STATE2,STATE2,STATE3 - VX STATE1,STATE1,STATE2 - VERLLF STATE1,STATE1,12 - - /* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 8) */ - VAF STATE0,STATE0,STATE1 - VX STATE3,STATE3,STATE0 - VERLLF STATE3,STATE3,8 - - /* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 7) */ - VAF STATE2,STATE2,STATE3 - VX STATE1,STATE1,STATE2 - VERLLF STATE1,STATE1,7 - - /* STATE1[0,1,2,3] = STATE1[1,2,3,0] */ - VSLDB STATE1,STATE1,STATE1,4 - /* STATE2[0,1,2,3] = STATE2[2,3,0,1] */ - VSLDB STATE2,STATE2,STATE2,8 - /* STATE3[0,1,2,3] = STATE3[3,0,1,2] */ - VSLDB STATE3,STATE3,STATE3,12 - - /* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 16) */ - VAF STATE0,STATE0,STATE1 - VX STATE3,STATE3,STATE0 - VERLLF STATE3,STATE3,16 - - /* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 12) */ - VAF STATE2,STATE2,STATE3 - VX STATE1,STATE1,STATE2 - VERLLF STATE1,STATE1,12 - - /* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 8) */ - VAF STATE0,STATE0,STATE1 - VX STATE3,STATE3,STATE0 - VERLLF STATE3,STATE3,8 - - /* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 7) */ - VAF STATE2,STATE2,STATE3 - VX STATE1,STATE1,STATE2 - VERLLF STATE1,STATE1,7 - - /* STATE1[0,1,2,3] = STATE1[3,0,1,2] */ - VSLDB STATE1,STATE1,STATE1,12 - /* STATE2[0,1,2,3] = STATE2[2,3,0,1] */ - VSLDB STATE2,STATE2,STATE2,8 - /* STATE3[0,1,2,3] = STATE3[1,2,3,0] */ - VSLDB STATE3,STATE3,STATE3,4 - brctg %r0,.Ldoubleround - - /* OUTPUT0 = STATE0 + COPY0 */ - VAF STATE0,STATE0,COPY0 - /* OUTPUT1 = STATE1 + COPY1 */ - VAF STATE1,STATE1,COPY1 - /* OUTPUT2 = STATE2 + COPY2 */ - VAF STATE2,STATE2,COPY2 - /* OUTPUT3 = STATE3 + COPY3 */ - VAF STATE3,STATE3,COPY3 - - ALTERNATIVE \ - __stringify( \ - /* Convert STATE to little endian and store to OUTPUT */\ - VPERM TMP0,STATE0,STATE0,BEPERM; \ - VPERM TMP1,STATE1,STATE1,BEPERM; \ - VPERM TMP2,STATE2,STATE2,BEPERM; \ - VPERM TMP3,STATE3,STATE3,BEPERM; \ - VSTM TMP0,TMP3,0,%r2), \ - __stringify( \ - /* 32 bit wise little endian store to OUTPUT */ \ - VSTBRF STATE0,0,,%r2; \ - VSTBRF STATE1,16,,%r2; \ - VSTBRF STATE2,32,,%r2; \ - VSTBRF STATE3,48,,%r2; \ - brcl 0,0), \ - ALT_FACILITY(148) - - /* ++COPY3.COUNTER */ - /* alsih %r3,1 */ - .insn rilu,0xcc0a00000000,%r3,1 - alcr %r3,%r1 - VLVGG COPY3,%r3,0 - - /* OUTPUT += 64, --NBLOCKS */ - aghi %r2,64 - brctg %r5,.Lblock - - /* COUNTER = COPY3.COUNTER */ - stg %r3,0(%r4) - - /* Zero out potentially sensitive regs */ - VZERO STATE0 - VZERO STATE1 - VZERO STATE2 - VZERO STATE3 - VZERO COPY1 - VZERO COPY2 - - /* Early exit if TMP0-TMP3 have not been used */ - ALTERNATIVE "nopr", "br %r14", ALT_FACILITY(148) - - VZERO TMP0 - VZERO TMP1 - VZERO TMP2 - VZERO TMP3 - - br %r14 - CFI_ENDPROC -SYM_FUNC_END(__arch_chacha20_blocks_nostack) diff --git a/arch/s390/kernel/vdso64/vgetrandom.c b/arch/s390/kernel/vdso64/vgetrandom.c deleted file mode 100644 index b5268b507fb5..000000000000 --- a/arch/s390/kernel/vdso64/vgetrandom.c +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include -#include "vdso.h" - -ssize_t __kernel_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state, size_t opaque_len) -{ - if (test_facility(129)) - return __cvdso_getrandom(buffer, len, flags, opaque_state, opaque_len); - if (unlikely(opaque_len == ~0UL && !buffer && !len && !flags)) - return -ENOSYS; - return getrandom_syscall(buffer, len, flags); -} -- cgit v1.2.3