diff options
Diffstat (limited to 'arch/s390')
212 files changed, 12166 insertions, 10229 deletions
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 0e5fad5f06ca..4b51bc6e8948 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -20,6 +20,9 @@ config ARCH_HAS_ILOG2_U64 config ARCH_PROC_KCORE_TEXT def_bool y +config ARCH_HAS_CPU_RELAX + def_bool y + config GENERIC_HWEIGHT def_bool !HAVE_MARCH_Z196_FEATURES @@ -29,12 +32,6 @@ config GENERIC_BUG config GENERIC_BUG_RELATIVE_POINTERS def_bool y -config GENERIC_LOCKBREAK - def_bool y if PREEMPTION - -config PGSTE - def_bool y if KVM - config AUDIT_ARCH def_bool y @@ -69,6 +66,12 @@ config CC_HAS_ASM_AOR_FORMAT_FLAGS Clang versions before 19.1.0 do not support A, O, and R inline assembly format flags. +config CC_HAS_ASM_IMMEDIATE_STRINGS + def_bool !(CC_IS_GCC && GCC_VERSION < 90000) + help + GCC versions before 9.0.0 cannot handle strings as immediate + input operands in inline assemblies. + config CC_HAS_STACKPROTECTOR_GLOBAL def_bool $(cc-option, -mstack-protector-guard=global -mstack-protector-guard-record) @@ -77,14 +80,14 @@ config S390 # # Note: keep this list sorted alphabetically # - imply IMA_SECURE_AND_OR_TRUSTED_BOOT select ALTERNATE_USER_ADDRESS_SPACE select ARCH_32BIT_USTAT_F_TINODE select ARCH_CORRECT_STACKTRACE_ON_KRETPROBE select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM - select ARCH_ENABLE_MEMORY_HOTREMOVE select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE + select ARCH_HAS_CC_CAN_LINK + select ARCH_HAS_CC_PLATFORM select ARCH_HAS_CPU_FINALIZE_INIT select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DEBUG_VIRTUAL @@ -145,12 +148,16 @@ config S390 select ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE select ARCH_STACKWALK select ARCH_SUPPORTS_ATOMIC_RMW + select ARCH_SUPPORTS_CFI select ARCH_SUPPORTS_DEBUG_PAGEALLOC select ARCH_SUPPORTS_HUGETLBFS select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && CC_IS_CLANG select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS + select ARCH_HAS_PTE_PROTNONE select ARCH_SUPPORTS_NUMA_BALANCING + select ARCH_SUPPORTS_PAGE_TABLE_CHECK select ARCH_SUPPORTS_PER_VMA_LOCK + select ARCH_USES_CFI_GENERIC_LLVM_PASS if CC_IS_CLANG select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF select ARCH_USE_SYM_ANNOTATIONS @@ -174,10 +181,11 @@ config S390 select GENERIC_ENTRY select GENERIC_GETTIMEOFDAY select GENERIC_SMP_IDLE_THREAD - select GENERIC_TIME_VSYSCALL select GENERIC_IOREMAP if PCI + select GLOB select HAVE_ALIGNED_STRUCT_PAGE select HAVE_ARCH_AUDITSYSCALL + select HAVE_ARCH_GET_SECUREBOOT select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_JUMP_LABEL_RELATIVE select HAVE_ARCH_KASAN @@ -215,7 +223,6 @@ config S390 select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_TRACER select HAVE_GCC_PLUGINS - select HAVE_GENERIC_VDSO select HAVE_IOREMAP_PROT if PCI select HAVE_KERNEL_BZIP2 select HAVE_KERNEL_GZIP @@ -244,6 +251,7 @@ config S390 select HAVE_RELIABLE_STACKTRACE select HAVE_RETHOOK select HAVE_RSEQ + select HAVE_RUST if !EXPOLINE select HAVE_SAMPLE_FTRACE_DIRECT select HAVE_SAMPLE_FTRACE_DIRECT_MULTI select HAVE_SETUP_PER_CPU_AREA @@ -272,9 +280,13 @@ config S390 select PCI_MSI if PCI select PCI_MSI_ARCH_FALLBACKS if PCI_MSI select PCI_QUIRKS if PCI + select HAS_SEPARATE_PREEMPT_RESCHED_BITS select SPARSE_IRQ select SWIOTLB + select SWIOTLB_DYNAMIC + select DMA_COHERENT_POOL select SYSCTL_EXCEPTION_TRACE + select SYSTEM_DATA_VERIFICATION if KEXEC_SIG select THREAD_INFO_IN_TASK select TRACE_IRQFLAGS_SUPPORT select TTY @@ -294,6 +306,14 @@ config PGTABLE_LEVELS source "kernel/livepatch/Kconfig" +config ARCH_CC_CAN_LINK + bool + default $(cc_can_link_user,-m64) + +config ARCH_USERFLAGS + string + default "-m64" + config ARCH_SUPPORTS_KEXEC def_bool y @@ -301,7 +321,7 @@ config ARCH_SUPPORTS_KEXEC_FILE def_bool y config ARCH_SUPPORTS_KEXEC_SIG - def_bool MODULE_SIG_FORMAT + def_bool y config ARCH_SUPPORTS_KEXEC_PURGATORY def_bool y @@ -565,7 +585,7 @@ config HIPERDISPATCH_ON def_bool y bool "Use hiperdispatch on vertical polarization by default" depends on SCHED_TOPOLOGY - depends on PROC_SYSCTL + depends on SYSCTL help Hiperdispatch aims to improve the CPU scheduler's decision making when using vertical polarization by adjusting CPU @@ -695,6 +715,8 @@ config KERNEL_IMAGE_BASE endmenu +source "drivers/cpuidle/Kconfig" + menu "Memory setup" config ARCH_SPARSEMEM_ENABLE @@ -704,6 +726,10 @@ config ARCH_SPARSEMEM_ENABLE config ARCH_SPARSEMEM_DEFAULT def_bool y +config ILLEGAL_POINTER_VALUE + hex + default 0xdead000000000000 + config MAX_PHYSMEM_BITS int "Maximum size of supported physical memory in bits (42-53)" range 42 53 @@ -879,7 +905,7 @@ config CMM_IUCV config APPLDATA_BASE def_bool n prompt "Linux - VM Monitor Stream, base infrastructure" - depends on PROC_SYSCTL + depends on SYSCTL help This provides a kernel interface for creating and updating z/VM APPLDATA monitor records. The monitor records are updated at certain time @@ -982,9 +1008,8 @@ config S390_MODULES_SANITY_TEST_HELPERS menu "Selftests" config S390_UNWIND_SELFTEST - def_tristate n + def_tristate KUNIT_ALL_TESTS depends on KUNIT - default KUNIT_ALL_TESTS prompt "Test unwind functions" help This option enables s390 specific stack unwinder testing kernel @@ -994,7 +1019,7 @@ config S390_UNWIND_SELFTEST Say N if you are unsure. config S390_KPROBES_SANITY_TEST - def_tristate n + def_tristate KUNIT_ALL_TESTS prompt "Enable s390 specific kprobes tests" depends on KPROBES depends on KUNIT @@ -1006,9 +1031,8 @@ config S390_KPROBES_SANITY_TEST Say N if you are unsure. config S390_MODULES_SANITY_TEST - def_tristate n - depends on KUNIT - default KUNIT_ALL_TESTS + def_tristate KUNIT_ALL_TESTS + depends on KUNIT && m prompt "Enable s390 specific modules tests" select S390_MODULES_SANITY_TEST_HELPERS help diff --git a/arch/s390/Makefile b/arch/s390/Makefile index d78ad6885ca2..8b712cd85fcd 100644 --- a/arch/s390/Makefile +++ b/arch/s390/Makefile @@ -12,8 +12,7 @@ LD_BFD := elf64-s390 KBUILD_LDFLAGS := -m elf64_s390 KBUILD_AFLAGS_MODULE += -fPIC KBUILD_CFLAGS_MODULE += -fPIC -KBUILD_AFLAGS += -m64 -KBUILD_CFLAGS += -m64 +KBUILD_CPPFLAGS += -m64 KBUILD_CFLAGS += -fPIC LDFLAGS_vmlinux := $(call ld-option,-no-pie) extra_tools := relocs @@ -22,7 +21,7 @@ KBUILD_AFLAGS_DECOMPRESSOR := $(CLANG_FLAGS) -m64 -D__ASSEMBLY__ ifndef CONFIG_AS_IS_LLVM KBUILD_AFLAGS_DECOMPRESSOR += $(if $(CONFIG_DEBUG_INFO),$(aflags_dwarf)) endif -KBUILD_CFLAGS_DECOMPRESSOR := $(CLANG_FLAGS) -m64 -O2 -mpacked-stack -std=gnu11 -fms-extensions +KBUILD_CFLAGS_DECOMPRESSOR := $(CLANG_FLAGS) -m64 -O2 -mpacked-stack $(CC_FLAGS_DIALECT) KBUILD_CFLAGS_DECOMPRESSOR += -DDISABLE_BRANCH_PROFILING -D__NO_FORTIFY KBUILD_CFLAGS_DECOMPRESSOR += -D__DECOMPRESSOR KBUILD_CFLAGS_DECOMPRESSOR += -Wno-pointer-sign @@ -35,7 +34,8 @@ KBUILD_CFLAGS_DECOMPRESSOR += $(call cc-disable-warning, address-of-packed-membe KBUILD_CFLAGS_DECOMPRESSOR += $(if $(CONFIG_DEBUG_INFO),-g) KBUILD_CFLAGS_DECOMPRESSOR += $(if $(CONFIG_DEBUG_INFO_DWARF4), $(call cc-option, -gdwarf-4,)) KBUILD_CFLAGS_DECOMPRESSOR += $(if $(CONFIG_CC_NO_ARRAY_BOUNDS),-Wno-array-bounds) -KBUILD_CFLAGS_DECOMPRESSOR += $(if $(CONFIG_CC_IS_CLANG),-Wno-microsoft-anon-tag) + +KBUILD_RUSTFLAGS += --target=s390x-unknown-none-softfloat -Zpacked-stack -Ctarget-feature=+backchain UTS_MACHINE := s390x STACK_SIZE := $(if $(CONFIG_KASAN),65536,$(if $(CONFIG_KMSAN),65536,16384)) @@ -43,19 +43,23 @@ CHECKFLAGS += -D__s390__ -D__s390x__ export LD_BFD -mflags-$(CONFIG_MARCH_Z10) := -march=z10 -mflags-$(CONFIG_MARCH_Z196) := -march=z196 -mflags-$(CONFIG_MARCH_ZEC12) := -march=zEC12 -mflags-$(CONFIG_MARCH_Z13) := -march=z13 -mflags-$(CONFIG_MARCH_Z14) := -march=z14 -mflags-$(CONFIG_MARCH_Z15) := -march=z15 -mflags-$(CONFIG_MARCH_Z16) := -march=z16 -mflags-$(CONFIG_MARCH_Z17) := -march=z17 +march-name-$(CONFIG_MARCH_Z10) := z10 +march-name-$(CONFIG_MARCH_Z196) := z196 +march-name-$(CONFIG_MARCH_ZEC12) := zEC12 +march-name-$(CONFIG_MARCH_Z13) := z13 +march-name-$(CONFIG_MARCH_Z14) := z14 +march-name-$(CONFIG_MARCH_Z15) := z15 +march-name-$(CONFIG_MARCH_Z16) := z16 +march-name-$(CONFIG_MARCH_Z17) := z17 + +mflags := -march=$(march-name-y) + +export CC_FLAGS_MARCH := $(mflags) -export CC_FLAGS_MARCH := $(mflags-y) +aflags-y += $(mflags) +cflags-y += $(mflags) -aflags-y += $(mflags-y) -cflags-y += $(mflags-y) +KBUILD_RUSTFLAGS += -Ctarget-cpu=$(march-name-y) cflags-$(CONFIG_MARCH_Z10_TUNE) += -mtune=z10 cflags-$(CONFIG_MARCH_Z196_TUNE) += -mtune=z196 diff --git a/arch/s390/appldata/Makefile b/arch/s390/appldata/Makefile index b06def4a4f2f..132381f16fb1 100644 --- a/arch/s390/appldata/Makefile +++ b/arch/s390/appldata/Makefile @@ -3,6 +3,8 @@ # Makefile for the Linux - z/VM Monitor Stream. # +CONTEXT_ANALYSIS := y + obj-$(CONFIG_APPLDATA_BASE) += appldata_base.o obj-$(CONFIG_APPLDATA_MEM) += appldata_mem.o obj-$(CONFIG_APPLDATA_OS) += appldata_os.o diff --git a/arch/s390/appldata/appldata_base.c b/arch/s390/appldata/appldata_base.c index feb43db63f30..9cba4633c3f3 100644 --- a/arch/s390/appldata/appldata_base.c +++ b/arch/s390/appldata/appldata_base.c @@ -51,7 +51,6 @@ static int appldata_timer_handler(const struct ctl_table *ctl, int write, static int appldata_interval_handler(const struct ctl_table *ctl, int write, void *buffer, size_t *lenp, loff_t *ppos); -static struct ctl_table_header *appldata_sysctl_header; static const struct ctl_table appldata_table[] = { { .procname = "timer", @@ -140,7 +139,7 @@ int appldata_diag(char record_nr, u16 function, unsigned long buffer, struct appldata_product_id *id; int rc; - parm_list = kmalloc(sizeof(*parm_list), GFP_KERNEL); + parm_list = kmalloc_obj(*parm_list); id = kmemdup(&appldata_id, sizeof(appldata_id), GFP_KERNEL); rc = -ENOMEM; if (parm_list && id) { @@ -350,7 +349,7 @@ int appldata_register_ops(struct appldata_ops *ops) if (ops->size > APPLDATA_MAX_REC_SIZE) return -EINVAL; - ops->ctl_table = kcalloc(1, sizeof(struct ctl_table), GFP_KERNEL); + ops->ctl_table = kzalloc_objs(struct ctl_table, 1); if (!ops->ctl_table) return -ENOMEM; @@ -406,7 +405,7 @@ static int __init appldata_init(void) appldata_wq = alloc_ordered_workqueue("appldata", 0); if (!appldata_wq) return -ENOMEM; - appldata_sysctl_header = register_sysctl(appldata_proc_name, appldata_table); + register_sysctl(appldata_proc_name, appldata_table); return 0; } diff --git a/arch/s390/appldata/appldata_mem.c b/arch/s390/appldata/appldata_mem.c index fc608f9b79ab..659af5545a38 100644 --- a/arch/s390/appldata/appldata_mem.c +++ b/arch/s390/appldata/appldata_mem.c @@ -130,7 +130,7 @@ static int __init appldata_mem_init(void) { int ret; - ops.data = kzalloc(sizeof(struct appldata_mem_data), GFP_KERNEL); + ops.data = kzalloc_obj(struct appldata_mem_data); if (!ops.data) return -ENOMEM; diff --git a/arch/s390/appldata/appldata_net_sum.c b/arch/s390/appldata/appldata_net_sum.c index 59c282ca002f..972f6d592c32 100644 --- a/arch/s390/appldata/appldata_net_sum.c +++ b/arch/s390/appldata/appldata_net_sum.c @@ -132,7 +132,7 @@ static int __init appldata_net_init(void) { int ret; - ops.data = kzalloc(sizeof(struct appldata_net_sum_data), GFP_KERNEL); + ops.data = kzalloc_obj(struct appldata_net_sum_data); if (!ops.data) return -ENOMEM; diff --git a/arch/s390/boot/Makefile b/arch/s390/boot/Makefile index 490167faba7a..10b75e053a6f 100644 --- a/arch/s390/boot/Makefile +++ b/arch/s390/boot/Makefile @@ -21,11 +21,17 @@ KBUILD_AFLAGS := $(filter-out $(CC_FLAGS_MARCH),$(KBUILD_AFLAGS_DECOMPRESSOR)) KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_MARCH),$(KBUILD_CFLAGS_DECOMPRESSOR)) KBUILD_AFLAGS += $(CC_FLAGS_MARCH_MINIMUM) -D__DISABLE_EXPORTS KBUILD_CFLAGS += $(CC_FLAGS_MARCH_MINIMUM) -D__DISABLE_EXPORTS +KBUILD_CFLAGS += $(call cc-option, -Wno-default-const-init-unsafe) CFLAGS_sclp_early_core.o += -I$(srctree)/drivers/s390/char +# string.o implements standard library functions like memset/memcpy etc. +# Use -ffreestanding to ensure that the compiler does not try to "optimize" +# them into calls to themselves. +CFLAGS_string.o = -ffreestanding + obj-y := head.o als.o startup.o physmem_info.o ipl_parm.o ipl_report.o vmem.o -obj-y += string.o ebcdic.o sclp_early_core.o mem.o ipl_vmparm.o cmdline.o +obj-y += string.o ebcdic.o sclp_early_core.o ipl_vmparm.o cmdline.o obj-y += version.o pgm_check.o ctype.o ipl_data.o relocs.o alternative.o obj-y += uv.o printk.o trampoline.o obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o diff --git a/arch/s390/boot/alternative.c b/arch/s390/boot/alternative.c index 19ea7934b918..77e8bad560c5 100644 --- a/arch/s390/boot/alternative.c +++ b/arch/s390/boot/alternative.c @@ -45,11 +45,12 @@ static void alt_debug_modify(int type, unsigned int nr, bool clear) static char *alt_debug_parse(int type, char *str) { - unsigned long val, endval; + unsigned long val, endval, limit; char *endp; bool clear; int i; + limit = type == ALT_TYPE_FACILITY ? MAX_FACILITY_BIT : MAX_MFEATURE_BIT; if (*str == ':') { str++; } else { @@ -73,7 +74,7 @@ static char *alt_debug_parse(int type, char *str) if (str == endp) break; str = endp; - while (val <= endval) { + while (val <= endval && val < limit) { alt_debug_modify(type, val, clear); val++; } diff --git a/arch/s390/boot/ipl_parm.c b/arch/s390/boot/ipl_parm.c index 6bc950b92be7..c1b43e5e688a 100644 --- a/arch/s390/boot/ipl_parm.c +++ b/arch/s390/boot/ipl_parm.c @@ -23,6 +23,7 @@ struct parmarea parmarea __section(".parmarea") = { }; char __bootdata(early_command_line)[COMMAND_LINE_SIZE]; +static char command_line_buf[COMMAND_LINE_SIZE]; unsigned int __bootdata_preserved(zlib_dfltcc_support) = ZLIB_DFLTCC_FULL; struct ipl_parameter_block __bootdata_preserved(ipl_block); @@ -135,31 +136,29 @@ out: static void append_ipl_block_parm(void) { - char *parm, *delim; - size_t len, rc = 0; + size_t len, extra = 0; + char *delim; len = strlen(early_command_line); - - delim = early_command_line + len; /* '\0' character position */ - parm = early_command_line + len + 1; /* append right after '\0' */ + delim = early_command_line + len; /* '\0' character position */ switch (ipl_block.pb0_hdr.pbt) { case IPL_PBT_CCW: - rc = ipl_block_get_ascii_vmparm( - parm, COMMAND_LINE_SIZE - len - 1, &ipl_block); + extra = ipl_block_get_ascii_vmparm(command_line_buf, sizeof(command_line_buf), &ipl_block); break; case IPL_PBT_FCP: case IPL_PBT_NVME: case IPL_PBT_ECKD: - rc = ipl_block_get_ascii_scpdata( - parm, COMMAND_LINE_SIZE - len - 1, &ipl_block); + extra = ipl_block_get_ascii_scpdata(command_line_buf, sizeof(command_line_buf), &ipl_block); break; } - if (rc) { - if (*parm == '=') - memmove(early_command_line, parm + 1, rc); - else + if (extra) { + if (command_line_buf[0] == '=') { + memmove(early_command_line, command_line_buf + 1, extra); + } else if (len < COMMAND_LINE_SIZE - 2) { *delim = ' '; /* replace '\0' with space */ + sized_strscpy(delim + 1, command_line_buf, COMMAND_LINE_SIZE - len - 1); + } } } @@ -231,7 +230,7 @@ static void modify_fac_list(char *str) if (str == endp) break; str = endp; - while (val <= endval) { + while (val <= endval && val < MAX_FACILITY_BIT) { modify_facility(val, clear); val++; } @@ -245,7 +244,6 @@ static void modify_fac_list(char *str) check_cleared_facilities(); } -static char command_line_buf[COMMAND_LINE_SIZE]; void parse_boot_command_line(void) { char *param, *val; diff --git a/arch/s390/boot/mem.S b/arch/s390/boot/mem.S deleted file mode 100644 index b33463633f03..000000000000 --- a/arch/s390/boot/mem.S +++ /dev/null @@ -1,2 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#include "../lib/mem.S" diff --git a/arch/s390/boot/physmem_info.c b/arch/s390/boot/physmem_info.c index 1f2ca5435838..0ebb2174713f 100644 --- a/arch/s390/boot/physmem_info.c +++ b/arch/s390/boot/physmem_info.c @@ -141,7 +141,7 @@ static int tprot(unsigned long addr) static unsigned long search_mem_end(void) { - unsigned long range = 1 << (MAX_PHYSMEM_BITS - 20); /* in 1MB blocks */ + unsigned long range = 1UL << (MAX_PHYSMEM_BITS - 20); /* in 1MB blocks */ unsigned long offset = 0; unsigned long pivot; diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index f77067dfc2a8..c1dcf8b1b579 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -336,6 +336,7 @@ static unsigned long setup_kernel_memory_layout(unsigned long kernel_size) BUILD_BUG_ON(!IS_ALIGNED(TEXT_OFFSET, THREAD_SIZE)); BUILD_BUG_ON(!IS_ALIGNED(__NO_KASLR_START_KERNEL, THREAD_SIZE)); BUILD_BUG_ON(__NO_KASLR_END_KERNEL > _REGION1_SIZE); + BUILD_BUG_ON(CONFIG_ILLEGAL_POINTER_VALUE < _REGION1_SIZE); vsize = get_vmem_size(ident_map_size, vmemmap_size, vmalloc_size, _REGION3_SIZE); boot_debug("vmem size estimated: 0x%016lx\n", vsize); if (IS_ENABLED(CONFIG_KASAN) || __NO_KASLR_END_KERNEL > _REGION2_SIZE || @@ -439,7 +440,8 @@ static unsigned long setup_kernel_memory_layout(unsigned long kernel_size) max_mappable = max(ident_map_size, MAX_DCSS_ADDR); max_mappable = min(max_mappable, vmemmap_start); #ifdef CONFIG_RANDOMIZE_IDENTITY_BASE - __identity_base = round_down(vmemmap_start - max_mappable, rte_size); + if (kaslr_enabled()) + __identity_base = round_down(vmemmap_start - max_mappable, rte_size); #endif boot_debug("identity map: 0x%016lx-0x%016lx\n", __identity_base, __identity_base + ident_map_size); diff --git a/arch/s390/boot/string.c b/arch/s390/boot/string.c index bd68161434a6..e4ad196cb720 100644 --- a/arch/s390/boot/string.c +++ b/arch/s390/boot/string.c @@ -43,11 +43,7 @@ ssize_t sized_strscpy(char *dst, const char *src, size_t count) void *memset64(uint64_t *s, uint64_t v, size_t count) { - uint64_t *xs = s; - - while (count--) - *xs++ = v; - return s; + return __memset64(s, v, count * sizeof(v)); } char *skip_spaces(const char *str) diff --git a/arch/s390/boot/vmem.c b/arch/s390/boot/vmem.c index 7d6cc4c85af0..ff6d58a476ba 100644 --- a/arch/s390/boot/vmem.c +++ b/arch/s390/boot/vmem.c @@ -338,7 +338,7 @@ static void pgtable_pte_populate(pmd_t *pmd, unsigned long addr, unsigned long e pte = pte_offset_kernel(pmd, addr); for (; addr < end; addr += PAGE_SIZE, pte++) { - if (pte_none(*pte)) { + if (pte_none(ptep_get(pte))) { if (kasan_pte_populate_zero_shadow(pte, mode)) continue; entry = __pte(resolve_pa_may_alloc(addr, PAGE_SIZE, mode)); @@ -355,26 +355,27 @@ static void pgtable_pmd_populate(pud_t *pud, unsigned long addr, unsigned long e enum populate_mode mode) { unsigned long pa, next, pages = 0; - pmd_t *pmd, entry; + pmd_t *pmd, entry, large_entry; pte_t *pte; pmd = pmd_offset(pud, addr); for (; addr < end; addr = next, pmd++) { next = pmd_addr_end(addr, end); - if (pmd_none(*pmd)) { + entry = pmdp_get(pmd); + if (pmd_none(entry)) { if (kasan_pmd_populate_zero_shadow(pmd, addr, next, mode)) continue; pa = try_get_large_pmd_pa(pmd, addr, next, mode); if (pa != INVALID_PHYS_ADDR) { - entry = __pmd(pa); - entry = set_pmd_bit(entry, SEGMENT_KERNEL); - set_pmd(pmd, entry); + large_entry = __pmd(pa); + large_entry = set_pmd_bit(large_entry, SEGMENT_KERNEL); + set_pmd(pmd, large_entry); pages++; continue; } pte = boot_pte_alloc(); pmd_populate(&init_mm, pmd, pte); - } else if (pmd_leaf(*pmd)) { + } else if (pmd_leaf(entry)) { continue; } pgtable_pte_populate(pmd, addr, next, mode); @@ -387,26 +388,27 @@ static void pgtable_pud_populate(p4d_t *p4d, unsigned long addr, unsigned long e enum populate_mode mode) { unsigned long pa, next, pages = 0; - pud_t *pud, entry; + pud_t *pud, entry, large_entry; pmd_t *pmd; pud = pud_offset(p4d, addr); for (; addr < end; addr = next, pud++) { next = pud_addr_end(addr, end); - if (pud_none(*pud)) { + entry = pudp_get(pud); + if (pud_none(entry)) { if (kasan_pud_populate_zero_shadow(pud, addr, next, mode)) continue; pa = try_get_large_pud_pa(pud, addr, next, mode); if (pa != INVALID_PHYS_ADDR) { - entry = __pud(pa); - entry = set_pud_bit(entry, REGION3_KERNEL); - set_pud(pud, entry); + large_entry = __pud(pa); + large_entry = set_pud_bit(large_entry, REGION3_KERNEL); + set_pud(pud, large_entry); pages++; continue; } pmd = boot_crst_alloc(_SEGMENT_ENTRY_EMPTY); pud_populate(&init_mm, pud, pmd); - } else if (pud_leaf(*pud)) { + } else if (pud_leaf(entry)) { continue; } pgtable_pmd_populate(pud, addr, next, mode); @@ -425,7 +427,7 @@ static void pgtable_p4d_populate(pgd_t *pgd, unsigned long addr, unsigned long e p4d = p4d_offset(pgd, addr); for (; addr < end; addr = next, p4d++) { next = p4d_addr_end(addr, end); - if (p4d_none(*p4d)) { + if (p4d_none(p4dp_get(p4d))) { if (kasan_p4d_populate_zero_shadow(p4d, addr, next, mode)) continue; pud = boot_crst_alloc(_REGION3_ENTRY_EMPTY); @@ -451,7 +453,7 @@ static void pgtable_populate(unsigned long addr, unsigned long end, enum populat pgd = pgd_offset(&init_mm, addr); for (; addr < end; addr = next, pgd++) { next = pgd_addr_end(addr, end); - if (pgd_none(*pgd)) { + if (pgd_none(pgdp_get(pgd))) { if (kasan_pgd_populate_zero_shadow(pgd, addr, next, mode)) continue; p4d = boot_crst_alloc(_REGION2_ENTRY_EMPTY); diff --git a/arch/s390/boot/vmlinux.lds.S b/arch/s390/boot/vmlinux.lds.S index 070bc18babd0..d44964592541 100644 --- a/arch/s390/boot/vmlinux.lds.S +++ b/arch/s390/boot/vmlinux.lds.S @@ -31,6 +31,7 @@ SECTIONS _text = .; /* Text */ *(.text) *(.text.*) + *(.noinstr.text) INIT_TEXT _etext = . ; } diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index 0713914b25b4..3dae71474333 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -29,6 +29,8 @@ CONFIG_CGROUP_PIDS=y CONFIG_CGROUP_RDMA=y CONFIG_CGROUP_FREEZER=y CONFIG_CGROUP_HUGETLB=y +CONFIG_CPU_IDLE=y +CONFIG_CPU_IDLE_GOV_TEO=y CONFIG_CPUSETS=y CONFIG_CGROUP_DEVICE=y CONFIG_CGROUP_CPUACCT=y @@ -56,6 +58,10 @@ CONFIG_EXPOLINE_AUTO=y CONFIG_CHSC_SCH=y CONFIG_VFIO_CCW=m CONFIG_VFIO_AP=m +CONFIG_VFIO_DEVICE_CDEV=y +CONFIG_IOMMUFD_DRIVER=y +CONFIG_IOMMUFD_DRIVER_CORE=y +CONFIG_IOMMUFD=y CONFIG_CMM=m CONFIG_APPLDATA_BASE=y CONFIG_S390_HYPFS_FS=y @@ -121,7 +127,6 @@ CONFIG_UNIX=y CONFIG_UNIX_DIAG=m CONFIG_TLS=m CONFIG_TLS_DEVICE=y -CONFIG_TLS_TOE=y CONFIG_XFRM_USER=m CONFIG_NET_KEY=m CONFIG_SMC=m @@ -446,6 +451,7 @@ CONFIG_BLK_DEV_RAM_SIZE=32768 CONFIG_VIRTIO_BLK=y CONFIG_BLK_DEV_RBD=m CONFIG_BLK_DEV_NVME=m +CONFIG_BLK_DEV_NULL_BLK=m CONFIG_ENCLOSURE_SERVICES=m CONFIG_GENWQE=m CONFIG_RAID_ATTRS=m @@ -523,7 +529,6 @@ CONFIG_NLMON=m # CONFIG_NET_VENDOR_ADAPTEC is not set # CONFIG_NET_VENDOR_AGERE is not set # CONFIG_NET_VENDOR_ALACRITECH is not set -# CONFIG_NET_VENDOR_ALTEON is not set # CONFIG_NET_VENDOR_AMAZON is not set # CONFIG_NET_VENDOR_AMD is not set # CONFIG_NET_VENDOR_AQUANTIA is not set @@ -560,7 +565,6 @@ CONFIG_MLX5_SF=y # CONFIG_NET_VENDOR_MYRI is not set # CONFIG_NET_VENDOR_NI is not set # CONFIG_NET_VENDOR_NATSEMI is not set -# CONFIG_NET_VENDOR_NETERION is not set # CONFIG_NET_VENDOR_NETRONOME is not set # CONFIG_NET_VENDOR_NVIDIA is not set # CONFIG_NET_VENDOR_OKI is not set @@ -770,14 +774,13 @@ CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECDSA=m CONFIG_CRYPTO_ECRDSA=m -CONFIG_CRYPTO_AES_TI=m +CONFIG_CRYPTO_AES=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_ARIA=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_DES=m -CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m CONFIG_CRYPTO_SEED=m CONFIG_CRYPTO_SERPENT=m @@ -788,16 +791,14 @@ CONFIG_CRYPTO_ADIANTUM=m CONFIG_CRYPTO_ARC4=m CONFIG_CRYPTO_HCTR2=m CONFIG_CRYPTO_LRW=m -CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_AEGIS128=m CONFIG_CRYPTO_GCM=y CONFIG_CRYPTO_SEQIV=y CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MD5=y -CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_CRC32=m @@ -809,9 +810,7 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_GHASH_S390=m CONFIG_CRYPTO_AES_S390=m -CONFIG_CRYPTO_DES_S390=m CONFIG_CRYPTO_HMAC_S390=m CONFIG_ZCRYPT=m CONFIG_PKEY=m @@ -870,7 +869,7 @@ CONFIG_DEBUG_PREEMPT=y CONFIG_PROVE_LOCKING=y CONFIG_LOCK_STAT=y CONFIG_LOCKDEP_BITS=16 -CONFIG_LOCKDEP_CHAINS_BITS=17 +CONFIG_LOCKDEP_CHAINS_BITS=18 CONFIG_DEBUG_ATOMIC_SLEEP=y CONFIG_DEBUG_LOCKING_API_SELFTESTS=y CONFIG_DEBUG_IRQFLAGS=y @@ -921,7 +920,7 @@ CONFIG_FAULT_INJECTION_DEBUG_FS=y CONFIG_FAULT_INJECTION_CONFIGFS=y CONFIG_FAULT_INJECTION_STACKTRACE_FILTER=y CONFIG_LKDTM=m -CONFIG_TEST_MIN_HEAP=y +CONFIG_MIN_HEAP_KUNIT_TEST=m CONFIG_KPROBES_SANITY_TEST=m CONFIG_RBTREE_TEST=y CONFIG_INTERVAL_TREE_TEST=m @@ -929,3 +928,5 @@ CONFIG_PERCPU_TEST=m CONFIG_ATOMIC64_SELFTEST=y CONFIG_TEST_BITOPS=m CONFIG_TEST_BPF=m +CONFIG_PAGE_TABLE_CHECK=y +CONFIG_PAGE_TABLE_CHECK_ENFORCED=y diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index c064e0cacc98..6f5722634b4d 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -27,6 +27,8 @@ CONFIG_CGROUP_PIDS=y CONFIG_CGROUP_RDMA=y CONFIG_CGROUP_FREEZER=y CONFIG_CGROUP_HUGETLB=y +CONFIG_CPU_IDLE=y +CONFIG_CPU_IDLE_GOV_TEO=y CONFIG_CPUSETS=y CONFIG_CGROUP_DEVICE=y CONFIG_CGROUP_CPUACCT=y @@ -54,6 +56,10 @@ CONFIG_EXPOLINE_AUTO=y CONFIG_CHSC_SCH=y CONFIG_VFIO_CCW=m CONFIG_VFIO_AP=m +CONFIG_VFIO_DEVICE_CDEV=y +CONFIG_IOMMUFD_DRIVER=y +CONFIG_IOMMUFD_DRIVER_CORE=y +CONFIG_IOMMUFD=y CONFIG_CMM=m CONFIG_APPLDATA_BASE=y CONFIG_S390_HYPFS_FS=y @@ -112,7 +118,6 @@ CONFIG_UNIX=y CONFIG_UNIX_DIAG=m CONFIG_TLS=m CONFIG_TLS_DEVICE=y -CONFIG_TLS_TOE=y CONFIG_XFRM_USER=m CONFIG_NET_KEY=m CONFIG_SMC=m @@ -436,6 +441,7 @@ CONFIG_BLK_DEV_RAM_SIZE=32768 CONFIG_VIRTIO_BLK=y CONFIG_BLK_DEV_RBD=m CONFIG_BLK_DEV_NVME=m +CONFIG_BLK_DEV_NULL_BLK=m CONFIG_ENCLOSURE_SERVICES=m CONFIG_GENWQE=m CONFIG_RAID_ATTRS=m @@ -513,7 +519,6 @@ CONFIG_NLMON=m # CONFIG_NET_VENDOR_ADAPTEC is not set # CONFIG_NET_VENDOR_AGERE is not set # CONFIG_NET_VENDOR_ALACRITECH is not set -# CONFIG_NET_VENDOR_ALTEON is not set # CONFIG_NET_VENDOR_AMAZON is not set # CONFIG_NET_VENDOR_AMD is not set # CONFIG_NET_VENDOR_AQUANTIA is not set @@ -550,7 +555,6 @@ CONFIG_MLX5_SF=y # CONFIG_NET_VENDOR_MYRI is not set # CONFIG_NET_VENDOR_NI is not set # CONFIG_NET_VENDOR_NATSEMI is not set -# CONFIG_NET_VENDOR_NETERION is not set # CONFIG_NET_VENDOR_NETRONOME is not set # CONFIG_NET_VENDOR_NVIDIA is not set # CONFIG_NET_VENDOR_OKI is not set @@ -754,14 +758,13 @@ CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECDSA=m CONFIG_CRYPTO_ECRDSA=m -CONFIG_CRYPTO_AES_TI=m +CONFIG_CRYPTO_AES=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_ARIA=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_DES=m -CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m CONFIG_CRYPTO_SEED=m CONFIG_CRYPTO_SERPENT=m @@ -772,16 +775,14 @@ CONFIG_CRYPTO_ADIANTUM=m CONFIG_CRYPTO_ARC4=m CONFIG_CRYPTO_HCTR2=m CONFIG_CRYPTO_LRW=m -CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_AEGIS128=m CONFIG_CRYPTO_GCM=y CONFIG_CRYPTO_SEQIV=y CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MD5=y -CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_CRC32=m @@ -794,9 +795,7 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_GHASH_S390=m CONFIG_CRYPTO_AES_S390=m -CONFIG_CRYPTO_DES_S390=m CONFIG_CRYPTO_HMAC_S390=m CONFIG_ZCRYPT=m CONFIG_PKEY=m diff --git a/arch/s390/crypto/Kconfig b/arch/s390/crypto/Kconfig index f838ca055f6d..228570a1b233 100644 --- a/arch/s390/crypto/Kconfig +++ b/arch/s390/crypto/Kconfig @@ -2,22 +2,11 @@ menu "Accelerated Cryptographic Algorithms for CPU (s390)" -config CRYPTO_GHASH_S390 - tristate "Hash functions: GHASH" - select CRYPTO_HASH - help - GCM GHASH hash function (NIST SP800-38D) - - Architecture: s390 - - It is available as of z196. - config CRYPTO_AES_S390 tristate "Ciphers: AES, modes: ECB, CBC, CTR, XTS, GCM" - select CRYPTO_ALGAPI + select CRYPTO_AEAD select CRYPTO_SKCIPHER help - Block cipher: AES cipher algorithms (FIPS 197) AEAD cipher: AES with GCM Length-preserving ciphers: AES with ECB, CBC, XTS, and CTR modes @@ -33,22 +22,6 @@ config CRYPTO_AES_S390 key sizes and XTS mode is hardware accelerated for 256 and 512 bit keys. -config CRYPTO_DES_S390 - tristate "Ciphers: DES and Triple DES EDE, modes: ECB, CBC, CTR" - select CRYPTO_ALGAPI - select CRYPTO_SKCIPHER - select CRYPTO_LIB_DES - help - Block ciphers: DES (FIPS 46-2) cipher algorithm - Block ciphers: Triple DES EDE (FIPS 46-3) cipher algorithm - Length-preserving ciphers: DES with ECB, CBC, and CTR modes - Length-preserving ciphers: Triple DES EDED with ECB, CBC, and CTR modes - - Architecture: s390 - - As of z990 the ECB and CBC mode are hardware accelerated. - As of z196 the CTR mode is hardware accelerated. - config CRYPTO_HMAC_S390 tristate "Keyed-hash message authentication code: HMAC" select CRYPTO_HASH diff --git a/arch/s390/crypto/Makefile b/arch/s390/crypto/Makefile index 387a229e1038..1d6420813935 100644 --- a/arch/s390/crypto/Makefile +++ b/arch/s390/crypto/Makefile @@ -3,11 +3,11 @@ # Cryptographic API # -obj-$(CONFIG_CRYPTO_DES_S390) += des_s390.o +CONTEXT_ANALYSIS := y + obj-$(CONFIG_CRYPTO_AES_S390) += aes_s390.o obj-$(CONFIG_CRYPTO_PAES_S390) += paes_s390.o obj-$(CONFIG_S390_PRNG) += prng.o -obj-$(CONFIG_CRYPTO_GHASH_S390) += ghash_s390.o obj-$(CONFIG_CRYPTO_HMAC_S390) += hmac_s390.o obj-$(CONFIG_CRYPTO_PHMAC_S390) += phmac_s390.o obj-y += arch_random.o diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index d0a295435680..0be6fa779d2c 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -20,21 +20,20 @@ #include <crypto/algapi.h> #include <crypto/ghash.h> #include <crypto/internal/aead.h> -#include <crypto/internal/cipher.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> #include <linux/err.h> #include <linux/module.h> #include <linux/cpufeature.h> #include <linux/init.h> -#include <linux/mutex.h> #include <linux/fips.h> +#include <linux/semaphore.h> #include <linux/string.h> #include <crypto/xts.h> #include <asm/cpacf.h> static u8 *ctrblk; -static DEFINE_MUTEX(ctrblk_lock); +static DEFINE_SEMAPHORE(ctrblk_sem, 1); static cpacf_mask_t km_functions, kmc_functions, kmctr_functions, kma_functions; @@ -45,7 +44,6 @@ struct s390_aes_ctx { unsigned long fc; union { struct crypto_skcipher *skcipher; - struct crypto_cipher *cip; } fallback; }; @@ -72,109 +70,6 @@ struct gcm_sg_walk { unsigned int nbytes; }; -static int setkey_fallback_cip(struct crypto_tfm *tfm, const u8 *in_key, - unsigned int key_len) -{ - struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm); - - sctx->fallback.cip->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; - sctx->fallback.cip->base.crt_flags |= (tfm->crt_flags & - CRYPTO_TFM_REQ_MASK); - - return crypto_cipher_setkey(sctx->fallback.cip, in_key, key_len); -} - -static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, - unsigned int key_len) -{ - struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm); - unsigned long fc; - - /* Pick the correct function code based on the key length */ - fc = (key_len == 16) ? CPACF_KM_AES_128 : - (key_len == 24) ? CPACF_KM_AES_192 : - (key_len == 32) ? CPACF_KM_AES_256 : 0; - - /* Check if the function code is available */ - sctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0; - if (!sctx->fc) - return setkey_fallback_cip(tfm, in_key, key_len); - - sctx->key_len = key_len; - memcpy(sctx->key, in_key, key_len); - return 0; -} - -static void crypto_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) -{ - struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm); - - if (unlikely(!sctx->fc)) { - crypto_cipher_encrypt_one(sctx->fallback.cip, out, in); - return; - } - cpacf_km(sctx->fc, &sctx->key, out, in, AES_BLOCK_SIZE); -} - -static void crypto_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) -{ - struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm); - - if (unlikely(!sctx->fc)) { - crypto_cipher_decrypt_one(sctx->fallback.cip, out, in); - return; - } - cpacf_km(sctx->fc | CPACF_DECRYPT, - &sctx->key, out, in, AES_BLOCK_SIZE); -} - -static int fallback_init_cip(struct crypto_tfm *tfm) -{ - const char *name = tfm->__crt_alg->cra_name; - struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm); - - sctx->fallback.cip = crypto_alloc_cipher(name, 0, - CRYPTO_ALG_NEED_FALLBACK); - - if (IS_ERR(sctx->fallback.cip)) { - pr_err("Allocating AES fallback algorithm %s failed\n", - name); - return PTR_ERR(sctx->fallback.cip); - } - - return 0; -} - -static void fallback_exit_cip(struct crypto_tfm *tfm) -{ - struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm); - - crypto_free_cipher(sctx->fallback.cip); - sctx->fallback.cip = NULL; -} - -static struct crypto_alg aes_alg = { - .cra_name = "aes", - .cra_driver_name = "aes-s390", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_CIPHER | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct s390_aes_ctx), - .cra_module = THIS_MODULE, - .cra_init = fallback_init_cip, - .cra_exit = fallback_exit_cip, - .cra_u = { - .cipher = { - .cia_min_keysize = AES_MIN_KEY_SIZE, - .cia_max_keysize = AES_MAX_KEY_SIZE, - .cia_setkey = aes_set_key, - .cia_encrypt = crypto_aes_encrypt, - .cia_decrypt = crypto_aes_decrypt, - } - } -}; - static int setkey_fallback_skcipher(struct crypto_skcipher *tfm, const u8 *key, unsigned int len) { @@ -234,7 +129,7 @@ static int ecb_aes_crypt(struct skcipher_request *req, unsigned long modifier) return fallback_skcipher_crypt(sctx, req, modifier); ret = skcipher_walk_virt(&walk, req, false); - while ((nbytes = walk.nbytes) != 0) { + while (!ret && ((nbytes = walk.nbytes) != 0)) { /* only use complete blocks */ n = nbytes & ~(AES_BLOCK_SIZE - 1); cpacf_km(sctx->fc | modifier, sctx->key, @@ -338,7 +233,7 @@ static int cbc_aes_crypt(struct skcipher_request *req, unsigned long modifier) return ret; memcpy(param.iv, walk.iv, AES_BLOCK_SIZE); memcpy(param.key, sctx->key, sctx->key_len); - while ((nbytes = walk.nbytes) != 0) { + while (!ret && ((nbytes = walk.nbytes) != 0)) { /* only use complete blocks */ n = nbytes & ~(AES_BLOCK_SIZE - 1); cpacf_kmc(sctx->fc | modifier, ¶m, @@ -464,7 +359,7 @@ static int xts_aes_crypt(struct skcipher_request *req, unsigned long modifier) memcpy(xts_param.key + offset, xts_ctx->key, xts_ctx->key_len); memcpy(xts_param.init, pcc_param.xts, 16); - while ((nbytes = walk.nbytes) != 0) { + while (!ret && ((nbytes = walk.nbytes) != 0)) { /* only use complete blocks */ n = nbytes & ~(AES_BLOCK_SIZE - 1); cpacf_km(xts_ctx->fc | modifier, xts_param.key + offset, @@ -592,7 +487,7 @@ static int fullxts_aes_crypt(struct skcipher_request *req, unsigned long modifi memcpy(fxts_param.tweak, req->iv, AES_BLOCK_SIZE); fxts_param.nap[0] = 0x01; /* initial alpha power (1, little-endian) */ - while ((nbytes = walk.nbytes) != 0) { + while (!ret && ((nbytes = walk.nbytes) != 0)) { /* only use complete blocks */ n = nbytes & ~(AES_BLOCK_SIZE - 1); cpacf_km(xts_ctx->fc | modifier, fxts_param.key + offset, @@ -667,48 +562,64 @@ static unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes) return n; } +static int __ctr_aes_crypt(struct s390_aes_ctx *sctx, + struct skcipher_walk *walk, bool locked) +{ + unsigned int n, nbytes; + int ret = 0; + u8 *ctrptr; + + while (!ret && ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE)) { + n = AES_BLOCK_SIZE; + if (nbytes >= 2 * AES_BLOCK_SIZE && locked) + n = __ctrblk_init(ctrblk, walk->iv, nbytes); + ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk->iv; + cpacf_kmctr(sctx->fc, sctx->key, walk->dst.virt.addr, + walk->src.virt.addr, n, ctrptr); + if (ctrptr == ctrblk) + memcpy(walk->iv, ctrptr + n - AES_BLOCK_SIZE, + AES_BLOCK_SIZE); + crypto_inc(walk->iv, AES_BLOCK_SIZE); + ret = skcipher_walk_done(walk, nbytes - n); + } + + return ret; +} + static int ctr_aes_crypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct s390_aes_ctx *sctx = crypto_skcipher_ctx(tfm); - u8 buf[AES_BLOCK_SIZE], *ctrptr; struct skcipher_walk walk; - unsigned int n, nbytes; - int ret, locked; + u8 buf[AES_BLOCK_SIZE]; + int ret; if (unlikely(!sctx->fc)) return fallback_skcipher_crypt(sctx, req, 0); - locked = mutex_trylock(&ctrblk_lock); - ret = skcipher_walk_virt(&walk, req, false); - while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { - n = AES_BLOCK_SIZE; + if (ret) + return ret; - if (nbytes >= 2*AES_BLOCK_SIZE && locked) - n = __ctrblk_init(ctrblk, walk.iv, nbytes); - ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk.iv; - cpacf_kmctr(sctx->fc, sctx->key, walk.dst.virt.addr, - walk.src.virt.addr, n, ctrptr); - if (ctrptr == ctrblk) - memcpy(walk.iv, ctrptr + n - AES_BLOCK_SIZE, - AES_BLOCK_SIZE); - crypto_inc(walk.iv, AES_BLOCK_SIZE); - ret = skcipher_walk_done(&walk, nbytes - n); + if (down_trylock(&ctrblk_sem) == 0) { + ret = __ctr_aes_crypt(sctx, &walk, true); + up(&ctrblk_sem); + } else { + ret = __ctr_aes_crypt(sctx, &walk, false); } - if (locked) - mutex_unlock(&ctrblk_lock); + /* * final block may be < AES_BLOCK_SIZE, copy only nbytes */ - if (nbytes) { + if (!ret && walk.nbytes > 0) { memset(buf, 0, AES_BLOCK_SIZE); - memcpy(buf, walk.src.virt.addr, nbytes); + memcpy(buf, walk.src.virt.addr, walk.nbytes); cpacf_kmctr(sctx->fc, sctx->key, buf, buf, AES_BLOCK_SIZE, walk.iv); - memcpy(walk.dst.virt.addr, buf, nbytes); + memcpy(walk.dst.virt.addr, buf, walk.nbytes); crypto_inc(walk.iv, AES_BLOCK_SIZE); ret = skcipher_walk_done(&walk, 0); + memzero_explicit(buf, sizeof(buf)); } return ret; @@ -1000,10 +911,14 @@ static int gcm_aes_crypt(struct aead_request *req, unsigned int flags) gw_in.ptr, aad_bytes); n = aad_bytes + pc_bytes; - if (gcm_in_walk_done(&gw_in, n) != n) - return -ENOMEM; - if (gcm_out_walk_done(&gw_out, n) != n) - return -ENOMEM; + if (gcm_in_walk_done(&gw_in, n) != n) { + ret = -ENOMEM; + goto out; + } + if (gcm_out_walk_done(&gw_out, n) != n) { + ret = -ENOMEM; + goto out; + } aadlen -= aad_bytes; pclen -= pc_bytes; } while (aadlen + pclen > 0); @@ -1015,7 +930,10 @@ static int gcm_aes_crypt(struct aead_request *req, unsigned int flags) } else scatterwalk_map_and_copy(param.t, req->dst, len, taglen, 1); +out: memzero_explicit(¶m, sizeof(param)); + memzero_explicit(gw_in.buf, sizeof(gw_in.buf)); + memzero_explicit(gw_out.buf, sizeof(gw_out.buf)); return ret; } @@ -1049,7 +967,6 @@ static struct aead_alg gcm_aes_aead = { }, }; -static struct crypto_alg *aes_s390_alg; static struct skcipher_alg *aes_s390_skcipher_algs[5]; static int aes_s390_skciphers_num; static struct aead_alg *aes_s390_aead_alg; @@ -1066,8 +983,6 @@ static int aes_s390_register_skcipher(struct skcipher_alg *alg) static void aes_s390_fini(void) { - if (aes_s390_alg) - crypto_unregister_alg(aes_s390_alg); while (aes_s390_skciphers_num--) crypto_unregister_skcipher(aes_s390_skcipher_algs[aes_s390_skciphers_num]); if (ctrblk) @@ -1090,10 +1005,6 @@ static int __init aes_s390_init(void) if (cpacf_test_func(&km_functions, CPACF_KM_AES_128) || cpacf_test_func(&km_functions, CPACF_KM_AES_192) || cpacf_test_func(&km_functions, CPACF_KM_AES_256)) { - ret = crypto_register_alg(&aes_alg); - if (ret) - goto out_err; - aes_s390_alg = &aes_alg; ret = aes_s390_register_skcipher(&ecb_aes_alg); if (ret) goto out_err; @@ -1156,4 +1067,3 @@ MODULE_ALIAS_CRYPTO("aes-all"); MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm"); MODULE_LICENSE("GPL"); -MODULE_IMPORT_NS("CRYPTO_INTERNAL"); diff --git a/arch/s390/crypto/des_s390.c b/arch/s390/crypto/des_s390.c deleted file mode 100644 index 8e75b83a5ddc..000000000000 --- a/arch/s390/crypto/des_s390.c +++ /dev/null @@ -1,502 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * Cryptographic API. - * - * s390 implementation of the DES Cipher Algorithm. - * - * Copyright IBM Corp. 2003, 2011 - * Author(s): Thomas Spatzier - * Jan Glauber (jan.glauber@de.ibm.com) - */ - -#include <linux/init.h> -#include <linux/module.h> -#include <linux/cpufeature.h> -#include <linux/crypto.h> -#include <linux/fips.h> -#include <linux/mutex.h> -#include <crypto/algapi.h> -#include <crypto/internal/des.h> -#include <crypto/internal/skcipher.h> -#include <asm/cpacf.h> - -#define DES3_KEY_SIZE (3 * DES_KEY_SIZE) - -static u8 *ctrblk; -static DEFINE_MUTEX(ctrblk_lock); - -static cpacf_mask_t km_functions, kmc_functions, kmctr_functions; - -struct s390_des_ctx { - u8 iv[DES_BLOCK_SIZE]; - u8 key[DES3_KEY_SIZE]; -}; - -static int des_setkey(struct crypto_tfm *tfm, const u8 *key, - unsigned int key_len) -{ - struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); - int err; - - err = crypto_des_verify_key(tfm, key); - if (err) - return err; - - memcpy(ctx->key, key, key_len); - return 0; -} - -static int des_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, - unsigned int key_len) -{ - return des_setkey(crypto_skcipher_tfm(tfm), key, key_len); -} - -static void s390_des_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) -{ - struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); - - cpacf_km(CPACF_KM_DEA, ctx->key, out, in, DES_BLOCK_SIZE); -} - -static void s390_des_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) -{ - struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); - - cpacf_km(CPACF_KM_DEA | CPACF_DECRYPT, - ctx->key, out, in, DES_BLOCK_SIZE); -} - -static struct crypto_alg des_alg = { - .cra_name = "des", - .cra_driver_name = "des-s390", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_CIPHER, - .cra_blocksize = DES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct s390_des_ctx), - .cra_module = THIS_MODULE, - .cra_u = { - .cipher = { - .cia_min_keysize = DES_KEY_SIZE, - .cia_max_keysize = DES_KEY_SIZE, - .cia_setkey = des_setkey, - .cia_encrypt = s390_des_encrypt, - .cia_decrypt = s390_des_decrypt, - } - } -}; - -static int ecb_desall_crypt(struct skcipher_request *req, unsigned long fc) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct s390_des_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_walk walk; - unsigned int nbytes, n; - int ret; - - ret = skcipher_walk_virt(&walk, req, false); - while ((nbytes = walk.nbytes) != 0) { - /* only use complete blocks */ - n = nbytes & ~(DES_BLOCK_SIZE - 1); - cpacf_km(fc, ctx->key, walk.dst.virt.addr, - walk.src.virt.addr, n); - ret = skcipher_walk_done(&walk, nbytes - n); - } - return ret; -} - -static int cbc_desall_crypt(struct skcipher_request *req, unsigned long fc) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct s390_des_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_walk walk; - unsigned int nbytes, n; - int ret; - struct { - u8 iv[DES_BLOCK_SIZE]; - u8 key[DES3_KEY_SIZE]; - } param; - - ret = skcipher_walk_virt(&walk, req, false); - if (ret) - return ret; - memcpy(param.iv, walk.iv, DES_BLOCK_SIZE); - memcpy(param.key, ctx->key, DES3_KEY_SIZE); - while ((nbytes = walk.nbytes) != 0) { - /* only use complete blocks */ - n = nbytes & ~(DES_BLOCK_SIZE - 1); - cpacf_kmc(fc, ¶m, walk.dst.virt.addr, - walk.src.virt.addr, n); - memcpy(walk.iv, param.iv, DES_BLOCK_SIZE); - ret = skcipher_walk_done(&walk, nbytes - n); - } - return ret; -} - -static int ecb_des_encrypt(struct skcipher_request *req) -{ - return ecb_desall_crypt(req, CPACF_KM_DEA); -} - -static int ecb_des_decrypt(struct skcipher_request *req) -{ - return ecb_desall_crypt(req, CPACF_KM_DEA | CPACF_DECRYPT); -} - -static struct skcipher_alg ecb_des_alg = { - .base.cra_name = "ecb(des)", - .base.cra_driver_name = "ecb-des-s390", - .base.cra_priority = 400, /* combo: des + ecb */ - .base.cra_blocksize = DES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct s390_des_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .setkey = des_setkey_skcipher, - .encrypt = ecb_des_encrypt, - .decrypt = ecb_des_decrypt, -}; - -static int cbc_des_encrypt(struct skcipher_request *req) -{ - return cbc_desall_crypt(req, CPACF_KMC_DEA); -} - -static int cbc_des_decrypt(struct skcipher_request *req) -{ - return cbc_desall_crypt(req, CPACF_KMC_DEA | CPACF_DECRYPT); -} - -static struct skcipher_alg cbc_des_alg = { - .base.cra_name = "cbc(des)", - .base.cra_driver_name = "cbc-des-s390", - .base.cra_priority = 400, /* combo: des + cbc */ - .base.cra_blocksize = DES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct s390_des_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, - .setkey = des_setkey_skcipher, - .encrypt = cbc_des_encrypt, - .decrypt = cbc_des_decrypt, -}; - -/* - * RFC2451: - * - * For DES-EDE3, there is no known need to reject weak or - * complementation keys. Any weakness is obviated by the use of - * multiple keys. - * - * However, if the first two or last two independent 64-bit keys are - * equal (k1 == k2 or k2 == k3), then the DES3 operation is simply the - * same as DES. Implementers MUST reject keys that exhibit this - * property. - * - * In fips mode additionally check for all 3 keys are unique. - * - */ -static int des3_setkey(struct crypto_tfm *tfm, const u8 *key, - unsigned int key_len) -{ - struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); - int err; - - err = crypto_des3_ede_verify_key(tfm, key); - if (err) - return err; - - memcpy(ctx->key, key, key_len); - return 0; -} - -static int des3_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, - unsigned int key_len) -{ - return des3_setkey(crypto_skcipher_tfm(tfm), key, key_len); -} - -static void des3_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) -{ - struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); - - cpacf_km(CPACF_KM_TDEA_192, ctx->key, dst, src, DES_BLOCK_SIZE); -} - -static void des3_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) -{ - struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); - - cpacf_km(CPACF_KM_TDEA_192 | CPACF_DECRYPT, - ctx->key, dst, src, DES_BLOCK_SIZE); -} - -static struct crypto_alg des3_alg = { - .cra_name = "des3_ede", - .cra_driver_name = "des3_ede-s390", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_CIPHER, - .cra_blocksize = DES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct s390_des_ctx), - .cra_module = THIS_MODULE, - .cra_u = { - .cipher = { - .cia_min_keysize = DES3_KEY_SIZE, - .cia_max_keysize = DES3_KEY_SIZE, - .cia_setkey = des3_setkey, - .cia_encrypt = des3_encrypt, - .cia_decrypt = des3_decrypt, - } - } -}; - -static int ecb_des3_encrypt(struct skcipher_request *req) -{ - return ecb_desall_crypt(req, CPACF_KM_TDEA_192); -} - -static int ecb_des3_decrypt(struct skcipher_request *req) -{ - return ecb_desall_crypt(req, CPACF_KM_TDEA_192 | CPACF_DECRYPT); -} - -static struct skcipher_alg ecb_des3_alg = { - .base.cra_name = "ecb(des3_ede)", - .base.cra_driver_name = "ecb-des3_ede-s390", - .base.cra_priority = 400, /* combo: des3 + ecb */ - .base.cra_blocksize = DES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct s390_des_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = DES3_KEY_SIZE, - .max_keysize = DES3_KEY_SIZE, - .setkey = des3_setkey_skcipher, - .encrypt = ecb_des3_encrypt, - .decrypt = ecb_des3_decrypt, -}; - -static int cbc_des3_encrypt(struct skcipher_request *req) -{ - return cbc_desall_crypt(req, CPACF_KMC_TDEA_192); -} - -static int cbc_des3_decrypt(struct skcipher_request *req) -{ - return cbc_desall_crypt(req, CPACF_KMC_TDEA_192 | CPACF_DECRYPT); -} - -static struct skcipher_alg cbc_des3_alg = { - .base.cra_name = "cbc(des3_ede)", - .base.cra_driver_name = "cbc-des3_ede-s390", - .base.cra_priority = 400, /* combo: des3 + cbc */ - .base.cra_blocksize = DES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct s390_des_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = DES3_KEY_SIZE, - .max_keysize = DES3_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, - .setkey = des3_setkey_skcipher, - .encrypt = cbc_des3_encrypt, - .decrypt = cbc_des3_decrypt, -}; - -static unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes) -{ - unsigned int i, n; - - /* align to block size, max. PAGE_SIZE */ - n = (nbytes > PAGE_SIZE) ? PAGE_SIZE : nbytes & ~(DES_BLOCK_SIZE - 1); - memcpy(ctrptr, iv, DES_BLOCK_SIZE); - for (i = (n / DES_BLOCK_SIZE) - 1; i > 0; i--) { - memcpy(ctrptr + DES_BLOCK_SIZE, ctrptr, DES_BLOCK_SIZE); - crypto_inc(ctrptr + DES_BLOCK_SIZE, DES_BLOCK_SIZE); - ctrptr += DES_BLOCK_SIZE; - } - return n; -} - -static int ctr_desall_crypt(struct skcipher_request *req, unsigned long fc) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct s390_des_ctx *ctx = crypto_skcipher_ctx(tfm); - u8 buf[DES_BLOCK_SIZE], *ctrptr; - struct skcipher_walk walk; - unsigned int n, nbytes; - int ret, locked; - - locked = mutex_trylock(&ctrblk_lock); - - ret = skcipher_walk_virt(&walk, req, false); - while ((nbytes = walk.nbytes) >= DES_BLOCK_SIZE) { - n = DES_BLOCK_SIZE; - if (nbytes >= 2*DES_BLOCK_SIZE && locked) - n = __ctrblk_init(ctrblk, walk.iv, nbytes); - ctrptr = (n > DES_BLOCK_SIZE) ? ctrblk : walk.iv; - cpacf_kmctr(fc, ctx->key, walk.dst.virt.addr, - walk.src.virt.addr, n, ctrptr); - if (ctrptr == ctrblk) - memcpy(walk.iv, ctrptr + n - DES_BLOCK_SIZE, - DES_BLOCK_SIZE); - crypto_inc(walk.iv, DES_BLOCK_SIZE); - ret = skcipher_walk_done(&walk, nbytes - n); - } - if (locked) - mutex_unlock(&ctrblk_lock); - /* final block may be < DES_BLOCK_SIZE, copy only nbytes */ - if (nbytes) { - cpacf_kmctr(fc, ctx->key, buf, walk.src.virt.addr, - DES_BLOCK_SIZE, walk.iv); - memcpy(walk.dst.virt.addr, buf, nbytes); - crypto_inc(walk.iv, DES_BLOCK_SIZE); - ret = skcipher_walk_done(&walk, 0); - } - return ret; -} - -static int ctr_des_crypt(struct skcipher_request *req) -{ - return ctr_desall_crypt(req, CPACF_KMCTR_DEA); -} - -static struct skcipher_alg ctr_des_alg = { - .base.cra_name = "ctr(des)", - .base.cra_driver_name = "ctr-des-s390", - .base.cra_priority = 400, /* combo: des + ctr */ - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct s390_des_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, - .setkey = des_setkey_skcipher, - .encrypt = ctr_des_crypt, - .decrypt = ctr_des_crypt, - .chunksize = DES_BLOCK_SIZE, -}; - -static int ctr_des3_crypt(struct skcipher_request *req) -{ - return ctr_desall_crypt(req, CPACF_KMCTR_TDEA_192); -} - -static struct skcipher_alg ctr_des3_alg = { - .base.cra_name = "ctr(des3_ede)", - .base.cra_driver_name = "ctr-des3_ede-s390", - .base.cra_priority = 400, /* combo: des3 + ede */ - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct s390_des_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = DES3_KEY_SIZE, - .max_keysize = DES3_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, - .setkey = des3_setkey_skcipher, - .encrypt = ctr_des3_crypt, - .decrypt = ctr_des3_crypt, - .chunksize = DES_BLOCK_SIZE, -}; - -static struct crypto_alg *des_s390_algs_ptr[2]; -static int des_s390_algs_num; -static struct skcipher_alg *des_s390_skciphers_ptr[6]; -static int des_s390_skciphers_num; - -static int des_s390_register_alg(struct crypto_alg *alg) -{ - int ret; - - ret = crypto_register_alg(alg); - if (!ret) - des_s390_algs_ptr[des_s390_algs_num++] = alg; - return ret; -} - -static int des_s390_register_skcipher(struct skcipher_alg *alg) -{ - int ret; - - ret = crypto_register_skcipher(alg); - if (!ret) - des_s390_skciphers_ptr[des_s390_skciphers_num++] = alg; - return ret; -} - -static void des_s390_exit(void) -{ - while (des_s390_algs_num--) - crypto_unregister_alg(des_s390_algs_ptr[des_s390_algs_num]); - while (des_s390_skciphers_num--) - crypto_unregister_skcipher(des_s390_skciphers_ptr[des_s390_skciphers_num]); - if (ctrblk) - free_page((unsigned long) ctrblk); -} - -static int __init des_s390_init(void) -{ - int ret; - - /* Query available functions for KM, KMC and KMCTR */ - cpacf_query(CPACF_KM, &km_functions); - cpacf_query(CPACF_KMC, &kmc_functions); - cpacf_query(CPACF_KMCTR, &kmctr_functions); - - if (cpacf_test_func(&km_functions, CPACF_KM_DEA)) { - ret = des_s390_register_alg(&des_alg); - if (ret) - goto out_err; - ret = des_s390_register_skcipher(&ecb_des_alg); - if (ret) - goto out_err; - } - if (cpacf_test_func(&kmc_functions, CPACF_KMC_DEA)) { - ret = des_s390_register_skcipher(&cbc_des_alg); - if (ret) - goto out_err; - } - if (cpacf_test_func(&km_functions, CPACF_KM_TDEA_192)) { - ret = des_s390_register_alg(&des3_alg); - if (ret) - goto out_err; - ret = des_s390_register_skcipher(&ecb_des3_alg); - if (ret) - goto out_err; - } - if (cpacf_test_func(&kmc_functions, CPACF_KMC_TDEA_192)) { - ret = des_s390_register_skcipher(&cbc_des3_alg); - if (ret) - goto out_err; - } - - if (cpacf_test_func(&kmctr_functions, CPACF_KMCTR_DEA) || - cpacf_test_func(&kmctr_functions, CPACF_KMCTR_TDEA_192)) { - ctrblk = (u8 *) __get_free_page(GFP_KERNEL); - if (!ctrblk) { - ret = -ENOMEM; - goto out_err; - } - } - - if (cpacf_test_func(&kmctr_functions, CPACF_KMCTR_DEA)) { - ret = des_s390_register_skcipher(&ctr_des_alg); - if (ret) - goto out_err; - } - if (cpacf_test_func(&kmctr_functions, CPACF_KMCTR_TDEA_192)) { - ret = des_s390_register_skcipher(&ctr_des3_alg); - if (ret) - goto out_err; - } - - return 0; -out_err: - des_s390_exit(); - return ret; -} - -module_cpu_feature_match(S390_CPU_FEATURE_MSA, des_s390_init); -module_exit(des_s390_exit); - -MODULE_ALIAS_CRYPTO("des"); -MODULE_ALIAS_CRYPTO("des3_ede"); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms"); diff --git a/arch/s390/crypto/ghash_s390.c b/arch/s390/crypto/ghash_s390.c deleted file mode 100644 index dcbcee37cb63..000000000000 --- a/arch/s390/crypto/ghash_s390.c +++ /dev/null @@ -1,144 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Cryptographic API. - * - * s390 implementation of the GHASH algorithm for GCM (Galois/Counter Mode). - * - * Copyright IBM Corp. 2011 - * Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com> - */ - -#include <asm/cpacf.h> -#include <crypto/ghash.h> -#include <crypto/internal/hash.h> -#include <linux/cpufeature.h> -#include <linux/err.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/string.h> - -struct s390_ghash_ctx { - u8 key[GHASH_BLOCK_SIZE]; -}; - -struct s390_ghash_desc_ctx { - u8 icv[GHASH_BLOCK_SIZE]; - u8 key[GHASH_BLOCK_SIZE]; -}; - -static int ghash_init(struct shash_desc *desc) -{ - struct s390_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); - struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - memset(dctx, 0, sizeof(*dctx)); - memcpy(dctx->key, ctx->key, GHASH_BLOCK_SIZE); - - return 0; -} - -static int ghash_setkey(struct crypto_shash *tfm, - const u8 *key, unsigned int keylen) -{ - struct s390_ghash_ctx *ctx = crypto_shash_ctx(tfm); - - if (keylen != GHASH_BLOCK_SIZE) - return -EINVAL; - - memcpy(ctx->key, key, GHASH_BLOCK_SIZE); - - return 0; -} - -static int ghash_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - unsigned int n; - - n = srclen & ~(GHASH_BLOCK_SIZE - 1); - cpacf_kimd(CPACF_KIMD_GHASH, dctx, src, n); - return srclen - n; -} - -static void ghash_flush(struct s390_ghash_desc_ctx *dctx, const u8 *src, - unsigned int len) -{ - if (len) { - u8 buf[GHASH_BLOCK_SIZE] = {}; - - memcpy(buf, src, len); - cpacf_kimd(CPACF_KIMD_GHASH, dctx, buf, GHASH_BLOCK_SIZE); - memzero_explicit(buf, sizeof(buf)); - } -} - -static int ghash_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *dst) -{ - struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - ghash_flush(dctx, src, len); - memcpy(dst, dctx->icv, GHASH_BLOCK_SIZE); - return 0; -} - -static int ghash_export(struct shash_desc *desc, void *out) -{ - struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - memcpy(out, dctx->icv, GHASH_DIGEST_SIZE); - return 0; -} - -static int ghash_import(struct shash_desc *desc, const void *in) -{ - struct s390_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); - struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - memcpy(dctx->icv, in, GHASH_DIGEST_SIZE); - memcpy(dctx->key, ctx->key, GHASH_BLOCK_SIZE); - return 0; -} - -static struct shash_alg ghash_alg = { - .digestsize = GHASH_DIGEST_SIZE, - .init = ghash_init, - .update = ghash_update, - .finup = ghash_finup, - .setkey = ghash_setkey, - .export = ghash_export, - .import = ghash_import, - .statesize = sizeof(struct ghash_desc_ctx), - .descsize = sizeof(struct s390_ghash_desc_ctx), - .base = { - .cra_name = "ghash", - .cra_driver_name = "ghash-s390", - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct s390_ghash_ctx), - .cra_module = THIS_MODULE, - }, -}; - -static int __init ghash_mod_init(void) -{ - if (!cpacf_query_func(CPACF_KIMD, CPACF_KIMD_GHASH)) - return -ENODEV; - - return crypto_register_shash(&ghash_alg); -} - -static void __exit ghash_mod_exit(void) -{ - crypto_unregister_shash(&ghash_alg); -} - -module_cpu_feature_match(S390_CPU_FEATURE_MSA, ghash_mod_init); -module_exit(ghash_mod_exit); - -MODULE_ALIAS_CRYPTO("ghash"); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("GHASH hash function, s390 implementation"); diff --git a/arch/s390/crypto/hmac_s390.c b/arch/s390/crypto/hmac_s390.c index f8cd09f341d4..445fa7bbd958 100644 --- a/arch/s390/crypto/hmac_s390.c +++ b/arch/s390/crypto/hmac_s390.c @@ -150,7 +150,10 @@ static int hash_data(const u8 *in, unsigned int inlen, #undef PARAM_INIT - cpacf_klmd(func, ¶m, in, inlen); + if (final) + cpacf_klmd(func, ¶m, in, inlen); + else + cpacf_kimd(func, ¶m, in, inlen); memcpy(digest, ¶m, digestsize); diff --git a/arch/s390/crypto/paes_s390.c b/arch/s390/crypto/paes_s390.c index 64aef7eb2030..f987bcbe8f35 100644 --- a/arch/s390/crypto/paes_s390.c +++ b/arch/s390/crypto/paes_s390.c @@ -19,7 +19,7 @@ #include <linux/init.h> #include <linux/miscdevice.h> #include <linux/module.h> -#include <linux/mutex.h> +#include <linux/semaphore.h> #include <linux/spinlock.h> #include <crypto/aes.h> #include <crypto/algapi.h> @@ -40,8 +40,12 @@ #define PAES_256_PROTKEY_SIZE (32 + 32) /* key + verification pattern */ #define PXTS_256_PROTKEY_SIZE (32 + 32 + 32) /* k1 + k2 + verification pattern */ +static bool pkey_clrkey_allowed; +module_param_named(clrkey, pkey_clrkey_allowed, bool, 0444); +MODULE_PARM_DESC(clrkey, "Allow clear key material (default N)"); + static u8 *ctrblk; -static DEFINE_MUTEX(ctrblk_lock); +static DEFINE_SEMAPHORE(ctrblk_sem, 1); static cpacf_mask_t km_functions, kmc_functions, kmctr_functions; @@ -192,10 +196,14 @@ static inline int pxts_ctx_setkey(struct s390_pxts_ctx *ctx, * This function may sleep - don't call in non-sleeping context. */ static inline int convert_key(const u8 *key, unsigned int keylen, - struct paes_protkey *pk) + struct paes_protkey *pk, bool tested) { + u32 xflags = PKEY_XFLAG_NOMEMALLOC; int rc, i; + if (tested && !pkey_clrkey_allowed) + xflags |= PKEY_XFLAG_NOCLEARKEY; + pk->len = sizeof(pk->protkey); /* @@ -209,9 +217,13 @@ static inline int convert_key(const u8 *key, unsigned int keylen, } rc = pkey_key2protkey(key, keylen, pk->protkey, &pk->len, &pk->type, - PKEY_XFLAG_NOMEMALLOC); + xflags); } + /* But finally map -EBUSY to -EIO to indicate an IO failure */ + if (rc == -EBUSY) + rc = -EIO; + out: pr_debug("rc=%d\n", rc); return rc; @@ -231,7 +243,7 @@ out: * unnecessary additional conversion but never to invalid data on en- * or decrypt operations. */ -static int paes_convert_key(struct s390_paes_ctx *ctx) +static int paes_convert_key(struct s390_paes_ctx *ctx, bool tested) { struct paes_protkey pk; int rc; @@ -240,7 +252,7 @@ static int paes_convert_key(struct s390_paes_ctx *ctx) ctx->pk_state = PK_STATE_CONVERT_IN_PROGRESS; spin_unlock_bh(&ctx->pk_lock); - rc = convert_key(ctx->keybuf, ctx->keylen, &pk); + rc = convert_key(ctx->keybuf, ctx->keylen, &pk, tested); /* update context */ spin_lock_bh(&ctx->pk_lock); @@ -263,7 +275,7 @@ static int paes_convert_key(struct s390_paes_ctx *ctx) * pk_type, pk_len and the protected key in the tfm context. * See also comments on function paes_convert_key. */ -static int pxts_convert_key(struct s390_pxts_ctx *ctx) +static int pxts_convert_key(struct s390_pxts_ctx *ctx, bool tested) { struct paes_protkey pk0, pk1; size_t split_keylen; @@ -273,7 +285,7 @@ static int pxts_convert_key(struct s390_pxts_ctx *ctx) ctx->pk_state = PK_STATE_CONVERT_IN_PROGRESS; spin_unlock_bh(&ctx->pk_lock); - rc = convert_key(ctx->keybuf, ctx->keylen, &pk0); + rc = convert_key(ctx->keybuf, ctx->keylen, &pk0, tested); if (rc) goto out; @@ -287,7 +299,7 @@ static int pxts_convert_key(struct s390_pxts_ctx *ctx) } split_keylen = ctx->keylen / 2; rc = convert_key(ctx->keybuf + split_keylen, - split_keylen, &pk1); + split_keylen, &pk1, tested); if (rc) goto out; if (pk0.type != pk1.type) { @@ -343,6 +355,7 @@ static int ecb_paes_setkey(struct crypto_skcipher *tfm, const u8 *in_key, unsigned int key_len) { struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); + bool tested = crypto_skcipher_tested(tfm); long fc; int rc; @@ -352,7 +365,7 @@ static int ecb_paes_setkey(struct crypto_skcipher *tfm, const u8 *in_key, goto out; /* convert key into protected key */ - rc = paes_convert_key(ctx); + rc = paes_convert_key(ctx, tested); if (rc) goto out; @@ -382,7 +395,7 @@ out: static int ecb_paes_do_crypt(struct s390_paes_ctx *ctx, struct s390_pecb_req_ctx *req_ctx, - bool maysleep) + bool tested, bool maysleep) { struct ecb_param *param = &req_ctx->param; struct skcipher_walk *walk = &req_ctx->walk; @@ -423,14 +436,17 @@ static int ecb_paes_do_crypt(struct s390_paes_ctx *ctx, n = nbytes & ~(AES_BLOCK_SIZE - 1); k = cpacf_km(ctx->fc | req_ctx->modifier, param, walk->dst.virt.addr, walk->src.virt.addr, n); - if (k) + if (k) { rc = skcipher_walk_done(walk, nbytes - k); + if (rc) + goto out; + } if (k < n) { if (!maysleep) { rc = -EKEYEXPIRED; goto out; } - rc = paes_convert_key(ctx); + rc = paes_convert_key(ctx, tested); if (rc) goto out; spin_lock_bh(&ctx->pk_lock); @@ -450,6 +466,8 @@ static int ecb_paes_crypt(struct skcipher_request *req, unsigned long modifier) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); struct skcipher_walk *walk = &req_ctx->walk; + bool tested = crypto_skcipher_tested(tfm); + bool cleanup = true; int rc; /* @@ -468,7 +486,7 @@ static int ecb_paes_crypt(struct skcipher_request *req, unsigned long modifier) /* Try synchronous operation if no active engine usage */ if (!atomic_read(&ctx->via_engine_ctr)) { - rc = ecb_paes_do_crypt(ctx, req_ctx, false); + rc = ecb_paes_do_crypt(ctx, req_ctx, tested, false); if (rc == 0) goto out; } @@ -481,15 +499,17 @@ static int ecb_paes_crypt(struct skcipher_request *req, unsigned long modifier) if (rc == 0 || rc == -EKEYEXPIRED) { atomic_inc(&ctx->via_engine_ctr); rc = crypto_transfer_skcipher_request_to_engine(paes_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) + if (cleanup && walk->nbytes) skcipher_walk_done(walk, rc); out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(&req_ctx->param, sizeof(req_ctx->param)); pr_debug("rc=%d\n", rc); return rc; @@ -531,23 +551,15 @@ static int ecb_paes_do_one_request(struct crypto_engine *engine, void *areq) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); struct skcipher_walk *walk = &req_ctx->walk; + bool tested = crypto_skcipher_tested(tfm); int rc; /* walk has already been prepared */ - rc = ecb_paes_do_crypt(ctx, req_ctx, true); + rc = ecb_paes_do_crypt(ctx, req_ctx, tested, true); if (rc == -EKEYEXPIRED) { - /* - * Protected key expired, conversion is in process. - * Trigger a re-schedule of this request by returning - * -ENOSPC ("hardware queue is full") to the crypto engine. - * To avoid immediately re-invocation of this callback, - * tell the scheduler to voluntarily give up the CPU here. - */ - cond_resched(); - pr_debug("rescheduling request\n"); - return -ENOSPC; - } else if (rc) { + return pkey_handle_expired(); + } else if (rc && walk->nbytes) { skcipher_walk_done(walk, rc); } @@ -557,7 +569,7 @@ static int ecb_paes_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&ctx->via_engine_ctr); crypto_finalize_skcipher_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } static struct skcipher_engine_alg ecb_paes_alg = { @@ -565,6 +577,7 @@ static struct skcipher_engine_alg ecb_paes_alg = { .base.cra_name = "ecb(paes)", .base.cra_driver_name = "ecb-paes-s390", .base.cra_priority = 401, /* combo: aes + ecb + 1 */ + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct s390_paes_ctx), .base.cra_module = THIS_MODULE, @@ -602,6 +615,7 @@ static int cbc_paes_setkey(struct crypto_skcipher *tfm, const u8 *in_key, unsigned int key_len) { struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); + bool tested = crypto_skcipher_tested(tfm); long fc; int rc; @@ -611,7 +625,7 @@ static int cbc_paes_setkey(struct crypto_skcipher *tfm, const u8 *in_key, goto out; /* convert raw key into protected key */ - rc = paes_convert_key(ctx); + rc = paes_convert_key(ctx, tested); if (rc) goto out; @@ -641,7 +655,7 @@ out: static int cbc_paes_do_crypt(struct s390_paes_ctx *ctx, struct s390_pcbc_req_ctx *req_ctx, - bool maysleep) + bool tested, bool maysleep) { struct cbc_param *param = &req_ctx->param; struct skcipher_walk *walk = &req_ctx->walk; @@ -687,13 +701,15 @@ static int cbc_paes_do_crypt(struct s390_paes_ctx *ctx, if (k) { memcpy(walk->iv, param->iv, AES_BLOCK_SIZE); rc = skcipher_walk_done(walk, nbytes - k); + if (rc) + goto out; } if (k < n) { if (!maysleep) { rc = -EKEYEXPIRED; goto out; } - rc = paes_convert_key(ctx); + rc = paes_convert_key(ctx, tested); if (rc) goto out; spin_lock_bh(&ctx->pk_lock); @@ -713,6 +729,8 @@ static int cbc_paes_crypt(struct skcipher_request *req, unsigned long modifier) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); struct skcipher_walk *walk = &req_ctx->walk; + bool tested = crypto_skcipher_tested(tfm); + bool cleanup = true; int rc; /* @@ -731,7 +749,7 @@ static int cbc_paes_crypt(struct skcipher_request *req, unsigned long modifier) /* Try synchronous operation if no active engine usage */ if (!atomic_read(&ctx->via_engine_ctr)) { - rc = cbc_paes_do_crypt(ctx, req_ctx, false); + rc = cbc_paes_do_crypt(ctx, req_ctx, tested, false); if (rc == 0) goto out; } @@ -744,15 +762,17 @@ static int cbc_paes_crypt(struct skcipher_request *req, unsigned long modifier) if (rc == 0 || rc == -EKEYEXPIRED) { atomic_inc(&ctx->via_engine_ctr); rc = crypto_transfer_skcipher_request_to_engine(paes_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) + if (cleanup && walk->nbytes) skcipher_walk_done(walk, rc); out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(&req_ctx->param, sizeof(req_ctx->param)); pr_debug("rc=%d\n", rc); return rc; @@ -794,23 +814,15 @@ static int cbc_paes_do_one_request(struct crypto_engine *engine, void *areq) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); struct skcipher_walk *walk = &req_ctx->walk; + bool tested = crypto_skcipher_tested(tfm); int rc; /* walk has already been prepared */ - rc = cbc_paes_do_crypt(ctx, req_ctx, true); + rc = cbc_paes_do_crypt(ctx, req_ctx, tested, true); if (rc == -EKEYEXPIRED) { - /* - * Protected key expired, conversion is in process. - * Trigger a re-schedule of this request by returning - * -ENOSPC ("hardware queue is full") to the crypto engine. - * To avoid immediately re-invocation of this callback, - * tell the scheduler to voluntarily give up the CPU here. - */ - cond_resched(); - pr_debug("rescheduling request\n"); - return -ENOSPC; - } else if (rc) { + return pkey_handle_expired(); + } else if (rc && walk->nbytes) { skcipher_walk_done(walk, rc); } @@ -820,7 +832,7 @@ static int cbc_paes_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&ctx->via_engine_ctr); crypto_finalize_skcipher_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } static struct skcipher_engine_alg cbc_paes_alg = { @@ -828,6 +840,7 @@ static struct skcipher_engine_alg cbc_paes_alg = { .base.cra_name = "cbc(paes)", .base.cra_driver_name = "cbc-paes-s390", .base.cra_priority = 402, /* cbc-paes-s390 + 1 */ + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct s390_paes_ctx), .base.cra_module = THIS_MODULE, @@ -865,6 +878,7 @@ static int ctr_paes_setkey(struct crypto_skcipher *tfm, const u8 *in_key, unsigned int key_len) { struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); + bool tested = crypto_skcipher_tested(tfm); long fc; int rc; @@ -874,7 +888,7 @@ static int ctr_paes_setkey(struct crypto_skcipher *tfm, const u8 *in_key, goto out; /* convert raw key into protected key */ - rc = paes_convert_key(ctx); + rc = paes_convert_key(ctx, tested); if (rc) goto out; @@ -917,41 +931,14 @@ static inline unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes return n; } -static int ctr_paes_do_crypt(struct s390_paes_ctx *ctx, - struct s390_pctr_req_ctx *req_ctx, - bool maysleep) +static int __ctr_paes_do_crypt(struct s390_paes_ctx *ctx, + struct ctr_param *param, + struct skcipher_walk *walk, + bool tested, bool maysleep, bool locked) { - struct ctr_param *param = &req_ctx->param; - struct skcipher_walk *walk = &req_ctx->walk; - u8 buf[AES_BLOCK_SIZE], *ctrptr; unsigned int nbytes, n, k; - int pk_state, locked, rc = 0; - - if (!req_ctx->param_init_done) { - /* fetch and check protected key state */ - spin_lock_bh(&ctx->pk_lock); - pk_state = ctx->pk_state; - switch (pk_state) { - case PK_STATE_NO_KEY: - rc = -ENOKEY; - break; - case PK_STATE_CONVERT_IN_PROGRESS: - rc = -EKEYEXPIRED; - break; - case PK_STATE_VALID: - memcpy(param->key, ctx->pk.protkey, sizeof(param->key)); - req_ctx->param_init_done = true; - break; - default: - rc = pk_state < 0 ? pk_state : -EIO; - break; - } - spin_unlock_bh(&ctx->pk_lock); - } - if (rc) - goto out; - - locked = mutex_trylock(&ctrblk_lock); + u8 *ctrptr; + int rc = 0; /* * Note that in case of partial processing or failure the walk @@ -971,32 +958,71 @@ static int ctr_paes_do_crypt(struct s390_paes_ctx *ctx, AES_BLOCK_SIZE); crypto_inc(walk->iv, AES_BLOCK_SIZE); rc = skcipher_walk_done(walk, nbytes - k); + if (rc) + goto out; } if (k < n) { if (!maysleep) { - if (locked) - mutex_unlock(&ctrblk_lock); rc = -EKEYEXPIRED; goto out; } - rc = paes_convert_key(ctx); - if (rc) { - if (locked) - mutex_unlock(&ctrblk_lock); + rc = paes_convert_key(ctx, tested); + if (rc) goto out; - } spin_lock_bh(&ctx->pk_lock); memcpy(param->key, ctx->pk.protkey, sizeof(param->key)); spin_unlock_bh(&ctx->pk_lock); } } - if (locked) - mutex_unlock(&ctrblk_lock); + +out: + return rc; +} + +static int ctr_paes_do_crypt(struct s390_paes_ctx *ctx, + struct s390_pctr_req_ctx *req_ctx, + bool tested, bool maysleep) +{ + struct ctr_param *param = &req_ctx->param; + struct skcipher_walk *walk = &req_ctx->walk; + u8 buf[AES_BLOCK_SIZE]; + int pk_state, rc = 0; + + if (!req_ctx->param_init_done) { + /* fetch and check protected key state */ + spin_lock_bh(&ctx->pk_lock); + pk_state = ctx->pk_state; + switch (pk_state) { + case PK_STATE_NO_KEY: + rc = -ENOKEY; + break; + case PK_STATE_CONVERT_IN_PROGRESS: + rc = -EKEYEXPIRED; + break; + case PK_STATE_VALID: + memcpy(param->key, ctx->pk.protkey, sizeof(param->key)); + req_ctx->param_init_done = true; + break; + default: + rc = pk_state < 0 ? pk_state : -EIO; + break; + } + spin_unlock_bh(&ctx->pk_lock); + } + if (rc) + goto out; + + if (down_trylock(&ctrblk_sem) == 0) { + rc = __ctr_paes_do_crypt(ctx, param, walk, tested, maysleep, true); + up(&ctrblk_sem); + } else { + rc = __ctr_paes_do_crypt(ctx, param, walk, tested, maysleep, false); + } /* final block may be < AES_BLOCK_SIZE, copy only nbytes */ - if (nbytes) { + if (!rc && walk->nbytes > 0) { memset(buf, 0, AES_BLOCK_SIZE); - memcpy(buf, walk->src.virt.addr, nbytes); + memcpy(buf, walk->src.virt.addr, walk->nbytes); while (1) { if (cpacf_kmctr(ctx->fc, param, buf, buf, AES_BLOCK_SIZE, @@ -1006,19 +1032,20 @@ static int ctr_paes_do_crypt(struct s390_paes_ctx *ctx, rc = -EKEYEXPIRED; goto out; } - rc = paes_convert_key(ctx); + rc = paes_convert_key(ctx, tested); if (rc) goto out; spin_lock_bh(&ctx->pk_lock); memcpy(param->key, ctx->pk.protkey, sizeof(param->key)); spin_unlock_bh(&ctx->pk_lock); } - memcpy(walk->dst.virt.addr, buf, nbytes); + memcpy(walk->dst.virt.addr, buf, walk->nbytes); crypto_inc(walk->iv, AES_BLOCK_SIZE); rc = skcipher_walk_done(walk, 0); } out: + memzero_explicit(buf, sizeof(buf)); pr_debug("rc=%d\n", rc); return rc; } @@ -1029,6 +1056,8 @@ static int ctr_paes_crypt(struct skcipher_request *req) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); struct skcipher_walk *walk = &req_ctx->walk; + bool tested = crypto_skcipher_tested(tfm); + bool cleanup = true; int rc; /* @@ -1046,7 +1075,7 @@ static int ctr_paes_crypt(struct skcipher_request *req) /* Try synchronous operation if no active engine usage */ if (!atomic_read(&ctx->via_engine_ctr)) { - rc = ctr_paes_do_crypt(ctx, req_ctx, false); + rc = ctr_paes_do_crypt(ctx, req_ctx, tested, false); if (rc == 0) goto out; } @@ -1059,15 +1088,17 @@ static int ctr_paes_crypt(struct skcipher_request *req) if (rc == 0 || rc == -EKEYEXPIRED) { atomic_inc(&ctx->via_engine_ctr); rc = crypto_transfer_skcipher_request_to_engine(paes_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) + if (cleanup && walk->nbytes) skcipher_walk_done(walk, rc); out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(&req_ctx->param, sizeof(req_ctx->param)); pr_debug("rc=%d\n", rc); return rc; @@ -1099,23 +1130,15 @@ static int ctr_paes_do_one_request(struct crypto_engine *engine, void *areq) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); struct skcipher_walk *walk = &req_ctx->walk; + bool tested = crypto_skcipher_tested(tfm); int rc; /* walk has already been prepared */ - rc = ctr_paes_do_crypt(ctx, req_ctx, true); + rc = ctr_paes_do_crypt(ctx, req_ctx, tested, true); if (rc == -EKEYEXPIRED) { - /* - * Protected key expired, conversion is in process. - * Trigger a re-schedule of this request by returning - * -ENOSPC ("hardware queue is full") to the crypto engine. - * To avoid immediately re-invocation of this callback, - * tell the scheduler to voluntarily give up the CPU here. - */ - cond_resched(); - pr_debug("rescheduling request\n"); - return -ENOSPC; - } else if (rc) { + return pkey_handle_expired(); + } else if (rc && walk->nbytes) { skcipher_walk_done(walk, rc); } @@ -1125,7 +1148,7 @@ static int ctr_paes_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&ctx->via_engine_ctr); crypto_finalize_skcipher_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } static struct skcipher_engine_alg ctr_paes_alg = { @@ -1133,6 +1156,7 @@ static struct skcipher_engine_alg ctr_paes_alg = { .base.cra_name = "ctr(paes)", .base.cra_driver_name = "ctr-paes-s390", .base.cra_priority = 402, /* ecb-paes-s390 + 1 */ + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct s390_paes_ctx), .base.cra_module = THIS_MODULE, @@ -1190,6 +1214,7 @@ static int xts_paes_setkey(struct crypto_skcipher *tfm, const u8 *in_key, unsigned int in_keylen) { struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm); + bool tested = crypto_skcipher_tested(tfm); u8 ckey[2 * AES_MAX_KEY_SIZE]; unsigned int ckey_len; long fc; @@ -1205,7 +1230,7 @@ static int xts_paes_setkey(struct crypto_skcipher *tfm, const u8 *in_key, goto out; /* convert raw key(s) into protected key(s) */ - rc = pxts_convert_key(ctx); + rc = pxts_convert_key(ctx, tested); if (rc) goto out; @@ -1255,7 +1280,7 @@ out: static int xts_paes_do_crypt_fullkey(struct s390_pxts_ctx *ctx, struct s390_pxts_req_ctx *req_ctx, - bool maysleep) + bool tested, bool maysleep) { struct xts_full_km_param *param = &req_ctx->param.full_km_param; struct skcipher_walk *walk = &req_ctx->walk; @@ -1292,14 +1317,17 @@ static int xts_paes_do_crypt_fullkey(struct s390_pxts_ctx *ctx, n = nbytes & ~(AES_BLOCK_SIZE - 1); k = cpacf_km(ctx->fc | req_ctx->modifier, param->key + offset, walk->dst.virt.addr, walk->src.virt.addr, n); - if (k) + if (k) { rc = skcipher_walk_done(walk, nbytes - k); + if (rc) + goto out; + } if (k < n) { if (!maysleep) { rc = -EKEYEXPIRED; goto out; } - rc = pxts_convert_key(ctx); + rc = pxts_convert_key(ctx, tested); if (rc) goto out; spin_lock_bh(&ctx->pk_lock); @@ -1318,7 +1346,8 @@ static inline int __xts_2keys_prep_param(struct s390_pxts_ctx *ctx, struct xts_km_param *param, struct skcipher_walk *walk, unsigned int keylen, - unsigned int offset, bool maysleep) + unsigned int offset, + bool tested, bool maysleep) { struct xts_pcc_param pcc_param; unsigned long cc = 1; @@ -1337,7 +1366,7 @@ static inline int __xts_2keys_prep_param(struct s390_pxts_ctx *ctx, rc = -EKEYEXPIRED; break; } - rc = pxts_convert_key(ctx); + rc = pxts_convert_key(ctx, tested); if (rc) break; continue; @@ -1345,13 +1374,13 @@ static inline int __xts_2keys_prep_param(struct s390_pxts_ctx *ctx, memcpy(param->init, pcc_param.xts, 16); } - memzero_explicit(pcc_param.key, sizeof(pcc_param.key)); + memzero_explicit(&pcc_param, sizeof(pcc_param)); return rc; } static int xts_paes_do_crypt_2keys(struct s390_pxts_ctx *ctx, struct s390_pxts_req_ctx *req_ctx, - bool maysleep) + bool tested, bool maysleep) { struct xts_km_param *param = &req_ctx->param.km_param; struct skcipher_walk *walk = &req_ctx->walk; @@ -1369,7 +1398,7 @@ static int xts_paes_do_crypt_2keys(struct s390_pxts_ctx *ctx, if (!req_ctx->param_init_done) { rc = __xts_2keys_prep_param(ctx, param, walk, - keylen, offset, maysleep); + keylen, offset, tested, maysleep); if (rc) goto out; req_ctx->param_init_done = true; @@ -1385,14 +1414,17 @@ static int xts_paes_do_crypt_2keys(struct s390_pxts_ctx *ctx, n = nbytes & ~(AES_BLOCK_SIZE - 1); k = cpacf_km(ctx->fc | req_ctx->modifier, param->key + offset, walk->dst.virt.addr, walk->src.virt.addr, n); - if (k) + if (k) { rc = skcipher_walk_done(walk, nbytes - k); + if (rc) + goto out; + } if (k < n) { if (!maysleep) { rc = -EKEYEXPIRED; goto out; } - rc = pxts_convert_key(ctx); + rc = pxts_convert_key(ctx, tested); if (rc) goto out; spin_lock_bh(&ctx->pk_lock); @@ -1408,7 +1440,7 @@ out: static int xts_paes_do_crypt(struct s390_pxts_ctx *ctx, struct s390_pxts_req_ctx *req_ctx, - bool maysleep) + bool tested, bool maysleep) { int pk_state, rc = 0; @@ -1436,11 +1468,11 @@ static int xts_paes_do_crypt(struct s390_pxts_ctx *ctx, switch (ctx->fc) { case CPACF_KM_PXTS_128: case CPACF_KM_PXTS_256: - rc = xts_paes_do_crypt_2keys(ctx, req_ctx, maysleep); + rc = xts_paes_do_crypt_2keys(ctx, req_ctx, tested, maysleep); break; case CPACF_KM_PXTS_128_FULL: case CPACF_KM_PXTS_256_FULL: - rc = xts_paes_do_crypt_fullkey(ctx, req_ctx, maysleep); + rc = xts_paes_do_crypt_fullkey(ctx, req_ctx, tested, maysleep); break; default: rc = -EINVAL; @@ -1457,6 +1489,8 @@ static inline int xts_paes_crypt(struct skcipher_request *req, unsigned long mod struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm); struct skcipher_walk *walk = &req_ctx->walk; + bool tested = crypto_skcipher_tested(tfm); + bool cleanup = true; int rc; /* @@ -1475,7 +1509,7 @@ static inline int xts_paes_crypt(struct skcipher_request *req, unsigned long mod /* Try synchronous operation if no active engine usage */ if (!atomic_read(&ctx->via_engine_ctr)) { - rc = xts_paes_do_crypt(ctx, req_ctx, false); + rc = xts_paes_do_crypt(ctx, req_ctx, tested, false); if (rc == 0) goto out; } @@ -1488,15 +1522,17 @@ static inline int xts_paes_crypt(struct skcipher_request *req, unsigned long mod if (rc == 0 || rc == -EKEYEXPIRED) { atomic_inc(&ctx->via_engine_ctr); rc = crypto_transfer_skcipher_request_to_engine(paes_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) + if (cleanup && walk->nbytes) skcipher_walk_done(walk, rc); out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(&req_ctx->param, sizeof(req_ctx->param)); pr_debug("rc=%d\n", rc); return rc; @@ -1538,23 +1574,15 @@ static int xts_paes_do_one_request(struct crypto_engine *engine, void *areq) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm); struct skcipher_walk *walk = &req_ctx->walk; + bool tested = crypto_skcipher_tested(tfm); int rc; /* walk has already been prepared */ - rc = xts_paes_do_crypt(ctx, req_ctx, true); + rc = xts_paes_do_crypt(ctx, req_ctx, tested, true); if (rc == -EKEYEXPIRED) { - /* - * Protected key expired, conversion is in process. - * Trigger a re-schedule of this request by returning - * -ENOSPC ("hardware queue is full") to the crypto engine. - * To avoid immediately re-invocation of this callback, - * tell the scheduler to voluntarily give up the CPU here. - */ - cond_resched(); - pr_debug("rescheduling request\n"); - return -ENOSPC; - } else if (rc) { + return pkey_handle_expired(); + } else if (rc && walk->nbytes) { skcipher_walk_done(walk, rc); } @@ -1564,7 +1592,7 @@ static int xts_paes_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&ctx->via_engine_ctr); crypto_finalize_skcipher_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } static struct skcipher_engine_alg xts_paes_alg = { @@ -1572,6 +1600,7 @@ static struct skcipher_engine_alg xts_paes_alg = { .base.cra_name = "xts(paes)", .base.cra_driver_name = "xts-paes-s390", .base.cra_priority = 402, /* ecb-paes-s390 + 1 */ + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct s390_pxts_ctx), .base.cra_module = THIS_MODULE, diff --git a/arch/s390/crypto/phmac_s390.c b/arch/s390/crypto/phmac_s390.c index 88342bd4c37a..bbf8a6809ecb 100644 --- a/arch/s390/crypto/phmac_s390.c +++ b/arch/s390/crypto/phmac_s390.c @@ -23,6 +23,10 @@ static struct crypto_engine *phmac_crypto_engine; #define MAX_QLEN 10 +static bool pkey_clrkey_allowed; +module_param_named(clrkey, pkey_clrkey_allowed, bool, 0444); +MODULE_PARM_DESC(clrkey, "Allow clear key material (default N)"); + /* * A simple hash walk helper */ @@ -58,8 +62,10 @@ static inline int hwh_prepare(struct ahash_request *req, */ static inline int hwh_advance(struct hash_walk_helper *hwh, int n) { - if (n < 0) + if (n < 0) { + hwh->walkbytes = n; return crypto_hash_walk_done(&hwh->walk, n); + } hwh->walkbytes -= n; hwh->walkaddr += n; @@ -311,10 +317,14 @@ static inline int phmac_tfm_ctx_setkey(struct phmac_tfm_ctx *tfm_ctx, * This function may sleep - don't call in non-sleeping context. */ static inline int convert_key(const u8 *key, unsigned int keylen, - struct phmac_protkey *pk) + struct phmac_protkey *pk, bool tested) { + u32 xflags = PKEY_XFLAG_NOMEMALLOC; int rc, i; + if (tested && !pkey_clrkey_allowed) + xflags |= PKEY_XFLAG_NOCLEARKEY; + pk->len = sizeof(pk->protkey); /* @@ -328,9 +338,13 @@ static inline int convert_key(const u8 *key, unsigned int keylen, } rc = pkey_key2protkey(key, keylen, pk->protkey, &pk->len, &pk->type, - PKEY_XFLAG_NOMEMALLOC); + xflags); } + /* But finally map -EBUSY to -EIO to indicate an IO failure */ + if (rc == -EBUSY) + rc = -EIO; + out: pr_debug("rc=%d\n", rc); return rc; @@ -350,7 +364,7 @@ out: * unnecessary additional conversion but never to invalid data on the * hash operation. */ -static int phmac_convert_key(struct phmac_tfm_ctx *tfm_ctx) +static int phmac_convert_key(struct phmac_tfm_ctx *tfm_ctx, bool tested) { struct phmac_protkey pk; int rc; @@ -359,7 +373,7 @@ static int phmac_convert_key(struct phmac_tfm_ctx *tfm_ctx) tfm_ctx->pk_state = PK_STATE_CONVERT_IN_PROGRESS; spin_unlock_bh(&tfm_ctx->pk_lock); - rc = convert_key(tfm_ctx->keybuf, tfm_ctx->keylen, &pk); + rc = convert_key(tfm_ctx->keybuf, tfm_ctx->keylen, &pk, tested); /* update context */ spin_lock_bh(&tfm_ctx->pk_lock); @@ -404,6 +418,7 @@ static int phmac_kmac_update(struct ahash_request *req, bool maysleep) struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx; struct hash_walk_helper *hwh = &req_ctx->hwh; unsigned int bs = crypto_ahash_blocksize(tfm); + bool tested = crypto_ahash_tested(tfm); unsigned int offset, k, n; int rc = 0; @@ -444,7 +459,7 @@ static int phmac_kmac_update(struct ahash_request *req, bool maysleep) rc = -EKEYEXPIRED; goto out; } - rc = phmac_convert_key(tfm_ctx); + rc = phmac_convert_key(tfm_ctx, tested); if (rc) goto out; spin_lock_bh(&tfm_ctx->pk_lock); @@ -480,7 +495,7 @@ static int phmac_kmac_update(struct ahash_request *req, bool maysleep) rc = -EKEYEXPIRED; goto out; } - rc = phmac_convert_key(tfm_ctx); + rc = phmac_convert_key(tfm_ctx, tested); if (rc) goto out; spin_lock_bh(&tfm_ctx->pk_lock); @@ -517,6 +532,7 @@ static int phmac_kmac_final(struct ahash_request *req, bool maysleep) struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx; unsigned int ds = crypto_ahash_digestsize(tfm); unsigned int bs = crypto_ahash_blocksize(tfm); + bool tested = crypto_ahash_tested(tfm); unsigned int k, n; int rc = 0; @@ -537,7 +553,7 @@ static int phmac_kmac_final(struct ahash_request *req, bool maysleep) rc = -EKEYEXPIRED; goto out; } - rc = phmac_convert_key(tfm_ctx); + rc = phmac_convert_key(tfm_ctx, tested); if (rc) goto out; spin_lock_bh(&tfm_ctx->pk_lock); @@ -596,6 +612,7 @@ static int phmac_update(struct ahash_request *req) struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm); struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx; struct hash_walk_helper *hwh = &req_ctx->hwh; + bool cleanup = true; int rc; /* prep the walk in the request context */ @@ -619,12 +636,15 @@ static int phmac_update(struct ahash_request *req) req_ctx->async_op = OP_UPDATE; atomic_inc(&tfm_ctx->via_engine_ctr); rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&tfm_ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) { - hwh_advance(hwh, rc); + if (cleanup) { + if (hwh->walkbytes > 0) + hwh_advance(hwh, rc); memzero_explicit(kmac_ctx, sizeof(*kmac_ctx)); } @@ -639,6 +659,7 @@ static int phmac_final(struct ahash_request *req) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm); struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx; + bool cleanup = true; int rc = 0; /* Try synchronous operation if no active engine usage */ @@ -657,12 +678,14 @@ static int phmac_final(struct ahash_request *req) req_ctx->async_op = OP_FINAL; atomic_inc(&tfm_ctx->via_engine_ctr); rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&tfm_ctx->via_engine_ctr); } out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(kmac_ctx, sizeof(*kmac_ctx)); pr_debug("rc=%d\n", rc); return rc; @@ -675,6 +698,7 @@ static int phmac_finup(struct ahash_request *req) struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm); struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx; struct hash_walk_helper *hwh = &req_ctx->hwh; + bool cleanup = true; int rc; /* prep the walk in the request context */ @@ -706,15 +730,17 @@ static int phmac_finup(struct ahash_request *req) /* req->async_op has been set to either OP_FINUP or OP_FINAL */ atomic_inc(&tfm_ctx->via_engine_ctr); rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&tfm_ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) + if (cleanup && hwh->walkbytes > 0) hwh_advance(hwh, rc); out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(kmac_ctx, sizeof(*kmac_ctx)); pr_debug("rc=%d\n", rc); return rc; @@ -741,11 +767,12 @@ static int phmac_setkey(struct crypto_ahash *tfm, struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm); unsigned int ds = crypto_ahash_digestsize(tfm); unsigned int bs = crypto_ahash_blocksize(tfm); + bool tested = crypto_ahash_tested(tfm); unsigned int tmpkeylen; u8 *tmpkey = NULL; int rc = 0; - if (!crypto_ahash_tested(tfm)) { + if (!tested) { /* * selftest running: key is a raw hmac clear key and needs * to get embedded into a 'clear key token' in order to have @@ -770,7 +797,7 @@ static int phmac_setkey(struct crypto_ahash *tfm, goto out; /* convert raw key into protected key */ - rc = phmac_convert_key(tfm_ctx); + rc = phmac_convert_key(tfm_ctx, tested); if (rc) goto out; @@ -876,16 +903,7 @@ static int phmac_do_one_request(struct crypto_engine *engine, void *areq) case OP_FINUP: rc = phmac_kmac_update(req, true); if (rc == -EKEYEXPIRED) { - /* - * Protected key expired, conversion is in process. - * Trigger a re-schedule of this request by returning - * -ENOSPC ("hardware queue full") to the crypto engine. - * To avoid immediately re-invocation of this callback, - * tell scheduler to voluntarily give up the CPU here. - */ - pr_debug("rescheduling request\n"); - cond_resched(); - return -ENOSPC; + return pkey_handle_expired(); } else if (rc) { hwh_advance(hwh, rc); goto out; @@ -896,18 +914,8 @@ static int phmac_do_one_request(struct crypto_engine *engine, void *areq) fallthrough; case OP_FINAL: rc = phmac_kmac_final(req, true); - if (rc == -EKEYEXPIRED) { - /* - * Protected key expired, conversion is in process. - * Trigger a re-schedule of this request by returning - * -ENOSPC ("hardware queue full") to the crypto engine. - * To avoid immediately re-invocation of this callback, - * tell scheduler to voluntarily give up the CPU here. - */ - pr_debug("rescheduling request\n"); - cond_resched(); - return -ENOSPC; - } + if (rc == -EKEYEXPIRED) + return pkey_handle_expired(); break; default: /* unknown/unsupported/unimplemented asynch op */ @@ -922,7 +930,7 @@ out: atomic_dec(&tfm_ctx->via_engine_ctr); crypto_finalize_hash_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } #define S390_ASYNC_PHMAC_ALG(x) \ diff --git a/arch/s390/hypfs/Makefile b/arch/s390/hypfs/Makefile index c34854d298f8..5a3daeafe926 100644 --- a/arch/s390/hypfs/Makefile +++ b/arch/s390/hypfs/Makefile @@ -3,6 +3,8 @@ # Makefile for the linux hypfs filesystem routines. # +CONTEXT_ANALYSIS := y + obj-$(CONFIG_S390_HYPFS) += hypfs_dbfs.o obj-$(CONFIG_S390_HYPFS) += hypfs_diag.o obj-$(CONFIG_S390_HYPFS) += hypfs_diag0c.o diff --git a/arch/s390/hypfs/hypfs_dbfs.c b/arch/s390/hypfs/hypfs_dbfs.c index 41a0d2066fa0..25a59c048857 100644 --- a/arch/s390/hypfs/hypfs_dbfs.c +++ b/arch/s390/hypfs/hypfs_dbfs.c @@ -16,7 +16,7 @@ static struct hypfs_dbfs_data *hypfs_dbfs_data_alloc(struct hypfs_dbfs_file *f) { struct hypfs_dbfs_data *data; - data = kmalloc(sizeof(*data), GFP_KERNEL); + data = kmalloc_obj(*data); if (!data) return NULL; data->dbfs_file = f; diff --git a/arch/s390/hypfs/hypfs_diag0c.c b/arch/s390/hypfs/hypfs_diag0c.c index 61220e717af0..8f726e6558a8 100644 --- a/arch/s390/hypfs/hypfs_diag0c.c +++ b/arch/s390/hypfs/hypfs_diag0c.c @@ -35,13 +35,12 @@ static void *diag0c_store(unsigned int *count) cpus_read_lock(); cpu_count = num_online_cpus(); - cpu_vec = kmalloc_array(num_possible_cpus(), sizeof(*cpu_vec), - GFP_KERNEL); + cpu_vec = kmalloc_objs(*cpu_vec, num_possible_cpus()); if (!cpu_vec) goto fail_unlock_cpus; /* Note: Diag 0c needs 8 byte alignment and real storage */ - diag0c_data = kzalloc(struct_size(diag0c_data, entry, cpu_count), - GFP_KERNEL | GFP_DMA); + diag0c_data = kzalloc_flex(*diag0c_data, entry, cpu_count, + GFP_KERNEL | GFP_DMA); if (!diag0c_data) goto fail_kfree_cpu_vec; i = 0; diff --git a/arch/s390/hypfs/hypfs_sprp.c b/arch/s390/hypfs/hypfs_sprp.c index a72576221cab..08042d3a9d56 100644 --- a/arch/s390/hypfs/hypfs_sprp.c +++ b/arch/s390/hypfs/hypfs_sprp.c @@ -74,7 +74,7 @@ static int __hypfs_sprp_ioctl(void __user *user_area) rc = -ENOMEM; data = (void *)get_zeroed_page(GFP_KERNEL); - diag304 = kzalloc(sizeof(*diag304), GFP_KERNEL); + diag304 = kzalloc_obj(*diag304); if (!data || !diag304) goto out; diff --git a/arch/s390/hypfs/inode.c b/arch/s390/hypfs/inode.c index 3a47c2e24b6e..559b011e6880 100644 --- a/arch/s390/hypfs/inode.c +++ b/arch/s390/hypfs/inode.c @@ -292,7 +292,7 @@ static int hypfs_init_fs_context(struct fs_context *fc) { struct hypfs_sb_info *sbi; - sbi = kzalloc(sizeof(struct hypfs_sb_info), GFP_KERNEL); + sbi = kzalloc_obj(struct hypfs_sb_info); if (!sbi) return -ENOMEM; diff --git a/arch/s390/include/asm/Kbuild b/arch/s390/include/asm/Kbuild index 297bf7157968..0c1fc47c3ba0 100644 --- a/arch/s390/include/asm/Kbuild +++ b/arch/s390/include/asm/Kbuild @@ -5,6 +5,6 @@ generated-y += syscall_table.h generated-y += unistd_nr.h generic-y += asm-offsets.h -generic-y += kvm_types.h generic-y += mcs_spinlock.h generic-y += mmzone.h +generic-y += ring_buffer.h diff --git a/arch/s390/include/asm/ap.h b/arch/s390/include/asm/ap.h index b24459f692fa..c91b6ace199d 100644 --- a/arch/s390/include/asm/ap.h +++ b/arch/s390/include/asm/ap.h @@ -78,7 +78,7 @@ union ap_queue_status_reg { }; /** - * ap_intructions_available() - Test if AP instructions are available. + * ap_instructions_available() - Test if AP instructions are available. * * Returns true if the AP instructions are installed, otherwise false. */ @@ -125,8 +125,8 @@ struct ap_tapq_hwinfo { unsigned int nd : 8; /* nr of domains */ unsigned int : 4; unsigned int ml : 4; /* apxl ml */ - unsigned int : 4; - unsigned int qd : 4; /* queue depth */ + unsigned int : 3; + unsigned int qd : 5; /* queue depth */ }; }; }; diff --git a/arch/s390/include/asm/asm-prototypes.h b/arch/s390/include/asm/asm-prototypes.h index f662eb4b9246..d4da4436d02b 100644 --- a/arch/s390/include/asm/asm-prototypes.h +++ b/arch/s390/include/asm/asm-prototypes.h @@ -3,12 +3,9 @@ #include <linux/kvm_host.h> #include <linux/ftrace.h> +#include <asm/bug.h> #include <asm/fpu.h> #include <asm/nospec-branch.h> #include <asm-generic/asm-prototypes.h> -__int128_t __ashlti3(__int128_t a, int b); -__int128_t __ashrti3(__int128_t a, int b); -__int128_t __lshrti3(__int128_t a, int b); - #endif /* _ASM_S390_PROTOTYPES_H */ diff --git a/arch/s390/include/asm/asm.h b/arch/s390/include/asm/asm.h index e9062b01e2a2..510901c2a5f9 100644 --- a/arch/s390/include/asm/asm.h +++ b/arch/s390/include/asm/asm.h @@ -30,7 +30,7 @@ */ #if defined(__GCC_ASM_FLAG_OUTPUTS__) && !(IS_ENABLED(CONFIG_CC_ASM_FLAG_OUTPUT_BROKEN)) -#define __HAVE_ASM_FLAG_OUTPUTS__ +#define __HAVE_ASM_FLAG_OUTPUTS__ 1 #define CC_IPM(sym) #define CC_OUT(sym, var) "=@cc" (var) diff --git a/arch/s390/include/asm/barrier.h b/arch/s390/include/asm/barrier.h index f3184073e754..d1e4201f3850 100644 --- a/arch/s390/include/asm/barrier.h +++ b/arch/s390/include/asm/barrier.h @@ -8,6 +8,7 @@ #ifndef __ASM_BARRIER_H #define __ASM_BARRIER_H +#include <asm/alternative.h> #include <asm/march.h> /* @@ -16,16 +17,11 @@ * to devices. */ -#ifdef MARCH_HAS_Z196_FEATURES -/* Fast-BCR without checkpoint synchronization */ -#define __ASM_BCR_SERIALIZE "bcr 14,0" -#else -#define __ASM_BCR_SERIALIZE "bcr 15,0" -#endif - static __always_inline void bcr_serialize(void) { - asm volatile(__ASM_BCR_SERIALIZE : : : "memory"); + asm_inline volatile( + ALTERNATIVE("bcr 15,0", "bcr 14,0", ALT_FACILITY(45)) + : : : "memory"); } #define __mb() bcr_serialize() @@ -62,8 +58,8 @@ do { \ * @size: number of elements in array */ #define array_index_mask_nospec array_index_mask_nospec -static inline unsigned long array_index_mask_nospec(unsigned long index, - unsigned long size) +static __always_inline unsigned long array_index_mask_nospec(unsigned long index, + unsigned long size) { unsigned long mask; diff --git a/arch/s390/include/asm/bug.h b/arch/s390/include/asm/bug.h index ee9221bb5d18..2010342c97b1 100644 --- a/arch/s390/include/asm/bug.h +++ b/arch/s390/include/asm/bug.h @@ -2,60 +2,139 @@ #ifndef _ASM_S390_BUG_H #define _ASM_S390_BUG_H +#include <linux/compiler.h> +#include <linux/const.h> #include <linux/stringify.h> -#ifdef CONFIG_BUG +#define MONCODE_BUG _AC(0, U) +#define MONCODE_BUG_ARG _AC(1, U) -#ifndef CONFIG_DEBUG_BUGVERBOSE -#define _BUGVERBOSE_LOCATION(file, line) +#ifndef __ASSEMBLER__ +#if defined(CONFIG_BUG) && defined(CONFIG_CC_HAS_ASM_IMMEDIATE_STRINGS) + +#ifdef CONFIG_DEBUG_BUGVERBOSE +#define __BUG_ENTRY_VERBOSE(file, line) \ + " .long " file " - . # bug_entry::file\n" \ + " .short " line " # bug_entry::line\n" #else -#define __BUGVERBOSE_LOCATION(file, line) \ - .pushsection .rodata.str, "aMS", @progbits, 1; \ - .align 2; \ - 10002: .ascii file "\0"; \ - .popsection; \ - \ - .long 10002b - .; \ - .short line; -#define _BUGVERBOSE_LOCATION(file, line) __BUGVERBOSE_LOCATION(file, line) +#define __BUG_ENTRY_VERBOSE(file, line) #endif -#ifndef CONFIG_GENERIC_BUG -#define __BUG_ENTRY(cond_str, flags) +#ifdef CONFIG_DEBUG_BUGVERBOSE_DETAILED +#define WARN_CONDITION_STR(cond_str) cond_str #else -#define __BUG_ENTRY(cond_str, flags) \ - .pushsection __bug_table, "aw"; \ - .align 4; \ - 10000: .long 10001f - .; \ - _BUGVERBOSE_LOCATION(WARN_CONDITION_STR(cond_str) __FILE__, __LINE__) \ - .short flags; \ - .popsection; \ - 10001: +#define WARN_CONDITION_STR(cond_str) "" #endif -#define ASM_BUG_FLAGS(cond_str, flags) \ - __BUG_ENTRY(cond_str, flags) \ - mc 0,0 +#define __BUG_ENTRY(format, file, line, flags, size) \ + " .section __bug_table,\"aw\"\n" \ + "1: .long 0b - . # bug_entry::bug_addr\n"\ + " .long " format " - . # bug_entry::format\n" \ + __BUG_ENTRY_VERBOSE(file, line) \ + " .short "flags" # bug_entry::flags\n" \ + " .org 1b+"size"\n" \ + " .previous" -#define ASM_BUG() ASM_BUG_FLAGS("", 0) +#define __BUG_ASM(cond_str, flags) \ +do { \ + asm_inline volatile("\n" \ + "0: mc %[monc](%%r0),0\n" \ + __BUG_ENTRY("%[frmt]", "%[file]", "%[line]", \ + "%[flgs]", "%[size]") \ + : \ + : [monc] "i" (MONCODE_BUG), \ + [frmt] "i" (WARN_CONDITION_STR(cond_str)), \ + [file] "i" (__FILE__), \ + [line] "i" (__LINE__), \ + [flgs] "i" (flags), \ + [size] "i" (sizeof(struct bug_entry))); \ +} while (0) -#define __BUG_FLAGS(cond_str, flags) \ - asm_inline volatile(__stringify(ASM_BUG_FLAGS(cond_str, flags))); +#define BUG() \ +do { \ + __BUG_ASM("", 0); \ + unreachable(); \ +} while (0) -#define __WARN_FLAGS(cond_str, flags) \ -do { \ - __BUG_FLAGS(cond_str, BUGFLAG_WARNING|(flags)); \ +#define __WARN_FLAGS(cond_str, flags) \ +do { \ + __BUG_ASM(cond_str, BUGFLAG_WARNING | (flags)); \ } while (0) -#define BUG() \ -do { \ - __BUG_FLAGS("", 0); \ - unreachable(); \ +#define __WARN_bug_entry(flags, format) \ +({ \ + struct bug_entry *bug; \ + \ + asm_inline volatile("\n" \ + "0: larl %[bug],1f\n" \ + __BUG_ENTRY("%[frmt]", "%[file]", "%[line]", \ + "%[flgs]", "%[size]") \ + : [bug] "=d" (bug) \ + : [frmt] "i" (format), \ + [file] "i" (__FILE__), \ + [line] "i" (__LINE__), \ + [flgs] "i" (flags), \ + [size] "i" (sizeof(struct bug_entry))); \ + bug; \ +}) + +/* + * Variable Argument List (va_list) as defined in ELF Application + * Binary Interface s390x Supplement documentation. + */ +struct arch_va_list { + long __gpr; + long __fpr; + void *__overflow_arg_area; + void *__reg_save_area; +}; + +struct bug_entry; +struct pt_regs; + +void *__warn_args(struct arch_va_list *args, struct pt_regs *regs); +void __WARN_trap(struct bug_entry *bug, ...); + +#define __WARN_print_arg(flags, format, arg...) \ +do { \ + int __flags = (flags) | BUGFLAG_WARNING | BUGFLAG_ARGS; \ + \ + __WARN_trap(__WARN_bug_entry(__flags, format), ## arg); \ + /* prevent tail-call optimization */ \ + asm(""); \ } while (0) +#define __WARN_printf(taint, fmt, arg...) \ + __WARN_print_arg(BUGFLAG_TAINT(taint), fmt, ## arg) + +#define WARN_ONCE(cond, format, arg...) \ +({ \ + int __ret_warn_on = !!(cond); \ + \ + if (unlikely(__ret_warn_on)) { \ + __WARN_print_arg(BUGFLAG_ONCE|BUGFLAG_TAINT(TAINT_WARN),\ + format, ## arg); \ + } \ + __ret_warn_on; \ +}) + #define HAVE_ARCH_BUG +#define HAVE_ARCH_BUG_FORMAT +#define HAVE_ARCH_BUG_FORMAT_ARGS + +#define ARCH_WARN_ASM(file, line, flags, size) \ + ".section .rodata.str,\"aMS\",@progbits,1\n" \ + "9:\n" \ + ".asciz \"\"\n" /* Empty string for compatibility */ \ + ".previous\n" \ + "0:\n" \ + __stringify(mc 0(%r0),0) "\n" \ + __BUG_ENTRY("9b", file, line, flags, size) + +#define ARCH_WARN_REACHABLE -#endif /* CONFIG_BUG */ +#endif /* CONFIG_BUG && CONFIG_CC_HAS_ASM_IMMEDIATE_STRINGS */ +#endif /* __ASSEMBLER__ */ #include <asm-generic/bug.h> diff --git a/arch/s390/include/asm/ccwdev.h b/arch/s390/include/asm/ccwdev.h index e3afcece375e..d77f26390c07 100644 --- a/arch/s390/include/asm/ccwdev.h +++ b/arch/s390/include/asm/ccwdev.h @@ -10,7 +10,7 @@ #define _S390_CCWDEV_H_ #include <linux/device.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/ccw.h> #include <asm/chsc.h> #include <asm/fcx.h> #include <asm/irq.h> diff --git a/arch/s390/include/asm/cfi.h b/arch/s390/include/asm/cfi.h new file mode 100644 index 000000000000..9af2c7cb70ca --- /dev/null +++ b/arch/s390/include/asm/cfi.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_S390_CFI_H +#define _ASM_S390_CFI_H + +#define __bpfcall + +#endif /* _ASM_S390_CFI_H */ diff --git a/arch/s390/include/asm/cmpxchg.h b/arch/s390/include/asm/cmpxchg.h index 008357996262..e6ac55cf3c17 100644 --- a/arch/s390/include/asm/cmpxchg.h +++ b/arch/s390/include/asm/cmpxchg.h @@ -35,7 +35,7 @@ static __always_inline u64 __csg_asm(u64 ptr, u64 old, u64 new) return old; } -static inline u8 __arch_cmpxchg1(u64 ptr, u8 old, u8 new) +static __no_sanitize_or_inline u8 __arch_cmpxchg1(u64 ptr, u8 old, u8 new) { union { u8 b[4]; @@ -58,7 +58,7 @@ static inline u8 __arch_cmpxchg1(u64 ptr, u8 old, u8 new) return old; } -static inline u16 __arch_cmpxchg2(u64 ptr, u16 old, u16 new) +static __no_sanitize_or_inline u16 __arch_cmpxchg2(u64 ptr, u16 old, u16 new) { union { u16 b[2]; @@ -173,7 +173,7 @@ static __always_inline u64 __arch_cmpxchg(u64 ptr, u64 old, u64 new, int size) void __xchg_called_with_bad_pointer(void); -static inline u8 __arch_xchg1(u64 ptr, u8 x) +static __no_sanitize_or_inline u8 __arch_xchg1(u64 ptr, u8 x) { int shift = (3 ^ (ptr & 3)) << 3; u32 mask, old, new; @@ -188,7 +188,7 @@ static inline u8 __arch_xchg1(u64 ptr, u8 x) return old >> shift; } -static inline u16 __arch_xchg2(u64 ptr, u16 x) +static __no_sanitize_or_inline u16 __arch_xchg2(u64 ptr, u16 x) { int shift = (2 ^ (ptr & 2)) << 3; u32 mask, old, new; diff --git a/arch/s390/include/asm/cpacf.h b/arch/s390/include/asm/cpacf.h index a83683169d98..6174552d856d 100644 --- a/arch/s390/include/asm/cpacf.h +++ b/arch/s390/include/asm/cpacf.h @@ -301,6 +301,7 @@ static __always_inline void __cpacf_query(unsigned int opcode, cpacf_mask_t *mask) { __cpacf_query_insn(opcode, mask, CPACF_FC_QUERY); + kmsan_unpoison_memory(mask, sizeof(*mask)); } static __always_inline int __cpacf_check_opcode(unsigned int opcode) @@ -370,6 +371,7 @@ static __always_inline int cpacf_query_func(unsigned int opcode, static __always_inline void __cpacf_qai(unsigned int opcode, cpacf_qai_t *qai) { __cpacf_query_insn(opcode, qai, CPACF_FC_QUERY_AUTH_INFO); + kmsan_unpoison_memory(qai, sizeof(*qai)); } /** @@ -422,6 +424,7 @@ static inline int cpacf_km(unsigned long func, void *param, [opc] "i" (CPACF_KM) : "cc", "memory", "0", "1"); + kmsan_unpoison_memory(dest, src_len - s.odd); return src_len - s.odd; } @@ -454,6 +457,7 @@ static inline int cpacf_kmc(unsigned long func, void *param, [opc] "i" (CPACF_KMC) : "cc", "memory", "0", "1"); + kmsan_unpoison_memory(dest, src_len - s.odd); return src_len - s.odd; } @@ -587,6 +591,7 @@ static inline int cpacf_kmctr(unsigned long func, void *param, u8 *dest, [opc] "i" (CPACF_KMCTR) : "cc", "memory", "0", "1"); + kmsan_unpoison_memory(dest, src_len - s.odd); return src_len - s.odd; } @@ -619,6 +624,7 @@ static inline void cpacf_prno(unsigned long func, void *param, : [fc] "d" (func), [pba] "d" ((unsigned long)param), [seed] "d" (s.pair), [opc] "i" (CPACF_PRNO) : "cc", "memory", "0", "1"); + kmsan_unpoison_memory(dest, dest_len); } /** diff --git a/arch/s390/include/asm/dat-bits.h b/arch/s390/include/asm/dat-bits.h index 8d65eec2f124..c40874e0e426 100644 --- a/arch/s390/include/asm/dat-bits.h +++ b/arch/s390/include/asm/dat-bits.h @@ -9,6 +9,32 @@ #ifndef _S390_DAT_BITS_H #define _S390_DAT_BITS_H +/* + * vaddress union in order to easily decode a virtual address into its + * region first index, region second index etc. parts. + */ +union vaddress { + unsigned long addr; + struct { + unsigned long rfx : 11; + unsigned long rsx : 11; + unsigned long rtx : 11; + unsigned long sx : 11; + unsigned long px : 8; + unsigned long bx : 12; + }; + struct { + unsigned long rfx01 : 2; + unsigned long : 9; + unsigned long rsx01 : 2; + unsigned long : 9; + unsigned long rtx01 : 2; + unsigned long : 9; + unsigned long sx01 : 2; + unsigned long : 29; + }; +}; + union asce { unsigned long val; struct { @@ -98,7 +124,8 @@ union region3_table_entry { struct { unsigned long : 53; unsigned long fc: 1; /* Format-Control */ - unsigned long : 4; + unsigned long p : 1; /* DAT-Protection Bit */ + unsigned long : 3; unsigned long i : 1; /* Region-Invalid Bit */ unsigned long cr: 1; /* Common-Region Bit */ unsigned long tt: 2; /* Table-Type Bits */ @@ -140,7 +167,8 @@ union segment_table_entry { struct { unsigned long : 53; unsigned long fc: 1; /* Format-Control */ - unsigned long : 4; + unsigned long p : 1; /* DAT-Protection Bit */ + unsigned long : 3; unsigned long i : 1; /* Segment-Invalid Bit */ unsigned long cs: 1; /* Common-Segment Bit */ unsigned long tt: 2; /* Table-Type Bits */ diff --git a/arch/s390/include/asm/debug.h b/arch/s390/include/asm/debug.h index 6375276d94ea..ad438d6352c8 100644 --- a/arch/s390/include/asm/debug.h +++ b/arch/s390/include/asm/debug.h @@ -45,7 +45,7 @@ typedef struct debug_info { struct debug_info *next; struct debug_info *prev; refcount_t ref_count; - spinlock_t lock; + raw_spinlock_t lock; int level; int nr_areas; int pages_per_area; @@ -440,7 +440,7 @@ static int VNAME(var, active_entries)[EARLY_AREAS] __initdata .next = NULL, \ .prev = NULL, \ .ref_count = REFCOUNT_INIT(1), \ - .lock = __SPIN_LOCK_UNLOCKED(var.lock), \ + .lock = __RAW_SPIN_LOCK_UNLOCKED(var.lock), \ .level = DEBUG_DEFAULT_LEVEL, \ .nr_areas = EARLY_AREAS, \ .pages_per_area = EARLY_PAGES, \ @@ -460,7 +460,11 @@ static int VNAME(var, active_entries)[EARLY_AREAS] __initdata #define __REGISTER_STATIC_DEBUG_INFO(var, name, pages, areas, view) \ static int __init VNAME(var, reg)(void) \ { \ - debug_register_static(&var, (pages), (areas)); \ + int rc; \ + \ + rc = debug_register_static(&var, (pages), (areas)); \ + if (rc) \ + return rc; \ debug_register_view(&var, (view)); \ return 0; \ } \ @@ -490,9 +494,10 @@ arch_initcall(VNAME(var, reg)) __DEFINE_STATIC_AREA(var); \ static debug_info_t __refdata var = \ __DEBUG_INFO_INIT(var, (name), (buf_size)); \ +static debug_info_t __used __section(".s390dbf_info") *VNAME(var, info) = &var; \ __REGISTER_STATIC_DEBUG_INFO(var, name, pages, nr_areas, view) -void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas); +int debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas); #endif /* MODULE */ diff --git a/arch/s390/include/asm/entry-common.h b/arch/s390/include/asm/entry-common.h index 979af986a8fe..35450a485323 100644 --- a/arch/s390/include/asm/entry-common.h +++ b/arch/s390/include/asm/entry-common.h @@ -51,14 +51,6 @@ static __always_inline void arch_exit_to_user_mode(void) #define arch_exit_to_user_mode arch_exit_to_user_mode -static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, - unsigned long ti_work) -{ - choose_random_kstack_offset(get_tod_clock_fast()); -} - -#define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare - static __always_inline bool arch_in_rcu_eqs(void) { if (IS_ENABLED(CONFIG_KVM)) diff --git a/arch/s390/include/asm/entry-percpu.h b/arch/s390/include/asm/entry-percpu.h new file mode 100644 index 000000000000..4ef47265cfce --- /dev/null +++ b/arch/s390/include/asm/entry-percpu.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ARCH_S390_ENTRY_PERCPU_H +#define ARCH_S390_ENTRY_PERCPU_H + +#include <linux/kprobes.h> +#include <linux/percpu.h> +#include <asm/lowcore.h> +#include <asm/ptrace.h> +#include <asm/asm-offsets.h> + +static __always_inline void percpu_entry(struct pt_regs *regs) +{ + struct lowcore *lc = get_lowcore(); + + if (user_mode(regs)) + return; + regs->cpu = lc->cpu_nr; + regs->percpu_register = lc->percpu_register; + lc->percpu_register = 0; +} + +static __always_inline bool percpu_code_check(struct pt_regs *regs) +{ + unsigned int insn, disp; + struct kprobe *p; + + if (likely(user_mode(regs) || !regs->percpu_register)) + return false; + /* + * Within a percpu code section - check if the percpu base register + * needs to be updated. This is the case if the PSW does not point to + * the ADD instruction within the section. + * - AG %rx,percpu_offset_in_lowcore(%r0,%r0) + * which adds the percpu offset to the percpu base register. + */ + lockdep_assert_preemption_disabled(); +again: + insn = READ_ONCE(*(u16 *)psw_bits(regs->psw).ia); + if (unlikely(insn == BREAKPOINT_INSTRUCTION)) { + p = get_kprobe((void *)psw_bits(regs->psw).ia); + /* + * If the kprobe is concurrently removed on a different CPU + * it might not be found anymore. However text must have + * been restored - try again. + */ + if (!p) + goto again; + insn = p->opcode; + } + if ((insn & 0xff0f) != 0xe300) + return true; + disp = offsetof(struct lowcore, percpu_offset); + if (machine_has_relocated_lowcore()) + disp += LOWCORE_ALT_ADDRESS; + insn = (disp & 0xff000) >> 4 | (disp & 0x00fff) << 16 | 0x8; + if (*(u32 *)(psw_bits(regs->psw).ia + 2) != insn) + return true; + return false; +} + +static __always_inline void percpu_exit(struct pt_regs *regs, bool needs_fixup) +{ + struct lowcore *lc = get_lowcore(); + unsigned char reg; + + if (user_mode(regs)) + return; + reg = regs->percpu_register; + lc->percpu_register = reg; + if (likely(!needs_fixup)) + return; + /* Check if process has been migrated to a different CPU. */ + if (regs->cpu == lc->cpu_nr) + return; + /* Fixup percpu base register */ + regs->gprs[reg] -= __per_cpu_offset[regs->cpu]; + regs->gprs[reg] += lc->percpu_offset; +} + +#endif diff --git a/arch/s390/include/asm/fpu-insn-asm.h b/arch/s390/include/asm/fpu-insn-asm.h index cc0468fdf2d0..4a1bfc0f578e 100644 --- a/arch/s390/include/asm/fpu-insn-asm.h +++ b/arch/s390/include/asm/fpu-insn-asm.h @@ -16,181 +16,9 @@ #error only <asm/fpu-insn.h> can be included directly #endif -#ifdef __ASSEMBLER__ - -/* Macros to generate vector instruction byte code */ +#include <asm/insn-common-asm.h> -/* GR_NUM - Retrieve general-purpose register number - * - * @opd: Operand to store register number - * @r64: String designation register in the format "%rN" - */ -.macro GR_NUM opd gr - \opd = 255 - .ifc \gr,%r0 - \opd = 0 - .endif - .ifc \gr,%r1 - \opd = 1 - .endif - .ifc \gr,%r2 - \opd = 2 - .endif - .ifc \gr,%r3 - \opd = 3 - .endif - .ifc \gr,%r4 - \opd = 4 - .endif - .ifc \gr,%r5 - \opd = 5 - .endif - .ifc \gr,%r6 - \opd = 6 - .endif - .ifc \gr,%r7 - \opd = 7 - .endif - .ifc \gr,%r8 - \opd = 8 - .endif - .ifc \gr,%r9 - \opd = 9 - .endif - .ifc \gr,%r10 - \opd = 10 - .endif - .ifc \gr,%r11 - \opd = 11 - .endif - .ifc \gr,%r12 - \opd = 12 - .endif - .ifc \gr,%r13 - \opd = 13 - .endif - .ifc \gr,%r14 - \opd = 14 - .endif - .ifc \gr,%r15 - \opd = 15 - .endif - .if \opd == 255 - \opd = \gr - .endif -.endm - -/* VX_NUM - Retrieve vector register number - * - * @opd: Operand to store register number - * @vxr: String designation register in the format "%vN" - * - * The vector register number is used for as input number to the - * instruction and, as well as, to compute the RXB field of the - * instruction. - */ -.macro VX_NUM opd vxr - \opd = 255 - .ifc \vxr,%v0 - \opd = 0 - .endif - .ifc \vxr,%v1 - \opd = 1 - .endif - .ifc \vxr,%v2 - \opd = 2 - .endif - .ifc \vxr,%v3 - \opd = 3 - .endif - .ifc \vxr,%v4 - \opd = 4 - .endif - .ifc \vxr,%v5 - \opd = 5 - .endif - .ifc \vxr,%v6 - \opd = 6 - .endif - .ifc \vxr,%v7 - \opd = 7 - .endif - .ifc \vxr,%v8 - \opd = 8 - .endif - .ifc \vxr,%v9 - \opd = 9 - .endif - .ifc \vxr,%v10 - \opd = 10 - .endif - .ifc \vxr,%v11 - \opd = 11 - .endif - .ifc \vxr,%v12 - \opd = 12 - .endif - .ifc \vxr,%v13 - \opd = 13 - .endif - .ifc \vxr,%v14 - \opd = 14 - .endif - .ifc \vxr,%v15 - \opd = 15 - .endif - .ifc \vxr,%v16 - \opd = 16 - .endif - .ifc \vxr,%v17 - \opd = 17 - .endif - .ifc \vxr,%v18 - \opd = 18 - .endif - .ifc \vxr,%v19 - \opd = 19 - .endif - .ifc \vxr,%v20 - \opd = 20 - .endif - .ifc \vxr,%v21 - \opd = 21 - .endif - .ifc \vxr,%v22 - \opd = 22 - .endif - .ifc \vxr,%v23 - \opd = 23 - .endif - .ifc \vxr,%v24 - \opd = 24 - .endif - .ifc \vxr,%v25 - \opd = 25 - .endif - .ifc \vxr,%v26 - \opd = 26 - .endif - .ifc \vxr,%v27 - \opd = 27 - .endif - .ifc \vxr,%v28 - \opd = 28 - .endif - .ifc \vxr,%v29 - \opd = 29 - .endif - .ifc \vxr,%v30 - \opd = 30 - .endif - .ifc \vxr,%v31 - \opd = 31 - .endif - .if \opd == 255 - \opd = \vxr - .endif -.endm +#ifdef __ASSEMBLER__ /* RXB - Compute most significant bit used vector registers * diff --git a/arch/s390/include/asm/fpu-insn.h b/arch/s390/include/asm/fpu-insn.h index 96727f3bd0dc..ae8b7033cfd2 100644 --- a/arch/s390/include/asm/fpu-insn.h +++ b/arch/s390/include/asm/fpu-insn.h @@ -15,6 +15,7 @@ #include <linux/kmsan.h> #include <asm/asm-extable.h> +asm(".include \"asm/insn-common-asm.h\"\n"); asm(".include \"asm/fpu-insn-asm.h\"\n"); /* diff --git a/arch/s390/include/asm/gmap.h b/arch/s390/include/asm/gmap.h deleted file mode 100644 index 66c5808fd011..000000000000 --- a/arch/s390/include/asm/gmap.h +++ /dev/null @@ -1,174 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * KVM guest address space mapping code - * - * Copyright IBM Corp. 2007, 2016 - * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> - */ - -#ifndef _ASM_S390_GMAP_H -#define _ASM_S390_GMAP_H - -#include <linux/radix-tree.h> -#include <linux/refcount.h> - -/* Generic bits for GMAP notification on DAT table entry changes. */ -#define GMAP_NOTIFY_SHADOW 0x2 -#define GMAP_NOTIFY_MPROT 0x1 - -/* Status bits only for huge segment entries */ -#define _SEGMENT_ENTRY_GMAP_IN 0x0800 /* invalidation notify bit */ -#define _SEGMENT_ENTRY_GMAP_UC 0x0002 /* dirty (migration) */ - -/** - * struct gmap_struct - guest address space - * @list: list head for the mm->context gmap list - * @mm: pointer to the parent mm_struct - * @guest_to_host: radix tree with guest to host address translation - * @host_to_guest: radix tree with pointer to segment table entries - * @guest_table_lock: spinlock to protect all entries in the guest page table - * @ref_count: reference counter for the gmap structure - * @table: pointer to the page directory - * @asce: address space control element for gmap page table - * @pfault_enabled: defines if pfaults are applicable for the guest - * @guest_handle: protected virtual machine handle for the ultravisor - * @host_to_rmap: radix tree with gmap_rmap lists - * @children: list of shadow gmap structures - * @shadow_lock: spinlock to protect the shadow gmap list - * @parent: pointer to the parent gmap for shadow guest address spaces - * @orig_asce: ASCE for which the shadow page table has been created - * @edat_level: edat level to be used for the shadow translation - * @removed: flag to indicate if a shadow guest address space has been removed - * @initialized: flag to indicate if a shadow guest address space can be used - */ -struct gmap { - struct list_head list; - struct mm_struct *mm; - struct radix_tree_root guest_to_host; - struct radix_tree_root host_to_guest; - spinlock_t guest_table_lock; - refcount_t ref_count; - unsigned long *table; - unsigned long asce; - unsigned long asce_end; - void *private; - bool pfault_enabled; - /* only set for protected virtual machines */ - unsigned long guest_handle; - /* Additional data for shadow guest address spaces */ - struct radix_tree_root host_to_rmap; - struct list_head children; - spinlock_t shadow_lock; - struct gmap *parent; - unsigned long orig_asce; - int edat_level; - bool removed; - bool initialized; -}; - -/** - * struct gmap_rmap - reverse mapping for shadow page table entries - * @next: pointer to next rmap in the list - * @raddr: virtual rmap address in the shadow guest address space - */ -struct gmap_rmap { - struct gmap_rmap *next; - unsigned long raddr; -}; - -#define gmap_for_each_rmap(pos, head) \ - for (pos = (head); pos; pos = pos->next) - -#define gmap_for_each_rmap_safe(pos, n, head) \ - for (pos = (head); n = pos ? pos->next : NULL, pos; pos = n) - -/** - * struct gmap_notifier - notify function block for page invalidation - * @notifier_call: address of callback function - */ -struct gmap_notifier { - struct list_head list; - struct rcu_head rcu; - void (*notifier_call)(struct gmap *gmap, unsigned long start, - unsigned long end); -}; - -static inline int gmap_is_shadow(struct gmap *gmap) -{ - return !!gmap->parent; -} - -struct gmap *gmap_create(struct mm_struct *mm, unsigned long limit); -void gmap_remove(struct gmap *gmap); -struct gmap *gmap_get(struct gmap *gmap); -void gmap_put(struct gmap *gmap); -void gmap_free(struct gmap *gmap); -struct gmap *gmap_alloc(unsigned long limit); - -int gmap_map_segment(struct gmap *gmap, unsigned long from, - unsigned long to, unsigned long len); -int gmap_unmap_segment(struct gmap *gmap, unsigned long to, unsigned long len); -unsigned long __gmap_translate(struct gmap *, unsigned long gaddr); -int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr); -void __gmap_zap(struct gmap *, unsigned long gaddr); -void gmap_unlink(struct mm_struct *, unsigned long *table, unsigned long vmaddr); - -int gmap_read_table(struct gmap *gmap, unsigned long gaddr, unsigned long *val); - -void gmap_unshadow(struct gmap *sg); -int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t, - int fake); -int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t, - int fake); -int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt, - int fake); -int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt, - int fake); -int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte); - -void gmap_register_pte_notifier(struct gmap_notifier *); -void gmap_unregister_pte_notifier(struct gmap_notifier *); - -int gmap_protect_one(struct gmap *gmap, unsigned long gaddr, int prot, unsigned long bits); - -void gmap_sync_dirty_log_pmd(struct gmap *gmap, unsigned long dirty_bitmap[4], - unsigned long gaddr, unsigned long vmaddr); -int s390_replace_asce(struct gmap *gmap); -void s390_uv_destroy_pfns(unsigned long count, unsigned long *pfns); -int __s390_uv_destroy_range(struct mm_struct *mm, unsigned long start, - unsigned long end, bool interruptible); -unsigned long *gmap_table_walk(struct gmap *gmap, unsigned long gaddr, int level); - -/** - * s390_uv_destroy_range - Destroy a range of pages in the given mm. - * @mm: the mm on which to operate on - * @start: the start of the range - * @end: the end of the range - * - * This function will call cond_sched, so it should not generate stalls, but - * it will otherwise only return when it completed. - */ -static inline void s390_uv_destroy_range(struct mm_struct *mm, unsigned long start, - unsigned long end) -{ - (void)__s390_uv_destroy_range(mm, start, end, false); -} - -/** - * s390_uv_destroy_range_interruptible - Destroy a range of pages in the - * given mm, but stop when a fatal signal is received. - * @mm: the mm on which to operate on - * @start: the start of the range - * @end: the end of the range - * - * This function will call cond_sched, so it should not generate stalls. If - * a fatal signal is received, it will return with -EINTR immediately, - * without finishing destroying the whole range. Upon successful - * completion, 0 is returned. - */ -static inline int s390_uv_destroy_range_interruptible(struct mm_struct *mm, unsigned long start, - unsigned long end) -{ - return __s390_uv_destroy_range(mm, start, end, true); -} -#endif /* _ASM_S390_GMAP_H */ diff --git a/arch/s390/include/asm/gmap_helpers.h b/arch/s390/include/asm/gmap_helpers.h index 5356446a61c4..d2b616604a46 100644 --- a/arch/s390/include/asm/gmap_helpers.h +++ b/arch/s390/include/asm/gmap_helpers.h @@ -11,5 +11,7 @@ void gmap_helper_zap_one_page(struct mm_struct *mm, unsigned long vmaddr); void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned long end); int gmap_helper_disable_cow_sharing(void); +void gmap_helper_try_set_pte_unused(struct mm_struct *mm, unsigned long vmaddr); +pte_t *try_get_locked_pte(struct mm_struct *mm, unsigned long addr, spinlock_t **ptl); #endif /* _ASM_S390_GMAP_HELPERS_H */ diff --git a/arch/s390/include/asm/hugetlb.h b/arch/s390/include/asm/hugetlb.h index 69131736daaa..aea754b67c89 100644 --- a/arch/s390/include/asm/hugetlb.h +++ b/arch/s390/include/asm/hugetlb.h @@ -37,20 +37,14 @@ static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm, return __huge_ptep_get_and_clear(mm, addr, ptep); } -static inline void arch_clear_hugetlb_flags(struct folio *folio) -{ - clear_bit(PG_arch_1, &folio->flags.f); -} -#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags - #define __HAVE_ARCH_HUGE_PTE_CLEAR static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned long sz) { - if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) - set_pte(ptep, __pte(_REGION3_ENTRY_EMPTY)); + if ((pte_val(ptep_get(ptep)) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) + set_pud((pud_t *)ptep, __pud(_REGION3_ENTRY_EMPTY)); else - set_pte(ptep, __pte(_SEGMENT_ENTRY_EMPTY)); + set_pmd((pmd_t *)ptep, __pmd(_SEGMENT_ENTRY_EMPTY)); } #define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH @@ -83,20 +77,20 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, __set_huge_pte_at(mm, addr, ptep, pte_wrprotect(pte)); } -#define __HAVE_ARCH_HUGE_PTE_MKUFFD_WP -static inline pte_t huge_pte_mkuffd_wp(pte_t pte) +#define __HAVE_ARCH_HUGE_PTE_MKUFFD +static inline pte_t huge_pte_mkuffd(pte_t pte) { return pte; } -#define __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD_WP -static inline pte_t huge_pte_clear_uffd_wp(pte_t pte) +#define __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD +static inline pte_t huge_pte_clear_uffd(pte_t pte) { return pte; } -#define __HAVE_ARCH_HUGE_PTE_UFFD_WP -static inline int huge_pte_uffd_wp(pte_t pte) +#define __HAVE_ARCH_HUGE_PTE_UFFD +static inline int huge_pte_uffd(pte_t pte) { return 0; } diff --git a/arch/s390/include/asm/idals.h b/arch/s390/include/asm/idals.h index e5000ee6cdc6..06e1ec2afd5a 100644 --- a/arch/s390/include/asm/idals.h +++ b/arch/s390/include/asm/idals.h @@ -94,7 +94,7 @@ static inline int set_normalized_cda(struct ccw1 *ccw, void *vaddr) return -EINVAL; nridaws = idal_nr_words(vaddr, ccw->count); if (nridaws > 0) { - idal = kcalloc(nridaws, sizeof(*idal), GFP_ATOMIC | GFP_DMA); + idal = kzalloc_objs(*idal, nridaws, GFP_ATOMIC | GFP_DMA); if (!idal) return -ENOMEM; idal_create_words(idal, vaddr, ccw->count); @@ -137,7 +137,7 @@ static inline struct idal_buffer *idal_buffer_alloc(size_t size, int page_order) nr_ptrs = (size + IDA_BLOCK_SIZE - 1) >> IDA_SIZE_SHIFT; nr_chunks = (PAGE_SIZE << page_order) >> IDA_SIZE_SHIFT; - ib = kmalloc(struct_size(ib, data, nr_ptrs), GFP_DMA | GFP_KERNEL); + ib = kmalloc_flex(*ib, data, nr_ptrs, GFP_DMA | GFP_KERNEL); if (!ib) return ERR_PTR(-ENOMEM); ib->size = size; @@ -195,7 +195,7 @@ static inline struct idal_buffer **idal_buffer_array_alloc(size_t size, int page int i; count = (size + CCW_MAX_BYTE_COUNT - 1) / CCW_MAX_BYTE_COUNT; - ibs = kmalloc_array(count + 1, sizeof(*ibs), GFP_KERNEL); + ibs = kmalloc_objs(*ibs, count + 1); for (i = 0; i < count; i++) { /* Determine size for the current idal buffer */ ib_size = min(size, CCW_MAX_BYTE_COUNT); diff --git a/arch/s390/include/asm/idle.h b/arch/s390/include/asm/idle.h index 09f763b9eb40..07819f11987c 100644 --- a/arch/s390/include/asm/idle.h +++ b/arch/s390/include/asm/idle.h @@ -8,20 +8,20 @@ #ifndef _S390_IDLE_H #define _S390_IDLE_H +#include <linux/percpu-defs.h> #include <linux/types.h> -#include <linux/device.h> +#include <asm/tod_types.h> struct s390_idle_data { - unsigned long idle_count; - unsigned long idle_time; - unsigned long clock_idle_enter; +#ifdef CONFIG_NO_HZ_COMMON + bool in_idle; +#endif unsigned long timer_idle_enter; unsigned long mt_cycles_enter[8]; + union tod_clock clock_idle_enter; + union tod_clock clock_idle_exit; }; -extern struct device_attribute dev_attr_idle_count; -extern struct device_attribute dev_attr_idle_time_us; - -void psw_idle(struct s390_idle_data *data, unsigned long psw_mask); +DECLARE_PER_CPU(struct s390_idle_data, s390_idle); #endif /* _S390_IDLE_H */ diff --git a/arch/s390/include/asm/insn-common-asm.h b/arch/s390/include/asm/insn-common-asm.h new file mode 100644 index 000000000000..fd9b3cacb7c5 --- /dev/null +++ b/arch/s390/include/asm/insn-common-asm.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Assembler helper macros to generate .byte/.word code for instructions + * that are unknown to older binutils versions. + */ + +#ifndef __ASM_S390_INSN_COMMON_ASM_H +#define __ASM_S390_INSN_COMMON_ASM_H + +#ifdef __ASSEMBLER__ + +/* + * GR_NUM - Retrieve general-purpose register number + * + * @opd: Operand to store register number + * @gr: String designation register in the format "%rN" + */ +.macro GR_NUM opd gr + \opd = 255 + .irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 + .ifc \gr,%r\rs + \opd = \rs + .endif + .endr + .if \opd == 255 + \opd = \gr + .endif +.endm + +/* + * VX_NUM - Retrieve vector register number + * + * @opd: Operand to store register number + * @vxr: String designation register in the format "%vN" + * + * The vector register number is used for as input number to the + * instruction and, as well as, to compute the RXB field of the + * instruction. + */ +.macro VX_NUM opd vxr + \opd = 255 + .irp vs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 + .ifc \vxr,%v\vs + \opd = \vs + .endif + .endr + .if \opd == 255 + \opd = \vxr + .endif +.endm + +#endif /* __ASSEMBLER__ */ +#endif /* __ASM_S390_INSN_COMMON_ASM_H */ diff --git a/arch/s390/include/asm/irqflags.h b/arch/s390/include/asm/irqflags.h index bcab456dfb80..6eb6de6a99f2 100644 --- a/arch/s390/include/asm/irqflags.h +++ b/arch/s390/include/asm/irqflags.h @@ -37,18 +37,24 @@ static __always_inline void __arch_local_irq_ssm(unsigned long flags) asm volatile("ssm %0" : : "Q" (flags) : "memory"); } -#ifdef CONFIG_KMSAN -#define arch_local_irq_attributes noinline notrace __no_sanitize_memory __maybe_unused +#if defined(CONFIG_KMSAN) && !defined(__DECOMPRESSOR) +unsigned long arch_local_save_flags(void); +unsigned long arch_local_irq_save(void); +void arch_local_irq_enable_external(void); +void arch_local_irq_enable(void); #else -#define arch_local_irq_attributes __always_inline +#define arch_local_save_flags __arch_local_save_flags +#define arch_local_irq_save __arch_local_irq_save +#define arch_local_irq_enable_external __arch_local_irq_enable_external +#define arch_local_irq_enable __arch_local_irq_enable #endif -static arch_local_irq_attributes unsigned long arch_local_save_flags(void) +static __always_inline unsigned long __arch_local_save_flags(void) { return __arch_local_irq_stnsm(0xff); } -static arch_local_irq_attributes unsigned long arch_local_irq_save(void) +static __always_inline unsigned long __arch_local_irq_save(void) { return __arch_local_irq_stnsm(0xfc); } @@ -58,12 +64,12 @@ static __always_inline void arch_local_irq_disable(void) arch_local_irq_save(); } -static arch_local_irq_attributes void arch_local_irq_enable_external(void) +static __always_inline void __arch_local_irq_enable_external(void) { __arch_local_irq_stosm(0x01); } -static arch_local_irq_attributes void arch_local_irq_enable(void) +static __always_inline void __arch_local_irq_enable(void) { __arch_local_irq_stosm(0x03); } diff --git a/arch/s390/include/asm/jump_label.h b/arch/s390/include/asm/jump_label.h index d9cbc18f6b2e..0e28c90dc242 100644 --- a/arch/s390/include/asm/jump_label.h +++ b/arch/s390/include/asm/jump_label.h @@ -19,19 +19,29 @@ #define JUMP_LABEL_STATIC_KEY_CONSTRAINT "jdd" #endif +#define ARCH_JUMP_TABLE_ENTRY(key, label, local_label) \ + ".pushsection __jump_table,\"aw\"\n" \ + ".balign 8\n" \ + ".long " local_label "-.," label "-.\n" \ + ".quad " key "-.\n" \ + ".popsection\n" + /* * We use a brcl 0,<offset> instruction for jump labels so it * can be easily distinguished from a hotpatch generated instruction. */ +#define ARCH_STATIC_BRANCH_ASM(key, label) \ + "0: brcl 0," label "\n" \ + ARCH_JUMP_TABLE_ENTRY(key, label, "0b") + +#define ARCH_STATIC_BRANCH_JUMP_ASM(key, label) \ + "0: brcl 15," label "\n" \ + ARCH_JUMP_TABLE_ENTRY(key, label, "0b") + static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { - asm goto("0: brcl 0,%l[label]\n" - ".pushsection __jump_table,\"aw\"\n" - ".balign 8\n" - ".long 0b-.,%l[label]-.\n" - ".quad %0+%1-.\n" - ".popsection\n" - : : JUMP_LABEL_STATIC_KEY_CONSTRAINT (key), "i" (branch) : : label); + asm goto(ARCH_STATIC_BRANCH_ASM("%0+%1", "%l[label]") + : : JUMP_LABEL_STATIC_KEY_CONSTRAINT (key), "i" (branch) : : label); return false; label: return true; @@ -39,13 +49,8 @@ label: static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) { - asm goto("0: brcl 15,%l[label]\n" - ".pushsection __jump_table,\"aw\"\n" - ".balign 8\n" - ".long 0b-.,%l[label]-.\n" - ".quad %0+%1-.\n" - ".popsection\n" - : : JUMP_LABEL_STATIC_KEY_CONSTRAINT (key), "i" (branch) : : label); + asm goto(ARCH_STATIC_BRANCH_JUMP_ASM("%0+%1", "%l[label]") + : : JUMP_LABEL_STATIC_KEY_CONSTRAINT (key), "i" (branch) : : label); return false; label: return true; diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index ae1223264d3c..147325978225 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -1,153 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* - * definition for kernel virtual machines on s390 - * - * Copyright IBM Corp. 2008, 2018 - * - * Author(s): Carsten Otte <cotte@de.ibm.com> - */ - #ifndef ASM_KVM_HOST_H #define ASM_KVM_HOST_H -#include <linux/types.h> -#include <linux/hrtimer.h> -#include <linux/interrupt.h> -#include <linux/kvm_types.h> -#include <linux/kvm.h> -#include <linux/seqlock.h> -#include <linux/module.h> -#include <linux/pci.h> -#include <linux/mmu_notifier.h> -#include <asm/kvm_host_types.h> -#include <asm/debug.h> -#include <asm/cpu.h> -#include <asm/fpu.h> -#include <asm/isc.h> -#include <asm/guarded_storage.h> - -#define KVM_MAX_VCPUS 255 - -#define KVM_INTERNAL_MEM_SLOTS 1 - -/* - * These seem to be used for allocating ->chip in the routing table, which we - * don't use. 1 is as small as we can get to reduce the needed memory. If we - * need to look at ->chip later on, we'll need to revisit this. - */ -#define KVM_NR_IRQCHIPS 1 -#define KVM_IRQCHIP_NUM_PINS 1 -#define KVM_HALT_POLL_NS_DEFAULT 50000 - -/* s390-specific vcpu->requests bit members */ -#define KVM_REQ_ENABLE_IBS KVM_ARCH_REQ(0) -#define KVM_REQ_DISABLE_IBS KVM_ARCH_REQ(1) -#define KVM_REQ_ICPT_OPEREXC KVM_ARCH_REQ(2) -#define KVM_REQ_START_MIGRATION KVM_ARCH_REQ(3) -#define KVM_REQ_STOP_MIGRATION KVM_ARCH_REQ(4) -#define KVM_REQ_VSIE_RESTART KVM_ARCH_REQ(5) -#define KVM_REQ_REFRESH_GUEST_PREFIX \ - KVM_ARCH_REQ_FLAGS(6, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) - -struct kvm_vcpu_stat { - struct kvm_vcpu_stat_generic generic; - u64 exit_userspace; - u64 exit_null; - u64 exit_external_request; - u64 exit_io_request; - u64 exit_external_interrupt; - u64 exit_stop_request; - u64 exit_validity; - u64 exit_instruction; - u64 exit_pei; - u64 halt_no_poll_steal; - u64 instruction_lctl; - u64 instruction_lctlg; - u64 instruction_stctl; - u64 instruction_stctg; - u64 exit_program_interruption; - u64 exit_instr_and_program; - u64 exit_operation_exception; - u64 deliver_ckc; - u64 deliver_cputm; - u64 deliver_external_call; - u64 deliver_emergency_signal; - u64 deliver_service_signal; - u64 deliver_virtio; - u64 deliver_stop_signal; - u64 deliver_prefix_signal; - u64 deliver_restart_signal; - u64 deliver_program; - u64 deliver_io; - u64 deliver_machine_check; - u64 exit_wait_state; - u64 inject_ckc; - u64 inject_cputm; - u64 inject_external_call; - u64 inject_emergency_signal; - u64 inject_mchk; - u64 inject_pfault_init; - u64 inject_program; - u64 inject_restart; - u64 inject_set_prefix; - u64 inject_stop_signal; - u64 instruction_epsw; - u64 instruction_gs; - u64 instruction_io_other; - u64 instruction_lpsw; - u64 instruction_lpswe; - u64 instruction_lpswey; - u64 instruction_pfmf; - u64 instruction_ptff; - u64 instruction_sck; - u64 instruction_sckpf; - u64 instruction_stidp; - u64 instruction_spx; - u64 instruction_stpx; - u64 instruction_stap; - u64 instruction_iske; - u64 instruction_ri; - u64 instruction_rrbe; - u64 instruction_sske; - u64 instruction_ipte_interlock; - u64 instruction_stsi; - u64 instruction_stfl; - u64 instruction_tb; - u64 instruction_tpi; - u64 instruction_tprot; - u64 instruction_tsch; - u64 instruction_sie; - u64 instruction_essa; - u64 instruction_sthyi; - u64 instruction_sigp_sense; - u64 instruction_sigp_sense_running; - u64 instruction_sigp_external_call; - u64 instruction_sigp_emergency; - u64 instruction_sigp_cond_emergency; - u64 instruction_sigp_start; - u64 instruction_sigp_stop; - u64 instruction_sigp_stop_store_status; - u64 instruction_sigp_store_status; - u64 instruction_sigp_store_adtl_status; - u64 instruction_sigp_arch; - u64 instruction_sigp_prefix; - u64 instruction_sigp_restart; - u64 instruction_sigp_init_cpu_reset; - u64 instruction_sigp_cpu_reset; - u64 instruction_sigp_unknown; - u64 instruction_diagnose_10; - u64 instruction_diagnose_44; - u64 instruction_diagnose_9c; - u64 diag_9c_ignored; - u64 diag_9c_forward; - u64 instruction_diagnose_258; - u64 instruction_diagnose_308; - u64 instruction_diagnose_500; - u64 instruction_diagnose_other; - u64 pfault_sync; - u64 signal_exits; -}; +#include <asm/kvm_host_s390.h> #define PGM_OPERATION 0x01 #define PGM_PRIVILEGED_OP 0x02 @@ -206,541 +62,4 @@ struct kvm_vcpu_stat { #define PGM_PER 0x80 #define PGM_CRYPTO_OPERATION 0x119 -/* irq types in ascend order of priorities */ -enum irq_types { - IRQ_PEND_SET_PREFIX = 0, - IRQ_PEND_RESTART, - IRQ_PEND_SIGP_STOP, - IRQ_PEND_IO_ISC_7, - IRQ_PEND_IO_ISC_6, - IRQ_PEND_IO_ISC_5, - IRQ_PEND_IO_ISC_4, - IRQ_PEND_IO_ISC_3, - IRQ_PEND_IO_ISC_2, - IRQ_PEND_IO_ISC_1, - IRQ_PEND_IO_ISC_0, - IRQ_PEND_VIRTIO, - IRQ_PEND_PFAULT_DONE, - IRQ_PEND_PFAULT_INIT, - IRQ_PEND_EXT_HOST, - IRQ_PEND_EXT_SERVICE, - IRQ_PEND_EXT_SERVICE_EV, - IRQ_PEND_EXT_TIMING, - IRQ_PEND_EXT_CPU_TIMER, - IRQ_PEND_EXT_CLOCK_COMP, - IRQ_PEND_EXT_EXTERNAL, - IRQ_PEND_EXT_EMERGENCY, - IRQ_PEND_EXT_MALFUNC, - IRQ_PEND_EXT_IRQ_KEY, - IRQ_PEND_MCHK_REP, - IRQ_PEND_PROG, - IRQ_PEND_SVC, - IRQ_PEND_MCHK_EX, - IRQ_PEND_COUNT -}; - -/* We have 2M for virtio device descriptor pages. Smallest amount of - * memory per page is 24 bytes (1 queue), so (2048*1024) / 24 = 87381 - */ -#define KVM_S390_MAX_VIRTIO_IRQS 87381 - -/* - * Repressible (non-floating) machine check interrupts - * subclass bits in MCIC - */ -#define MCHK_EXTD_BIT 58 -#define MCHK_DEGR_BIT 56 -#define MCHK_WARN_BIT 55 -#define MCHK_REP_MASK ((1UL << MCHK_DEGR_BIT) | \ - (1UL << MCHK_EXTD_BIT) | \ - (1UL << MCHK_WARN_BIT)) - -/* Exigent machine check interrupts subclass bits in MCIC */ -#define MCHK_SD_BIT 63 -#define MCHK_PD_BIT 62 -#define MCHK_EX_MASK ((1UL << MCHK_SD_BIT) | (1UL << MCHK_PD_BIT)) - -#define IRQ_PEND_EXT_MASK ((1UL << IRQ_PEND_EXT_IRQ_KEY) | \ - (1UL << IRQ_PEND_EXT_CLOCK_COMP) | \ - (1UL << IRQ_PEND_EXT_CPU_TIMER) | \ - (1UL << IRQ_PEND_EXT_MALFUNC) | \ - (1UL << IRQ_PEND_EXT_EMERGENCY) | \ - (1UL << IRQ_PEND_EXT_EXTERNAL) | \ - (1UL << IRQ_PEND_EXT_TIMING) | \ - (1UL << IRQ_PEND_EXT_HOST) | \ - (1UL << IRQ_PEND_EXT_SERVICE) | \ - (1UL << IRQ_PEND_EXT_SERVICE_EV) | \ - (1UL << IRQ_PEND_VIRTIO) | \ - (1UL << IRQ_PEND_PFAULT_INIT) | \ - (1UL << IRQ_PEND_PFAULT_DONE)) - -#define IRQ_PEND_IO_MASK ((1UL << IRQ_PEND_IO_ISC_0) | \ - (1UL << IRQ_PEND_IO_ISC_1) | \ - (1UL << IRQ_PEND_IO_ISC_2) | \ - (1UL << IRQ_PEND_IO_ISC_3) | \ - (1UL << IRQ_PEND_IO_ISC_4) | \ - (1UL << IRQ_PEND_IO_ISC_5) | \ - (1UL << IRQ_PEND_IO_ISC_6) | \ - (1UL << IRQ_PEND_IO_ISC_7)) - -#define IRQ_PEND_MCHK_MASK ((1UL << IRQ_PEND_MCHK_REP) | \ - (1UL << IRQ_PEND_MCHK_EX)) - -#define IRQ_PEND_EXT_II_MASK ((1UL << IRQ_PEND_EXT_CPU_TIMER) | \ - (1UL << IRQ_PEND_EXT_CLOCK_COMP) | \ - (1UL << IRQ_PEND_EXT_EMERGENCY) | \ - (1UL << IRQ_PEND_EXT_EXTERNAL) | \ - (1UL << IRQ_PEND_EXT_SERVICE) | \ - (1UL << IRQ_PEND_EXT_SERVICE_EV)) - -struct kvm_s390_interrupt_info { - struct list_head list; - u64 type; - union { - struct kvm_s390_io_info io; - struct kvm_s390_ext_info ext; - struct kvm_s390_pgm_info pgm; - struct kvm_s390_emerg_info emerg; - struct kvm_s390_extcall_info extcall; - struct kvm_s390_prefix_info prefix; - struct kvm_s390_stop_info stop; - struct kvm_s390_mchk_info mchk; - }; -}; - -struct kvm_s390_irq_payload { - struct kvm_s390_io_info io; - struct kvm_s390_ext_info ext; - struct kvm_s390_pgm_info pgm; - struct kvm_s390_emerg_info emerg; - struct kvm_s390_extcall_info extcall; - struct kvm_s390_prefix_info prefix; - struct kvm_s390_stop_info stop; - struct kvm_s390_mchk_info mchk; -}; - -struct kvm_s390_local_interrupt { - spinlock_t lock; - DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS); - struct kvm_s390_irq_payload irq; - unsigned long pending_irqs; -}; - -#define FIRQ_LIST_IO_ISC_0 0 -#define FIRQ_LIST_IO_ISC_1 1 -#define FIRQ_LIST_IO_ISC_2 2 -#define FIRQ_LIST_IO_ISC_3 3 -#define FIRQ_LIST_IO_ISC_4 4 -#define FIRQ_LIST_IO_ISC_5 5 -#define FIRQ_LIST_IO_ISC_6 6 -#define FIRQ_LIST_IO_ISC_7 7 -#define FIRQ_LIST_PFAULT 8 -#define FIRQ_LIST_VIRTIO 9 -#define FIRQ_LIST_COUNT 10 -#define FIRQ_CNTR_IO 0 -#define FIRQ_CNTR_SERVICE 1 -#define FIRQ_CNTR_VIRTIO 2 -#define FIRQ_CNTR_PFAULT 3 -#define FIRQ_MAX_COUNT 4 - -/* mask the AIS mode for a given ISC */ -#define AIS_MODE_MASK(isc) (0x80 >> isc) - -#define KVM_S390_AIS_MODE_ALL 0 -#define KVM_S390_AIS_MODE_SINGLE 1 - -struct kvm_s390_float_interrupt { - unsigned long pending_irqs; - unsigned long masked_irqs; - spinlock_t lock; - struct list_head lists[FIRQ_LIST_COUNT]; - int counters[FIRQ_MAX_COUNT]; - struct kvm_s390_mchk_info mchk; - struct kvm_s390_ext_info srv_signal; - int last_sleep_cpu; - struct mutex ais_lock; - u8 simm; - u8 nimm; -}; - -struct kvm_hw_wp_info_arch { - unsigned long addr; - unsigned long phys_addr; - int len; - char *old_data; -}; - -struct kvm_hw_bp_info_arch { - unsigned long addr; - int len; -}; - -/* - * Only the upper 16 bits of kvm_guest_debug->control are arch specific. - * Further KVM_GUESTDBG flags which an be used from userspace can be found in - * arch/s390/include/uapi/asm/kvm.h - */ -#define KVM_GUESTDBG_EXIT_PENDING 0x10000000 - -#define guestdbg_enabled(vcpu) \ - (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) -#define guestdbg_sstep_enabled(vcpu) \ - (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) -#define guestdbg_hw_bp_enabled(vcpu) \ - (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) -#define guestdbg_exit_pending(vcpu) (guestdbg_enabled(vcpu) && \ - (vcpu->guest_debug & KVM_GUESTDBG_EXIT_PENDING)) - -#define KVM_GUESTDBG_VALID_MASK \ - (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP |\ - KVM_GUESTDBG_USE_HW_BP | KVM_GUESTDBG_EXIT_PENDING) - -struct kvm_guestdbg_info_arch { - unsigned long cr0; - unsigned long cr9; - unsigned long cr10; - unsigned long cr11; - struct kvm_hw_bp_info_arch *hw_bp_info; - struct kvm_hw_wp_info_arch *hw_wp_info; - int nr_hw_bp; - int nr_hw_wp; - unsigned long last_bp; -}; - -struct kvm_s390_pv_vcpu { - u64 handle; - unsigned long stor_base; -}; - -struct kvm_vcpu_arch { - struct kvm_s390_sie_block *sie_block; - /* if vsie is active, currently executed shadow sie control block */ - struct kvm_s390_sie_block *vsie_block; - unsigned int host_acrs[NUM_ACRS]; - struct gs_cb *host_gscb; - struct kvm_s390_local_interrupt local_int; - struct hrtimer ckc_timer; - struct kvm_s390_pgm_info pgm; - struct gmap *gmap; - struct kvm_guestdbg_info_arch guestdbg; - unsigned long pfault_token; - unsigned long pfault_select; - unsigned long pfault_compare; - bool cputm_enabled; - /* - * The seqcount protects updates to cputm_start and sie_block.cputm, - * this way we can have non-blocking reads with consistent values. - * Only the owning VCPU thread (vcpu->cpu) is allowed to change these - * values and to start/stop/enable/disable cpu timer accounting. - */ - seqcount_t cputm_seqcount; - __u64 cputm_start; - bool gs_enabled; - bool skey_enabled; - /* Indicator if the access registers have been loaded from guest */ - bool acrs_loaded; - struct kvm_s390_pv_vcpu pv; - union diag318_info diag318_info; -}; - -struct kvm_vm_stat { - struct kvm_vm_stat_generic generic; - u64 inject_io; - u64 inject_float_mchk; - u64 inject_pfault_done; - u64 inject_service_signal; - u64 inject_virtio; - u64 aen_forward; - u64 gmap_shadow_create; - u64 gmap_shadow_reuse; - u64 gmap_shadow_r1_entry; - u64 gmap_shadow_r2_entry; - u64 gmap_shadow_r3_entry; - u64 gmap_shadow_sg_entry; - u64 gmap_shadow_pg_entry; -}; - -struct kvm_arch_memory_slot { -}; - -struct s390_map_info { - struct list_head list; - __u64 guest_addr; - __u64 addr; - struct page *page; -}; - -struct s390_io_adapter { - unsigned int id; - int isc; - bool maskable; - bool masked; - bool swap; - bool suppressible; -}; - -#define MAX_S390_IO_ADAPTERS ((MAX_ISC + 1) * 8) -#define MAX_S390_ADAPTER_MAPS 256 - -/* maximum size of facilities and facility mask is 2k bytes */ -#define S390_ARCH_FAC_LIST_SIZE_BYTE (1<<11) -#define S390_ARCH_FAC_LIST_SIZE_U64 \ - (S390_ARCH_FAC_LIST_SIZE_BYTE / sizeof(u64)) -#define S390_ARCH_FAC_MASK_SIZE_BYTE S390_ARCH_FAC_LIST_SIZE_BYTE -#define S390_ARCH_FAC_MASK_SIZE_U64 \ - (S390_ARCH_FAC_MASK_SIZE_BYTE / sizeof(u64)) - -struct kvm_s390_cpu_model { - /* facility mask supported by kvm & hosting machine */ - __u64 fac_mask[S390_ARCH_FAC_MASK_SIZE_U64]; - struct kvm_s390_vm_cpu_subfunc subfuncs; - /* facility list requested by guest (in dma page) */ - __u64 *fac_list; - u64 cpuid; - unsigned short ibc; - /* subset of available UV-features for pv-guests enabled by user space */ - struct kvm_s390_vm_cpu_uv_feat uv_feat_guest; -}; - -typedef int (*crypto_hook)(struct kvm_vcpu *vcpu); - -struct kvm_s390_crypto { - struct kvm_s390_crypto_cb *crycb; - struct rw_semaphore pqap_hook_rwsem; - crypto_hook *pqap_hook; - __u32 crycbd; - __u8 aes_kw; - __u8 dea_kw; - __u8 apie; -}; - -#define APCB0_MASK_SIZE 1 -struct kvm_s390_apcb0 { - __u64 apm[APCB0_MASK_SIZE]; /* 0x0000 */ - __u64 aqm[APCB0_MASK_SIZE]; /* 0x0008 */ - __u64 adm[APCB0_MASK_SIZE]; /* 0x0010 */ - __u64 reserved18; /* 0x0018 */ -}; - -#define APCB1_MASK_SIZE 4 -struct kvm_s390_apcb1 { - __u64 apm[APCB1_MASK_SIZE]; /* 0x0000 */ - __u64 aqm[APCB1_MASK_SIZE]; /* 0x0020 */ - __u64 adm[APCB1_MASK_SIZE]; /* 0x0040 */ - __u64 reserved60[4]; /* 0x0060 */ -}; - -struct kvm_s390_crypto_cb { - struct kvm_s390_apcb0 apcb0; /* 0x0000 */ - __u8 reserved20[0x0048 - 0x0020]; /* 0x0020 */ - __u8 dea_wrapping_key_mask[24]; /* 0x0048 */ - __u8 aes_wrapping_key_mask[32]; /* 0x0060 */ - struct kvm_s390_apcb1 apcb1; /* 0x0080 */ -}; - -struct kvm_s390_gisa { - union { - struct { /* common to all formats */ - u32 next_alert; - u8 ipm; - u8 reserved01[2]; - u8 iam; - }; - struct { /* format 0 */ - u32 next_alert; - u8 ipm; - u8 reserved01; - u8 : 6; - u8 g : 1; - u8 c : 1; - u8 iam; - u8 reserved02[4]; - u32 airq_count; - } g0; - struct { /* format 1 */ - u32 next_alert; - u8 ipm; - u8 simm; - u8 nimm; - u8 iam; - u8 aism[8]; - u8 : 6; - u8 g : 1; - u8 c : 1; - u8 reserved03[11]; - u32 airq_count; - } g1; - struct { - u64 word[4]; - } u64; - }; -}; - -struct kvm_s390_gib { - u32 alert_list_origin; - u32 reserved01; - u8:5; - u8 nisc:3; - u8 reserved03[3]; - u32 reserved04[5]; -}; - -/* - * sie_page2 has to be allocated as DMA because fac_list, crycb and - * gisa need 31bit addresses in the sie control block. - */ -struct sie_page2 { - __u64 fac_list[S390_ARCH_FAC_LIST_SIZE_U64]; /* 0x0000 */ - struct kvm_s390_crypto_cb crycb; /* 0x0800 */ - struct kvm_s390_gisa gisa; /* 0x0900 */ - struct kvm *kvm; /* 0x0920 */ - u8 reserved928[0x1000 - 0x928]; /* 0x0928 */ -}; - -struct vsie_page; - -struct kvm_s390_vsie { - struct mutex mutex; - struct radix_tree_root addr_to_page; - int page_count; - int next; - struct vsie_page *pages[KVM_MAX_VCPUS]; -}; - -struct kvm_s390_gisa_iam { - u8 mask; - spinlock_t ref_lock; - u32 ref_count[MAX_ISC + 1]; -}; - -struct kvm_s390_gisa_interrupt { - struct kvm_s390_gisa *origin; - struct kvm_s390_gisa_iam alert; - struct hrtimer timer; - u64 expires; - DECLARE_BITMAP(kicked_mask, KVM_MAX_VCPUS); -}; - -struct kvm_s390_pv { - u64 handle; - u64 guest_len; - unsigned long stor_base; - void *stor_var; - bool dumping; - void *set_aside; - struct list_head need_cleanup; - struct mmu_notifier mmu_notifier; -}; - -struct kvm_arch { - struct esca_block *sca; - debug_info_t *dbf; - struct kvm_s390_float_interrupt float_int; - struct kvm_device *flic; - struct gmap *gmap; - unsigned long mem_limit; - int css_support; - int use_irqchip; - int use_cmma; - int use_pfmfi; - int use_skf; - int use_zpci_interp; - int user_cpu_state_ctrl; - int user_sigp; - int user_stsi; - int user_instr0; - int user_operexec; - struct s390_io_adapter *adapters[MAX_S390_IO_ADAPTERS]; - wait_queue_head_t ipte_wq; - int ipte_lock_count; - struct mutex ipte_mutex; - spinlock_t start_stop_lock; - struct sie_page2 *sie_page2; - struct kvm_s390_cpu_model model; - struct kvm_s390_crypto crypto; - struct kvm_s390_vsie vsie; - u8 epdx; - u64 epoch; - int migration_mode; - atomic64_t cmma_dirty_pages; - /* subset of available cpu features enabled by user space */ - DECLARE_BITMAP(cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS); - /* indexed by vcpu_idx */ - DECLARE_BITMAP(idle_mask, KVM_MAX_VCPUS); - struct kvm_s390_gisa_interrupt gisa_int; - struct kvm_s390_pv pv; - struct list_head kzdev_list; - spinlock_t kzdev_list_lock; -}; - -#define KVM_HVA_ERR_BAD (-1UL) -#define KVM_HVA_ERR_RO_BAD (-2UL) - -static inline bool kvm_is_error_hva(unsigned long addr) -{ - return IS_ERR_VALUE(addr); -} - -#define ASYNC_PF_PER_VCPU 64 -struct kvm_arch_async_pf { - unsigned long pfault_token; -}; - -bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu); - -void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, - struct kvm_async_pf *work); - -bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, - struct kvm_async_pf *work); - -void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, - struct kvm_async_pf *work); - -static inline void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu) {} - -void kvm_arch_crypto_clear_masks(struct kvm *kvm); -void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm, - unsigned long *aqm, unsigned long *adm); - -int __sie64a(phys_addr_t sie_block_phys, struct kvm_s390_sie_block *sie_block, u64 *rsa, - unsigned long gasce); - -static inline int sie64a(struct kvm_s390_sie_block *sie_block, u64 *rsa, unsigned long gasce) -{ - return __sie64a(virt_to_phys(sie_block), sie_block, rsa, gasce); -} - -extern char sie_exit; - -bool kvm_s390_pv_is_protected(struct kvm *kvm); -bool kvm_s390_pv_cpu_is_protected(struct kvm_vcpu *vcpu); - -extern int kvm_s390_enter_exit_sie(struct kvm_s390_sie_block *scb, - u64 *gprs, unsigned long gasce); - -extern int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc); -extern int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc); - -bool kvm_s390_is_gpa_in_memslot(struct kvm *kvm, gpa_t gpa); - -static inline void kvm_arch_free_memslot(struct kvm *kvm, - struct kvm_memory_slot *slot) {} -static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} -static inline void kvm_arch_flush_shadow_all(struct kvm *kvm) {} -static inline void kvm_arch_flush_shadow_memslot(struct kvm *kvm, - struct kvm_memory_slot *slot) {} -static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} -static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} - -#define __KVM_HAVE_ARCH_VM_FREE -void kvm_arch_free_vm(struct kvm *kvm); - -struct zpci_kvm_hook { - int (*kvm_register)(void *opaque, struct kvm *kvm); - void (*kvm_unregister)(void *opaque); -}; - -extern struct zpci_kvm_hook zpci_kvm_hook; - -#endif +#endif /* ASM_KVM_HOST_H */ diff --git a/arch/s390/include/asm/kvm_host_s390.h b/arch/s390/include/asm/kvm_host_s390.h new file mode 100644 index 000000000000..cd692f8fb764 --- /dev/null +++ b/arch/s390/include/asm/kvm_host_s390.h @@ -0,0 +1,729 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * definition for kernel virtual machines on s390 + * + * Copyright IBM Corp. 2008, 2018 + * + * Author(s): Carsten Otte <cotte@de.ibm.com> + */ + + +#ifndef ASM_KVM_HOST_S390_H +#define ASM_KVM_HOST_S390_H + +#include <linux/types.h> +#include <linux/hrtimer.h> +#include <linux/interrupt.h> +#include <linux/kvm_types.h> +#include <linux/kvm.h> +#include <linux/seqlock.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/mmu_notifier.h> +#include <asm/kvm_host_types.h> +#include <asm/debug.h> +#include <asm/cpu.h> +#include <asm/fpu.h> +#include <asm/isc.h> +#include <asm/guarded_storage.h> + +#define KVM_HAVE_MMU_RWLOCK +#define KVM_MAX_VCPUS 255 + +#define KVM_INTERNAL_MEM_SLOTS 1 + +#define KVM_S390_MANAGES_S390_GUEST 1 + +/* + * These seem to be used for allocating ->chip in the routing table, which we + * don't use. 1 is as small as we can get to reduce the needed memory. If we + * need to look at ->chip later on, we'll need to revisit this. + */ +#define KVM_NR_IRQCHIPS 1 +#define KVM_IRQCHIP_NUM_PINS 1 +#define KVM_HALT_POLL_NS_DEFAULT 50000 + +/* s390-specific vcpu->requests bit members */ +#define KVM_REQ_ENABLE_IBS KVM_ARCH_REQ(0) +#define KVM_REQ_DISABLE_IBS KVM_ARCH_REQ(1) +#define KVM_REQ_ICPT_OPEREXC KVM_ARCH_REQ(2) +#define KVM_REQ_START_MIGRATION KVM_ARCH_REQ(3) +#define KVM_REQ_STOP_MIGRATION KVM_ARCH_REQ(4) +#define KVM_REQ_VSIE_RESTART KVM_ARCH_REQ(5) +#define KVM_REQ_REFRESH_GUEST_PREFIX \ + KVM_ARCH_REQ_FLAGS(6, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) + +struct kvm_vcpu_stat { + struct kvm_vcpu_stat_generic generic; + u64 exit_userspace; + u64 exit_null; + u64 exit_external_request; + u64 exit_io_request; + u64 exit_external_interrupt; + u64 exit_stop_request; + u64 exit_validity; + u64 exit_instruction; + u64 exit_pei; + u64 halt_no_poll_steal; + u64 instruction_lctl; + u64 instruction_lctlg; + u64 instruction_stctl; + u64 instruction_stctg; + u64 exit_program_interruption; + u64 exit_instr_and_program; + u64 exit_operation_exception; + u64 deliver_ckc; + u64 deliver_cputm; + u64 deliver_external_call; + u64 deliver_emergency_signal; + u64 deliver_service_signal; + u64 deliver_virtio; + u64 deliver_stop_signal; + u64 deliver_prefix_signal; + u64 deliver_restart_signal; + u64 deliver_program; + u64 deliver_io; + u64 deliver_machine_check; + u64 exit_wait_state; + u64 inject_ckc; + u64 inject_cputm; + u64 inject_external_call; + u64 inject_emergency_signal; + u64 inject_mchk; + u64 inject_pfault_init; + u64 inject_program; + u64 inject_restart; + u64 inject_set_prefix; + u64 inject_stop_signal; + u64 instruction_epsw; + u64 instruction_gs; + u64 instruction_io_other; + u64 instruction_lpsw; + u64 instruction_lpswe; + u64 instruction_lpswey; + u64 instruction_pfmf; + u64 instruction_ptff; + u64 instruction_sck; + u64 instruction_sckpf; + u64 instruction_stidp; + u64 instruction_spx; + u64 instruction_stpx; + u64 instruction_stap; + u64 instruction_iske; + u64 instruction_ri; + u64 instruction_rrbe; + u64 instruction_sske; + u64 instruction_ipte_interlock; + u64 instruction_stsi; + u64 instruction_stfl; + u64 instruction_tb; + u64 instruction_tpi; + u64 instruction_tprot; + u64 instruction_tsch; + u64 instruction_sie; + u64 instruction_essa; + u64 instruction_sthyi; + u64 instruction_sigp_sense; + u64 instruction_sigp_sense_running; + u64 instruction_sigp_external_call; + u64 instruction_sigp_emergency; + u64 instruction_sigp_cond_emergency; + u64 instruction_sigp_start; + u64 instruction_sigp_stop; + u64 instruction_sigp_stop_store_status; + u64 instruction_sigp_store_status; + u64 instruction_sigp_store_adtl_status; + u64 instruction_sigp_arch; + u64 instruction_sigp_prefix; + u64 instruction_sigp_restart; + u64 instruction_sigp_init_cpu_reset; + u64 instruction_sigp_cpu_reset; + u64 instruction_sigp_unknown; + u64 instruction_diagnose_10; + u64 instruction_diagnose_44; + u64 instruction_diagnose_9c; + u64 diag_9c_ignored; + u64 diag_9c_forward; + u64 instruction_diagnose_258; + u64 instruction_diagnose_308; + u64 instruction_diagnose_500; + u64 instruction_diagnose_other; + u64 pfault_sync; + u64 signal_exits; +}; + +/* irq types in ascend order of priorities */ +enum irq_types { + IRQ_PEND_SET_PREFIX = 0, + IRQ_PEND_RESTART, + IRQ_PEND_SIGP_STOP, + IRQ_PEND_IO_ISC_7, + IRQ_PEND_IO_ISC_6, + IRQ_PEND_IO_ISC_5, + IRQ_PEND_IO_ISC_4, + IRQ_PEND_IO_ISC_3, + IRQ_PEND_IO_ISC_2, + IRQ_PEND_IO_ISC_1, + IRQ_PEND_IO_ISC_0, + IRQ_PEND_VIRTIO, + IRQ_PEND_PFAULT_DONE, + IRQ_PEND_PFAULT_INIT, + IRQ_PEND_EXT_HOST, + IRQ_PEND_EXT_SERVICE, + IRQ_PEND_EXT_SERVICE_EV, + IRQ_PEND_EXT_TIMING, + IRQ_PEND_EXT_CPU_TIMER, + IRQ_PEND_EXT_CLOCK_COMP, + IRQ_PEND_EXT_EXTERNAL, + IRQ_PEND_EXT_EMERGENCY, + IRQ_PEND_EXT_MALFUNC, + IRQ_PEND_EXT_IRQ_KEY, + IRQ_PEND_MCHK_REP, + IRQ_PEND_PROG, + IRQ_PEND_SVC, + IRQ_PEND_MCHK_EX, + IRQ_PEND_COUNT +}; + +/* We have 2M for virtio device descriptor pages. Smallest amount of + * memory per page is 24 bytes (1 queue), so (2048*1024) / 24 = 87381 + */ +#define KVM_S390_MAX_VIRTIO_IRQS 87381 + +/* + * Repressible (non-floating) machine check interrupts + * subclass bits in MCIC + */ +#define MCHK_EXTD_BIT 58 +#define MCHK_DEGR_BIT 56 +#define MCHK_WARN_BIT 55 +#define MCHK_REP_MASK ((1UL << MCHK_DEGR_BIT) | \ + (1UL << MCHK_EXTD_BIT) | \ + (1UL << MCHK_WARN_BIT)) + +/* Exigent machine check interrupts subclass bits in MCIC */ +#define MCHK_SD_BIT 63 +#define MCHK_PD_BIT 62 +#define MCHK_EX_MASK ((1UL << MCHK_SD_BIT) | (1UL << MCHK_PD_BIT)) + +#define IRQ_PEND_EXT_MASK ((1UL << IRQ_PEND_EXT_IRQ_KEY) | \ + (1UL << IRQ_PEND_EXT_CLOCK_COMP) | \ + (1UL << IRQ_PEND_EXT_CPU_TIMER) | \ + (1UL << IRQ_PEND_EXT_MALFUNC) | \ + (1UL << IRQ_PEND_EXT_EMERGENCY) | \ + (1UL << IRQ_PEND_EXT_EXTERNAL) | \ + (1UL << IRQ_PEND_EXT_TIMING) | \ + (1UL << IRQ_PEND_EXT_HOST) | \ + (1UL << IRQ_PEND_EXT_SERVICE) | \ + (1UL << IRQ_PEND_EXT_SERVICE_EV) | \ + (1UL << IRQ_PEND_VIRTIO) | \ + (1UL << IRQ_PEND_PFAULT_INIT) | \ + (1UL << IRQ_PEND_PFAULT_DONE)) + +#define IRQ_PEND_IO_MASK ((1UL << IRQ_PEND_IO_ISC_0) | \ + (1UL << IRQ_PEND_IO_ISC_1) | \ + (1UL << IRQ_PEND_IO_ISC_2) | \ + (1UL << IRQ_PEND_IO_ISC_3) | \ + (1UL << IRQ_PEND_IO_ISC_4) | \ + (1UL << IRQ_PEND_IO_ISC_5) | \ + (1UL << IRQ_PEND_IO_ISC_6) | \ + (1UL << IRQ_PEND_IO_ISC_7)) + +#define IRQ_PEND_MCHK_MASK ((1UL << IRQ_PEND_MCHK_REP) | \ + (1UL << IRQ_PEND_MCHK_EX)) + +#define IRQ_PEND_EXT_II_MASK ((1UL << IRQ_PEND_EXT_CPU_TIMER) | \ + (1UL << IRQ_PEND_EXT_CLOCK_COMP) | \ + (1UL << IRQ_PEND_EXT_EMERGENCY) | \ + (1UL << IRQ_PEND_EXT_EXTERNAL) | \ + (1UL << IRQ_PEND_EXT_SERVICE) | \ + (1UL << IRQ_PEND_EXT_SERVICE_EV)) + +struct kvm_s390_interrupt_info { + struct list_head list; + u64 type; + union { + struct kvm_s390_io_info io; + struct kvm_s390_ext_info ext; + struct kvm_s390_pgm_info pgm; + struct kvm_s390_emerg_info emerg; + struct kvm_s390_extcall_info extcall; + struct kvm_s390_prefix_info prefix; + struct kvm_s390_stop_info stop; + struct kvm_s390_mchk_info mchk; + }; +}; + +struct kvm_s390_irq_payload { + struct kvm_s390_io_info io; + struct kvm_s390_ext_info ext; + struct kvm_s390_pgm_info pgm; + struct kvm_s390_emerg_info emerg; + struct kvm_s390_extcall_info extcall; + struct kvm_s390_prefix_info prefix; + struct kvm_s390_stop_info stop; + struct kvm_s390_mchk_info mchk; +}; + +struct kvm_s390_local_interrupt { + spinlock_t lock; + DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS); + struct kvm_s390_irq_payload irq; + unsigned long pending_irqs; +}; + +#define FIRQ_LIST_IO_ISC_0 0 +#define FIRQ_LIST_IO_ISC_1 1 +#define FIRQ_LIST_IO_ISC_2 2 +#define FIRQ_LIST_IO_ISC_3 3 +#define FIRQ_LIST_IO_ISC_4 4 +#define FIRQ_LIST_IO_ISC_5 5 +#define FIRQ_LIST_IO_ISC_6 6 +#define FIRQ_LIST_IO_ISC_7 7 +#define FIRQ_LIST_PFAULT 8 +#define FIRQ_LIST_VIRTIO 9 +#define FIRQ_LIST_COUNT 10 +#define FIRQ_CNTR_IO 0 +#define FIRQ_CNTR_SERVICE 1 +#define FIRQ_CNTR_VIRTIO 2 +#define FIRQ_CNTR_PFAULT 3 +#define FIRQ_MAX_COUNT 4 + +/* mask the AIS mode for a given ISC */ +#define AIS_MODE_MASK(isc) (0x80 >> isc) + +#define KVM_S390_AIS_MODE_ALL 0 +#define KVM_S390_AIS_MODE_SINGLE 1 + +struct kvm_s390_float_interrupt { + unsigned long pending_irqs; + unsigned long masked_irqs; + spinlock_t lock; + struct list_head lists[FIRQ_LIST_COUNT]; + int counters[FIRQ_MAX_COUNT]; + struct kvm_s390_mchk_info mchk; + struct kvm_s390_ext_info srv_signal; + int last_sleep_cpu; + spinlock_t ais_lock; + u8 simm; + u8 nimm; +}; + +struct kvm_hw_wp_info_arch { + unsigned long addr; + unsigned long phys_addr; + int len; + char *old_data; +}; + +struct kvm_hw_bp_info_arch { + unsigned long addr; + int len; +}; + +/* + * Only the upper 16 bits of kvm_guest_debug->control are arch specific. + * Further KVM_GUESTDBG flags which an be used from userspace can be found in + * arch/s390/include/uapi/asm/kvm.h + */ +#define KVM_GUESTDBG_EXIT_PENDING 0x10000000 + +#define guestdbg_enabled(vcpu) \ + (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) +#define guestdbg_sstep_enabled(vcpu) \ + (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) +#define guestdbg_hw_bp_enabled(vcpu) \ + (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) +#define guestdbg_exit_pending(vcpu) (guestdbg_enabled(vcpu) && \ + (vcpu->guest_debug & KVM_GUESTDBG_EXIT_PENDING)) + +#define KVM_GUESTDBG_VALID_MASK \ + (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP |\ + KVM_GUESTDBG_USE_HW_BP | KVM_GUESTDBG_EXIT_PENDING) + +struct kvm_guestdbg_info_arch { + unsigned long cr0; + unsigned long cr9; + unsigned long cr10; + unsigned long cr11; + struct kvm_hw_bp_info_arch *hw_bp_info; + struct kvm_hw_wp_info_arch *hw_wp_info; + int nr_hw_bp; + int nr_hw_wp; + unsigned long last_bp; +}; + +struct kvm_s390_pv_vcpu { + u64 handle; + unsigned long stor_base; +}; + +struct kvm_vcpu_arch { + struct kvm_s390_sie_block *sie_block; + /* if vsie is active, currently executed shadow sie control block */ + struct kvm_s390_sie_block *vsie_block; + unsigned int host_acrs[NUM_ACRS]; + struct gs_cb *host_gscb; + struct kvm_s390_local_interrupt local_int; + struct hrtimer ckc_timer; + struct kvm_s390_pgm_info pgm; + struct gmap *gmap; + struct kvm_guestdbg_info_arch guestdbg; + unsigned long pfault_token; + unsigned long pfault_select; + unsigned long pfault_compare; + bool cputm_enabled; + /* + * The seqcount protects updates to cputm_start and sie_block.cputm, + * this way we can have non-blocking reads with consistent values. + * Only the owning VCPU thread (vcpu->cpu) is allowed to change these + * values and to start/stop/enable/disable cpu timer accounting. + */ + seqcount_t cputm_seqcount; + __u64 cputm_start; + bool gs_enabled; + bool skey_enabled; + /* Indicator if the access registers have been loaded from guest */ + bool acrs_loaded; + bool initialized; + struct kvm_s390_pv_vcpu pv; + union diag318_info diag318_info; + struct kvm_s390_mmu_cache *mc; +}; + +struct kvm_vm_stat { + struct kvm_vm_stat_generic generic; + u64 inject_io; + u64 io_390_adapter_map; + u64 io_390_adapter_unmap; + u64 io_390_inatomic; + u64 io_flic_inject_airq; + u64 io_set_adapter_int; + u64 io_390_inatomic_no_inject; + u64 inject_float_mchk; + u64 inject_pfault_done; + u64 inject_service_signal; + u64 inject_virtio; + u64 aen_forward; + u64 gmap_shadow_create; + u64 gmap_shadow_reuse; + u64 gmap_shadow_r1_entry; + u64 gmap_shadow_r2_entry; + u64 gmap_shadow_r3_entry; + u64 gmap_shadow_sg_entry; + u64 gmap_shadow_pg_entry; +}; + +struct kvm_arch_memory_slot { +}; + +struct s390_map_info { + struct list_head list; + __u64 guest_addr; + __u64 addr; + struct page *page; + /* + * True if the page is long-term pinned. False if long-term pinning + * failed and this entry exists only to preserve MAP/UNMAP symmetry. + */ + bool pinned; +}; + +struct s390_io_adapter { + unsigned int id; + int isc; + bool maskable; + bool masked; + bool swap; + bool suppressible; + spinlock_t maps_lock; + struct list_head maps; + unsigned int nr_maps; +}; + +#define MAX_S390_IO_ADAPTERS ((MAX_ISC + 1) * 8) +#define MAX_S390_ADAPTER_MAPS 256 + +/* maximum size of facilities and facility mask is 2k bytes */ +#define S390_ARCH_FAC_LIST_SIZE_BYTE (1<<11) +#define S390_ARCH_FAC_LIST_SIZE_U64 \ + (S390_ARCH_FAC_LIST_SIZE_BYTE / sizeof(u64)) +#define S390_ARCH_FAC_MASK_SIZE_BYTE S390_ARCH_FAC_LIST_SIZE_BYTE +#define S390_ARCH_FAC_MASK_SIZE_U64 \ + (S390_ARCH_FAC_MASK_SIZE_BYTE / sizeof(u64)) + +struct kvm_s390_cpu_model { + /* facility mask supported by kvm & hosting machine */ + __u64 fac_mask[S390_ARCH_FAC_MASK_SIZE_U64]; + struct kvm_s390_vm_cpu_subfunc subfuncs; + /* facility list requested by guest (in dma page) */ + __u64 *fac_list; + u64 cpuid; + unsigned short ibc; + /* subset of available UV-features for pv-guests enabled by user space */ + struct kvm_s390_vm_cpu_uv_feat uv_feat_guest; +}; + +#define S390_ARCH_FAC_FORMAT_2 2 +struct kvm_s390_flcb2 { + union { + struct { + u8 reserved0[7]; + u8 length; + }; + u64 header_val; + }; + u64 facilities[S390_ARCH_FAC_LIST_SIZE_U64]; +}; + +typedef int (*crypto_hook)(struct kvm_vcpu *vcpu); + +struct kvm_s390_crypto { + struct kvm_s390_crypto_cb *crycb; + struct rw_semaphore pqap_hook_rwsem; + crypto_hook *pqap_hook; + __u32 crycbd; + __u8 aes_kw; + __u8 dea_kw; + __u8 apie; +}; + +#define APCB0_MASK_SIZE 1 +struct kvm_s390_apcb0 { + __u64 apm[APCB0_MASK_SIZE]; /* 0x0000 */ + __u64 aqm[APCB0_MASK_SIZE]; /* 0x0008 */ + __u64 adm[APCB0_MASK_SIZE]; /* 0x0010 */ + __u64 reserved18; /* 0x0018 */ +}; + +#define APCB1_MASK_SIZE 4 +struct kvm_s390_apcb1 { + __u64 apm[APCB1_MASK_SIZE]; /* 0x0000 */ + __u64 aqm[APCB1_MASK_SIZE]; /* 0x0020 */ + __u64 adm[APCB1_MASK_SIZE]; /* 0x0040 */ + __u64 reserved60[4]; /* 0x0060 */ +}; + +struct kvm_s390_crypto_cb { + struct kvm_s390_apcb0 apcb0; /* 0x0000 */ + __u8 reserved20[0x0048 - 0x0020]; /* 0x0020 */ + __u8 dea_wrapping_key_mask[24]; /* 0x0048 */ + __u8 aes_wrapping_key_mask[32]; /* 0x0060 */ + struct kvm_s390_apcb1 apcb1; /* 0x0080 */ +}; + +struct kvm_s390_gisa { + union { + struct { /* common to all formats */ + u32 next_alert; + u8 ipm; + u8 reserved01[2]; + u8 iam; + }; + struct { /* format 0 */ + u32 next_alert; + u8 ipm; + u8 reserved01; + u8 : 6; + u8 g : 1; + u8 c : 1; + u8 iam; + u8 reserved02[4]; + u32 airq_count; + } g0; + struct { /* format 1 */ + u32 next_alert; + u8 ipm; + u8 simm; + u8 nimm; + u8 iam; + u8 aism[8]; + u8 : 6; + u8 g : 1; + u8 c : 1; + u8 reserved03[11]; + u32 airq_count; + } g1; + struct { + u64 word[4]; + } u64; + }; +}; + +struct kvm_s390_gib { + u32 alert_list_origin; + u32 reserved01; + u8:5; + u8 nisc:3; + u8 reserved03[3]; + u32 reserved04[5]; +}; + +/* + * sie_page2 has to be allocated as DMA because fac_list, crycb and + * gisa need 31bit addresses in the sie control block. + */ +struct sie_page2 { + __u64 fac_list[S390_ARCH_FAC_LIST_SIZE_U64]; /* 0x0000 */ + struct kvm_s390_crypto_cb crycb; /* 0x0800 */ + struct kvm_s390_gisa gisa; /* 0x0900 */ + struct kvm *kvm; /* 0x0920 */ + u8 reserved928[0x1000 - 0x928]; /* 0x0928 */ +}; + +struct vsie_page; + +struct kvm_s390_vsie { + struct mutex mutex; + struct radix_tree_root addr_to_page; + int page_count; + int next; + struct vsie_page *pages[KVM_MAX_VCPUS]; +}; + +struct kvm_s390_gisa_iam { + u8 mask; + spinlock_t ref_lock; + u32 ref_count[MAX_ISC + 1]; +}; + +struct kvm_s390_gisa_interrupt { + struct kvm_s390_gisa *origin; + struct kvm_s390_gisa_iam alert; + struct hrtimer timer; + u64 expires; + DECLARE_BITMAP(kicked_mask, KVM_MAX_VCPUS); +}; + +struct kvm_s390_pv { + u64 handle; + u64 guest_len; + unsigned long stor_base; + void *stor_var; + bool dumping; + void *set_aside; + struct list_head need_cleanup; + struct mmu_notifier mmu_notifier; + /* Protects against concurrent import-like operations */ + struct mutex import_lock; +}; + +struct kvm_s390_mmu_cache; + +struct kvm_arch { + struct esca_block *sca; + debug_info_t *dbf; + struct kvm_s390_float_interrupt float_int; + struct kvm_device *flic; + struct gmap *gmap; + unsigned long mem_limit; + int css_support; + int use_irqchip; + int use_cmma; + int use_pfmfi; + int use_skf; + int use_zpci_interp; + int user_cpu_state_ctrl; + int user_sigp; + int user_stsi; + int user_instr0; + int user_operexec; + int allow_vsie_esamode; + struct s390_io_adapter *adapters[MAX_S390_IO_ADAPTERS]; + wait_queue_head_t ipte_wq; + int ipte_lock_count; + struct mutex ipte_mutex; + spinlock_t start_stop_lock; + struct sie_page2 *sie_page2; + struct kvm_s390_cpu_model model; + struct kvm_s390_crypto crypto; + struct kvm_s390_vsie vsie; + u8 epdx; + u64 epoch; + int migration_mode; + atomic64_t cmma_dirty_pages; + /* subset of available cpu features enabled by user space */ + DECLARE_BITMAP(cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS); + /* indexed by vcpu_idx */ + DECLARE_BITMAP(idle_mask, KVM_MAX_VCPUS); + struct kvm_s390_gisa_interrupt gisa_int; + struct kvm_s390_pv pv; + struct list_head kzdev_list; + spinlock_t kzdev_list_lock; + struct kvm_s390_mmu_cache *mc; +}; + +#define KVM_HVA_ERR_BAD (-1UL) +#define KVM_HVA_ERR_RO_BAD (-2UL) + +static inline bool kvm_is_error_hva(unsigned long addr) +{ + return IS_ERR_VALUE(addr); +} + +#define ASYNC_PF_PER_VCPU 64 +struct kvm_arch_async_pf { + unsigned long pfault_token; +}; + +bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu); + +void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work); + +bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work); + +void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work); + +static inline void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu) {} + +void kvm_arch_crypto_clear_masks(struct kvm *kvm); +void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm, + unsigned long *aqm, unsigned long *adm); + +#define SIE64_RETURN_NORMAL 0 +#define SIE64_RETURN_MCCK 1 + +int __sie64a(phys_addr_t sie_block_phys, struct kvm_s390_sie_block *sie_block, u64 *rsa, + unsigned long gasce); + +static inline int sie64a(struct kvm_s390_sie_block *sie_block, u64 *rsa, unsigned long gasce) +{ + return __sie64a(virt_to_phys(sie_block), sie_block, rsa, gasce); +} + +extern char sie_exit; + +bool kvm_s390_pv_is_protected(struct kvm *kvm); +bool kvm_s390_pv_cpu_is_protected(struct kvm_vcpu *vcpu); + +extern int kvm_s390_enter_exit_sie(struct kvm_s390_sie_block *scb, + u64 *gprs, unsigned long gasce); + +extern int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc); +extern int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc); + +bool kvm_s390_is_gpa_in_memslot(struct kvm *kvm, gpa_t gpa); + +static inline void kvm_arch_free_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot) {} +static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} +static inline void kvm_arch_flush_shadow_all(struct kvm *kvm) {} +static inline void kvm_arch_flush_shadow_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot) {} +static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} +static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} + +#define __KVM_HAVE_ARCH_VM_FREE +void kvm_arch_free_vm(struct kvm *kvm); + +struct zpci_kvm_hook { + int (*kvm_register)(void *opaque, struct kvm *kvm); + void (*kvm_unregister)(void *opaque); +}; + +extern struct zpci_kvm_hook zpci_kvm_hook; + +#endif /* ASM_KVM_HOST_S390_H */ diff --git a/arch/s390/include/asm/kvm_host_s390_types.h b/arch/s390/include/asm/kvm_host_s390_types.h new file mode 100644 index 000000000000..9e348a530421 --- /dev/null +++ b/arch/s390/include/asm/kvm_host_s390_types.h @@ -0,0 +1,349 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_KVM_HOST_S390_TYPES_H +#define _ASM_KVM_HOST_S390_TYPES_H + +#include <linux/atomic.h> +#include <linux/types.h> + +#define KVM_S390_BSCA_CPU_SLOTS 64 +#define KVM_S390_ESCA_CPU_SLOTS 248 + +#define SCB_ALIGNMENT_SHIFT 9 + +#define SIGP_CTRL_C 0x80 +#define SIGP_CTRL_SCN_MASK 0x3f + +union bsca_sigp_ctrl { + __u8 value; + struct { + __u8 c : 1; + __u8 r : 1; + __u8 scn : 6; + }; +}; + +union esca_sigp_ctrl { + __u16 value; + struct { + __u8 c : 1; + __u8 reserved: 7; + __u8 scn; + }; +}; + +struct esca_entry { + union esca_sigp_ctrl sigp_ctrl; + __u16 reserved1[3]; + __u64 sda; + __u64 reserved2[6]; +}; + +struct bsca_entry { + __u8 reserved0; + union bsca_sigp_ctrl sigp_ctrl; + __u16 reserved[3]; + __u64 sda; + __u64 reserved2[2]; +}; + +union ipte_control { + unsigned long val; + struct { + unsigned long k : 1; + unsigned long kh : 31; + unsigned long kg : 32; + }; +}; + +/* + * Utility is defined as two bytes but having it four bytes wide + * generates more efficient code. Since the following bytes are + * reserved this makes no functional difference. + */ +union sca_utility { + __u32 val; + struct { + __u32 mtcr : 1; + __u32 : 31; + }; +}; + +struct bsca_block { + union ipte_control ipte_control; + __u64 reserved[5]; + __u64 mcn; + union sca_utility utility; + __u8 reserved2[4]; + struct bsca_entry cpu[KVM_S390_BSCA_CPU_SLOTS]; +}; + +struct esca_block { + union ipte_control ipte_control; + __u64 reserved1[6]; + union sca_utility utility; + __u8 reserved2[4]; + __u64 mcn[4]; + __u64 reserved3[20]; + struct esca_entry cpu[KVM_S390_ESCA_CPU_SLOTS]; +}; + +/* + * This struct is used to store some machine check info from lowcore + * for machine checks that happen while the guest is running. + * This info in host's lowcore might be overwritten by a second machine + * check from host when host is in the machine check's high-level handling. + * The size is 24 bytes. + */ +struct mcck_volatile_info { + __u64 mcic; + __u64 failing_storage_address; + __u32 ext_damage_code; + __u32 reserved; +}; + +#define CR0_INITIAL_MASK (CR0_UNUSED_56 | CR0_INTERRUPT_KEY_SUBMASK | \ + CR0_MEASUREMENT_ALERT_SUBMASK) +#define CR14_INITIAL_MASK (CR14_UNUSED_32 | CR14_UNUSED_33 | \ + CR14_EXTERNAL_DAMAGE_SUBMASK) + +#define SIDAD_SIZE_MASK 0xff +#define sida_addr(sie_block) phys_to_virt((sie_block)->sidad & PAGE_MASK) +#define sida_size(sie_block) \ + ((((sie_block)->sidad & SIDAD_SIZE_MASK) + 1) * PAGE_SIZE) + +#define CPUSTAT_STOPPED 0x80000000 +#define CPUSTAT_WAIT 0x10000000 +#define CPUSTAT_ECALL_PEND 0x08000000 +#define CPUSTAT_STOP_INT 0x04000000 +#define CPUSTAT_IO_INT 0x02000000 +#define CPUSTAT_EXT_INT 0x01000000 +#define CPUSTAT_RUNNING 0x00800000 +#define CPUSTAT_RETAINED 0x00400000 +#define CPUSTAT_TIMING_SUB 0x00020000 +#define CPUSTAT_SIE_SUB 0x00010000 +#define CPUSTAT_RRF 0x00008000 +#define CPUSTAT_SLSV 0x00004000 +#define CPUSTAT_SLSR 0x00002000 +#define CPUSTAT_ZARCH 0x00000800 +#define CPUSTAT_MCDS 0x00000100 +#define CPUSTAT_KSS 0x00000200 +#define CPUSTAT_SM 0x00000080 +#define CPUSTAT_IBS 0x00000040 +#define CPUSTAT_GED2 0x00000010 +#define CPUSTAT_G 0x00000008 +#define CPUSTAT_GED 0x00000004 +#define CPUSTAT_J 0x00000002 +#define CPUSTAT_P 0x00000001 + +struct kvm_s390_sie_block { + atomic_t cpuflags; /* 0x0000 */ + __u32 : 1; /* 0x0004 */ + __u32 prefix : 19; + __u32 ibc : 12; + __u8 reserved08[4]; /* 0x0008 */ +#define PROG_IN_SIE (1<<0) + __u32 prog0c; /* 0x000c */ + union { + __u8 reserved10[16]; /* 0x0010 */ + struct { + __u64 pv_handle_cpu; + __u64 pv_handle_config; + }; + }; +#define PROG_BLOCK_SIE (1<<0) +#define PROG_REQUEST (1<<1) + atomic_t prog20; /* 0x0020 */ + __u8 reserved24[4]; /* 0x0024 */ + __u64 cputm; /* 0x0028 */ + __u64 ckc; /* 0x0030 */ + __u64 epoch; /* 0x0038 */ + __u32 svcc; /* 0x0040 */ +#define LCTL_CR0 0x8000 +#define LCTL_CR6 0x0200 +#define LCTL_CR9 0x0040 +#define LCTL_CR10 0x0020 +#define LCTL_CR11 0x0010 +#define LCTL_CR14 0x0002 + __u16 lctl; /* 0x0044 */ + __s16 icpua; /* 0x0046 */ +#define ICTL_OPEREXC 0x80000000 +#define ICTL_PINT 0x20000000 +#define ICTL_LPSW 0x00400000 +#define ICTL_STCTL 0x00040000 +#define ICTL_ISKE 0x00004000 +#define ICTL_SSKE 0x00002000 +#define ICTL_RRBE 0x00001000 +#define ICTL_TPROT 0x00000200 + __u32 ictl; /* 0x0048 */ +#define ECA_CEI 0x80000000 +#define ECA_IB 0x40000000 +#define ECA_SIGPI 0x10000000 +#define ECA_MVPGI 0x01000000 +#define ECA_AIV 0x00200000 +#define ECA_VX 0x00020000 +#define ECA_PROTEXCI 0x00002000 +#define ECA_APIE 0x00000008 +#define ECA_SII 0x00000001 + __u32 eca; /* 0x004c */ +#define ICPT_INST 0x04 +#define ICPT_PROGI 0x08 +#define ICPT_INSTPROGI 0x0C +#define ICPT_EXTREQ 0x10 +#define ICPT_EXTINT 0x14 +#define ICPT_IOREQ 0x18 +#define ICPT_WAIT 0x1c +#define ICPT_VALIDITY 0x20 +#define ICPT_STOP 0x28 +#define ICPT_OPEREXC 0x2C +#define ICPT_PARTEXEC 0x38 +#define ICPT_IOINST 0x40 +#define ICPT_KSS 0x5c +#define ICPT_MCHKREQ 0x60 +#define ICPT_INT_ENABLE 0x64 +#define ICPT_PV_INSTR 0x68 +#define ICPT_PV_NOTIFY 0x6c +#define ICPT_PV_PREF 0x70 + __u8 icptcode; /* 0x0050 */ + __u8 icptstatus; /* 0x0051 */ + __u16 ihcpu; /* 0x0052 */ + __u8 reserved54; /* 0x0054 */ +#define IICTL_CODE_NONE 0x00 +#define IICTL_CODE_MCHK 0x01 +#define IICTL_CODE_EXT 0x02 +#define IICTL_CODE_IO 0x03 +#define IICTL_CODE_RESTART 0x04 +#define IICTL_CODE_SPECIFICATION 0x10 +#define IICTL_CODE_OPERAND 0x11 + __u8 iictl; /* 0x0055 */ + __u16 ipa; /* 0x0056 */ + __u32 ipb; /* 0x0058 */ + __u32 scaoh; /* 0x005c */ +#define FPF_BPBC 0x20 + __u8 fpf; /* 0x0060 */ +#define ECB_GS 0x40 +#define ECB_TE 0x10 +#define ECB_SPECI 0x08 +#define ECB_SRSI 0x04 +#define ECB_HOSTPROTINT 0x02 +#define ECB_PTF 0x01 + __u8 ecb; /* 0x0061 */ +#define ECB2_CMMA 0x80 +#define ECB2_IEP 0x20 +#define ECB2_PFMFI 0x08 +#define ECB2_ESCA 0x04 +#define ECB2_ZPCI_LSI 0x02 + __u8 ecb2; /* 0x0062 */ +#define ECB3_AISI 0x20 +#define ECB3_AISII 0x10 +#define ECB3_DEA 0x08 +#define ECB3_AES 0x04 +#define ECB3_RI 0x01 + __u8 ecb3; /* 0x0063 */ +#define ESCA_SCAOL_MASK ~0x3fU + __u32 scaol; /* 0x0064 */ + __u8 sdf; /* 0x0068 */ + __u8 epdx; /* 0x0069 */ + __u8 cpnc; /* 0x006a */ + __u8 reserved6b; /* 0x006b */ + __u32 todpr; /* 0x006c */ +#define GISA_FORMAT1 0x00000001 + __u32 gd; /* 0x0070 */ + __u8 reserved74[12]; /* 0x0074 */ + __u64 mso; /* 0x0080 */ + __u64 msl; /* 0x0088 */ + psw_t gpsw; /* 0x0090 */ + __u64 gg14; /* 0x00a0 */ + __u64 gg15; /* 0x00a8 */ + __u8 reservedb0[8]; /* 0x00b0 */ +#define HPID_KVM 0x4 +#define HPID_VSIE 0x5 + __u8 hpid; /* 0x00b8 */ + __u8 reservedb9[7]; /* 0x00b9 */ + union { + struct { + __u32 eiparams; /* 0x00c0 */ + __u16 extcpuaddr; /* 0x00c4 */ + __u16 eic; /* 0x00c6 */ + }; + __u64 mcic; /* 0x00c0 */ + } __packed; + __u32 reservedc8; /* 0x00c8 */ + union { + struct { + __u16 pgmilc; /* 0x00cc */ + __u16 iprcc; /* 0x00ce */ + }; + __u32 edc; /* 0x00cc */ + } __packed; + union { + struct { + __u32 dxc; /* 0x00d0 */ + __u16 mcn; /* 0x00d4 */ + __u8 perc; /* 0x00d6 */ + __u8 peratmid; /* 0x00d7 */ + }; + __u64 faddr; /* 0x00d0 */ + } __packed; + __u64 peraddr; /* 0x00d8 */ + __u8 eai; /* 0x00e0 */ + __u8 peraid; /* 0x00e1 */ + __u8 oai; /* 0x00e2 */ + __u8 armid; /* 0x00e3 */ + __u8 reservede4[4]; /* 0x00e4 */ + union { + __u64 tecmc; /* 0x00e8 */ + struct { + __u16 subchannel_id; /* 0x00e8 */ + __u16 subchannel_nr; /* 0x00ea */ + __u32 io_int_parm; /* 0x00ec */ + __u32 io_int_word; /* 0x00f0 */ + }; + } __packed; + __u8 reservedf4[8]; /* 0x00f4 */ +#define CRYCB_FORMAT_MASK 0x00000003 +#define CRYCB_FORMAT0 0x00000000 +#define CRYCB_FORMAT1 0x00000001 +#define CRYCB_FORMAT2 0x00000003 + __u32 crycbd; /* 0x00fc */ + __u64 gcr[16]; /* 0x0100 */ + union { + __u64 gbea; /* 0x0180 */ + __u64 sidad; + }; + __u8 reserved188[8]; /* 0x0188 */ + __u64 sdnxo; /* 0x0190 */ + __u8 reserved198[8]; /* 0x0198 */ + __u32 fac; /* 0x01a0 */ + __u8 reserved1a4[20]; /* 0x01a4 */ + __u64 cbrlo; /* 0x01b8 */ + __u8 reserved1c0[8]; /* 0x01c0 */ +#define ECD_HOSTREGMGMT 0x20000000 +#define ECD_MEF 0x08000000 +#define ECD_ETOKENF 0x02000000 +#define ECD_ECC 0x00200000 +#define ECD_HMAC 0x00004000 + __u32 ecd; /* 0x01c8 */ + __u8 reserved1cc[18]; /* 0x01cc */ + __u64 pp; /* 0x01de */ + __u8 reserved1e6[2]; /* 0x01e6 */ + __u64 itdba; /* 0x01e8 */ + __u64 riccbd; /* 0x01f0 */ + __u64 gvrd; /* 0x01f8 */ +} __packed __aligned(512); + +struct kvm_s390_itdb { + __u8 data[256]; +}; + +struct sie_page { + struct kvm_s390_sie_block sie_block; + struct mcck_volatile_info mcck_info; /* 0x0200 */ + __u8 reserved218[360]; /* 0x0218 */ + __u64 pv_grregs[16]; /* 0x0380 */ + __u8 reserved400[512]; /* 0x0400 */ + struct kvm_s390_itdb itdb; /* 0x0600 */ + __u8 reserved700[2304]; /* 0x0700 */ +}; + +#endif /* _ASM_KVM_HOST_S390_TYPES_H */ diff --git a/arch/s390/include/asm/kvm_host_types.h b/arch/s390/include/asm/kvm_host_types.h index 1394d3fb648f..e5bdba07cab0 100644 --- a/arch/s390/include/asm/kvm_host_types.h +++ b/arch/s390/include/asm/kvm_host_types.h @@ -1,348 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_KVM_HOST_TYPES_H -#define _ASM_KVM_HOST_TYPES_H +#ifndef ASM_KVM_HOST_TYPES_H +#define ASM_KVM_HOST_TYPES_H -#include <linux/atomic.h> -#include <linux/types.h> +#include <asm/kvm_host_s390_types.h> -#define KVM_S390_BSCA_CPU_SLOTS 64 -#define KVM_S390_ESCA_CPU_SLOTS 248 - -#define SIGP_CTRL_C 0x80 -#define SIGP_CTRL_SCN_MASK 0x3f - -union bsca_sigp_ctrl { - __u8 value; - struct { - __u8 c : 1; - __u8 r : 1; - __u8 scn : 6; - }; -}; - -union esca_sigp_ctrl { - __u16 value; - struct { - __u8 c : 1; - __u8 reserved: 7; - __u8 scn; - }; -}; - -struct esca_entry { - union esca_sigp_ctrl sigp_ctrl; - __u16 reserved1[3]; - __u64 sda; - __u64 reserved2[6]; -}; - -struct bsca_entry { - __u8 reserved0; - union bsca_sigp_ctrl sigp_ctrl; - __u16 reserved[3]; - __u64 sda; - __u64 reserved2[2]; -}; - -union ipte_control { - unsigned long val; - struct { - unsigned long k : 1; - unsigned long kh : 31; - unsigned long kg : 32; - }; -}; - -/* - * Utility is defined as two bytes but having it four bytes wide - * generates more efficient code. Since the following bytes are - * reserved this makes no functional difference. - */ -union sca_utility { - __u32 val; - struct { - __u32 mtcr : 1; - __u32 : 31; - }; -}; - -struct bsca_block { - union ipte_control ipte_control; - __u64 reserved[5]; - __u64 mcn; - union sca_utility utility; - __u8 reserved2[4]; - struct bsca_entry cpu[KVM_S390_BSCA_CPU_SLOTS]; -}; - -struct esca_block { - union ipte_control ipte_control; - __u64 reserved1[6]; - union sca_utility utility; - __u8 reserved2[4]; - __u64 mcn[4]; - __u64 reserved3[20]; - struct esca_entry cpu[KVM_S390_ESCA_CPU_SLOTS]; -}; - -/* - * This struct is used to store some machine check info from lowcore - * for machine checks that happen while the guest is running. - * This info in host's lowcore might be overwritten by a second machine - * check from host when host is in the machine check's high-level handling. - * The size is 24 bytes. - */ -struct mcck_volatile_info { - __u64 mcic; - __u64 failing_storage_address; - __u32 ext_damage_code; - __u32 reserved; -}; - -#define CR0_INITIAL_MASK (CR0_UNUSED_56 | CR0_INTERRUPT_KEY_SUBMASK | \ - CR0_MEASUREMENT_ALERT_SUBMASK) -#define CR14_INITIAL_MASK (CR14_UNUSED_32 | CR14_UNUSED_33 | \ - CR14_EXTERNAL_DAMAGE_SUBMASK) - -#define SIDAD_SIZE_MASK 0xff -#define sida_addr(sie_block) phys_to_virt((sie_block)->sidad & PAGE_MASK) -#define sida_size(sie_block) \ - ((((sie_block)->sidad & SIDAD_SIZE_MASK) + 1) * PAGE_SIZE) - -#define CPUSTAT_STOPPED 0x80000000 -#define CPUSTAT_WAIT 0x10000000 -#define CPUSTAT_ECALL_PEND 0x08000000 -#define CPUSTAT_STOP_INT 0x04000000 -#define CPUSTAT_IO_INT 0x02000000 -#define CPUSTAT_EXT_INT 0x01000000 -#define CPUSTAT_RUNNING 0x00800000 -#define CPUSTAT_RETAINED 0x00400000 -#define CPUSTAT_TIMING_SUB 0x00020000 -#define CPUSTAT_SIE_SUB 0x00010000 -#define CPUSTAT_RRF 0x00008000 -#define CPUSTAT_SLSV 0x00004000 -#define CPUSTAT_SLSR 0x00002000 -#define CPUSTAT_ZARCH 0x00000800 -#define CPUSTAT_MCDS 0x00000100 -#define CPUSTAT_KSS 0x00000200 -#define CPUSTAT_SM 0x00000080 -#define CPUSTAT_IBS 0x00000040 -#define CPUSTAT_GED2 0x00000010 -#define CPUSTAT_G 0x00000008 -#define CPUSTAT_GED 0x00000004 -#define CPUSTAT_J 0x00000002 -#define CPUSTAT_P 0x00000001 - -struct kvm_s390_sie_block { - atomic_t cpuflags; /* 0x0000 */ - __u32 : 1; /* 0x0004 */ - __u32 prefix : 18; - __u32 : 1; - __u32 ibc : 12; - __u8 reserved08[4]; /* 0x0008 */ -#define PROG_IN_SIE (1<<0) - __u32 prog0c; /* 0x000c */ - union { - __u8 reserved10[16]; /* 0x0010 */ - struct { - __u64 pv_handle_cpu; - __u64 pv_handle_config; - }; - }; -#define PROG_BLOCK_SIE (1<<0) -#define PROG_REQUEST (1<<1) - atomic_t prog20; /* 0x0020 */ - __u8 reserved24[4]; /* 0x0024 */ - __u64 cputm; /* 0x0028 */ - __u64 ckc; /* 0x0030 */ - __u64 epoch; /* 0x0038 */ - __u32 svcc; /* 0x0040 */ -#define LCTL_CR0 0x8000 -#define LCTL_CR6 0x0200 -#define LCTL_CR9 0x0040 -#define LCTL_CR10 0x0020 -#define LCTL_CR11 0x0010 -#define LCTL_CR14 0x0002 - __u16 lctl; /* 0x0044 */ - __s16 icpua; /* 0x0046 */ -#define ICTL_OPEREXC 0x80000000 -#define ICTL_PINT 0x20000000 -#define ICTL_LPSW 0x00400000 -#define ICTL_STCTL 0x00040000 -#define ICTL_ISKE 0x00004000 -#define ICTL_SSKE 0x00002000 -#define ICTL_RRBE 0x00001000 -#define ICTL_TPROT 0x00000200 - __u32 ictl; /* 0x0048 */ -#define ECA_CEI 0x80000000 -#define ECA_IB 0x40000000 -#define ECA_SIGPI 0x10000000 -#define ECA_MVPGI 0x01000000 -#define ECA_AIV 0x00200000 -#define ECA_VX 0x00020000 -#define ECA_PROTEXCI 0x00002000 -#define ECA_APIE 0x00000008 -#define ECA_SII 0x00000001 - __u32 eca; /* 0x004c */ -#define ICPT_INST 0x04 -#define ICPT_PROGI 0x08 -#define ICPT_INSTPROGI 0x0C -#define ICPT_EXTREQ 0x10 -#define ICPT_EXTINT 0x14 -#define ICPT_IOREQ 0x18 -#define ICPT_WAIT 0x1c -#define ICPT_VALIDITY 0x20 -#define ICPT_STOP 0x28 -#define ICPT_OPEREXC 0x2C -#define ICPT_PARTEXEC 0x38 -#define ICPT_IOINST 0x40 -#define ICPT_KSS 0x5c -#define ICPT_MCHKREQ 0x60 -#define ICPT_INT_ENABLE 0x64 -#define ICPT_PV_INSTR 0x68 -#define ICPT_PV_NOTIFY 0x6c -#define ICPT_PV_PREF 0x70 - __u8 icptcode; /* 0x0050 */ - __u8 icptstatus; /* 0x0051 */ - __u16 ihcpu; /* 0x0052 */ - __u8 reserved54; /* 0x0054 */ -#define IICTL_CODE_NONE 0x00 -#define IICTL_CODE_MCHK 0x01 -#define IICTL_CODE_EXT 0x02 -#define IICTL_CODE_IO 0x03 -#define IICTL_CODE_RESTART 0x04 -#define IICTL_CODE_SPECIFICATION 0x10 -#define IICTL_CODE_OPERAND 0x11 - __u8 iictl; /* 0x0055 */ - __u16 ipa; /* 0x0056 */ - __u32 ipb; /* 0x0058 */ - __u32 scaoh; /* 0x005c */ -#define FPF_BPBC 0x20 - __u8 fpf; /* 0x0060 */ -#define ECB_GS 0x40 -#define ECB_TE 0x10 -#define ECB_SPECI 0x08 -#define ECB_SRSI 0x04 -#define ECB_HOSTPROTINT 0x02 -#define ECB_PTF 0x01 - __u8 ecb; /* 0x0061 */ -#define ECB2_CMMA 0x80 -#define ECB2_IEP 0x20 -#define ECB2_PFMFI 0x08 -#define ECB2_ESCA 0x04 -#define ECB2_ZPCI_LSI 0x02 - __u8 ecb2; /* 0x0062 */ -#define ECB3_AISI 0x20 -#define ECB3_AISII 0x10 -#define ECB3_DEA 0x08 -#define ECB3_AES 0x04 -#define ECB3_RI 0x01 - __u8 ecb3; /* 0x0063 */ -#define ESCA_SCAOL_MASK ~0x3fU - __u32 scaol; /* 0x0064 */ - __u8 sdf; /* 0x0068 */ - __u8 epdx; /* 0x0069 */ - __u8 cpnc; /* 0x006a */ - __u8 reserved6b; /* 0x006b */ - __u32 todpr; /* 0x006c */ -#define GISA_FORMAT1 0x00000001 - __u32 gd; /* 0x0070 */ - __u8 reserved74[12]; /* 0x0074 */ - __u64 mso; /* 0x0080 */ - __u64 msl; /* 0x0088 */ - psw_t gpsw; /* 0x0090 */ - __u64 gg14; /* 0x00a0 */ - __u64 gg15; /* 0x00a8 */ - __u8 reservedb0[8]; /* 0x00b0 */ -#define HPID_KVM 0x4 -#define HPID_VSIE 0x5 - __u8 hpid; /* 0x00b8 */ - __u8 reservedb9[7]; /* 0x00b9 */ - union { - struct { - __u32 eiparams; /* 0x00c0 */ - __u16 extcpuaddr; /* 0x00c4 */ - __u16 eic; /* 0x00c6 */ - }; - __u64 mcic; /* 0x00c0 */ - } __packed; - __u32 reservedc8; /* 0x00c8 */ - union { - struct { - __u16 pgmilc; /* 0x00cc */ - __u16 iprcc; /* 0x00ce */ - }; - __u32 edc; /* 0x00cc */ - } __packed; - union { - struct { - __u32 dxc; /* 0x00d0 */ - __u16 mcn; /* 0x00d4 */ - __u8 perc; /* 0x00d6 */ - __u8 peratmid; /* 0x00d7 */ - }; - __u64 faddr; /* 0x00d0 */ - } __packed; - __u64 peraddr; /* 0x00d8 */ - __u8 eai; /* 0x00e0 */ - __u8 peraid; /* 0x00e1 */ - __u8 oai; /* 0x00e2 */ - __u8 armid; /* 0x00e3 */ - __u8 reservede4[4]; /* 0x00e4 */ - union { - __u64 tecmc; /* 0x00e8 */ - struct { - __u16 subchannel_id; /* 0x00e8 */ - __u16 subchannel_nr; /* 0x00ea */ - __u32 io_int_parm; /* 0x00ec */ - __u32 io_int_word; /* 0x00f0 */ - }; - } __packed; - __u8 reservedf4[8]; /* 0x00f4 */ -#define CRYCB_FORMAT_MASK 0x00000003 -#define CRYCB_FORMAT0 0x00000000 -#define CRYCB_FORMAT1 0x00000001 -#define CRYCB_FORMAT2 0x00000003 - __u32 crycbd; /* 0x00fc */ - __u64 gcr[16]; /* 0x0100 */ - union { - __u64 gbea; /* 0x0180 */ - __u64 sidad; - }; - __u8 reserved188[8]; /* 0x0188 */ - __u64 sdnxo; /* 0x0190 */ - __u8 reserved198[8]; /* 0x0198 */ - __u32 fac; /* 0x01a0 */ - __u8 reserved1a4[20]; /* 0x01a4 */ - __u64 cbrlo; /* 0x01b8 */ - __u8 reserved1c0[8]; /* 0x01c0 */ -#define ECD_HOSTREGMGMT 0x20000000 -#define ECD_MEF 0x08000000 -#define ECD_ETOKENF 0x02000000 -#define ECD_ECC 0x00200000 -#define ECD_HMAC 0x00004000 - __u32 ecd; /* 0x01c8 */ - __u8 reserved1cc[18]; /* 0x01cc */ - __u64 pp; /* 0x01de */ - __u8 reserved1e6[2]; /* 0x01e6 */ - __u64 itdba; /* 0x01e8 */ - __u64 riccbd; /* 0x01f0 */ - __u64 gvrd; /* 0x01f8 */ -} __packed __aligned(512); - -struct kvm_s390_itdb { - __u8 data[256]; -}; - -struct sie_page { - struct kvm_s390_sie_block sie_block; - struct mcck_volatile_info mcck_info; /* 0x0200 */ - __u8 reserved218[360]; /* 0x0218 */ - __u64 pv_grregs[16]; /* 0x0380 */ - __u8 reserved400[512]; /* 0x0400 */ - struct kvm_s390_itdb itdb; /* 0x0600 */ - __u8 reserved700[2304]; /* 0x0700 */ -}; - -#endif /* _ASM_KVM_HOST_TYPES_H */ +#endif /* ASM_KVM_HOST_TYPES_H */ diff --git a/arch/s390/include/asm/linkage.h b/arch/s390/include/asm/linkage.h index df3fb7d8227b..1b3ac553a642 100644 --- a/arch/s390/include/asm/linkage.h +++ b/arch/s390/include/asm/linkage.h @@ -7,4 +7,6 @@ #define __ALIGN .balign CONFIG_FUNCTION_ALIGNMENT, 0x07 #define __ALIGN_STR __stringify(__ALIGN) +#define _THIS_IP_ ({ unsigned long __ip; asm volatile("larl %0, ." : "=d" (__ip)); __ip; }) + #endif diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 50ffe75adeb4..5cef215d30e7 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -10,6 +10,7 @@ #define _ASM_S390_LOWCORE_H #include <linux/types.h> +#include <asm/tod_types.h> #include <asm/machine.h> #include <asm/ptrace.h> #include <asm/ctlreg.h> @@ -125,8 +126,7 @@ struct lowcore { __u64 avg_steal_timer; /* 0x0300 */ __u64 last_update_timer; /* 0x0308 */ __u64 last_update_clock; /* 0x0310 */ - __u64 int_clock; /* 0x0318 */ - __u8 pad_0x0320[0x0328-0x0320]; /* 0x0320 */ + union tod_clock int_clock; /* 0x0318 */ __u64 clock_comparator; /* 0x0328 */ __u8 pad_0x0330[0x0340-0x0330]; /* 0x0330 */ @@ -160,12 +160,18 @@ struct lowcore { /* SMP info area */ __u32 cpu_nr; /* 0x03a0 */ __u32 softirq_pending; /* 0x03a4 */ - __s32 preempt_count; /* 0x03a8 */ - __u32 spinlock_lockval; /* 0x03ac */ - __u32 spinlock_index; /* 0x03b0 */ - __u8 pad_0x03b4[0x03b8-0x03b4]; /* 0x03b4 */ + union { + struct { + __u32 need_resched; /* 0x03a8 */ + __u32 count; /* 0x03ac */ + } preempt; + __u64 preempt_count; /* 0x03a8 */ + }; + __u32 spinlock_lockval; /* 0x03b0 */ + __u32 spinlock_index; /* 0x03b4 */ __u64 percpu_offset; /* 0x03b8 */ - __u8 pad_0x03c0[0x0400-0x03c0]; /* 0x03c0 */ + __u8 percpu_register; /* 0x03c0 */ + __u8 pad_0x03c1[0x0400-0x03c1]; /* 0x03c1 */ __u32 return_lpswe; /* 0x0400 */ __u32 return_mcck_lpswe; /* 0x0404 */ diff --git a/arch/s390/include/asm/mmu.h b/arch/s390/include/asm/mmu.h index f07e49b419ab..d4fd7bf3692e 100644 --- a/arch/s390/include/asm/mmu.h +++ b/arch/s390/include/asm/mmu.h @@ -19,23 +19,10 @@ typedef struct { /* The mmu context belongs to a secure guest. */ atomic_t protected_count; /* - * The following bitfields need a down_write on the mm - * semaphore when they are written to. As they are only - * written once, they can be read without a lock. - */ - /* The mmu context uses extended page tables. */ - unsigned int has_pgste:1; - /* The mmu context uses storage keys. */ - unsigned int uses_skeys:1; - /* The mmu context uses CMM. */ - unsigned int uses_cmm:1; - /* * The mmu context allows COW-sharing of memory pages (KSM, zeropage). * Note that COW-sharing during fork() is currently always allowed. */ unsigned int allow_cow_sharing:1; - /* The gmaps associated with this context are allowed to use huge pages. */ - unsigned int allow_gmap_hpage_1m:1; } mm_context_t; #define INIT_MM_CONTEXT(name) \ diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index d9b8501bc93d..bd1ef5e2d2eb 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -29,12 +29,8 @@ static inline int init_new_context(struct task_struct *tsk, atomic_set(&mm->context.protected_count, 0); mm->context.gmap_asce = 0; mm->context.flush_mm = 0; -#ifdef CONFIG_PGSTE - mm->context.has_pgste = 0; - mm->context.uses_skeys = 0; - mm->context.uses_cmm = 0; +#if IS_ENABLED(CONFIG_KVM) mm->context.allow_cow_sharing = 1; - mm->context.allow_gmap_hpage_1m = 0; #endif switch (mm->context.asce_limit) { default: diff --git a/arch/s390/include/asm/nmi.h b/arch/s390/include/asm/nmi.h index 6454c1531854..7919b2b9ac9c 100644 --- a/arch/s390/include/asm/nmi.h +++ b/arch/s390/include/asm/nmi.h @@ -22,6 +22,7 @@ #define MCCK_CODE_SYSTEM_DAMAGE BIT(63) #define MCCK_CODE_EXT_DAMAGE BIT(63 - 5) #define MCCK_CODE_CP BIT(63 - 9) +#define MCCK_CODE_CK BIT(63 - 11) #define MCCK_CODE_STG_ERROR BIT(63 - 16) #define MCCK_CODE_STG_KEY_ERROR BIT(63 - 18) #define MCCK_CODE_STG_DEGRAD BIT(63 - 19) @@ -33,6 +34,8 @@ #define MCCK_CODE_FC_VALID BIT(63 - 43) #define MCCK_CODE_CPU_TIMER_VALID BIT(63 - 46) +#define MCCK_CODE_NO_GUEST (MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE | MCCK_CODE_CK) + #ifndef __ASSEMBLER__ union mci { diff --git a/arch/s390/include/asm/page.h b/arch/s390/include/asm/page.h index c1d63b613bf9..56da819a79e6 100644 --- a/arch/s390/include/asm/page.h +++ b/arch/s390/include/asm/page.h @@ -65,7 +65,6 @@ static inline void copy_page(void *to, void *from) : : "memory", "cc"); } -#define clear_user_page(page, vaddr, pg) clear_page(page) #define copy_user_page(to, from, vaddr, pg) copy_page(to, from) #define vma_alloc_zeroed_movable_folio(vma, vaddr) \ @@ -78,7 +77,6 @@ static inline void copy_page(void *to, void *from) #ifdef STRICT_MM_TYPECHECKS typedef struct { unsigned long pgprot; } pgprot_t; -typedef struct { unsigned long pgste; } pgste_t; typedef struct { unsigned long pte; } pte_t; typedef struct { unsigned long pmd; } pmd_t; typedef struct { unsigned long pud; } pud_t; @@ -94,7 +92,6 @@ static __always_inline unsigned long name ## _val(name ## _t name) \ #else /* STRICT_MM_TYPECHECKS */ typedef unsigned long pgprot_t; -typedef unsigned long pgste_t; typedef unsigned long pte_t; typedef unsigned long pmd_t; typedef unsigned long pud_t; @@ -110,7 +107,6 @@ static __always_inline unsigned long name ## _val(name ## _t name) \ #endif /* STRICT_MM_TYPECHECKS */ DEFINE_PGVAL_FUNC(pgprot) -DEFINE_PGVAL_FUNC(pgste) DEFINE_PGVAL_FUNC(pte) DEFINE_PGVAL_FUNC(pmd) DEFINE_PGVAL_FUNC(pud) @@ -120,7 +116,6 @@ DEFINE_PGVAL_FUNC(pgd) typedef pte_t *pgtable_t; #define __pgprot(x) ((pgprot_t) { (x) } ) -#define __pgste(x) ((pgste_t) { (x) } ) #define __pte(x) ((pte_t) { (x) } ) #define __pmd(x) ((pmd_t) { (x) } ) #define __pud(x) ((pud_t) { (x) } ) @@ -282,7 +277,7 @@ static inline unsigned long virt_to_pfn(const void *kaddr) #define virt_addr_valid(kaddr) pfn_valid(phys_to_pfn(__pa_nodebug((unsigned long)(kaddr)))) -#define VM_DATA_DEFAULT_FLAGS VM_DATA_FLAGS_NON_EXEC +#define VMA_DATA_DEFAULT_FLAGS VMA_DATA_FLAGS_NON_EXEC #endif /* !__ASSEMBLER__ */ diff --git a/arch/s390/include/asm/pai.h b/arch/s390/include/asm/pai.h index 534d0320e2aa..a3456a36aaa7 100644 --- a/arch/s390/include/asm/pai.h +++ b/arch/s390/include/asm/pai.h @@ -76,7 +76,6 @@ static __always_inline void pai_kernel_exit(struct pt_regs *regs) } #define PAI_SAVE_AREA(x) ((x)->hw.event_base) -#define PAI_CPU_MASK(x) ((x)->hw.addr_filters) #define PAI_PMU_IDX(x) ((x)->hw.last_tag) #define PAI_SWLIST(x) (&(x)->hw.tp_list) diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h index c0ff19dab580..88a125b92bdd 100644 --- a/arch/s390/include/asm/pci.h +++ b/arch/s390/include/asm/pci.h @@ -118,6 +118,32 @@ struct zpci_bus { enum pci_bus_speed max_bus_speed; }; +/* Content Code Description for PCI Function Error */ +struct zpci_ccdf_err { + u32 reserved1; + u32 fh; /* function handle */ + u32 fid; /* function id */ + u32 ett : 4; /* expected table type */ + u32 mvn : 12; /* MSI vector number */ + u32 dmaas : 8; /* DMA address space */ + u32 reserved2 : 6; + u32 q : 1; /* event qualifier */ + u32 rw : 1; /* read/write */ + u64 faddr; /* failing address */ + u32 reserved3; + u16 reserved4; + u16 pec; /* PCI event code */ +} __packed; + +#define ZPCI_ERR_PENDING_MAX 4 +struct zpci_ccdf_pending { + bool mediated_recovery; + u8 count; + u8 head; + u8 tail; + struct zpci_ccdf_err err[ZPCI_ERR_PENDING_MAX]; +}; + /* Private data per function */ struct zpci_dev { struct zpci_bus *zbus; @@ -192,6 +218,8 @@ struct zpci_dev { struct iommu_domain *s390_domain; /* attached IOMMU domain */ struct kvm_zdev *kzdev; struct mutex kzdev_lock; + struct zpci_ccdf_pending pending_errs; + struct mutex pending_errs_lock; spinlock_t dom_lock; /* protect s390_domain change */ }; @@ -208,6 +236,10 @@ extern const struct attribute_group zpci_ident_attr_group; &pfip_attr_group, \ &zpci_ident_attr_group, +extern const struct attribute_group zpci_slot_attr_group; + +#define ARCH_PCI_SLOT_GROUPS (&zpci_slot_attr_group) + extern unsigned int s390_pci_force_floating __initdata; extern unsigned int s390_pci_no_rid; @@ -330,6 +362,10 @@ void zpci_debug_exit_device(struct zpci_dev *); int zpci_report_error(struct pci_dev *, struct zpci_report_error_header *); int zpci_clear_error_state(struct zpci_dev *zdev); int zpci_reset_load_store_blocked(struct zpci_dev *zdev); +void zpci_start_mediated_recovery(struct zpci_dev *zdev); +void zpci_stop_mediated_recovery(struct zpci_dev *zdev); +int zpci_get_pending_error(struct zpci_dev *zdev, + struct zpci_ccdf_err *ccdf); #ifdef CONFIG_NUMA diff --git a/arch/s390/include/asm/pci_io.h b/arch/s390/include/asm/pci_io.h index 43a5ea4ee20f..f3bef5bc7223 100644 --- a/arch/s390/include/asm/pci_io.h +++ b/arch/s390/include/asm/pci_io.h @@ -18,6 +18,7 @@ #define ZPCI_IOMAP_SHIFT 48 #define ZPCI_IOMAP_ADDR_SHIFT 62 #define ZPCI_IOMAP_ADDR_BASE (1UL << ZPCI_IOMAP_ADDR_SHIFT) +#define ZPCI_IOMAP_ADDR_MAX ((1UL << (ZPCI_IOMAP_ADDR_SHIFT + 1)) - 1) #define ZPCI_IOMAP_ADDR_OFF_MASK ((1UL << ZPCI_IOMAP_SHIFT) - 1) #define ZPCI_IOMAP_MAX_ENTRIES \ (1UL << (ZPCI_IOMAP_ADDR_SHIFT - ZPCI_IOMAP_SHIFT)) diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index 5899f57f17d1..feb58acf87d6 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -12,6 +12,24 @@ */ #define __my_cpu_offset get_lowcore()->percpu_offset +#define arch_raw_cpu_ptr(_ptr) \ +({ \ + unsigned long lc_percpu, tcp_ptr__; \ + \ + tcp_ptr__ = (__force unsigned long)(_ptr); \ + lc_percpu = offsetof(struct lowcore, percpu_offset); \ + asm_inline volatile( \ + ALTERNATIVE("ag %[__ptr__],%[offzero](%%r0)\n", \ + "ag %[__ptr__],%[offalt](%%r0)\n", \ + ALT_FEATURE(MFEATURE_LOWCORE)) \ + : [__ptr__] "+d" (tcp_ptr__) \ + : [offzero] "i" (lc_percpu), \ + [offalt] "i" (lc_percpu + LOWCORE_ALT_ADDRESS), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "cc"); \ + (TYPEOF_UNQUAL(*(_ptr)) __force __kernel *)tcp_ptr__; \ +}) + /* * We use a compare-and-swap loop since that uses less cpu cycles than * disabling and enabling interrupts like the generic variant would do. @@ -42,6 +60,68 @@ #define this_cpu_or_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) #define this_cpu_or_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) +/* + * Macros to be used for percpu code section based on atomic instructions. + * + * Avoid the need to use preempt_disable() / preempt_disable() pairs and the + * conditional preempt_schedule_notrace() function calls which come with + * this. The idea is that this_cpu operations based on atomic instructions are + * guarded with mviy instructions: + * + * - The first mviy instruction writes the register number, which contains the + * percpu address variable to lowcore. This also indicates that a percpu + * code section is executed. + * + * - The first mviy instruction following the mviy instruction must be the ag + * instruction which adds the percpu offset to the percpu address register. + * + * - Afterwards the atomic percpu operation follows. + * + * - Then a second mviy instruction writes a zero to lowcore, which indicates + * the end of the percpu code section. + * + * - In case of an interrupt/exception/nmi the register number which was + * written to lowcore is copied to the exception frame (pt_regs), and a zero + * is written to lowcore. + * + * - On return to the previous context it is checked if a percpu code section + * was executed (saved register number not zero), and if the process was + * migrated to a different cpu. If the percpu offset was already added to + * the percpu address register (instruction address does _not_ point to the + * ag instruction) the content of the percpu address register is adjusted so + * it points to percpu variable of the new cpu. + * + * Inline assemblies making use of this typically have a code sequence like: + * + * MVIY_PERCPU(...) <- start of percpu code section + * AG_ALT(...) <- add percpu offset; must be the second instruction + * atomic_op <- atomic op + * MVIY_ALT(...) <- end of percpu code section + */ + +#define MVIY_PERCPU(disp, dispalt, reg) \ + ".macro GEN_MVIY disp reg\n" \ + ".irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\n" \ + " .ifc \\reg,%%r\\rs\n" \ + " mviy \\disp(%%r0),\\rs\n" \ + " .endif\n" \ + ".endr\n" \ + ".endm\n" \ + ALTERNATIVE("GEN_MVIY " disp ", " reg "\n", \ + "GEN_MVIY " dispalt ", " reg "\n", \ + ALT_FEATURE(MFEATURE_LOWCORE)) \ + ".purgem GEN_MVIY\n" + +#define MVIY_ALT(disp, dispalt) \ + ALTERNATIVE(" mviy " disp "(%%r0),0\n", \ + " mviy " dispalt "(%%r0),0\n", \ + ALT_FEATURE(MFEATURE_LOWCORE)) + +#define AG_ALT(disp, dispalt, reg) \ + ALTERNATIVE(" ag " reg ", " disp "(%%r0)\n", \ + " ag " reg ", " dispalt "(%%r0)\n", \ + ALT_FEATURE(MFEATURE_LOWCORE)) + #ifndef MARCH_HAS_Z196_FEATURES #define this_cpu_add_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) @@ -55,46 +135,79 @@ #else /* MARCH_HAS_Z196_FEATURES */ -#define arch_this_cpu_add(pcp, val, op1, op2, szcast) \ -{ \ - typedef typeof(pcp) pcp_op_T__; \ - pcp_op_T__ val__ = (val); \ - pcp_op_T__ old__, *ptr__; \ - preempt_disable_notrace(); \ - ptr__ = raw_cpu_ptr(&(pcp)); \ - if (__builtin_constant_p(val__) && \ - ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ - asm volatile( \ - op2 " %[ptr__],%[val__]" \ - : [ptr__] "+Q" (*ptr__) \ - : [val__] "i" ((szcast)val__) \ - : "cc"); \ - } else { \ - asm volatile( \ - op1 " %[old__],%[val__],%[ptr__]" \ - : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ - : [val__] "d" (val__) \ - : "cc"); \ - } \ - preempt_enable_notrace(); \ -} +#define arch_this_cpu_add(pcp, val, op1, op2, szcast) \ +do { \ + unsigned long lc_pcpr, lc_pcpo; \ + typedef typeof(pcp) pcp_op_T__; \ + pcp_op_T__ val__ = (val); \ + pcp_op_T__ old__, *ptr__; \ + \ + lc_pcpr = offsetof(struct lowcore, percpu_register); \ + lc_pcpo = offsetof(struct lowcore, percpu_offset); \ + ptr__ = PERCPU_PTR(&(pcp)); \ + if (__builtin_constant_p(val__) && \ + ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ + asm volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op2 " 0(%[ptr__]),%[val__]\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [val__] "i" ((szcast)val__), \ + [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "cc"); \ + } else { \ + asm volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op1 " %[old__],%[val__],0(%[ptr__])\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [old__] "=&d" (old__), \ + [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [val__] "d" (val__), \ + [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "cc"); \ + } \ +} while (0) #define this_cpu_add_4(pcp, val) arch_this_cpu_add(pcp, val, "laa", "asi", int) #define this_cpu_add_8(pcp, val) arch_this_cpu_add(pcp, val, "laag", "agsi", long) #define arch_this_cpu_add_return(pcp, val, op) \ ({ \ + unsigned long lc_pcpr, lc_pcpo; \ typedef typeof(pcp) pcp_op_T__; \ pcp_op_T__ val__ = (val); \ pcp_op_T__ old__, *ptr__; \ - preempt_disable_notrace(); \ - ptr__ = raw_cpu_ptr(&(pcp)); \ - asm volatile( \ - op " %[old__],%[val__],%[ptr__]" \ - : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ - : [val__] "d" (val__) \ + \ + lc_pcpr = offsetof(struct lowcore, percpu_register); \ + lc_pcpo = offsetof(struct lowcore, percpu_offset); \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op " %[old__],%[val__],0(%[ptr__])\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [old__] "=&d" (old__), \ + [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [val__] "d" (val__), \ + [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ - preempt_enable_notrace(); \ old__ + val__; \ }) @@ -102,19 +215,31 @@ #define this_cpu_add_return_8(pcp, val) arch_this_cpu_add_return(pcp, val, "laag") #define arch_this_cpu_to_op(pcp, val, op) \ -{ \ +do { \ + unsigned long lc_pcpr, lc_pcpo; \ typedef typeof(pcp) pcp_op_T__; \ pcp_op_T__ val__ = (val); \ pcp_op_T__ old__, *ptr__; \ - preempt_disable_notrace(); \ - ptr__ = raw_cpu_ptr(&(pcp)); \ - asm volatile( \ - op " %[old__],%[val__],%[ptr__]" \ - : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ - : [val__] "d" (val__) \ + \ + lc_pcpr = offsetof(struct lowcore, percpu_register); \ + lc_pcpo = offsetof(struct lowcore, percpu_offset); \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op " %[old__],%[val__],0(%[ptr__])\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [old__] "=&d" (old__), \ + [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [val__] "d" (val__), \ + [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ - preempt_enable_notrace(); \ -} +} while (0) #define this_cpu_and_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lan") #define this_cpu_and_8(pcp, val) arch_this_cpu_to_op(pcp, val, "lang") @@ -123,6 +248,67 @@ #endif /* MARCH_HAS_Z196_FEATURES */ +#define arch_this_cpu_read(pcp, op) \ +({ \ + unsigned long lc_pcpr, lc_pcpo, res__; \ + typedef typeof(pcp) pcp_op_T__; \ + pcp_op_T__ *ptr__; \ + \ + lc_pcpr = offsetof(struct lowcore, percpu_register); \ + lc_pcpo = offsetof(struct lowcore, percpu_offset); \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op " %[res__],0(%[ptr__])\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [res__] "=&d" (res__), [ptr__] "+&a" (ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (*ptr__), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "cc"); \ + (pcp_op_T__)res__; \ +}) + +#define this_cpu_read_1(pcp) arch_this_cpu_read(pcp, "llgc") +#define this_cpu_read_2(pcp) arch_this_cpu_read(pcp, "llgh") +#define this_cpu_read_4(pcp) arch_this_cpu_read(pcp, "llgf") +#define this_cpu_read_8(pcp) arch_this_cpu_read(pcp, "lg") + +#define arch_this_cpu_write(pcp, val, op) \ +do { \ + unsigned long lc_pcpr, lc_pcpo; \ + typedef typeof(pcp) pcp_op_T__; \ + pcp_op_T__ *ptr__, val__ = (val); \ + \ + lc_pcpr = offsetof(struct lowcore, percpu_register); \ + lc_pcpo = offsetof(struct lowcore, percpu_offset); \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op " %[val__],0(%[ptr__])\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [ptr__] "+&a" (ptr__), "=m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [val__] "d" (val__), \ + [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "cc"); \ +} while (0) + +#define this_cpu_write_1(pcp, val) arch_this_cpu_write(pcp, val, "stc") +#define this_cpu_write_2(pcp, val) arch_this_cpu_write(pcp, val, "sth") +#define this_cpu_write_4(pcp, val) arch_this_cpu_write(pcp, val, "st") +#define this_cpu_write_8(pcp, val) arch_this_cpu_write(pcp, val, "stg") + #define arch_this_cpu_cmpxchg(pcp, oval, nval) \ ({ \ typedef typeof(pcp) pcp_op_T__; \ diff --git a/arch/s390/include/asm/pgalloc.h b/arch/s390/include/asm/pgalloc.h index a16e65072371..a5de9e61ea9e 100644 --- a/arch/s390/include/asm/pgalloc.h +++ b/arch/s390/include/asm/pgalloc.h @@ -27,10 +27,6 @@ unsigned long *page_table_alloc_noprof(struct mm_struct *); #define page_table_alloc(...) alloc_hooks(page_table_alloc_noprof(__VA_ARGS__)) void page_table_free(struct mm_struct *, unsigned long *); -struct ptdesc *page_table_alloc_pgste_noprof(struct mm_struct *mm); -#define page_table_alloc_pgste(...) alloc_hooks(page_table_alloc_pgste_noprof(__VA_ARGS__)) -void page_table_free_pgste(struct ptdesc *ptdesc); - static inline void crst_table_init(unsigned long *crst, unsigned long entry) { memset64((u64 *)crst, entry, _CRST_ENTRIES); diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index bca9b29778c3..e882663a58e7 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -16,8 +16,10 @@ #include <linux/mm_types.h> #include <linux/cpufeature.h> #include <linux/page-flags.h> +#include <linux/page_table_check.h> #include <linux/radix-tree.h> #include <linux/atomic.h> +#include <linux/mmap_lock.h> #include <asm/ctlreg.h> #include <asm/bug.h> #include <asm/page.h> @@ -413,28 +415,6 @@ void setup_protection_map(void); * SW-bits: y young, d dirty, r read, w write */ -/* Page status table bits for virtualization */ -#define PGSTE_ACC_BITS 0xf000000000000000UL -#define PGSTE_FP_BIT 0x0800000000000000UL -#define PGSTE_PCL_BIT 0x0080000000000000UL -#define PGSTE_HR_BIT 0x0040000000000000UL -#define PGSTE_HC_BIT 0x0020000000000000UL -#define PGSTE_GR_BIT 0x0004000000000000UL -#define PGSTE_GC_BIT 0x0002000000000000UL -#define PGSTE_ST2_MASK 0x0000ffff00000000UL -#define PGSTE_UC_BIT 0x0000000000008000UL /* user dirty (migration) */ -#define PGSTE_IN_BIT 0x0000000000004000UL /* IPTE notify bit */ -#define PGSTE_VSIE_BIT 0x0000000000002000UL /* ref'd in a shadow table */ - -/* Guest Page State used for virtualization */ -#define _PGSTE_GPS_ZERO 0x0000000080000000UL -#define _PGSTE_GPS_NODAT 0x0000000040000000UL -#define _PGSTE_GPS_USAGE_MASK 0x0000000003000000UL -#define _PGSTE_GPS_USAGE_STABLE 0x0000000000000000UL -#define _PGSTE_GPS_USAGE_UNUSED 0x0000000001000000UL -#define _PGSTE_GPS_USAGE_POT_VOLATILE 0x0000000002000000UL -#define _PGSTE_GPS_USAGE_VOLATILE _PGSTE_GPS_USAGE_MASK - /* * A user page table pointer has the space-switch-event bit, the * private-space-control bit and the storage-alteration-event-control @@ -566,34 +546,15 @@ static inline bool mm_pmd_folded(struct mm_struct *mm) } #define mm_pmd_folded(mm) mm_pmd_folded(mm) -static inline int mm_has_pgste(struct mm_struct *mm) -{ -#ifdef CONFIG_PGSTE - if (unlikely(mm->context.has_pgste)) - return 1; -#endif - return 0; -} - static inline int mm_is_protected(struct mm_struct *mm) { -#ifdef CONFIG_PGSTE +#if IS_ENABLED(CONFIG_KVM) if (unlikely(atomic_read(&mm->context.protected_count))) return 1; #endif return 0; } -static inline pgste_t clear_pgste_bit(pgste_t pgste, unsigned long mask) -{ - return __pgste(pgste_val(pgste) & ~mask); -} - -static inline pgste_t set_pgste_bit(pgste_t pgste, unsigned long mask) -{ - return __pgste(pgste_val(pgste) | mask); -} - static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot) { return __pte(pte_val(pte) & ~pgprot_val(prot)); @@ -632,22 +593,13 @@ static inline pud_t set_pud_bit(pud_t pud, pgprot_t prot) #define mm_forbids_zeropage mm_forbids_zeropage static inline int mm_forbids_zeropage(struct mm_struct *mm) { -#ifdef CONFIG_PGSTE +#if IS_ENABLED(CONFIG_KVM) if (!mm->context.allow_cow_sharing) return 1; #endif return 0; } -static inline int mm_uses_skeys(struct mm_struct *mm) -{ -#ifdef CONFIG_PGSTE - if (mm->context.uses_skeys) - return 1; -#endif - return 0; -} - /** * cspg() - Compare and Swap and Purge (CSPG) * @ptr: Pointer to the value to be exchanged @@ -890,7 +842,7 @@ static inline int pte_same(pte_t a, pte_t b) return pte_val(a) == pte_val(b); } -#ifdef CONFIG_NUMA_BALANCING +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pte_protnone(pte_t pte) { return pte_present(pte) && !(pte_val(pte) & _PAGE_READ); @@ -901,7 +853,7 @@ static inline int pmd_protnone(pmd_t pmd) /* pmd_leaf(pmd) implies pmd_present(pmd) */ return pmd_leaf(pmd) && !(pmd_val(pmd) & _SEGMENT_ENTRY_READ); } -#endif +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ static inline bool pte_swp_exclusive(pte_t pte) { @@ -951,7 +903,7 @@ static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) return clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_SOFT_DIRTY)); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES #define pmd_swp_soft_dirty(pmd) pmd_soft_dirty(pmd) #define pmd_swp_mksoft_dirty(pmd) pmd_mksoft_dirty(pmd) #define pmd_swp_clear_soft_dirty(pmd) pmd_clear_soft_dirty(pmd) @@ -1028,30 +980,39 @@ static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) static inline void set_pte(pte_t *ptep, pte_t pte) { + if (pte_present(pte)) + pte = clear_pte_bit(pte, __pgprot(_PAGE_UNUSED)); WRITE_ONCE(*ptep, pte); } -static inline void pgd_clear(pgd_t *pgd) +#define ptep_get ptep_get +static inline pte_t ptep_get(pte_t *ptep) { - if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R1) - set_pgd(pgd, __pgd(_REGION1_ENTRY_EMPTY)); + return READ_ONCE(*ptep); } -static inline void p4d_clear(p4d_t *p4d) +#define pmdp_get pmdp_get +static inline pmd_t pmdp_get(pmd_t *pmdp) { - if ((p4d_val(*p4d) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2) - set_p4d(p4d, __p4d(_REGION2_ENTRY_EMPTY)); + return READ_ONCE(*pmdp); } -static inline void pud_clear(pud_t *pud) +#define pudp_get pudp_get +static inline pud_t pudp_get(pud_t *pudp) { - if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) - set_pud(pud, __pud(_REGION3_ENTRY_EMPTY)); + return READ_ONCE(*pudp); } -static inline void pmd_clear(pmd_t *pmdp) +#define p4dp_get p4dp_get +static inline p4d_t p4dp_get(p4d_t *p4dp) { - set_pmd(pmdp, __pmd(_SEGMENT_ENTRY_EMPTY)); + return READ_ONCE(*p4dp); +} + +#define pgdp_get pgdp_get +static inline pgd_t pgdp_get(pgd_t *pgdp) +{ + return READ_ONCE(*pgdp); } static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) @@ -1059,6 +1020,29 @@ static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *pt set_pte(ptep, __pte(_PAGE_INVALID)); } +static inline void pmd_clear(pmd_t *pmdp) +{ + set_pmd(pmdp, __pmd(_SEGMENT_ENTRY_EMPTY)); +} + +static inline void pud_clear(pud_t *pud) +{ + if ((pud_val(pudp_get(pud)) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) + set_pud(pud, __pud(_REGION3_ENTRY_EMPTY)); +} + +static inline void p4d_clear(p4d_t *p4d) +{ + if ((p4d_val(p4dp_get(p4d)) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2) + set_p4d(p4d, __p4d(_REGION2_ENTRY_EMPTY)); +} + +static inline void pgd_clear(pgd_t *pgd) +{ + if ((pgd_val(pgdp_get(pgd)) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R1) + set_pgd(pgd, __pgd(_REGION1_ENTRY_EMPTY)); +} + /* * The following pte modification functions only work if * pte_present() is true. Undefined behaviour if not.. @@ -1136,6 +1120,13 @@ static inline pte_t pte_mkhuge(pte_t pte) } #endif +static inline unsigned long sske_frame(unsigned long addr, unsigned char skey) +{ + asm volatile("sske %[skey],%[addr],1" + : [addr] "+a" (addr) : [skey] "d" (skey)); + return addr; +} + #define IPTE_GLOBAL 0 #define IPTE_LOCAL 1 @@ -1207,18 +1198,18 @@ pte_t ptep_xchg_direct(struct mm_struct *, unsigned long, pte_t *, pte_t); pte_t ptep_xchg_lazy(struct mm_struct *, unsigned long, pte_t *, pte_t); #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG -static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, - unsigned long addr, pte_t *ptep) +static inline bool ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) { - pte_t pte = *ptep; + pte_t pte = ptep_get(ptep); pte = ptep_xchg_direct(vma->vm_mm, addr, ptep, pte_mkold(pte)); return pte_young(pte); } #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH -static inline int ptep_clear_flush_young(struct vm_area_struct *vma, - unsigned long address, pte_t *ptep) +static inline bool ptep_clear_flush_young(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep) { return ptep_test_and_clear_young(vma, address, ptep); } @@ -1230,9 +1221,10 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, pte_t res; res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID)); + page_table_check_pte_clear(mm, addr, res); /* At this point the reference through the mapping is still present */ if (mm_is_protected(mm) && pte_present(res)) - uv_convert_from_secure_pte(res); + WARN_ON_ONCE(uv_convert_from_secure_pte(res)); return res; } @@ -1248,9 +1240,10 @@ static inline pte_t ptep_clear_flush(struct vm_area_struct *vma, pte_t res; res = ptep_xchg_direct(vma->vm_mm, addr, ptep, __pte(_PAGE_INVALID)); + page_table_check_pte_clear(vma->vm_mm, addr, res); /* At this point the reference through the mapping is still present */ if (mm_is_protected(vma->vm_mm) && pte_present(res)) - uv_convert_from_secure_pte(res); + WARN_ON_ONCE(uv_convert_from_secure_pte(res)); return res; } @@ -1269,27 +1262,28 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, pte_t res; if (full) { - res = *ptep; + res = ptep_get(ptep); set_pte(ptep, __pte(_PAGE_INVALID)); } else { res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID)); } - /* Nothing to do */ - if (!mm_is_protected(mm) || !pte_present(res)) - return res; - /* - * At this point the reference through the mapping is still present. - * The notifier should have destroyed all protected vCPUs at this - * point, so the destroy should be successful. - */ - if (full && !uv_destroy_pte(res)) - return res; - /* - * If something went wrong and the page could not be destroyed, or - * if this is not a mm teardown, the slower export is used as - * fallback instead. - */ - uv_convert_from_secure_pte(res); + page_table_check_pte_clear(mm, addr, res); + /* At this point the reference through the mapping is still present */ + if (mm_is_protected(mm) && pte_present(res)) { + /* + * The notifier should have destroyed all protected vCPUs at + * this point, so the destroy should be successful. + */ + if (full && !uv_destroy_pte(res)) + return res; + /* + * If something went wrong and the page could not be destroyed, + * or if this is not a mm teardown, the slower export is used + * as fallback instead. If even that fails, print a warning and + * leak the page, to avoid crashing the whole system. + */ + WARN_ON_ONCE(uv_convert_from_secure_pte(res)); + } return res; } @@ -1297,7 +1291,7 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - pte_t pte = *ptep; + pte_t pte = ptep_get(ptep); if (pte_write(pte)) ptep_xchg_lazy(mm, addr, ptep, pte_wrprotect(pte)); @@ -1333,7 +1327,7 @@ static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma, * PTE does not have _PAGE_PROTECT set, to avoid unnecessary overhead. * A local RDP can be used to do the flush. */ - if (cpu_has_rdp() && !(pte_val(*ptep) & _PAGE_PROTECT)) + if (cpu_has_rdp() && !(pte_val(ptep_get(ptep)) & _PAGE_PROTECT)) __ptep_rdp(address, ptep, 1); } #define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault @@ -1348,50 +1342,13 @@ static inline int ptep_set_access_flags(struct vm_area_struct *vma, { if (pte_same(*ptep, entry)) return 0; - if (cpu_has_rdp() && !mm_has_pgste(vma->vm_mm) && pte_allow_rdp(*ptep, entry)) + if (cpu_has_rdp() && pte_allow_rdp(*ptep, entry)) ptep_reset_dat_prot(vma->vm_mm, addr, ptep, entry); else ptep_xchg_direct(vma->vm_mm, addr, ptep, entry); return 1; } -/* - * Additional functions to handle KVM guest page tables - */ -void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t entry); -void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep); -void ptep_notify(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, unsigned long bits); -int ptep_force_prot(struct mm_struct *mm, unsigned long gaddr, - pte_t *ptep, int prot, unsigned long bit); -void ptep_zap_unused(struct mm_struct *mm, unsigned long addr, - pte_t *ptep , int reset); -void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep); -int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr, - pte_t *sptep, pte_t *tptep, pte_t pte); -void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep); - -bool ptep_test_and_clear_uc(struct mm_struct *mm, unsigned long address, - pte_t *ptep); -int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, - unsigned char key, bool nq); -int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr, - unsigned char key, unsigned char *oldkey, - bool nq, bool mr, bool mc); -int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr); -int get_guest_storage_key(struct mm_struct *mm, unsigned long addr, - unsigned char *key); - -int set_pgste_bits(struct mm_struct *mm, unsigned long addr, - unsigned long bits, unsigned long value); -int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep); -int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc, - unsigned long *oldpte, unsigned long *oldpgste); -void gmap_pmdp_invalidate(struct mm_struct *mm, unsigned long vmaddr); -void gmap_pmdp_idte_local(struct mm_struct *mm, unsigned long vmaddr); -void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr); - #define pgprot_writecombine pgprot_writecombine pgprot_t pgprot_writecombine(pgprot_t prot); @@ -1404,25 +1361,13 @@ pgprot_t pgprot_writecombine(pgprot_t prot); static inline void set_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t entry, unsigned int nr) { - if (pte_present(entry)) - entry = clear_pte_bit(entry, __pgprot(_PAGE_UNUSED)); - if (mm_has_pgste(mm)) { - for (;;) { - ptep_set_pte_at(mm, addr, ptep, entry); - if (--nr == 0) - break; - ptep++; - entry = __pte(pte_val(entry) + PAGE_SIZE); - addr += PAGE_SIZE; - } - } else { - for (;;) { - set_pte(ptep, entry); - if (--nr == 0) - break; - ptep++; - entry = __pte(pte_val(entry) + PAGE_SIZE); - } + page_table_check_ptes_set(mm, addr, ptep, entry, nr); + for (;;) { + set_pte(ptep, entry); + if (--nr == 0) + break; + ptep++; + entry = __pte(pte_val(entry) + PAGE_SIZE); } } #define set_ptes set_ptes @@ -1773,8 +1718,8 @@ static inline int pmdp_set_access_flags(struct vm_area_struct *vma, } #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG -static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, - unsigned long addr, pmd_t *pmdp) +static inline bool pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long addr, pmd_t *pmdp) { pmd_t pmd = *pmdp; @@ -1783,8 +1728,8 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, } #define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH -static inline int pmdp_clear_flush_young(struct vm_area_struct *vma, - unsigned long addr, pmd_t *pmdp) +static inline bool pmdp_clear_flush_young(struct vm_area_struct *vma, + unsigned long addr, pmd_t *pmdp) { VM_BUG_ON(addr & ~HPAGE_MASK); return pmdp_test_and_clear_young(vma, addr, pmdp); @@ -1793,6 +1738,7 @@ static inline int pmdp_clear_flush_young(struct vm_area_struct *vma, static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t entry) { + page_table_check_pmd_set(mm, addr, pmdp, entry); set_pmd(pmdp, entry); } @@ -1807,7 +1753,11 @@ static inline pmd_t pmd_mkhuge(pmd_t pmd) static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) { - return pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY)); + pmd_t pmd; + + pmd = pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY)); + page_table_check_pmd_clear(mm, addr, pmd); + return pmd; } #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL @@ -1815,12 +1765,17 @@ static inline pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp, int full) { + pmd_t pmd; + if (full) { - pmd_t pmd = *pmdp; + pmd = *pmdp; set_pmd(pmdp, __pmd(_SEGMENT_ENTRY_EMPTY)); + page_table_check_pmd_clear(vma->vm_mm, addr, pmd); return pmd; } - return pmdp_xchg_lazy(vma->vm_mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY)); + pmd = pmdp_xchg_lazy(vma->vm_mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY)); + page_table_check_pmd_clear(vma->vm_mm, addr, pmd); + return pmd; } #define __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH @@ -1834,11 +1789,16 @@ static inline pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, static inline pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp) { - pmd_t pmd; + pmd_t pmd = *pmdp; - VM_WARN_ON_ONCE(!pmd_present(*pmdp)); - pmd = __pmd(pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID); - return pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd); + VM_WARN_ON_ONCE(!pmd_present(pmd)); + pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_INVALID)); +#ifdef CONFIG_PAGE_TABLE_CHECK + pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_READ)); +#endif + page_table_check_pmd_set(vma->vm_mm, addr, pmdp, pmd); + pmd = pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd); + return pmd; } #define __HAVE_ARCH_PMDP_SET_WRPROTECT @@ -1873,6 +1833,29 @@ static inline int has_transparent_hugepage(void) } #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +#ifdef CONFIG_PAGE_TABLE_CHECK +static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte) +{ + VM_BUG_ON(mm == &init_mm); + + return pte_present(pte); +} + +static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd) +{ + VM_BUG_ON(mm == &init_mm); + + return pmd_leaf(pmd) && (pmd_val(pmd) & _SEGMENT_ENTRY_READ); +} + +static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud) +{ + VM_BUG_ON(mm == &init_mm); + + return pud_leaf(pud); +} +#endif + /* * 64 bit swap entry format: * A page-table entry has some bits we have to treat in a special way. @@ -2015,9 +1998,6 @@ extern int __vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t p extern int vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot); extern void vmem_unmap_4k_page(unsigned long addr); extern pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc); -extern int s390_enable_sie(void); -extern int s390_enable_skey(void); -extern void s390_reset_cmma(struct mm_struct *mm); /* s390 has a private copy of get unmapped area to deal with cache synonyms */ #define HAVE_ARCH_UNMAPPED_AREA @@ -2026,40 +2006,4 @@ extern void s390_reset_cmma(struct mm_struct *mm); #define pmd_pgtable(pmd) \ ((pgtable_t)__va(pmd_val(pmd) & -sizeof(pte_t)*PTRS_PER_PTE)) -static inline unsigned long gmap_pgste_get_pgt_addr(unsigned long *pgt) -{ - unsigned long *pgstes, res; - - pgstes = pgt + _PAGE_ENTRIES; - - res = (pgstes[0] & PGSTE_ST2_MASK) << 16; - res |= pgstes[1] & PGSTE_ST2_MASK; - res |= (pgstes[2] & PGSTE_ST2_MASK) >> 16; - res |= (pgstes[3] & PGSTE_ST2_MASK) >> 32; - - return res; -} - -static inline pgste_t pgste_get_lock(pte_t *ptep) -{ - unsigned long value = 0; -#ifdef CONFIG_PGSTE - unsigned long *ptr = (unsigned long *)(ptep + PTRS_PER_PTE); - - do { - value = __atomic64_or_barrier(PGSTE_PCL_BIT, ptr); - } while (value & PGSTE_PCL_BIT); - value |= PGSTE_PCL_BIT; -#endif - return __pgste(value); -} - -static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste) -{ -#ifdef CONFIG_PGSTE - barrier(); - WRITE_ONCE(*(unsigned long *)(ptep + PTRS_PER_PTE), pgste_val(pgste) & ~PGSTE_PCL_BIT); -#endif -} - #endif /* _S390_PAGE_H */ diff --git a/arch/s390/include/asm/pkey.h b/arch/s390/include/asm/pkey.h index b7b59faf16f4..82e403322e89 100644 --- a/arch/s390/include/asm/pkey.h +++ b/arch/s390/include/asm/pkey.h @@ -13,6 +13,7 @@ #include <linux/ioctl.h> #include <linux/types.h> +#include <linux/delay.h> #include <uapi/asm/pkey.h> /* @@ -21,7 +22,8 @@ * @param keylen size of the key blob in bytes * @param protkey pointer to buffer receiving the protected key * @param xflags additional execution flags (see PKEY_XFLAG_* definitions below) - * As of now the only supported flag is PKEY_XFLAG_NOMEMALLOC. + * As of now the only supported flags are PKEY_XFLAG_NOMEMALLOC + * and PKEY_XFLAG_NOCLEARKEY. * @return 0 on success, negative errno value on failure */ int pkey_key2protkey(const u8 *key, u32 keylen, @@ -38,4 +40,24 @@ int pkey_key2protkey(const u8 *key, u32 keylen, */ #define PKEY_XFLAG_NOMEMALLOC 0x0001 +/* + * Do not accept a clear key token as source for a protected key. + */ +#define PKEY_XFLAG_NOCLEARKEY 0x0002 + +static inline int pkey_handle_expired(void) +{ + /* + * Protected key expired due to relocation to another host. The long + * running re-wrap has no asynchronous completion notification, so + * polling is required. Trigger a re-schedule of this request by + * returning -ENOSPC ("hardware queue full") to the crypto engine. + * To avoid immediately re-invocation of this callback, + * tell the scheduler to voluntarily give up the CPU here. + */ + msleep(1); + pr_debug("rescheduling request\n"); + return -ENOSPC; +} + #endif /* _KAPI_PKEY_H */ diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preempt.h index 6ccd033acfe5..5560d5fca2a3 100644 --- a/arch/s390/include/asm/preempt.h +++ b/arch/s390/include/asm/preempt.h @@ -8,8 +8,8 @@ #include <asm/cmpxchg.h> #include <asm/march.h> -/* We use the MSB mostly because its available */ -#define PREEMPT_NEED_RESCHED 0x80000000 +/* Use MSB for PREEMPT_NEED_RESCHED mostly because it is available. */ +#define PREEMPT_NEED_RESCHED 0x8000000000000000UL /* * We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such @@ -23,12 +23,25 @@ */ static __always_inline int preempt_count(void) { - return READ_ONCE(get_lowcore()->preempt_count) & ~PREEMPT_NEED_RESCHED; + unsigned long lc_preempt; + int count; + + lc_preempt = offsetof(struct lowcore, preempt.count); + /* READ_ONCE(get_lowcore()->preempt.count) (without PREEMPT_NEED_RESCHED) */ + asm_inline( + ALTERNATIVE("ly %[count],%[offzero](%%r0)\n", + "ly %[count],%[offalt](%%r0)\n", + ALT_FEATURE(MFEATURE_LOWCORE)) + : [count] "=d" (count) + : [offzero] "i" (lc_preempt), + [offalt] "i" (lc_preempt + LOWCORE_ALT_ADDRESS), + "m" (((struct lowcore *)0)->preempt.count)); + return count; } -static __always_inline void preempt_count_set(int pc) +static __always_inline void preempt_count_set(unsigned long pc) { - int old, new; + unsigned long old, new; old = READ_ONCE(get_lowcore()->preempt_count); do { @@ -47,12 +60,12 @@ static __always_inline void preempt_count_set(int pc) static __always_inline void set_preempt_need_resched(void) { - __atomic_and(~PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count); + __atomic64_and(~PREEMPT_NEED_RESCHED, (long *)&get_lowcore()->preempt_count); } static __always_inline void clear_preempt_need_resched(void) { - __atomic_or(PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count); + __atomic64_or(PREEMPT_NEED_RESCHED, (long *)&get_lowcore()->preempt_count); } static __always_inline bool test_preempt_need_resched(void) @@ -68,11 +81,21 @@ static __always_inline void __preempt_count_add(int val) */ if (!IS_ENABLED(CONFIG_PROFILE_ALL_BRANCHES)) { if (__builtin_constant_p(val) && (val >= -128) && (val <= 127)) { - __atomic_add_const(val, &get_lowcore()->preempt_count); + unsigned long lc_preempt; + + lc_preempt = offsetof(struct lowcore, preempt_count); + asm_inline( + ALTERNATIVE("agsi %[offzero](%%r0),%[val]\n", + "agsi %[offalt](%%r0),%[val]\n", + ALT_FEATURE(MFEATURE_LOWCORE)) + : "+m" (((struct lowcore *)0)->preempt_count) + : [offzero] "i" (lc_preempt), [val] "i" (val), + [offalt] "i" (lc_preempt + LOWCORE_ALT_ADDRESS) + : "cc"); return; } } - __atomic_add(val, &get_lowcore()->preempt_count); + __atomic64_add(val, (long *)&get_lowcore()->preempt_count); } static __always_inline void __preempt_count_sub(int val) @@ -87,7 +110,22 @@ static __always_inline void __preempt_count_sub(int val) */ static __always_inline bool __preempt_count_dec_and_test(void) { - return __atomic_add_const_and_test(-1, &get_lowcore()->preempt_count); +#ifdef __HAVE_ASM_FLAG_OUTPUTS__ + unsigned long lc_preempt; + int cc; + + lc_preempt = offsetof(struct lowcore, preempt_count); + asm_inline( + ALTERNATIVE("algsi %[offzero](%%r0),%[val]\n", + "algsi %[offalt](%%r0),%[val]\n", + ALT_FEATURE(MFEATURE_LOWCORE)) + : "=@cc" (cc), "+m" (((struct lowcore *)0)->preempt_count) + : [offzero] "i" (lc_preempt), [val] "i" (-1), + [offalt] "i" (lc_preempt + LOWCORE_ALT_ADDRESS)); + return (cc == 0) || (cc == 2); +#else + return __atomic64_add_const_and_test(-1, (long *)&get_lowcore()->preempt_count); +#endif } /* @@ -98,6 +136,16 @@ static __always_inline bool should_resched(int preempt_offset) return unlikely(READ_ONCE(get_lowcore()->preempt_count) == preempt_offset); } +static __always_inline int __preempt_count_add_return(int val) +{ + return val + __atomic64_add(val, (long *)&get_lowcore()->preempt_count); +} + +static __always_inline int __preempt_count_sub_return(int val) +{ + return __preempt_count_add_return(-val); +} + #define init_task_preempt_count(p) do { } while (0) /* Deferred to CPU bringup time */ #define init_idle_preempt_count(p, cpu) do { } while (0) diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 3affba95845b..9434c76c25b8 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -14,12 +14,10 @@ #include <linux/bits.h> -#define CIF_NOHZ_DELAY 2 /* delay HZ disable for a tick */ #define CIF_ENABLED_WAIT 5 /* in enabled wait state */ #define CIF_MCCK_GUEST 6 /* machine check happening in guest */ #define CIF_DEDICATED_CPU 7 /* this CPU is dedicated */ -#define _CIF_NOHZ_DELAY BIT(CIF_NOHZ_DELAY) #define _CIF_ENABLED_WAIT BIT(CIF_ENABLED_WAIT) #define _CIF_MCCK_GUEST BIT(CIF_MCCK_GUEST) #define _CIF_DEDICATED_CPU BIT(CIF_DEDICATED_CPU) @@ -31,7 +29,9 @@ #include <linux/cpumask.h> #include <linux/linkage.h> #include <linux/irqflags.h> +#include <linux/instruction_pointer.h> #include <linux/bitops.h> +#include <asm/vdso/processor.h> #include <asm/fpu-types.h> #include <asm/cpu.h> #include <asm/page.h> @@ -46,7 +46,6 @@ struct pcpu { unsigned long ec_mask; /* bit mask for ec_xxx functions */ unsigned long ec_clk; /* sigp timestamp for ec_xxx */ unsigned long flags; /* per CPU flags */ - unsigned long capacity; /* cpu capacity for scheduler */ signed char state; /* physical cpu state */ signed char polarization; /* physical polarization */ u16 address; /* physical cpu address */ @@ -95,8 +94,6 @@ static __always_inline bool test_cpu_flag_of(int flag, int cpu) return test_bit(flag, &per_cpu(pcpu_devices, cpu).flags); } -#define arch_needs_cpu() test_cpu_flag(CIF_NOHZ_DELAY) - static inline void get_cpu_id(struct cpuid *ptr) { asm volatile("stidp %0" : "=Q" (*ptr)); @@ -158,7 +155,7 @@ static __always_inline void __stackleak_poison(unsigned long erase_low, " j 4f\n" "3: mvc 8(1,%[addr]),0(%[addr])\n" "4:" - : [addr] "+&a" (erase_low), [count] "+&d" (count), [tmp] "=&a" (tmp) + : [addr] "+&a" (erase_low), [count] "+&a" (count), [tmp] "=&a" (tmp) : [poison] "d" (poison) : "memory", "cc" ); @@ -281,8 +278,6 @@ static __always_inline unsigned short stap(void) return cpu_address; } -#define cpu_relax() barrier() - #define ECAG_CACHE_ATTRIBUTE 0 #define ECAG_CPU_ATTRIBUTE 1 diff --git a/arch/s390/include/asm/ptrace.h b/arch/s390/include/asm/ptrace.h index 962cf042c66d..6411e3584283 100644 --- a/arch/s390/include/asm/ptrace.h +++ b/arch/s390/include/asm/ptrace.h @@ -16,13 +16,11 @@ #define PIF_SYSCALL 0 /* inside a system call */ #define PIF_PSW_ADDR_ADJUSTED 1 /* psw address has been adjusted */ #define PIF_SYSCALL_RET_SET 2 /* return value was set via ptrace */ -#define PIF_GUEST_FAULT 3 /* indicates program check in sie64a */ -#define PIF_FTRACE_FULL_REGS 4 /* all register contents valid (ftrace) */ +#define PIF_FTRACE_FULL_REGS 3 /* all register contents valid (ftrace) */ #define _PIF_SYSCALL BIT(PIF_SYSCALL) #define _PIF_ADDR_PSW_ADJUSTED BIT(PIF_PSW_ADDR_ADJUSTED) #define _PIF_SYSCALL_RET_SET BIT(PIF_SYSCALL_RET_SET) -#define _PIF_GUEST_FAULT BIT(PIF_GUEST_FAULT) #define _PIF_FTRACE_FULL_REGS BIT(PIF_FTRACE_FULL_REGS) #define PSW32_MASK_PER _AC(0x40000000, UL) @@ -120,7 +118,10 @@ struct pt_regs { unsigned long gprs[NUM_GPRS]; }; }; - unsigned long orig_gpr2; + union { + unsigned long orig_gpr2; + unsigned long monitor_code; + }; union { struct { unsigned int int_code; @@ -131,6 +132,8 @@ struct pt_regs { }; unsigned long flags; unsigned long last_break; + unsigned int cpu; + unsigned char percpu_register; }; /* @@ -214,16 +217,23 @@ void update_cr_regs(struct task_struct *task); #define arch_has_single_step() (1) #define arch_has_block_step() (1) -#define user_mode(regs) (((regs)->psw.mask & PSW_MASK_PSTATE) != 0) -#define instruction_pointer(regs) ((regs)->psw.addr) -#define user_stack_pointer(regs)((regs)->gprs[15]) #define profile_pc(regs) instruction_pointer(regs) -static inline long regs_return_value(struct pt_regs *regs) +static __always_inline bool user_mode(const struct pt_regs *regs) +{ + return psw_bits(regs->psw).pstate; +} + +static inline long regs_return_value(const struct pt_regs *regs) { return regs->gprs[2]; } +static __always_inline unsigned long instruction_pointer(const struct pt_regs *regs) +{ + return regs->psw.addr; +} + static inline void instruction_pointer_set(struct pt_regs *regs, unsigned long val) { @@ -233,19 +243,26 @@ static inline void instruction_pointer_set(struct pt_regs *regs, int regs_query_register_offset(const char *name); const char *regs_query_register_name(unsigned int offset); -static __always_inline unsigned long kernel_stack_pointer(struct pt_regs *regs) +static __always_inline unsigned long kernel_stack_pointer(const struct pt_regs *regs) +{ + return regs->gprs[15]; +} + +static __always_inline unsigned long user_stack_pointer(const struct pt_regs *regs) { return regs->gprs[15]; } -static __always_inline unsigned long regs_get_register(struct pt_regs *regs, unsigned int offset) +static __always_inline unsigned long regs_get_register(const struct pt_regs *regs, + unsigned int offset) { if (offset >= NUM_GPRS) return 0; return regs->gprs[offset]; } -static __always_inline int regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr) +static __always_inline int regs_within_kernel_stack(const struct pt_regs *regs, + unsigned long addr) { unsigned long ksp = kernel_stack_pointer(regs); @@ -261,7 +278,8 @@ static __always_inline int regs_within_kernel_stack(struct pt_regs *regs, unsign * is specifined by @regs. If the @n th entry is NOT in the kernel stack, * this returns 0. */ -static __always_inline unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n) +static __always_inline unsigned long regs_get_kernel_stack_nth(const struct pt_regs *regs, + unsigned int n) { unsigned long addr; @@ -278,8 +296,8 @@ static __always_inline unsigned long regs_get_kernel_stack_nth(struct pt_regs *r * * regs_get_kernel_argument() returns @n th argument of the function call. */ -static inline unsigned long regs_get_kernel_argument(struct pt_regs *regs, - unsigned int n) +static __always_inline unsigned long regs_get_kernel_argument(const struct pt_regs *regs, + unsigned int n) { unsigned int argoffset = STACK_FRAME_OVERHEAD / sizeof(long); @@ -290,7 +308,7 @@ static inline unsigned long regs_get_kernel_argument(struct pt_regs *regs, return regs_get_kernel_stack_nth(regs, argoffset + n); } -static inline void regs_set_return_value(struct pt_regs *regs, unsigned long rc) +static __always_inline void regs_set_return_value(struct pt_regs *regs, unsigned long rc) { regs->gprs[2] = rc; } diff --git a/arch/s390/include/asm/runtime-const.h b/arch/s390/include/asm/runtime-const.h index 17878b1d048c..7b71156031ec 100644 --- a/arch/s390/include/asm/runtime-const.h +++ b/arch/s390/include/asm/runtime-const.h @@ -33,6 +33,20 @@ __ret; \ }) +#define runtime_const_mask_32(val, sym) \ +({ \ + unsigned int __ret = (val); \ + \ + asm_inline( \ + "0: nilf %[__ret],12\n" \ + ".pushsection runtime_mask_" #sym ",\"a\"\n" \ + ".long 0b - .\n" \ + ".popsection" \ + : [__ret] "+d" (__ret) \ + : : "cc"); \ + __ret; \ +}) + #define runtime_const_init(type, sym) do { \ extern s32 __start_runtime_##type##_##sym[]; \ extern s32 __stop_runtime_##type##_##sym[]; \ @@ -43,12 +57,12 @@ __stop_runtime_##type##_##sym); \ } while (0) -/* 32-bit immediate for iihf and iilf in bits in I2 field */ static inline void __runtime_fixup_32(u32 *p, unsigned int val) { s390_kernel_write(p, &val, sizeof(val)); } +/* 32-bit immediate for iihf and iilf in bits in I2 field */ static inline void __runtime_fixup_ptr(void *where, unsigned long val) { __runtime_fixup_32(where + 2, val >> 32); @@ -65,6 +79,12 @@ static inline void __runtime_fixup_shift(void *where, unsigned long val) s390_kernel_write(where, &insn, sizeof(insn)); } +/* 32-bit immediate for nilf in bits in I2 field */ +static inline void __runtime_fixup_mask(void *where, unsigned long val) +{ + __runtime_fixup_32(where + 2, val); +} + static inline void runtime_const_fixup(void (*fn)(void *, unsigned long), unsigned long val, s32 *start, s32 *end) { diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h index 0f184dbdbe5e..44066550b9b1 100644 --- a/arch/s390/include/asm/sclp.h +++ b/arch/s390/include/asm/sclp.h @@ -20,6 +20,10 @@ #define SCLP_ERRNOTIFY_AQ_REPAIR 1 #define SCLP_ERRNOTIFY_AQ_INFO_LOG 2 #define SCLP_ERRNOTIFY_AQ_OPTICS_DATA 3 +#define SCLP_ERRNOTIFY_AQ_NVME_SMART_LOG 4 +#define SCLP_ERRNOTIFY_AQ_ADAPTER_INITIALIZED 5 +#define SCLP_ERRNOTIFY_AQ_RECOVERABLE_ERROR 6 +#define SCLP_ERRNOTIFY_AQ_TELEMETRY_DATA 7 #ifndef __ASSEMBLER__ #include <linux/uio.h> @@ -104,6 +108,7 @@ struct sclp_info { unsigned char has_aisii : 1; unsigned char has_aeni : 1; unsigned char has_aisi : 1; + unsigned char has_astfleie2 : 1; unsigned int ibc; unsigned int mtid; unsigned int mtid_cp; diff --git a/arch/s390/include/asm/setup.h b/arch/s390/include/asm/setup.h index 7c57ac968bf6..cbf60ade741d 100644 --- a/arch/s390/include/asm/setup.h +++ b/arch/s390/include/asm/setup.h @@ -52,7 +52,6 @@ extern unsigned int zlib_dfltcc_support; #define ZLIB_DFLTCC_INFLATE_ONLY 3 #define ZLIB_DFLTCC_FULL_DEBUG 4 -extern unsigned long ident_map_size; extern unsigned long max_mappable; /* The Write Back bit position in the physaddr is given by the SLPC PCI */ diff --git a/arch/s390/include/asm/smp.h b/arch/s390/include/asm/smp.h index fb2bdbf35da5..a6c621e0491c 100644 --- a/arch/s390/include/asm/smp.h +++ b/arch/s390/include/asm/smp.h @@ -30,7 +30,7 @@ static __always_inline unsigned int raw_smp_processor_id(void) return cpu; } -#define arch_scale_cpu_capacity smp_cpu_get_capacity +#define arch_scale_cpu_capacity topology_get_cpu_scale extern struct mutex smp_cpu_state_mutex; extern unsigned int smp_cpu_mt_shift; @@ -53,9 +53,7 @@ extern void smp_save_dump_secondary_cpus(void); extern void smp_yield_cpu(int cpu); extern void smp_cpu_set_polarization(int cpu, int val); extern int smp_cpu_get_polarization(int cpu); -extern void smp_cpu_set_capacity(int cpu, unsigned long val); extern void smp_set_core_capacity(int cpu, unsigned long val); -extern unsigned long smp_cpu_get_capacity(int cpu); extern int smp_cpu_get_cpu_address(int cpu); extern void smp_fill_possible_mask(void); extern void smp_detect_cpus(void); diff --git a/arch/s390/include/asm/stacktrace.h b/arch/s390/include/asm/stacktrace.h index c9ae680a28af..ac3606c3babe 100644 --- a/arch/s390/include/asm/stacktrace.h +++ b/arch/s390/include/asm/stacktrace.h @@ -62,7 +62,7 @@ struct stack_frame { struct { unsigned long sie_control_block; unsigned long sie_savearea; - unsigned long sie_reason; + unsigned long sie_return; unsigned long sie_flags; unsigned long sie_control_block_phys; unsigned long sie_guest_asce; diff --git a/arch/s390/include/asm/string.h b/arch/s390/include/asm/string.h index 238e721e5a22..378f85304ef5 100644 --- a/arch/s390/include/asm/string.h +++ b/arch/s390/include/asm/string.h @@ -26,7 +26,6 @@ void *memmove(void *dest, const void *src, size_t n); #define __HAVE_ARCH_MEMSCAN /* inline & arch function */ #define __HAVE_ARCH_STRCAT /* inline & arch function */ #define __HAVE_ARCH_STRCMP /* arch function */ -#define __HAVE_ARCH_STRLCAT /* arch function */ #define __HAVE_ARCH_STRLEN /* inline & arch function */ #define __HAVE_ARCH_STRNCAT /* arch function */ #define __HAVE_ARCH_STRNLEN /* inline & arch function */ @@ -38,7 +37,6 @@ void *memmove(void *dest, const void *src, size_t n); /* Prototypes for non-inlined arch strings functions. */ int memcmp(const void *s1, const void *s2, size_t n); int strcmp(const char *s1, const char *s2); -size_t strlcat(char *dest, const char *src, size_t n); char *strncat(char *dest, const char *src, size_t n); char *strstr(const char *s1, const char *s2); #endif /* !defined(CONFIG_KASAN) && !defined(CONFIG_KMSAN) */ diff --git a/arch/s390/include/asm/syscall.h b/arch/s390/include/asm/syscall.h index 4271e4169f45..5f310caad1fc 100644 --- a/arch/s390/include/asm/syscall.h +++ b/arch/s390/include/asm/syscall.h @@ -89,11 +89,6 @@ static inline int syscall_get_arch(struct task_struct *task) return AUDIT_ARCH_S390X; } -static inline bool arch_syscall_is_vdso_sigreturn(struct pt_regs *regs) -{ - return false; -} - #define SYSCALL_FMT_0 #define SYSCALL_FMT_1 , "0" (r2) #define SYSCALL_FMT_2 , "d" (r3) SYSCALL_FMT_1 diff --git a/arch/s390/include/asm/thread_info.h b/arch/s390/include/asm/thread_info.h index 6a548a819400..2554cef3a6a5 100644 --- a/arch/s390/include/asm/thread_info.h +++ b/arch/s390/include/asm/thread_info.h @@ -61,6 +61,7 @@ void arch_setup_new_exec(void); */ #define HAVE_TIF_NEED_RESCHED_LAZY #define HAVE_TIF_RESTORE_SIGMASK +#define HAVE_TIF_POLLING_NRFLAG #include <asm-generic/thread_info_tif.h> diff --git a/arch/s390/include/asm/timex.h b/arch/s390/include/asm/timex.h index 49447b40f038..23421b6eb225 100644 --- a/arch/s390/include/asm/timex.h +++ b/arch/s390/include/asm/timex.h @@ -12,6 +12,7 @@ #include <linux/preempt.h> #include <linux/time64.h> +#include <asm/tod_types.h> #include <asm/lowcore.h> #include <asm/machine.h> #include <asm/asm.h> @@ -21,25 +22,6 @@ extern u64 clock_comparator_max; -union tod_clock { - __uint128_t val; - struct { - __uint128_t ei : 8; /* epoch index */ - __uint128_t tod : 64; /* bits 0-63 of tod clock */ - __uint128_t : 40; - __uint128_t pf : 16; /* programmable field */ - }; - struct { - __uint128_t eitod : 72; /* epoch index + bits 0-63 tod clock */ - __uint128_t : 56; - }; - struct { - __uint128_t us : 60; /* micro-seconds */ - __uint128_t sus : 12; /* sub-microseconds */ - __uint128_t : 56; - }; -} __packed; - /* Inline functions for clock register access. */ static inline int set_tod_clock(__u64 time) { @@ -177,8 +159,6 @@ static inline void local_tick_enable(unsigned long comp) set_clock_comparator(get_lowcore()->clock_comparator); } -#define CLOCK_TICK_RATE 1193180 /* Underlying HZ */ - typedef unsigned long cycles_t; static __always_inline unsigned long get_tod_clock(void) diff --git a/arch/s390/include/asm/tlb.h b/arch/s390/include/asm/tlb.h index 1e50f6f1ad9d..619fd41e710e 100644 --- a/arch/s390/include/asm/tlb.h +++ b/arch/s390/include/asm/tlb.h @@ -24,7 +24,7 @@ static inline void tlb_flush(struct mmu_gather *tlb); static inline bool __tlb_remove_page_size(struct mmu_gather *tlb, - struct page *page, bool delay_rmap, int page_size); + struct page *page, int page_size); static inline bool __tlb_remove_folio_pages(struct mmu_gather *tlb, struct page *page, unsigned int nr_pages, bool delay_rmap); @@ -36,7 +36,6 @@ static inline bool __tlb_remove_folio_pages(struct mmu_gather *tlb, #include <asm/tlbflush.h> #include <asm-generic/tlb.h> -#include <asm/gmap.h> /* * Release the page cache reference for a pte removed by @@ -46,10 +45,8 @@ static inline bool __tlb_remove_folio_pages(struct mmu_gather *tlb, * s390 doesn't delay rmap removal. */ static inline bool __tlb_remove_page_size(struct mmu_gather *tlb, - struct page *page, bool delay_rmap, int page_size) + struct page *page, int page_size) { - VM_WARN_ON_ONCE(delay_rmap); - free_folio_and_swap_cache(page_folio(page)); return false; } @@ -85,8 +82,6 @@ static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, tlb->mm->context.flush_mm = 1; tlb->freed_tables = 1; tlb->cleared_pmds = 1; - if (mm_has_pgste(tlb->mm)) - gmap_unlink(tlb->mm, (unsigned long *)pte, address); tlb_remove_ptdesc(tlb, virt_to_ptdesc(pte)); } diff --git a/arch/s390/include/asm/tod_types.h b/arch/s390/include/asm/tod_types.h new file mode 100644 index 000000000000..976fa0a1e895 --- /dev/null +++ b/arch/s390/include/asm/tod_types.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_S390_TOD_TYPES_H +#define _ASM_S390_TOD_TYPES_H + +#include <linux/types.h> + +#ifndef __ASSEMBLER__ + +union tod_clock { + __uint128_t val; + struct { + __uint128_t ei : 8; /* epoch index */ + __uint128_t tod : 64; /* bits 0-63 of tod clock */ + __uint128_t : 40; + __uint128_t pf : 16; /* programmable field */ + }; + struct { + __uint128_t eitod : 72; /* epoch index + bits 0-63 tod clock */ + __uint128_t : 56; + }; + struct { + __uint128_t us : 60; /* micro-seconds */ + __uint128_t sus : 12; /* sub-microseconds */ + __uint128_t : 56; + }; +} __packed; + +#endif +#endif diff --git a/arch/s390/include/asm/trace_clock.h b/arch/s390/include/asm/trace_clock.h new file mode 100644 index 000000000000..273e05cbdae0 --- /dev/null +++ b/arch/s390/include/asm/trace_clock.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_S390_TRACE_CLOCK_H +#define _ASM_S390_TRACE_CLOCK_H + +#include <linux/compiler.h> +#include <linux/types.h> + +u64 notrace trace_clock_s390_tod(void); + +#define ARCH_TRACE_CLOCKS \ + { trace_clock_s390_tod, "s390-tod", .in_ns = 0 }, + +#endif /* _ASM_S390_TRACE_CLOCK_H */ diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index c5e02addcd67..a9f32c53f699 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -30,8 +30,7 @@ void debug_user_asce(int exit); #define uaccess_kmsan_or_inline __always_inline #endif -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER static uaccess_kmsan_or_inline __must_check unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long size) @@ -471,65 +470,15 @@ do { \ #define arch_get_kernel_nofault __mvc_kernel_nofault #define arch_put_kernel_nofault __mvc_kernel_nofault -void __cmpxchg_user_key_called_with_bad_pointer(void); - -int __cmpxchg_user_key1(unsigned long address, unsigned char *uval, - unsigned char old, unsigned char new, unsigned long key); -int __cmpxchg_user_key2(unsigned long address, unsigned short *uval, - unsigned short old, unsigned short new, unsigned long key); -int __cmpxchg_user_key4(unsigned long address, unsigned int *uval, - unsigned int old, unsigned int new, unsigned long key); -int __cmpxchg_user_key8(unsigned long address, unsigned long *uval, - unsigned long old, unsigned long new, unsigned long key); -int __cmpxchg_user_key16(unsigned long address, __uint128_t *uval, - __uint128_t old, __uint128_t new, unsigned long key); - -static __always_inline int _cmpxchg_user_key(unsigned long address, void *uval, - __uint128_t old, __uint128_t new, - unsigned long key, int size) -{ - switch (size) { - case 1: return __cmpxchg_user_key1(address, uval, old, new, key); - case 2: return __cmpxchg_user_key2(address, uval, old, new, key); - case 4: return __cmpxchg_user_key4(address, uval, old, new, key); - case 8: return __cmpxchg_user_key8(address, uval, old, new, key); - case 16: return __cmpxchg_user_key16(address, uval, old, new, key); - default: __cmpxchg_user_key_called_with_bad_pointer(); - } - return 0; -} - -/** - * cmpxchg_user_key() - cmpxchg with user space target, honoring storage keys - * @ptr: User space address of value to compare to @old and exchange with - * @new. Must be aligned to sizeof(*@ptr). - * @uval: Address where the old value of *@ptr is written to. - * @old: Old value. Compared to the content pointed to by @ptr in order to - * determine if the exchange occurs. The old value read from *@ptr is - * written to *@uval. - * @new: New value to place at *@ptr. - * @key: Access key to use for checking storage key protection. - * - * Perform a cmpxchg on a user space target, honoring storage key protection. - * @key alone determines how key checking is performed, neither - * storage-protection-override nor fetch-protection-override apply. - * The caller must compare *@uval and @old to determine if values have been - * exchanged. In case of an exception *@uval is set to zero. - * - * Return: 0: cmpxchg executed - * -EFAULT: an exception happened when trying to access *@ptr - * -EAGAIN: maxed out number of retries (byte and short only) - */ -#define cmpxchg_user_key(ptr, uval, old, new, key) \ -({ \ - __typeof__(ptr) __ptr = (ptr); \ - __typeof__(uval) __uval = (uval); \ - \ - BUILD_BUG_ON(sizeof(*(__ptr)) != sizeof(*(__uval))); \ - might_fault(); \ - __chk_user_ptr(__ptr); \ - _cmpxchg_user_key((unsigned long)(__ptr), (void *)(__uval), \ - (old), (new), (key), sizeof(*(__ptr))); \ -}) +int __cmpxchg_key1(void *address, unsigned char *uval, unsigned char old, + unsigned char new, unsigned long key); +int __cmpxchg_key2(void *address, unsigned short *uval, unsigned short old, + unsigned short new, unsigned long key); +int __cmpxchg_key4(void *address, unsigned int *uval, unsigned int old, + unsigned int new, unsigned long key); +int __cmpxchg_key8(void *address, unsigned long *uval, unsigned long old, + unsigned long new, unsigned long key); +int __cmpxchg_key16(void *address, __uint128_t *uval, __uint128_t old, + __uint128_t new, unsigned long key); #endif /* __S390_UACCESS_H */ diff --git a/arch/s390/include/asm/uv.h b/arch/s390/include/asm/uv.h index 8018549a1ad2..508075e9e8a9 100644 --- a/arch/s390/include/asm/uv.h +++ b/arch/s390/include/asm/uv.h @@ -454,18 +454,6 @@ static inline int uv_call(unsigned long r1, unsigned long r2) return cc; } -/* Low level uv_call that avoids stalls for long running busy conditions */ -static inline int uv_call_sched(unsigned long r1, unsigned long r2) -{ - int cc; - - do { - cc = __uv_call(r1, r2); - cond_resched(); - } while (cc > 1); - return cc; -} - /* * special variant of uv_call that only transports the cpu or guest * handle and the command, like destroy or verify. @@ -480,7 +468,7 @@ static inline int uv_cmd_nodata(u64 handle, u16 cmd, u16 *rc, u16 *rrc) int cc; WARN(!handle, "No handle provided to Ultravisor call cmd %x\n", cmd); - cc = uv_call_sched(0, (u64)&uvcb); + cc = uv_call(0, (u64)&uvcb); *rc = uvcb.header.rc; *rrc = uvcb.header.rrc; return cc ? -EINVAL : 0; @@ -518,7 +506,7 @@ static inline int uv_list_secrets(struct uv_secret_list *buf, u16 start_idx, .start_idx = start_idx, .list_addr = (u64)buf, }; - int cc = uv_call_sched(0, (u64)&uvcb); + int cc = uv_call(0, (u64)&uvcb); if (rc) *rc = uvcb.header.rc; @@ -631,9 +619,12 @@ int uv_pin_shared(unsigned long paddr); int uv_destroy_folio(struct folio *folio); int uv_destroy_pte(pte_t pte); int uv_convert_from_secure_pte(pte_t pte); -int make_hva_secure(struct mm_struct *mm, unsigned long hva, struct uv_cb_header *uvcb); +int s390_wiggle_split_folio(struct mm_struct *mm, struct folio *folio); +int __make_folio_secure(struct folio *folio, struct uv_cb_header *uvcb); int uv_convert_from_secure(unsigned long paddr); int uv_convert_from_secure_folio(struct folio *folio); +void *uv_alloc_stor_var(unsigned long size); +void uv_free_stor_var(void *stor_var); void setup_uv(void); diff --git a/arch/s390/include/asm/vdso/processor.h b/arch/s390/include/asm/vdso/processor.h index cfcc3e117c4c..6775621e5a5a 100644 --- a/arch/s390/include/asm/vdso/processor.h +++ b/arch/s390/include/asm/vdso/processor.h @@ -2,6 +2,8 @@ #ifndef __ASM_VDSO_PROCESSOR_H #define __ASM_VDSO_PROCESSOR_H -#define cpu_relax() barrier() +#include <asm/barrier.h> + +#define cpu_relax() bcr_serialize() #endif /* __ASM_VDSO_PROCESSOR_H */ diff --git a/arch/s390/include/asm/vtime.h b/arch/s390/include/asm/vtime.h index 9d25fb35a042..da116a93d3b6 100644 --- a/arch/s390/include/asm/vtime.h +++ b/arch/s390/include/asm/vtime.h @@ -2,6 +2,12 @@ #ifndef _S390_VTIME_H #define _S390_VTIME_H +#include <asm/lowcore.h> +#include <asm/cpu_mf.h> +#include <asm/idle.h> + +DECLARE_PER_CPU(u64, mt_cycles[8]); + static inline void update_timer_sys(void) { struct lowcore *lc = get_lowcore(); @@ -20,4 +26,32 @@ static inline void update_timer_mcck(void) lc->last_update_timer = lc->mcck_enter_timer; } +static inline void update_timer_idle(void) +{ + struct s390_idle_data *idle = this_cpu_ptr(&s390_idle); + struct lowcore *lc = get_lowcore(); + u64 cycles_new[8]; + int i, mtid; + + mtid = smp_cpu_mtid; + if (mtid) { + stcctm(MT_DIAG, mtid, cycles_new); + for (i = 0; i < mtid; i++) + __this_cpu_add(mt_cycles[i], cycles_new[i] - idle->mt_cycles_enter[i]); + } + /* + * This is a bit subtle: Forward last_update_clock so it excludes idle + * time. For correct steal time calculation in do_account_vtime() add + * passed wall time before idle_enter to steal_timer: + * During the passed wall time before idle_enter CPU time may have + * been accounted to system, hardirq, softirq, etc. lowcore fields. + * The accounted CPU times will be subtracted again from steal_timer + * when accumulated steal time is calculated in do_account_vtime(). + */ + lc->steal_timer += idle->clock_idle_enter.tod - lc->last_update_clock; + lc->last_update_clock = lc->int_clock.tod; + lc->system_timer += lc->last_update_timer - idle->timer_idle_enter; + lc->last_update_timer = lc->sys_enter_timer; +} + #endif /* _S390_VTIME_H */ diff --git a/arch/s390/include/asm/xor.h b/arch/s390/include/asm/xor.h deleted file mode 100644 index 857d6759b67f..000000000000 --- a/arch/s390/include/asm/xor.h +++ /dev/null @@ -1,21 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Optimited xor routines - * - * Copyright IBM Corp. 2016 - * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> - */ -#ifndef _ASM_S390_XOR_H -#define _ASM_S390_XOR_H - -extern struct xor_block_template xor_block_xc; - -#undef XOR_TRY_TEMPLATES -#define XOR_TRY_TEMPLATES \ -do { \ - xor_speed(&xor_block_xc); \ -} while (0) - -#define XOR_SELECT_TEMPLATE(FASTEST) (&xor_block_xc) - -#endif /* _ASM_S390_XOR_H */ diff --git a/arch/s390/include/uapi/asm/Kbuild b/arch/s390/include/uapi/asm/Kbuild index 46c1ff0b842a..378151cef3cc 100644 --- a/arch/s390/include/uapi/asm/Kbuild +++ b/arch/s390/include/uapi/asm/Kbuild @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 -generated-y += unistd_32.h generated-y += unistd_64.h diff --git a/arch/s390/include/uapi/asm/kvm.h b/arch/s390/include/uapi/asm/kvm.h index 60345dd2cba2..4192769b5ce0 100644 --- a/arch/s390/include/uapi/asm/kvm.h +++ b/arch/s390/include/uapi/asm/kvm.h @@ -444,6 +444,7 @@ struct kvm_s390_vm_cpu_machine { #define KVM_S390_VM_CPU_FEAT_PFMFI 11 #define KVM_S390_VM_CPU_FEAT_SIGPIF 12 #define KVM_S390_VM_CPU_FEAT_KSS 13 +#define KVM_S390_VM_CPU_FEAT_ASTFLEIE2 14 struct kvm_s390_vm_cpu_feat { __u64 feat[16]; }; diff --git a/arch/s390/include/uapi/asm/tape390.h b/arch/s390/include/uapi/asm/tape390.h deleted file mode 100644 index 90266c696486..000000000000 --- a/arch/s390/include/uapi/asm/tape390.h +++ /dev/null @@ -1,103 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/************************************************************************* - * - * enables user programs to display messages and control encryption - * on s390 tape devices - * - * Copyright IBM Corp. 2001, 2006 - * Author(s): Michael Holzheu <holzheu@de.ibm.com> - * - *************************************************************************/ - -#ifndef _TAPE390_H -#define _TAPE390_H - -#define TAPE390_DISPLAY _IOW('d', 1, struct display_struct) - -/* - * The TAPE390_DISPLAY ioctl calls the Load Display command - * which transfers 17 bytes of data from the channel to the subsystem: - * - 1 format control byte, and - * - two 8-byte messages - * - * Format control byte: - * 0-2: New Message Overlay - * 3: Alternate Messages - * 4: Blink Message - * 5: Display Low/High Message - * 6: Reserved - * 7: Automatic Load Request - * - */ - -typedef struct display_struct { - char cntrl; - char message1[8]; - char message2[8]; -} display_struct; - -/* - * Tape encryption support - */ - -struct tape390_crypt_info { - char capability; - char status; - char medium_status; -} __attribute__ ((packed)); - - -/* Macros for "capable" field */ -#define TAPE390_CRYPT_SUPPORTED_MASK 0x01 -#define TAPE390_CRYPT_SUPPORTED(x) \ - ((x.capability & TAPE390_CRYPT_SUPPORTED_MASK)) - -/* Macros for "status" field */ -#define TAPE390_CRYPT_ON_MASK 0x01 -#define TAPE390_CRYPT_ON(x) (((x.status) & TAPE390_CRYPT_ON_MASK)) - -/* Macros for "medium status" field */ -#define TAPE390_MEDIUM_LOADED_MASK 0x01 -#define TAPE390_MEDIUM_ENCRYPTED_MASK 0x02 -#define TAPE390_MEDIUM_ENCRYPTED(x) \ - (((x.medium_status) & TAPE390_MEDIUM_ENCRYPTED_MASK)) -#define TAPE390_MEDIUM_LOADED(x) \ - (((x.medium_status) & TAPE390_MEDIUM_LOADED_MASK)) - -/* - * The TAPE390_CRYPT_SET ioctl is used to switch on/off encryption. - * The "encryption_capable" and "tape_status" fields are ignored for this ioctl! - */ -#define TAPE390_CRYPT_SET _IOW('d', 2, struct tape390_crypt_info) - -/* - * The TAPE390_CRYPT_QUERY ioctl is used to query the encryption state. - */ -#define TAPE390_CRYPT_QUERY _IOR('d', 3, struct tape390_crypt_info) - -/* Values for "kekl1/2_type" and "kekl1/2_type_on_tape" fields */ -#define TAPE390_KEKL_TYPE_NONE 0 -#define TAPE390_KEKL_TYPE_LABEL 1 -#define TAPE390_KEKL_TYPE_HASH 2 - -struct tape390_kekl { - unsigned char type; - unsigned char type_on_tape; - char label[65]; -} __attribute__ ((packed)); - -struct tape390_kekl_pair { - struct tape390_kekl kekl[2]; -} __attribute__ ((packed)); - -/* - * The TAPE390_KEKL_SET ioctl is used to set Key Encrypting Key labels. - */ -#define TAPE390_KEKL_SET _IOW('d', 4, struct tape390_kekl_pair) - -/* - * The TAPE390_KEKL_QUERY ioctl is used to query Key Encrypting Key labels. - */ -#define TAPE390_KEKL_QUERY _IOR('d', 5, struct tape390_kekl_pair) - -#endif diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 42c83d60d6fa..f84b8fb41331 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -3,6 +3,8 @@ # Makefile for the linux kernel. # +CONTEXT_ANALYSIS := y + ifdef CONFIG_FUNCTION_TRACER # Do not trace tracer code @@ -62,6 +64,7 @@ obj-$(CONFIG_KPROBES) += mcount.o obj-$(CONFIG_RETHOOK) += rethook.o obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o obj-$(CONFIG_FUNCTION_TRACER) += mcount.o +obj-$(CONFIG_TRACING) += trace_clock.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o obj-$(CONFIG_KEXEC_CORE) += machine_kexec.o relocate_kernel.o obj-$(CONFIG_VMCORE_INFO) += vmcore_info.o @@ -71,7 +74,7 @@ obj-$(CONFIG_STACKPROTECTOR) += stackprotector.o obj-$(CONFIG_KEXEC_FILE) += machine_kexec_file.o kexec_image.o obj-$(CONFIG_KEXEC_FILE) += kexec_elf.o obj-$(CONFIG_CERT_STORE) += cert_store.o -obj-$(CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT) += ima_arch.o +obj-$(CONFIG_KMSAN) += irqflags.o obj-$(CONFIG_PERF_EVENTS) += perf_event.o obj-$(CONFIG_PERF_EVENTS) += perf_cpum_cf.o perf_cpum_sf.o @@ -80,5 +83,7 @@ obj-$(CONFIG_PERF_EVENTS) += perf_pai.o obj-$(CONFIG_TRACEPOINTS) += trace.o +obj-$(CONFIG_BPF_SYSCALL) += bpf.o + # vdso obj-y += vdso/ diff --git a/arch/s390/kernel/alternative.c b/arch/s390/kernel/alternative.c index 90c0e6408992..02d04ae621ba 100644 --- a/arch/s390/kernel/alternative.c +++ b/arch/s390/kernel/alternative.c @@ -4,6 +4,7 @@ #define pr_fmt(fmt) "alt: " fmt #endif +#include <linux/hex.h> #include <linux/uaccess.h> #include <linux/printk.h> #include <asm/nospec-branch.h> diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c index e1a5b5b54e4f..f6dd2b67dcee 100644 --- a/arch/s390/kernel/asm-offsets.c +++ b/arch/s390/kernel/asm-offsets.c @@ -11,9 +11,11 @@ #include <linux/purgatory.h> #include <linux/pgtable.h> #include <linux/ftrace_regs.h> +#include <linux/kernel_stat.h> #include <asm/kvm_host_types.h> #include <asm/stacktrace.h> #include <asm/ptrace.h> +#include <asm/idle.h> int main(void) { @@ -63,7 +65,7 @@ int main(void) OFFSET(__SF_EMPTY, stack_frame, empty[0]); OFFSET(__SF_SIE_CONTROL, stack_frame, sie_control_block); OFFSET(__SF_SIE_SAVEAREA, stack_frame, sie_savearea); - OFFSET(__SF_SIE_REASON, stack_frame, sie_reason); + OFFSET(__SF_SIE_RETURN, stack_frame, sie_return); OFFSET(__SF_SIE_FLAGS, stack_frame, sie_flags); OFFSET(__SF_SIE_CONTROL_PHYS, stack_frame, sie_control_block_phys); OFFSET(__SF_SIE_GUEST_ASCE, stack_frame, sie_guest_asce); @@ -128,6 +130,7 @@ int main(void) OFFSET(__LC_LAST_UPDATE_CLOCK, lowcore, last_update_clock); OFFSET(__LC_INT_CLOCK, lowcore, int_clock); OFFSET(__LC_CURRENT, lowcore, current_task); + OFFSET(__LC_PERCPU_OFFSET, lowcore, percpu_offset); OFFSET(__LC_KERNEL_STACK, lowcore, kernel_stack); OFFSET(__LC_ASYNC_STACK, lowcore, async_stack); OFFSET(__LC_NODAT_STACK, lowcore, nodat_stack); @@ -180,6 +183,10 @@ int main(void) DEFINE(OLDMEM_SIZE, PARMAREA + offsetof(struct parmarea, oldmem_size)); DEFINE(COMMAND_LINE, PARMAREA + offsetof(struct parmarea, command_line)); DEFINE(MAX_COMMAND_LINE_SIZE, PARMAREA + offsetof(struct parmarea, max_command_line_size)); + OFFSET(__IDLE_CLOCK_EXIT, s390_idle_data, clock_idle_exit); +#ifdef CONFIG_NO_HZ_COMMON + OFFSET(__KCPUSTAT_SEQUENCE, kernel_cpustat, idle_sleeptime_seq); +#endif OFFSET(__FTRACE_REGS_PT_REGS, __arch_ftrace_regs, regs); DEFINE(__FTRACE_REGS_SIZE, sizeof(struct __arch_ftrace_regs)); diff --git a/arch/s390/kernel/bpf.c b/arch/s390/kernel/bpf.c new file mode 100644 index 000000000000..713337fae626 --- /dev/null +++ b/arch/s390/kernel/bpf.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <asm/lowcore.h> +#include <linux/btf.h> + +__bpf_kfunc_start_defs(); + +__bpf_kfunc struct lowcore *bpf_get_lowcore(void) +{ + return get_lowcore(); +} + +__bpf_kfunc_end_defs(); diff --git a/arch/s390/kernel/cert_store.c b/arch/s390/kernel/cert_store.c index c217a5e64094..dc1992a675de 100644 --- a/arch/s390/kernel/cert_store.c +++ b/arch/s390/kernel/cert_store.c @@ -322,7 +322,7 @@ static int invalidate_keyring_keys(struct key *keyring) keyring_payload_len = key_type_keyring.read(keyring, NULL, 0); num_keys = keyring_payload_len / sizeof(key_serial_t); - key_array = kcalloc(num_keys, sizeof(key_serial_t), GFP_KERNEL); + key_array = kzalloc_objs(key_serial_t, num_keys); if (!key_array) return -ENOMEM; diff --git a/arch/s390/kernel/debug.c b/arch/s390/kernel/debug.c index 71cdb6845dd7..3b68f5c58333 100644 --- a/arch/s390/kernel/debug.c +++ b/arch/s390/kernel/debug.c @@ -26,6 +26,8 @@ #include <linux/math.h> #include <linux/minmax.h> #include <linux/debugfs.h> +#include <linux/glob.h> +#include <linux/stringify.h> #include <asm/debug.h> @@ -154,6 +156,7 @@ static unsigned int __used debug_feature_version = __DEBUG_FEATURE_VERSION; static debug_info_t *debug_area_first; static debug_info_t *debug_area_last; static DEFINE_MUTEX(debug_mutex); +extern debug_info_t *__s390dbf_info[], *__s390dbf_info_end[]; static int initialized; static int debug_critical; @@ -168,7 +171,93 @@ static const struct file_operations debug_file_ops = { static struct dentry *debug_debugfs_root_entry; +/* List of debug area parameters to override */ +#define PARAM_UNSET -2 +#define PARAM_NUM 16 +static struct debug_param_t { + char name[DEBUG_MAX_NAME_LEN + 1]; + int level; + int pages; +} debug_param[PARAM_NUM]; +static int debug_param_num; + /* functions */ +static void debug_get_param(const char *name, int *level, int *pages, bool quiet) +{ + struct debug_param_t *p; + int i; + + for (i = 0; i < debug_param_num; i++) { + p = &debug_param[i]; + if (!glob_match(p->name, name)) + continue; + if (level && p->level != PARAM_UNSET) { + if (!quiet) + pr_info("%s: override level to %d\n", name, p->level); + *level = p->level; + } + if (pages && p->pages != PARAM_UNSET) { + if (!quiet) + pr_info("%s: override pages to %d\n", name, p->pages); + *pages = p->pages; + } + } +} + +#define LVL_LEN 10 +#define LVL_FMT "%" __stringify(LVL_LEN) "[^:]" +#define NAME_FMT "%" __stringify(DEBUG_MAX_NAME_LEN) "[^:]" + +static bool __init s390dbf_parse_one(const char *arg, struct debug_param_t *param) +{ + struct debug_param_t p = { { 0 }, PARAM_UNSET, PARAM_UNSET }; + char level[LVL_LEN + 1] = { 0 }; + + /* arg: <name|pattern>:[<level>|-]:[<pages>] */ + if (sscanf(arg, NAME_FMT ":" LVL_FMT ":%d", p.name, level, &p.pages) > 1) { + if (strcmp(level, "-") == 0) + p.level = DEBUG_OFF_LEVEL; + else if (kstrtoint(level, 0, &p.level) != 0) + return false; + } else if (sscanf(arg, NAME_FMT "::%d", p.name, &p.pages) != 2) { + return false; + } + + if (p.level != PARAM_UNSET && p.level != DEBUG_OFF_LEVEL && + (p.level < 0 || p.level > DEBUG_MAX_LEVEL)) + return false; + if (p.pages != PARAM_UNSET && p.pages < 0) + return false; + *param = p; + + return true; +} + +static int __init s390dbf_parse(char *arg) +{ + debug_info_t **id; + int i, rc = 0; + + while (arg && debug_param_num < PARAM_NUM) { + if (s390dbf_parse_one(arg, &debug_param[debug_param_num])) + debug_param_num++; + else + rc = -EINVAL; + arg = strchr(arg, ','); + if (arg) + arg++; + } + + /* + * Apply level to static debug areas, delay buffer size changes until + * regular memory allocations are possible. + */ + for (i = 0, id = __s390dbf_info; &id[i] < __s390dbf_info_end; i++) + debug_get_param(id[i]->name, &id[i]->level, NULL, false); + + return rc; +} +early_param("s390dbf", s390dbf_parse); /* * debug_areas_alloc @@ -181,14 +270,13 @@ static debug_entry_t ***debug_areas_alloc(int pages_per_area, int nr_areas) debug_entry_t ***areas; int i, j; - areas = kmalloc_array(nr_areas, sizeof(debug_entry_t **), GFP_KERNEL); + areas = kmalloc_objs(debug_entry_t **, nr_areas); if (!areas) goto fail_malloc_areas; for (i = 0; i < nr_areas; i++) { /* GFP_NOWARN to avoid user triggerable WARN, we handle fails */ - areas[i] = kmalloc_array(pages_per_area, - sizeof(debug_entry_t *), - GFP_KERNEL | __GFP_NOWARN); + areas[i] = kmalloc_objs(debug_entry_t *, pages_per_area, + GFP_KERNEL | __GFP_NOWARN); if (!areas[i]) goto fail_malloc_areas2; for (j = 0; j < pages_per_area; j++) { @@ -225,13 +313,13 @@ static debug_info_t *debug_info_alloc(const char *name, int pages_per_area, debug_info_t *rc; /* alloc everything */ - rc = kmalloc(sizeof(debug_info_t), GFP_KERNEL); + rc = kmalloc_obj(debug_info_t); if (!rc) goto fail_malloc_rc; - rc->active_entries = kcalloc(nr_areas, sizeof(int), GFP_KERNEL); + rc->active_entries = kzalloc_objs(int, nr_areas); if (!rc->active_entries) goto fail_malloc_active_entries; - rc->active_pages = kcalloc(nr_areas, sizeof(int), GFP_KERNEL); + rc->active_pages = kzalloc_objs(int, nr_areas); if (!rc->active_pages) goto fail_malloc_active_pages; if ((mode == ALL_AREAS) && (pages_per_area != 0)) { @@ -243,7 +331,7 @@ static debug_info_t *debug_info_alloc(const char *name, int pages_per_area, } /* initialize members */ - spin_lock_init(&rc->lock); + raw_spin_lock_init(&rc->lock); rc->pages_per_area = pages_per_area; rc->nr_areas = nr_areas; rc->active_area = 0; @@ -306,10 +394,11 @@ static void debug_info_free(debug_info_t *db_info) static debug_info_t *debug_info_create(const char *name, int pages_per_area, int nr_areas, int buf_size, umode_t mode) { + int level = DEBUG_DEFAULT_LEVEL; debug_info_t *rc; - rc = debug_info_alloc(name, pages_per_area, nr_areas, buf_size, - DEBUG_DEFAULT_LEVEL, ALL_AREAS); + debug_get_param(name, &level, &pages_per_area, false); + rc = debug_info_alloc(name, pages_per_area, nr_areas, buf_size, level, ALL_AREAS); if (!rc) goto out; @@ -333,7 +422,7 @@ static debug_info_t *debug_info_copy(debug_info_t *in, int mode) do { rc = debug_info_alloc(in->name, in->pages_per_area, in->nr_areas, in->buf_size, in->level, mode); - spin_lock_irqsave(&in->lock, flags); + raw_spin_lock_irqsave(&in->lock, flags); if (!rc) goto out; /* has something changed in the meantime ? */ @@ -341,11 +430,12 @@ static debug_info_t *debug_info_copy(debug_info_t *in, int mode) (rc->nr_areas == in->nr_areas)) { break; } - spin_unlock_irqrestore(&in->lock, flags); + raw_spin_unlock_irqrestore(&in->lock, flags); debug_info_free(rc); } while (1); - if (mode == NO_AREAS) + /* debug_register_static() failure leaves areas NULL, bounds intact */ + if (mode == NO_AREAS || !in->areas) goto out; for (i = 0; i < in->nr_areas; i++) { @@ -356,7 +446,7 @@ static debug_info_t *debug_info_copy(debug_info_t *in, int mode) } rc->active_area = in->active_area; out: - spin_unlock_irqrestore(&in->lock, flags); + raw_spin_unlock_irqrestore(&in->lock, flags); return rc; } @@ -631,7 +721,7 @@ static file_private_info_t *debug_file_private_alloc(debug_info_t *debug_info, if (!debug_info_snapshot) return NULL; - p_info = kmalloc(sizeof(file_private_info_t), GFP_KERNEL); + p_info = kmalloc_obj(file_private_info_t); if (!p_info) { debug_info_free(debug_info_snapshot); return NULL; @@ -734,6 +824,9 @@ ssize_t debug_dump(debug_info_t *id, struct debug_view *view, file_private_info_t *p_info; size_t size, offset = 0; + if (!id) + return -EINVAL; + /* Need space for '\0' byte */ if (buf_size < 1) return 0; @@ -861,8 +954,12 @@ EXPORT_SYMBOL(debug_register); * * Note: This function is called automatically via an initcall generated by * DEFINE_STATIC_DEBUG_INFO. + * + * Return: + * - 0 on success + * - negative error code on failure */ -void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas) +int debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas) { unsigned long flags; debug_info_t *copy; @@ -870,29 +967,30 @@ void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas) if (!initialized) { pr_err("Tried to register debug feature %s too early\n", id->name); - return; + return -EINVAL; } + debug_get_param(id->name, &id->level, &pages_per_area, false); copy = debug_info_alloc("", pages_per_area, nr_areas, id->buf_size, id->level, ALL_AREAS); if (!copy) { pr_err("Registering debug feature %s failed\n", id->name); /* Clear pointers to prevent tracing into released initdata. */ - spin_lock_irqsave(&id->lock, flags); + raw_spin_lock_irqsave(&id->lock, flags); id->areas = NULL; id->active_pages = NULL; id->active_entries = NULL; - spin_unlock_irqrestore(&id->lock, flags); + raw_spin_unlock_irqrestore(&id->lock, flags); - return; + return -ENOMEM; } /* Replace static trace area with dynamic copy. */ - spin_lock_irqsave(&id->lock, flags); + raw_spin_lock_irqsave(&id->lock, flags); debug_events_append(copy, id); debug_areas_swap(id, copy); - spin_unlock_irqrestore(&id->lock, flags); + raw_spin_unlock_irqrestore(&id->lock, flags); /* Clear pointers to initdata and discard copy. */ copy->areas = NULL; @@ -903,10 +1001,12 @@ void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas) mutex_lock(&debug_mutex); _debug_register(id); mutex_unlock(&debug_mutex); + + return 0; } -/* Remove debugfs entries and remove from internal list. */ -static void _debug_unregister(debug_info_t *id) +/* Remove debugfs entries. */ +static void _debug_unregister_debugfs(debug_info_t *id) { int i; @@ -916,6 +1016,11 @@ static void _debug_unregister(debug_info_t *id) debugfs_remove(id->debugfs_entries[i]); } debugfs_remove(id->debugfs_root_entry); +} + +/* Remove from internal list. */ +static void _debug_unregister(debug_info_t *id) +{ if (id == debug_area_first) debug_area_first = id->next; if (id == debug_area_last) @@ -941,6 +1046,7 @@ void debug_unregister(debug_info_t *id) mutex_lock(&debug_mutex); _debug_unregister(id); mutex_unlock(&debug_mutex); + _debug_unregister_debugfs(id); debug_info_put(id); } @@ -966,16 +1072,33 @@ static int debug_set_size(debug_info_t *id, int nr_areas, int pages_per_area) return -ENOMEM; } - spin_lock_irqsave(&id->lock, flags); + raw_spin_lock_irqsave(&id->lock, flags); debug_events_append(new_id, id); debug_areas_swap(new_id, id); + raw_spin_unlock_irqrestore(&id->lock, flags); debug_info_free(new_id); - spin_unlock_irqrestore(&id->lock, flags); pr_info("%s: set new size (%i pages)\n", id->name, pages_per_area); return 0; } +static void _debug_set_level(debug_info_t *id, int new_level) +{ + unsigned long flags; + + if (new_level == DEBUG_OFF_LEVEL) { + pr_info("%s: switched off\n", id->name); + } else if ((new_level > DEBUG_MAX_LEVEL) || (new_level < 0)) { + pr_info("%s: level %i is out of range (%i - %i)\n", + id->name, new_level, 0, DEBUG_MAX_LEVEL); + return; + } + + raw_spin_lock_irqsave(&id->lock, flags); + id->level = new_level; + raw_spin_unlock_irqrestore(&id->lock, flags); +} + /** * debug_set_level() - Sets new actual debug level if new_level is valid. * @@ -987,22 +1110,16 @@ static int debug_set_size(debug_info_t *id, int nr_areas, int pages_per_area) */ void debug_set_level(debug_info_t *id, int new_level) { - unsigned long flags; - if (!id) return; - if (new_level == DEBUG_OFF_LEVEL) { - pr_info("%s: switched off\n", id->name); - } else if ((new_level > DEBUG_MAX_LEVEL) || (new_level < 0)) { - pr_info("%s: level %i is out of range (%i - %i)\n", - id->name, new_level, 0, DEBUG_MAX_LEVEL); - return; - } + /* + * Level specified via kernel parameter takes precedence. The override + * was already announced during registration, so stay quiet here. + */ + debug_get_param(id->name, &new_level, NULL, true); - spin_lock_irqsave(&id->lock, flags); - id->level = new_level; - spin_unlock_irqrestore(&id->lock, flags); + _debug_set_level(id, new_level); } EXPORT_SYMBOL(debug_set_level); @@ -1138,8 +1255,6 @@ static const struct ctl_table s390dbf_table[] = { }, }; -static struct ctl_table_header *s390dbf_sysctl_header; - /** * debug_stop_all() - stops the debug feature if stopping is allowed. * @@ -1178,17 +1293,19 @@ void debug_set_critical(void) debug_entry_t *debug_event_common(debug_info_t *id, int level, const void *buf, int len) { - debug_entry_t *active; + debug_entry_t *active = NULL; unsigned long flags; if (!debug_active || !id->areas) return NULL; if (debug_critical) { - if (!spin_trylock_irqsave(&id->lock, flags)) + if (!raw_spin_trylock_irqsave(&id->lock, flags)) return NULL; } else { - spin_lock_irqsave(&id->lock, flags); + raw_spin_lock_irqsave(&id->lock, flags); } + if (!id->areas) + goto out; do { active = get_active_entry(id); memcpy(DEBUG_DATA(active), buf, min(len, id->buf_size)); @@ -1198,8 +1315,8 @@ debug_entry_t *debug_event_common(debug_info_t *id, int level, const void *buf, len -= id->buf_size; buf += id->buf_size; } while (len > 0); - - spin_unlock_irqrestore(&id->lock, flags); +out: + raw_spin_unlock_irqrestore(&id->lock, flags); return active; } EXPORT_SYMBOL(debug_event_common); @@ -1211,17 +1328,19 @@ EXPORT_SYMBOL(debug_event_common); debug_entry_t *debug_exception_common(debug_info_t *id, int level, const void *buf, int len) { - debug_entry_t *active; + debug_entry_t *active = NULL; unsigned long flags; if (!debug_active || !id->areas) return NULL; if (debug_critical) { - if (!spin_trylock_irqsave(&id->lock, flags)) + if (!raw_spin_trylock_irqsave(&id->lock, flags)) return NULL; } else { - spin_lock_irqsave(&id->lock, flags); + raw_spin_lock_irqsave(&id->lock, flags); } + if (!id->areas) + goto out; do { active = get_active_entry(id); memcpy(DEBUG_DATA(active), buf, min(len, id->buf_size)); @@ -1231,8 +1350,8 @@ debug_entry_t *debug_exception_common(debug_info_t *id, int level, len -= id->buf_size; buf += id->buf_size; } while (len > 0); - - spin_unlock_irqrestore(&id->lock, flags); +out: + raw_spin_unlock_irqrestore(&id->lock, flags); return active; } EXPORT_SYMBOL(debug_exception_common); @@ -1257,7 +1376,7 @@ static inline int debug_count_numargs(char *string) debug_entry_t *__debug_sprintf_event(debug_info_t *id, int level, char *string, ...) { debug_sprintf_entry_t *curr_event; - debug_entry_t *active; + debug_entry_t *active = NULL; unsigned long flags; int numargs, idx; va_list ap; @@ -1267,11 +1386,13 @@ debug_entry_t *__debug_sprintf_event(debug_info_t *id, int level, char *string, numargs = debug_count_numargs(string); if (debug_critical) { - if (!spin_trylock_irqsave(&id->lock, flags)) + if (!raw_spin_trylock_irqsave(&id->lock, flags)) return NULL; } else { - spin_lock_irqsave(&id->lock, flags); + raw_spin_lock_irqsave(&id->lock, flags); } + if (!id->areas) + goto out; active = get_active_entry(id); curr_event = (debug_sprintf_entry_t *) DEBUG_DATA(active); va_start(ap, string); @@ -1280,7 +1401,8 @@ debug_entry_t *__debug_sprintf_event(debug_info_t *id, int level, char *string, curr_event->args[idx] = va_arg(ap, long); va_end(ap); debug_finish_entry(id, active, level, 0); - spin_unlock_irqrestore(&id->lock, flags); +out: + raw_spin_unlock_irqrestore(&id->lock, flags); return active; } @@ -1292,7 +1414,7 @@ EXPORT_SYMBOL(__debug_sprintf_event); debug_entry_t *__debug_sprintf_exception(debug_info_t *id, int level, char *string, ...) { debug_sprintf_entry_t *curr_event; - debug_entry_t *active; + debug_entry_t *active = NULL; unsigned long flags; int numargs, idx; va_list ap; @@ -1303,11 +1425,13 @@ debug_entry_t *__debug_sprintf_exception(debug_info_t *id, int level, char *stri numargs = debug_count_numargs(string); if (debug_critical) { - if (!spin_trylock_irqsave(&id->lock, flags)) + if (!raw_spin_trylock_irqsave(&id->lock, flags)) return NULL; } else { - spin_lock_irqsave(&id->lock, flags); + raw_spin_lock_irqsave(&id->lock, flags); } + if (!id->areas) + goto out; active = get_active_entry(id); curr_event = (debug_sprintf_entry_t *)DEBUG_DATA(active); va_start(ap, string); @@ -1316,7 +1440,8 @@ debug_entry_t *__debug_sprintf_exception(debug_info_t *id, int level, char *stri curr_event->args[idx] = va_arg(ap, long); va_end(ap); debug_finish_entry(id, active, level, 1); - spin_unlock_irqrestore(&id->lock, flags); +out: + raw_spin_unlock_irqrestore(&id->lock, flags); return active; } @@ -1350,7 +1475,7 @@ int debug_register_view(debug_info_t *id, struct debug_view *view) mode &= ~(S_IWUSR | S_IWGRP | S_IWOTH); pde = debugfs_create_file(view->name, mode, id->debugfs_root_entry, id, &debug_file_ops); - spin_lock_irqsave(&id->lock, flags); + raw_spin_lock_irqsave(&id->lock, flags); for (i = 0; i < DEBUG_MAX_VIEWS; i++) { if (!id->views[i]) break; @@ -1361,7 +1486,7 @@ int debug_register_view(debug_info_t *id, struct debug_view *view) id->views[i] = view; id->debugfs_entries[i] = pde; } - spin_unlock_irqrestore(&id->lock, flags); + raw_spin_unlock_irqrestore(&id->lock, flags); if (rc) { pr_err("Registering view %s/%s would exceed the maximum " "number of views %i\n", id->name, view->name, i); @@ -1391,7 +1516,7 @@ int debug_unregister_view(debug_info_t *id, struct debug_view *view) if (!id) goto out; - spin_lock_irqsave(&id->lock, flags); + raw_spin_lock_irqsave(&id->lock, flags); for (i = 0; i < DEBUG_MAX_VIEWS; i++) { if (id->views[i] == view) break; @@ -1403,7 +1528,7 @@ int debug_unregister_view(debug_info_t *id, struct debug_view *view) id->views[i] = NULL; id->debugfs_entries[i] = NULL; } - spin_unlock_irqrestore(&id->lock, flags); + raw_spin_unlock_irqrestore(&id->lock, flags); debugfs_remove(dentry); out: return rc; @@ -1415,6 +1540,9 @@ static inline char *debug_get_user_string(const char __user *user_buf, { char *buffer; + if (!user_len) + return ERR_PTR(-EINVAL); + buffer = memdup_user_nul(user_buf, user_len); if (IS_ERR(buffer)) return buffer; @@ -1527,7 +1655,7 @@ static int debug_input_level_fn(debug_info_t *id, struct debug_view *view, goto out; } if (str[0] == '-') { - debug_set_level(id, DEBUG_OFF_LEVEL); + _debug_set_level(id, DEBUG_OFF_LEVEL); rc = user_len; goto free_str; } else { @@ -1537,7 +1665,7 @@ static int debug_input_level_fn(debug_info_t *id, struct debug_view *view, pr_warn("%s is not a valid level for a debug feature\n", str); rc = -EINVAL; } else { - debug_set_level(id, new_level); + _debug_set_level(id, new_level); rc = user_len; } free_str: @@ -1555,9 +1683,11 @@ static void debug_flush(debug_info_t *id, int area) unsigned long flags; int i, j; - if (!id || !id->areas) + if (!id) return; - spin_lock_irqsave(&id->lock, flags); + raw_spin_lock_irqsave(&id->lock, flags); + if (!id->areas) + goto out; if (area == DEBUG_FLUSH_ALL) { id->active_area = 0; memset(id->active_entries, 0, id->nr_areas * sizeof(int)); @@ -1572,7 +1702,8 @@ static void debug_flush(debug_info_t *id, int area) for (i = 0; i < id->pages_per_area; i++) memset(id->areas[area][i], 0, PAGE_SIZE); } - spin_unlock_irqrestore(&id->lock, flags); +out: + raw_spin_unlock_irqrestore(&id->lock, flags); } /* @@ -1585,6 +1716,11 @@ static int debug_input_flush_fn(debug_info_t *id, struct debug_view *view, char input_buf[1]; int rc = user_len; + if (!user_len) { + rc = -EINVAL; + goto out; + } + if (user_len > 0x10000) user_len = 0x10000; if (*offset != 0) { @@ -1721,7 +1857,7 @@ EXPORT_SYMBOL(debug_sprintf_format_fn); */ static int __init debug_init(void) { - s390dbf_sysctl_header = register_sysctl("s390dbf", s390dbf_table); + register_sysctl("s390dbf", s390dbf_table); mutex_lock(&debug_mutex); debug_debugfs_root_entry = debugfs_create_dir(DEBUG_DIR_ROOT, NULL); initialized = 1; diff --git a/arch/s390/kernel/diag/diag310.c b/arch/s390/kernel/diag/diag310.c index f411562aa7f6..c68ec9d28513 100644 --- a/arch/s390/kernel/diag/diag310.c +++ b/arch/s390/kernel/diag/diag310.c @@ -190,17 +190,18 @@ static int memtop_get_stride_len(unsigned long *res) static int memtop_get_page_count(unsigned long *res, unsigned long level) { static unsigned long memtop_pages[DIAG310_LEVELMAX]; - unsigned long pages; + unsigned long pages, idx; int rc; if (level > DIAG310_LEVELMAX || level < DIAG310_LEVELMIN) return -EINVAL; - pages = READ_ONCE(memtop_pages[level - 1]); + idx = array_index_nospec(level - 1, ARRAY_SIZE(memtop_pages)); + pages = READ_ONCE(memtop_pages[idx]); if (!pages) { rc = diag310_get_memtop_size(&pages, level); if (rc) return rc; - WRITE_ONCE(memtop_pages[level - 1], pages); + WRITE_ONCE(memtop_pages[idx], pages); } *res = pages; return 0; diff --git a/arch/s390/kernel/diag/diag324.c b/arch/s390/kernel/diag/diag324.c index fe325c2a2d0d..3eec0cc8fb9e 100644 --- a/arch/s390/kernel/diag/diag324.c +++ b/arch/s390/kernel/diag/diag324.c @@ -182,8 +182,7 @@ long diag324_pibbuf(unsigned long arg) goto out; rc = copy_to_user((void __user *)address, data->pib, data->pib->len); rc |= put_user(data->sequence, &udata->sequence); - if (rc) - rc = -EFAULT; + rc = rc ? -EFAULT : data->rc; out: mutex_unlock(&pibmutex); return rc; diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index b7f1553d9ee5..10dd9bbdf985 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -23,6 +23,7 @@ #include <asm/unistd.h> #include <asm/page.h> #include <asm/sigp.h> +#include <asm/bug.h> #include <asm/irq.h> #include <asm/fpu-insn.h> #include <asm/setup.h> @@ -30,6 +31,7 @@ #include <asm/nospec-insn.h> #include <asm/lowcore.h> #include <asm/machine.h> +#include "entry.h" _LPP_OFFSET = __LC_LPP @@ -173,6 +175,16 @@ SYM_FUNC_START(__switch_to_asm) BR_EX %r14 SYM_FUNC_END(__switch_to_asm) +#if defined(CONFIG_BUG) && defined(CONFIG_CC_HAS_ASM_IMMEDIATE_STRINGS) + +SYM_FUNC_START(__WARN_trap) + mc MONCODE_BUG_ARG(%r0),0 + BR_EX %r14 +SYM_FUNC_END(__WARN_trap) +EXPORT_SYMBOL(__WARN_trap) + +#endif /* CONFIG_BUG && CONFIG_CC_HAS_ASM_IMMEDIATE_STRINGS */ + #if IS_ENABLED(CONFIG_KVM) /* * __sie64a calling convention: @@ -189,7 +201,7 @@ SYM_FUNC_START(__sie64a) stg %r3,__SF_SIE_CONTROL(%r15) # ...and virtual addresses stg %r4,__SF_SIE_SAVEAREA(%r15) # save guest register save area stg %r5,__SF_SIE_GUEST_ASCE(%r15) # save guest asce - xc __SF_SIE_REASON(8,%r15),__SF_SIE_REASON(%r15) # reason code = 0 + xc __SF_SIE_RETURN(8,%r15),__SF_SIE_RETURN(%r15) # return code = 0 mvc __SF_SIE_FLAGS(8,%r15),__TI_flags(%r14) # copy thread flags lmg %r0,%r13,0(%r4) # load guest gprs 0-13 mvi __TI_sie(%r14),1 @@ -226,7 +238,7 @@ SYM_INNER_LABEL(sie_exit, SYM_L_GLOBAL) xgr %r4,%r4 xgr %r5,%r5 lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers - lg %r2,__SF_SIE_REASON(%r15) # return exit reason code + lg %r2,__SF_SIE_RETURN(%r15) # return sie return code BR_EX %r14 SYM_FUNC_END(__sie64a) EXPORT_SYMBOL(__sie64a) @@ -260,6 +272,7 @@ SYM_CODE_START(system_call) xgr %r9,%r9 xgr %r10,%r10 xgr %r11,%r11 + xgr %r12,%r12 la %r2,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs mvc __PT_R8(64,%r2),__LC_SAVE_AREA(%r13) MBEAR %r2,%r13 @@ -309,7 +322,7 @@ SYM_CODE_START(pgm_check_handler) jz 1f BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST SIEEXIT __SF_SIE_CONTROL(%r15),%r13 - lghi %r10,_PIF_GUEST_FAULT + lghi %r10,PGM_FLAG_GUEST_FAULT #endif 1: tmhh %r8,0x4000 # PER bit set in old PSW ? jnz 2f # -> enabled, can't be a double fault @@ -320,7 +333,7 @@ SYM_CODE_START(pgm_check_handler) CHECK_VMAP_STACK __LC_SAVE_AREA,%r13,4f 3: lg %r15,__LC_KERNEL_STACK(%r13) 4: la %r11,STACK_FRAME_OVERHEAD(%r15) - stg %r10,__PT_FLAGS(%r11) + xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) stmg %r0,%r7,__PT_R0(%r11) mvc __PT_R8(64,%r11),__LC_SAVE_AREA(%r13) @@ -329,13 +342,13 @@ SYM_CODE_START(pgm_check_handler) # clear user controlled registers to prevent speculative use xgr %r0,%r0 xgr %r1,%r1 - xgr %r3,%r3 xgr %r4,%r4 xgr %r5,%r5 xgr %r6,%r6 xgr %r7,%r7 xgr %r12,%r12 lgr %r2,%r11 + lgr %r3,%r10 brasl %r14,__do_pgm_check tmhh %r8,0x0001 # returning to user space? jno .Lpgm_exit_kernel @@ -355,7 +368,7 @@ SYM_CODE_START(pgm_check_handler) mvc __LC_RETURN_PSW(8,%r13),__LC_SVC_NEW_PSW(%r13) larl %r14,.Lsysc_per stg %r14,__LC_RETURN_PSW+8(%r13) - lghi %r14,1 + lghi %r14,SYSCALL_FLAG_PER_TRAP LBEAR __LC_PGM_LAST_BREAK(%r13) LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE # branch to .Lsysc_per SYM_CODE_END(pgm_check_handler) @@ -367,8 +380,19 @@ SYM_CODE_END(pgm_check_handler) SYM_CODE_START(\name) STMG_LC %r8,%r15,__LC_SAVE_AREA GET_LC %r13 - stckf __LC_INT_CLOCK(%r13) +#ifdef CONFIG_NO_HZ_COMMON + larl %r12,kernel_cpustat + ag %r12,__LC_PERCPU_OFFSET(%r13) + asi __KCPUSTAT_SEQUENCE(%r12),1 +#endif + stcke __LC_INT_CLOCK(%r13) stpt __LC_SYS_ENTER_TIMER(%r13) + larl %r10,s390_idle + ag %r10,__LC_PERCPU_OFFSET(%r13) + mvc __IDLE_CLOCK_EXIT(16,%r10),__LC_INT_CLOCK(%r13) +#ifdef CONFIG_NO_HZ_COMMON + asi __KCPUSTAT_SEQUENCE(%r12),1 +#endif STBEAR __LC_LAST_BREAK(%r13) BPOFF lmg %r8,%r9,\lc_old_psw(%r13) @@ -395,7 +419,7 @@ SYM_CODE_START(\name) xgr %r5,%r5 xgr %r6,%r6 xgr %r7,%r7 - xgr %r10,%r10 + xgr %r12,%r12 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) mvc __PT_R8(64,%r11),__LC_SAVE_AREA(%r13) MBEAR %r11,%r13 @@ -485,6 +509,7 @@ SYM_CODE_START(mcck_int_handler) xgr %r6,%r6 xgr %r7,%r7 xgr %r10,%r10 + xgr %r12,%r12 stmg %r8,%r9,__PT_PSW(%r11) xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h index dd55cc6bbc28..dc84ab497251 100644 --- a/arch/s390/kernel/entry.h +++ b/arch/s390/kernel/entry.h @@ -2,6 +2,11 @@ #ifndef _ENTRY_H #define _ENTRY_H +#define PGM_FLAG_GUEST_FAULT 1 +#define SYSCALL_FLAG_PER_TRAP 1 + +#ifndef __ASSEMBLER__ + #include <linux/percpu.h> #include <linux/types.h> #include <linux/signal.h> @@ -21,8 +26,8 @@ void early_pgm_check_handler(void); struct task_struct *__switch_to_asm(struct task_struct *prev, struct task_struct *next); void __ret_from_fork(struct task_struct *prev, struct pt_regs *regs); -void __do_pgm_check(struct pt_regs *regs); -void __do_syscall(struct pt_regs *regs, int per_trap); +void __do_pgm_check(struct pt_regs *regs, unsigned long flags); +void __do_syscall(struct pt_regs *regs, unsigned long flags); void __do_early_pgm_check(struct pt_regs *regs); void do_protection_exception(struct pt_regs *regs); @@ -56,8 +61,6 @@ long sys_s390_pci_mmio_write(unsigned long, const void __user *, size_t); long sys_s390_pci_mmio_read(unsigned long, void __user *, size_t); long sys_s390_sthyi(unsigned long function_code, void __user *buffer, u64 __user *return_code, unsigned long flags); -DECLARE_PER_CPU(u64, mt_cycles[8]); - unsigned long stack_alloc(void); void stack_free(unsigned long stack); @@ -72,4 +75,5 @@ extern struct exception_table_entry _stop_amode31_ex_table[]; #define __amode31_ref __section(".amode31.refs") extern long _start_amode31_refs[], _end_amode31_refs[]; +#endif /* __ASSEMBLER__ */ #endif /* _ENTRY_H */ diff --git a/arch/s390/kernel/guarded_storage.c b/arch/s390/kernel/guarded_storage.c index cf26d7a37425..cab8ac25a010 100644 --- a/arch/s390/kernel/guarded_storage.c +++ b/arch/s390/kernel/guarded_storage.c @@ -24,7 +24,7 @@ static int gs_enable(void) struct gs_cb *gs_cb; if (!current->thread.gs_cb) { - gs_cb = kzalloc(sizeof(*gs_cb), GFP_KERNEL); + gs_cb = kzalloc_obj(*gs_cb); if (!gs_cb) return -ENOMEM; gs_cb->gsd = 25; @@ -55,7 +55,7 @@ static int gs_set_bc_cb(struct gs_cb __user *u_gs_cb) gs_cb = current->thread.gs_bc_cb; if (!gs_cb) { - gs_cb = kzalloc(sizeof(*gs_cb), GFP_KERNEL); + gs_cb = kzalloc_obj(*gs_cb); if (!gs_cb) return -ENOMEM; current->thread.gs_bc_cb = gs_cb; diff --git a/arch/s390/kernel/idle.c b/arch/s390/kernel/idle.c index 39cb8d0ae348..dd12efb58201 100644 --- a/arch/s390/kernel/idle.c +++ b/arch/s390/kernel/idle.c @@ -9,46 +9,125 @@ #include <linux/kernel.h> #include <linux/kernel_stat.h> +#include <linux/sched/stat.h> #include <linux/notifier.h> #include <linux/init.h> #include <linux/cpu.h> +#include <linux/export.h> #include <trace/events/power.h> #include <asm/cpu_mf.h> #include <asm/cputime.h> +#include <asm/idle.h> #include <asm/nmi.h> #include <asm/smp.h> -#include "entry.h" -static DEFINE_PER_CPU(struct s390_idle_data, s390_idle); +DEFINE_PER_CPU(struct s390_idle_data, s390_idle); -void account_idle_time_irq(void) +static __always_inline void __account_idle_time_irq(void) { struct s390_idle_data *idle = this_cpu_ptr(&s390_idle); - struct lowcore *lc = get_lowcore(); unsigned long idle_time; - u64 cycles_new[8]; - int i; - if (smp_cpu_mtid) { - stcctm(MT_DIAG, smp_cpu_mtid, cycles_new); - for (i = 0; i < smp_cpu_mtid; i++) - this_cpu_add(mt_cycles[i], cycles_new[i] - idle->mt_cycles_enter[i]); - } + idle_time = idle->clock_idle_exit.tod - idle->clock_idle_enter.tod; + account_idle_time(cputime_to_nsecs(idle_time)); +} + +static __always_inline void __account_idle_time_setup(void) +{ + struct s390_idle_data *idle = this_cpu_ptr(&s390_idle); - idle_time = lc->int_clock - idle->clock_idle_enter; + store_tod_clock_ext(&idle->clock_idle_enter); + idle->timer_idle_enter = get_cpu_timer(); + idle->clock_idle_exit = idle->clock_idle_enter; +} - lc->steal_timer += idle->clock_idle_enter - lc->last_update_clock; - lc->last_update_clock = lc->int_clock; +#ifdef CONFIG_NO_HZ_COMMON - lc->system_timer += lc->last_update_timer - idle->timer_idle_enter; - lc->last_update_timer = lc->sys_enter_timer; +static u64 arch_cpu_in_idle_time(int cpu) +{ + struct s390_idle_data *idle = &per_cpu(s390_idle, cpu); + union tod_clock now; + u64 idle_time; + + if (!idle->in_idle) + return 0; + store_tod_clock_ext(&now); + if (tod_after(idle->clock_idle_exit.tod, idle->clock_idle_enter.tod)) + idle_time = idle->clock_idle_exit.tod - idle->clock_idle_enter.tod; + else + idle_time = now.tod - idle->clock_idle_enter.tod; + return cputime_to_nsecs(idle_time); +} - /* Account time spent with enabled wait psw loaded as idle time. */ - WRITE_ONCE(idle->idle_time, READ_ONCE(idle->idle_time) + idle_time); - WRITE_ONCE(idle->idle_count, READ_ONCE(idle->idle_count) + 1); - account_idle_time(cputime_to_nsecs(idle_time)); +static u64 arch_cpu_idle_time(int cpu, enum cpu_usage_stat idx, bool compute_delta) +{ + struct kernel_cpustat *kc = &kcpustat_cpu(cpu); + u64 *cpustat = kc->cpustat; + unsigned int seq; + u64 idle_time; + + /* + * The open coded seqcount writer in entry.S relies on the + * raw counting mechanism without any writer protection. + */ + typecheck(typeof(kc->idle_sleeptime_seq), seqcount_t); + do { + seq = read_seqcount_begin(&kc->idle_sleeptime_seq); + idle_time = cpustat[idx]; + if (compute_delta) + idle_time += arch_cpu_in_idle_time(cpu); + } while (read_seqcount_retry(&kc->idle_sleeptime_seq, seq)); + return idle_time; +} + +u64 arch_kcpustat_field_idle(int cpu) +{ + return arch_cpu_idle_time(cpu, CPUTIME_IDLE, !nr_iowait_cpu(cpu)); +} +EXPORT_SYMBOL_GPL(arch_kcpustat_field_idle); + +u64 arch_kcpustat_field_iowait(int cpu) +{ + return arch_cpu_idle_time(cpu, CPUTIME_IOWAIT, nr_iowait_cpu(cpu)); +} +EXPORT_SYMBOL_GPL(arch_kcpustat_field_iowait); + +void account_idle_time_irq(void) +{ + struct s390_idle_data *idle = this_cpu_ptr(&s390_idle); + struct kernel_cpustat *kc = kcpustat_this_cpu; + + write_seqcount_begin(&kc->idle_sleeptime_seq); + idle->in_idle = false; + __account_idle_time_irq(); + write_seqcount_end(&kc->idle_sleeptime_seq); } +static __always_inline void account_idle_time_setup(void) +{ + struct s390_idle_data *idle = this_cpu_ptr(&s390_idle); + struct kernel_cpustat *kc = kcpustat_this_cpu; + + raw_write_seqcount_begin(&kc->idle_sleeptime_seq); + idle->in_idle = true; + __account_idle_time_setup(); + raw_write_seqcount_end(&kc->idle_sleeptime_seq); +} + +#else /* CONFIG_NO_HZ_COMMON */ + +void account_idle_time_irq(void) +{ + __account_idle_time_irq(); +} + +static __always_inline void account_idle_time_setup(void) +{ + __account_idle_time_setup(); +} + +#endif /* CONFIG_NO_HZ_COMMON */ + void noinstr arch_cpu_idle(void) { struct s390_idle_data *idle = this_cpu_ptr(&s390_idle); @@ -57,34 +136,14 @@ void noinstr arch_cpu_idle(void) /* Wait for external, I/O or machine check interrupt. */ psw_mask = PSW_KERNEL_BITS | PSW_MASK_WAIT | PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK; - clear_cpu_flag(CIF_NOHZ_DELAY); set_cpu_flag(CIF_ENABLED_WAIT); if (smp_cpu_mtid) stcctm(MT_DIAG, smp_cpu_mtid, (u64 *)&idle->mt_cycles_enter); - idle->clock_idle_enter = get_tod_clock_fast(); - idle->timer_idle_enter = get_cpu_timer(); + account_idle_time_setup(); bpon(); __load_psw_mask(psw_mask); } -static ssize_t show_idle_count(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct s390_idle_data *idle = &per_cpu(s390_idle, dev->id); - - return sysfs_emit(buf, "%lu\n", READ_ONCE(idle->idle_count)); -} -DEVICE_ATTR(idle_count, 0444, show_idle_count, NULL); - -static ssize_t show_idle_time(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct s390_idle_data *idle = &per_cpu(s390_idle, dev->id); - - return sysfs_emit(buf, "%lu\n", READ_ONCE(idle->idle_time) >> 12); -} -DEVICE_ATTR(idle_time_us, 0444, show_idle_time, NULL); - void arch_cpu_idle_enter(void) { } diff --git a/arch/s390/kernel/ima_arch.c b/arch/s390/kernel/ima_arch.c deleted file mode 100644 index f3c3e6e1c5d3..000000000000 --- a/arch/s390/kernel/ima_arch.c +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include <linux/ima.h> -#include <asm/boot_data.h> - -bool arch_ima_get_secureboot(void) -{ - return ipl_secure_flag; -} - -const char * const *arch_get_ima_policy(void) -{ - return NULL; -} diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index dcdc7e274848..b1e798f8e1dd 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -21,6 +21,7 @@ #include <linux/crash_dump.h> #include <linux/debug_locks.h> #include <linux/vmalloc.h> +#include <linux/secure_boot.h> #include <asm/asm-extable.h> #include <asm/machine.h> #include <asm/diag.h> @@ -1156,6 +1157,8 @@ static struct attribute_group reipl_nss_attr_group = { void set_os_info_reipl_block(void) { + if (!reipl_block_actual) + return; os_info_entry_add_data(OS_INFO_REIPL_BLOCK, reipl_block_actual, reipl_block_actual->hdr.len); } @@ -1926,7 +1929,8 @@ static struct shutdown_action __refdata dump_action = { static void dump_reipl_run(struct shutdown_trigger *trigger) { struct lowcore *abs_lc; - unsigned int csum; + unsigned long ipib = 0; + unsigned int csum = 0; /* * Set REIPL_CLEAR flag in os_info flags entry indicating @@ -1942,9 +1946,12 @@ static void dump_reipl_run(struct shutdown_trigger *trigger) reipl_type == IPL_TYPE_UNKNOWN) os_info_flags |= OS_INFO_FLAG_REIPL_CLEAR; os_info_entry_add_data(OS_INFO_FLAGS_ENTRY, &os_info_flags, sizeof(os_info_flags)); - csum = (__force unsigned int)cksm(reipl_block_actual, reipl_block_actual->hdr.len, 0); + if (reipl_block_actual) { + ipib = __pa(reipl_block_actual); + csum = (__force unsigned int)cksm(reipl_block_actual, reipl_block_actual->hdr.len, 0); + } abs_lc = get_abs_lowcore(); - abs_lc->ipib = __pa(reipl_block_actual); + abs_lc->ipib = ipib; abs_lc->ipib_checksum = csum; put_abs_lowcore(abs_lc); dump_run(trigger); @@ -2020,8 +2027,11 @@ static int vmcmd_init(void) return sysfs_create_group(&vmcmd_kset->kobj, &vmcmd_attr_group); } -static struct shutdown_action vmcmd_action = {SHUTDOWN_ACTION_VMCMD_STR, - vmcmd_run, vmcmd_init}; +static struct shutdown_action vmcmd_action = { + .name = SHUTDOWN_ACTION_VMCMD_STR, + .fn = vmcmd_run, + .init = vmcmd_init +}; /* * stop shutdown action: Stop Linux on shutdown. @@ -2035,15 +2045,21 @@ static void stop_run(struct shutdown_trigger *trigger) smp_stop_cpu(); } -static struct shutdown_action stop_action = {SHUTDOWN_ACTION_STOP_STR, - stop_run, NULL}; +static struct shutdown_action stop_action = { + .name = SHUTDOWN_ACTION_STOP_STR, + .fn = stop_run +}; /* action list */ static struct shutdown_action *shutdown_actions_list[] = { - &ipl_action, &reipl_action, &dump_reipl_action, &dump_action, - &vmcmd_action, &stop_action}; -#define SHUTDOWN_ACTIONS_COUNT (sizeof(shutdown_actions_list) / sizeof(void *)) + &ipl_action, + &reipl_action, + &dump_reipl_action, + &dump_action, + &vmcmd_action, + &stop_action +}; /* * Trigger section @@ -2056,7 +2072,7 @@ static int set_trigger(const char *buf, struct shutdown_trigger *trigger, { int i; - for (i = 0; i < SHUTDOWN_ACTIONS_COUNT; i++) { + for (i = 0; i < ARRAY_SIZE(shutdown_actions_list); i++) { if (sysfs_streq(buf, shutdown_actions_list[i]->name)) { if (shutdown_actions_list[i]->init_rc) { return shutdown_actions_list[i]->init_rc; @@ -2071,8 +2087,10 @@ static int set_trigger(const char *buf, struct shutdown_trigger *trigger, /* on reipl */ -static struct shutdown_trigger on_reboot_trigger = {ON_REIPL_STR, - &reipl_action}; +static struct shutdown_trigger on_reboot_trigger = { + .name = ON_REIPL_STR, + .action = &reipl_action +}; static ssize_t on_reboot_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) @@ -2097,8 +2115,10 @@ static void do_machine_restart(char *__unused) void (*_machine_restart)(char *command) = do_machine_restart; /* on panic */ - -static struct shutdown_trigger on_panic_trigger = {ON_PANIC_STR, &stop_action}; +static struct shutdown_trigger on_panic_trigger = { + .name = ON_PANIC_STR, + .action = &stop_action +}; static ssize_t on_panic_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) @@ -2122,9 +2142,10 @@ static void do_panic(void) } /* on restart */ - -static struct shutdown_trigger on_restart_trigger = {ON_RESTART_STR, - &stop_action}; +static struct shutdown_trigger on_restart_trigger = { + .name = ON_RESTART_STR, + .action = &stop_action +}; static ssize_t on_restart_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) @@ -2159,8 +2180,10 @@ void do_restart(void *arg) } /* on halt */ - -static struct shutdown_trigger on_halt_trigger = {ON_HALT_STR, &stop_action}; +static struct shutdown_trigger on_halt_trigger = { + .name = ON_HALT_STR, + .action = &stop_action +}; static ssize_t on_halt_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) @@ -2185,8 +2208,10 @@ static void do_machine_halt(void) void (*_machine_halt)(void) = do_machine_halt; /* on power off */ - -static struct shutdown_trigger on_poff_trigger = {ON_POFF_STR, &stop_action}; +static struct shutdown_trigger on_poff_trigger = { + .name = ON_POFF_STR, + .action = &stop_action +}; static ssize_t on_poff_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) @@ -2241,7 +2266,7 @@ static void __init shutdown_actions_init(void) { int i; - for (i = 0; i < SHUTDOWN_ACTIONS_COUNT; i++) { + for (i = 0; i < ARRAY_SIZE(shutdown_actions_list); i++) { if (!shutdown_actions_list[i]->init) continue; shutdown_actions_list[i]->init_rc = @@ -2377,7 +2402,7 @@ void __init setup_ipl(void) atomic_notifier_chain_register(&panic_notifier_list, &on_panic_nb); } -void s390_reset_system(void) +void __no_stack_protector s390_reset_system(void) { /* Disable prefixing */ set_prefix(0); @@ -2387,6 +2412,11 @@ void s390_reset_system(void) diag_amode31_ops.diag308_reset(); } +bool arch_get_secureboot(void) +{ + return ipl_secure_flag; +} + #ifdef CONFIG_KEXEC_FILE int ipl_report_add_component(struct ipl_report *report, struct kexec_buf *kbuf, diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index bdf9c7cb5685..c923496aa7b4 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -33,6 +33,7 @@ #include <asm/softirq_stack.h> #include <asm/vtime.h> #include <asm/asm.h> +#include <asm/entry-percpu.h> #include "entry.h" DEFINE_PER_CPU_SHARED_ALIGNED(struct irq_stat, irq_stat); @@ -103,9 +104,10 @@ static const struct irq_class irqclass_sub_desc[] = { static void do_IRQ(struct pt_regs *regs, int irq) { - if (tod_after_eq(get_lowcore()->int_clock, - get_lowcore()->clock_comparator)) - /* Serve timer interrupts first. */ + struct lowcore *lc = get_lowcore(); + + /* Serve timer interrupts first */ + if (tod_after_eq(lc->int_clock.tod, lc->clock_comparator)) clock_comparator_work(); generic_handle_irq(irq); } @@ -142,9 +144,16 @@ static int irq_pending(struct pt_regs *regs) void noinstr do_io_irq(struct pt_regs *regs) { - irqentry_state_t state = irqentry_enter(regs); - struct pt_regs *old_regs = set_irq_regs(regs); - bool from_idle; + bool from_idle, percpu_needs_fixup; + struct pt_regs *old_regs; + irqentry_state_t state; + + percpu_entry(regs); + state = irqentry_enter(regs); + old_regs = set_irq_regs(regs); + from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); + if (from_idle) + update_timer_idle(); irq_enter_rcu(); @@ -154,11 +163,9 @@ void noinstr do_io_irq(struct pt_regs *regs) current->thread.last_break = regs->last_break; } - from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); if (from_idle) account_idle_time_irq(); - set_cpu_flag(CIF_NOHZ_DELAY); do { regs->tpi_info = get_lowcore()->tpi_info; if (get_lowcore()->tpi_info.adapter_IO) @@ -167,20 +174,28 @@ void noinstr do_io_irq(struct pt_regs *regs) do_irq_async(regs, IO_INTERRUPT); } while (machine_is_lpar() && irq_pending(regs)); + percpu_needs_fixup = percpu_code_check(regs); irq_exit_rcu(); - set_irq_regs(old_regs); irqentry_exit(regs, state); if (from_idle) regs->psw.mask &= ~(PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_WAIT); + percpu_exit(regs, percpu_needs_fixup); } void noinstr do_ext_irq(struct pt_regs *regs) { - irqentry_state_t state = irqentry_enter(regs); - struct pt_regs *old_regs = set_irq_regs(regs); - bool from_idle; + bool from_idle, percpu_needs_fixup; + struct pt_regs *old_regs; + irqentry_state_t state; + + percpu_entry(regs); + state = irqentry_enter(regs); + old_regs = set_irq_regs(regs); + from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); + if (from_idle) + update_timer_idle(); irq_enter_rcu(); @@ -194,18 +209,19 @@ void noinstr do_ext_irq(struct pt_regs *regs) regs->int_parm = get_lowcore()->ext_params; regs->int_parm_long = get_lowcore()->ext_params2; - from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); if (from_idle) account_idle_time_irq(); do_irq_async(regs, EXT_INTERRUPT); + percpu_needs_fixup = percpu_code_check(regs); irq_exit_rcu(); set_irq_regs(old_regs); irqentry_exit(regs, state); if (from_idle) regs->psw.mask &= ~(PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_WAIT); + percpu_exit(regs, percpu_needs_fixup); } static void show_msi_interrupt(struct seq_file *p, int irq) @@ -312,7 +328,7 @@ int register_external_irq(u16 code, ext_int_handler_t handler) unsigned long flags; int index; - p = kmalloc(sizeof(*p), GFP_ATOMIC); + p = kmalloc_obj(*p, GFP_ATOMIC); if (!p) return -ENOMEM; p->code = code; @@ -352,9 +368,6 @@ static irqreturn_t do_ext_interrupt(int irq, void *dummy) int index; ext_code.int_code = regs->int_code; - if (ext_code.code != EXT_IRQ_CLK_COMP) - set_cpu_flag(CIF_NOHZ_DELAY); - index = ext_hash(ext_code.code); rcu_read_lock(); hlist_for_each_entry_rcu(p, &ext_int_hash[index], entry) { diff --git a/arch/s390/kernel/irqflags.c b/arch/s390/kernel/irqflags.c new file mode 100644 index 000000000000..e192f59f8918 --- /dev/null +++ b/arch/s390/kernel/irqflags.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/export.h> +#include <asm/irqflags.h> + +noinstr unsigned long arch_local_save_flags(void) +{ + return __arch_local_save_flags(); +} +EXPORT_SYMBOL(arch_local_save_flags); + +noinstr unsigned long arch_local_irq_save(void) +{ + return __arch_local_irq_save(); +} +EXPORT_SYMBOL(arch_local_irq_save); + +noinstr void arch_local_irq_enable_external(void) +{ + __arch_local_irq_enable_external(); +} +EXPORT_SYMBOL(arch_local_irq_enable_external); + +noinstr void arch_local_irq_enable(void) +{ + __arch_local_irq_enable(); +} +EXPORT_SYMBOL(arch_local_irq_enable); diff --git a/arch/s390/kernel/machine_kexec_file.c b/arch/s390/kernel/machine_kexec_file.c index a36d7311c668..6f0852d5a3a9 100644 --- a/arch/s390/kernel/machine_kexec_file.c +++ b/arch/s390/kernel/machine_kexec_file.c @@ -28,7 +28,7 @@ const struct kexec_file_ops * const kexec_file_loaders[] = { #ifdef CONFIG_KEXEC_SIG int s390_verify_sig(const char *kernel, unsigned long kernel_len) { - const unsigned long marker_len = sizeof(MODULE_SIG_STRING) - 1; + const unsigned long marker_len = sizeof(MODULE_SIGNATURE_MARKER) - 1; struct module_signature *ms; unsigned long sig_len; int ret; @@ -40,7 +40,7 @@ int s390_verify_sig(const char *kernel, unsigned long kernel_len) if (marker_len > kernel_len) return -EKEYREJECTED; - if (memcmp(kernel + kernel_len - marker_len, MODULE_SIG_STRING, + if (memcmp(kernel + kernel_len - marker_len, MODULE_SIGNATURE_MARKER, marker_len)) return -EKEYREJECTED; kernel_len -= marker_len; @@ -53,7 +53,7 @@ int s390_verify_sig(const char *kernel, unsigned long kernel_len) return -EKEYREJECTED; kernel_len -= sig_len; - if (ms->id_type != PKEY_ID_PKCS7) + if (ms->id_type != MODULE_SIGNATURE_TYPE_PKCS7) return -EKEYREJECTED; if (ms->algo != 0 || @@ -270,8 +270,10 @@ void *kexec_file_add_components(struct kimage *image, if (image->kernel_buf_len < minsize + max_command_line_size) goto out; - if (image->cmdline_buf_len >= max_command_line_size) + if (image->cmdline_buf_len >= max_command_line_size) { + pr_err("Kernel command line exceeds supported limit of %lu", max_command_line_size); goto out; + } memcpy(data.parm->command_line, image->cmdline_buf, image->cmdline_buf_len); diff --git a/arch/s390/kernel/mcount.S b/arch/s390/kernel/mcount.S index 1fec370fecf4..6bc44c767642 100644 --- a/arch/s390/kernel/mcount.S +++ b/arch/s390/kernel/mcount.S @@ -4,6 +4,7 @@ * */ +#include <linux/cfi_types.h> #include <linux/linkage.h> #include <asm/asm-offsets.h> #include <asm/ftrace.h> @@ -34,10 +35,16 @@ .section .kprobes.text, "ax" -SYM_FUNC_START(ftrace_stub) +SYM_TYPED_FUNC_START(ftrace_stub) BR_EX %r14 SYM_FUNC_END(ftrace_stub) +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +SYM_TYPED_FUNC_START(ftrace_stub_graph) + BR_EX %r14 +SYM_FUNC_END(ftrace_stub_graph) +#endif + SYM_CODE_START(ftrace_stub_direct_tramp) lgr %r1, %r0 BR_EX %r1 diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index a55abbf65333..17297a8b63d9 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -22,6 +22,7 @@ #include <linux/module.h> #include <linux/sched/signal.h> #include <linux/kvm_host.h> +#include <asm/entry-percpu.h> #include <asm/lowcore.h> #include <asm/ctlreg.h> #include <asm/fpu.h> @@ -343,8 +344,7 @@ static void notrace s390_backup_mcck_info(struct pt_regs *regs) sie_page = container_of(sie_block, struct sie_page, sie_block); mcck_backup = &sie_page->mcck_info; - mcck_backup->mcic = get_lowcore()->mcck_interruption_code & - ~(MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE); + mcck_backup->mcic = get_lowcore()->mcck_interruption_code & ~MCCK_CODE_NO_GUEST; mcck_backup->ext_damage_code = get_lowcore()->external_damage_code; mcck_backup->failing_storage_address = get_lowcore()->failing_storage_address; } @@ -356,13 +356,12 @@ NOKPROBE_SYMBOL(s390_backup_mcck_info); #define ED_STP_ISLAND 6 /* External damage STP island check */ #define ED_STP_SYNC 7 /* External damage STP sync check */ -#define MCCK_CODE_NO_GUEST (MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE) - /* * machine check handler. */ void notrace s390_do_machine_check(struct pt_regs *regs) { + bool percpu_needs_fixup; static int ipd_count; static DEFINE_SPINLOCK(ipd_lock); static unsigned long long last_ipd; @@ -374,6 +373,7 @@ void notrace s390_do_machine_check(struct pt_regs *regs) unsigned long mcck_dam_code; int mcck_pending = 0; + percpu_entry(regs); irq_state = irqentry_nmi_enter(regs); if (user_mode(regs)) @@ -487,15 +487,17 @@ void notrace s390_do_machine_check(struct pt_regs *regs) mcck_dam_code = (mci.val & MCIC_SUBCLASS_MASK); if (test_cpu_flag(CIF_MCCK_GUEST) && (mcck_dam_code & MCCK_CODE_NO_GUEST) != mcck_dam_code) { - /* Set exit reason code for host's later handling */ - *((long *)(regs->gprs[15] + __SF_SIE_REASON)) = -EINTR; + /* Set sie return code for host's later handling */ + ((struct stack_frame *)regs->gprs[15])->sie_return = SIE64_RETURN_MCCK; } clear_cpu_flag(CIF_MCCK_GUEST); if (mcck_pending) schedule_mcck_handler(); + percpu_needs_fixup = percpu_code_check(regs); irqentry_nmi_exit(regs, irq_state); + percpu_exit(regs, percpu_needs_fixup); } NOKPROBE_SYMBOL(s390_do_machine_check); diff --git a/arch/s390/kernel/os_info.c b/arch/s390/kernel/os_info.c index 94fa44776d0c..4a30c10ae63b 100644 --- a/arch/s390/kernel/os_info.c +++ b/arch/s390/kernel/os_info.c @@ -154,7 +154,7 @@ static void os_info_old_init(void) goto fail; if (addr == 0 || addr % PAGE_SIZE) goto fail; - os_info_old = kzalloc(sizeof(*os_info_old), GFP_KERNEL); + os_info_old = kzalloc_obj(*os_info_old); if (!os_info_old) goto fail; if (copy_oldmem_kernel(os_info_old, addr, sizeof(*os_info_old))) diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index 408ab93112bf..69fa303cd479 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -15,6 +15,7 @@ #include <linux/init.h> #include <linux/miscdevice.h> #include <linux/perf_event.h> +#include <linux/nospec.h> #include <asm/cpu_mf.h> #include <asm/hwctrset.h> @@ -109,6 +110,7 @@ struct cpu_cf_ptr { static struct cpu_cf_root { /* Anchor to per CPU data */ refcount_t refcnt; /* Overall active events */ + unsigned int tskctx; /* Users tracking all CPUs (cpu == -1) */ struct cpu_cf_ptr __percpu *cfptr; } cpu_cf_root; @@ -117,13 +119,15 @@ static struct cpu_cf_root { /* Anchor to per CPU data */ * user space in task context with perf_event_open() and close() * system calls. * - * This mutex serializes functions cpum_cf_alloc_cpu() called at event - * initialization via cpumf_pmu_event_init() and function cpum_cf_free_cpu() - * called at event removal via call back function hw_perf_event_destroy() - * when the event is deleted. They are serialized to enforce correct - * bookkeeping of pointer and reference counts anchored by - * struct cpu_cf_root and the access to cpu_cf_root::refcnt and the - * per CPU pointers stored in cpu_cf_root::cfptr. + * This mutex serializes the allocation and removal of the per CPU counter + * data via cpum_cf_alloc_cpu() and cpum_cf_free_cpu(). They are called with + * this mutex held at event initialization via cpumf_pmu_event_init(), at + * event removal via call back function hw_perf_event_destroy() when the + * event is deleted, and from the CPU hotplug prepare/dead callbacks. The + * mutex enforces correct bookkeeping of pointer and reference counts + * anchored by struct cpu_cf_root and protects the access to + * cpu_cf_root::refcnt, cpu_cf_root::tskctx and the per CPU pointers + * stored in cpu_cf_root::cfptr. */ static DEFINE_MUTEX(pmc_reserve_mutex); @@ -166,12 +170,14 @@ static void cpum_cf_reset_cpu(void *flags) } /* Free per CPU data when the last event is removed. */ -static void cpum_cf_free_root(void) +static void cpum_cf_free_root(unsigned int num) { - if (!refcount_dec_and_test(&cpu_cf_root.refcnt)) + struct cpu_cf_ptr __percpu *p = cpu_cf_root.cfptr; + + if (!refcount_sub_and_test(num, &cpu_cf_root.refcnt)) return; - free_percpu(cpu_cf_root.cfptr); cpu_cf_root.cfptr = NULL; + free_percpu(p); irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT); on_each_cpu(cpum_cf_reset_cpu, NULL, 1); debug_sprintf_event(cf_dbg, 4, "%s root.refcnt %u cfptr %d\n", @@ -185,17 +191,17 @@ static void cpum_cf_free_root(void) * CPUs possible, which might be larger than the number of CPUs currently * online. */ -static int cpum_cf_alloc_root(void) +static int cpum_cf_alloc_root(unsigned int num) { int rc = 0; - if (refcount_inc_not_zero(&cpu_cf_root.refcnt)) + if (refcount_add_not_zero(num, &cpu_cf_root.refcnt)) return rc; /* The memory is already zeroed. */ cpu_cf_root.cfptr = alloc_percpu(struct cpu_cf_ptr); if (cpu_cf_root.cfptr) { - refcount_set(&cpu_cf_root.refcnt, 1); + refcount_set(&cpu_cf_root.refcnt, num); on_each_cpu(cpum_cf_reset_cpu, NULL, 1); irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT); } else { @@ -205,20 +211,23 @@ static int cpum_cf_alloc_root(void) return rc; } -/* Free CPU counter data structure for a PMU */ -static void cpum_cf_free_cpu(int cpu) +/* + * Remove num references to the CPU counter data structure of a PMU. + * Called with pmc_reserve_mutex held. + */ +static void cpum_cf_free_cpu(int cpu, unsigned int num) { struct cpu_cf_events *cpuhw; struct cpu_cf_ptr *p; - mutex_lock(&pmc_reserve_mutex); + lockdep_assert_held(&pmc_reserve_mutex); /* * When invoked via CPU hotplug handler, there might be no events * installed or that particular CPU might not have an * event installed. This anchor pointer can be NULL! */ if (!cpu_cf_root.cfptr) - goto out; + return; p = per_cpu_ptr(cpu_cf_root.cfptr, cpu); cpuhw = p->cpucf; /* @@ -226,41 +235,42 @@ static void cpum_cf_free_cpu(int cpu) * installed on that CPU, but on different CPUs. */ if (!cpuhw) - goto out; + return; - if (refcount_dec_and_test(&cpuhw->refcnt)) { - kfree(cpuhw); + if (refcount_sub_and_test(num, &cpuhw->refcnt)) { p->cpucf = NULL; + kfree(cpuhw); } - cpum_cf_free_root(); -out: - mutex_unlock(&pmc_reserve_mutex); + cpum_cf_free_root(num); } -/* Allocate CPU counter data structure for a PMU. Called under mutex lock. */ -static int cpum_cf_alloc_cpu(int cpu) +/* + * Add num references to the CPU counter data structure of a PMU and + * allocate it when necessary. Called with pmc_reserve_mutex held. + */ +static int cpum_cf_alloc_cpu(int cpu, unsigned int num) { struct cpu_cf_events *cpuhw; struct cpu_cf_ptr *p; int rc; - mutex_lock(&pmc_reserve_mutex); - rc = cpum_cf_alloc_root(); + lockdep_assert_held(&pmc_reserve_mutex); + rc = cpum_cf_alloc_root(num); if (rc) - goto unlock; + return rc; p = per_cpu_ptr(cpu_cf_root.cfptr, cpu); cpuhw = p->cpucf; if (!cpuhw) { - cpuhw = kzalloc(sizeof(*cpuhw), GFP_KERNEL); + cpuhw = kzalloc_obj(*cpuhw); if (cpuhw) { p->cpucf = cpuhw; - refcount_set(&cpuhw->refcnt, 1); + refcount_set(&cpuhw->refcnt, num); } else { rc = -ENOMEM; } } else { - refcount_inc(&cpuhw->refcnt); + refcount_add(num, &cpuhw->refcnt); } if (rc) { /* @@ -268,10 +278,8 @@ static int cpum_cf_alloc_cpu(int cpu) * cpu_cf_event in not created, its destroy() function is not * invoked. Adjust the reference counter for the anchor. */ - cpum_cf_free_root(); + cpum_cf_free_root(num); } -unlock: - mutex_unlock(&pmc_reserve_mutex); return rc; } @@ -283,39 +291,70 @@ unlock: * perf_event_open() with task context and /dev/hwctr interface. * If cpu is non-zero install event on this CPU only. This setup handles * perf_event_open() with CPU context. + * Users with cpu == -1 are counted in cpu_cf_root::tskctx. The CPU hotplug + * prepare and dead callbacks use this count to install and remove the per + * CPU counter data on a new or dying CPU. */ -static int cpum_cf_alloc(int cpu) +static int cpum_cf_alloc_cpuslocked(int cpu) { cpumask_var_t mask; int rc; + lockdep_assert_cpus_held(); if (cpu == -1) { if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) return -ENOMEM; + mutex_lock(&pmc_reserve_mutex); for_each_online_cpu(cpu) { - rc = cpum_cf_alloc_cpu(cpu); + rc = cpum_cf_alloc_cpu(cpu, 1); if (rc) { for_each_cpu(cpu, mask) - cpum_cf_free_cpu(cpu); + cpum_cf_free_cpu(cpu, 1); break; } cpumask_set_cpu(cpu, mask); } + if (!rc) + cpu_cf_root.tskctx++; + mutex_unlock(&pmc_reserve_mutex); free_cpumask_var(mask); } else { - rc = cpum_cf_alloc_cpu(cpu); + mutex_lock(&pmc_reserve_mutex); + rc = cpum_cf_alloc_cpu(cpu, 1); + mutex_unlock(&pmc_reserve_mutex); } return rc; } -static void cpum_cf_free(int cpu) +static int cpum_cf_alloc(int cpu) +{ + int rc; + + cpus_read_lock(); + rc = cpum_cf_alloc_cpuslocked(cpu); + cpus_read_unlock(); + return rc; +} + +static void cpum_cf_free_cpuslocked(int cpu) { + lockdep_assert_cpus_held(); + mutex_lock(&pmc_reserve_mutex); if (cpu == -1) { + cpu_cf_root.tskctx--; for_each_online_cpu(cpu) - cpum_cf_free_cpu(cpu); + cpum_cf_free_cpu(cpu, 1); } else { - cpum_cf_free_cpu(cpu); + cpum_cf_free_cpu(cpu, 1); } + mutex_unlock(&pmc_reserve_mutex); +} + +static void cpum_cf_free(int cpu) +{ + cpus_read_lock(); + cpum_cf_free_cpuslocked(cpu); + cpus_read_unlock(); } #define CF_DIAG_CTRSET_DEF 0xfeef /* Counter set header mark */ @@ -768,6 +807,7 @@ static int __hw_perf_event_init(struct perf_event *event, unsigned int type) if (!is_userspace_event(ev)) { if (ev >= ARRAY_SIZE(cpumf_generic_events_user)) return -EOPNOTSUPP; + ev = array_index_nospec(ev, ARRAY_SIZE(cpumf_generic_events_user)); ev = cpumf_generic_events_user[ev]; } } else if (!attr->exclude_kernel && attr->exclude_user) { @@ -778,6 +818,7 @@ static int __hw_perf_event_init(struct perf_event *event, unsigned int type) if (!is_userspace_event(ev)) { if (ev >= ARRAY_SIZE(cpumf_generic_events_basic)) return -EOPNOTSUPP; + ev = array_index_nospec(ev, ARRAY_SIZE(cpumf_generic_events_basic)); ev = cpumf_generic_events_basic[ev]; } } @@ -1087,53 +1128,67 @@ static refcount_t cfset_opencnt = REFCOUNT_INIT(0); /* Access count */ static DEFINE_MUTEX(cfset_ctrset_mutex); /* - * CPU hotplug handles only /dev/hwctr device. - * For perf_event_open() the CPU hotplug handling is done on kernel common - * code: + * CPU hotplug handling: + * + * cpum_cf_prepare_cpu() and cpum_cf_dead_cpu() run while the new or dying + * CPU is offline. They create and remove the per CPU counter data for all + * users tracking every CPU (cpu == -1), that is perf_event_open() events + * with task context and /dev/hwctr device sessions. Each such user holds + * one reference to the per CPU counter data of each CPU. Therefore install + * and remove one reference per user, tracked in cpu_cf_root::tskctx. This + * guarantees the per CPU counter data exists before the new CPU executes + * its first task and is removed only after the dying CPU is gone. + * + * cpum_cf_online_cpu() and cpum_cf_offline_cpu() run while the new or + * dying CPU is online. They handle only the counter set state of open + * /dev/hwctr device sessions on that CPU. For perf_event_open() events + * nothing is done: * - CPU add: Nothing is done since a file descriptor can not be created * and returned to the user. * - CPU delete: Handled by common code via pmu_disable(), pmu_stop() and - * pmu_delete(). The event itself is removed when the file descriptor is - * closed. + * pmu_delete(). During task exit processing of grouped perf events + * triggered by CPU hotplug processing, pmu_disable() is called as part + * of perf context removal process. The event itself is removed when the + * event file descriptor is closed. */ +static int cpum_cf_prepare_cpu(unsigned int cpu) +{ + int rc = 0; + + mutex_lock(&pmc_reserve_mutex); + if (cpu_cf_root.tskctx) + rc = cpum_cf_alloc_cpu(cpu, cpu_cf_root.tskctx); + mutex_unlock(&pmc_reserve_mutex); + return rc; +} + +static int cpum_cf_dead_cpu(unsigned int cpu) +{ + mutex_lock(&pmc_reserve_mutex); + if (cpu_cf_root.tskctx) + cpum_cf_free_cpu(cpu, cpu_cf_root.tskctx); + mutex_unlock(&pmc_reserve_mutex); + return 0; +} + static int cfset_online_cpu(unsigned int cpu); static int cpum_cf_online_cpu(unsigned int cpu) { - int rc = 0; - - /* - * Ignore notification for perf_event_open(). - * Handle only /dev/hwctr device sessions. - */ mutex_lock(&cfset_ctrset_mutex); - if (refcount_read(&cfset_opencnt)) { - rc = cpum_cf_alloc_cpu(cpu); - if (!rc) - cfset_online_cpu(cpu); - } + if (refcount_read(&cfset_opencnt)) + cfset_online_cpu(cpu); mutex_unlock(&cfset_ctrset_mutex); - return rc; + return 0; } static int cfset_offline_cpu(unsigned int cpu); static int cpum_cf_offline_cpu(unsigned int cpu) { - /* - * During task exit processing of grouped perf events triggered by CPU - * hotplug processing, pmu_disable() is called as part of perf context - * removal process. Therefore do not trigger event removal now for - * perf_event_open() created events. Perf common code triggers event - * destruction when the event file descriptor is closed. - * - * Handle only /dev/hwctr device sessions. - */ mutex_lock(&cfset_ctrset_mutex); - if (refcount_read(&cfset_opencnt)) { + if (refcount_read(&cfset_opencnt)) cfset_offline_cpu(cpu); - cpum_cf_free_cpu(cpu); - } mutex_unlock(&cfset_ctrset_mutex); return 0; } @@ -1180,7 +1235,7 @@ static void cpumf_measurement_alert(struct ext_code ext_code, static int cfset_init(void); static int __init cpumf_pmu_init(void) { - int rc; + int state, rc; /* Extract counter measurement facility information */ if (!cpum_cf_avail() || qctri(&cpumf_ctr_info)) @@ -1222,11 +1277,24 @@ static int __init cpumf_pmu_init(void) cfset_init(); } + rc = cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, + "perf/s390/cf:prepare", + cpum_cf_prepare_cpu, cpum_cf_dead_cpu); + if (rc < 0) + goto out3; + state = rc; + rc = cpuhp_setup_state(CPUHP_AP_PERF_S390_CF_ONLINE, "perf/s390/cf:online", cpum_cf_online_cpu, cpum_cf_offline_cpu); - return rc; + if (rc < 0) + goto out4; + return 0; +out4: + cpuhp_remove_state(state); +out3: + perf_pmu_unregister(&cpumf_pmu); out2: debug_unregister_view(cf_dbg, &debug_sprintf_view); debug_unregister(cf_dbg); @@ -1382,6 +1450,7 @@ static void cfset_all_stop(struct cfset_request *req) */ static int cfset_release(struct inode *inode, struct file *file) { + cpus_read_lock(); mutex_lock(&cfset_ctrset_mutex); /* Open followed by close/exit has no private_data */ if (file->private_data) { @@ -1392,9 +1461,10 @@ static int cfset_release(struct inode *inode, struct file *file) } if (refcount_dec_and_test(&cfset_opencnt)) { /* Last close */ on_each_cpu(cfset_release_cpu, NULL, 1); - cpum_cf_free(-1); + cpum_cf_free_cpuslocked(-1); } mutex_unlock(&cfset_ctrset_mutex); + cpus_read_unlock(); return 0; } @@ -1413,15 +1483,17 @@ static int cfset_open(struct inode *inode, struct file *file) return -EPERM; file->private_data = NULL; + cpus_read_lock(); mutex_lock(&cfset_ctrset_mutex); if (!refcount_inc_not_zero(&cfset_opencnt)) { /* First open */ - rc = cpum_cf_alloc(-1); + rc = cpum_cf_alloc_cpuslocked(-1); if (!rc) { cfset_session_init(); refcount_set(&cfset_opencnt, 1); } } mutex_unlock(&cfset_ctrset_mutex); + cpus_read_unlock(); /* nonseekable_open() never fails */ return rc ?: nonseekable_open(inode, file); @@ -1493,7 +1565,6 @@ static int cfset_all_copy(unsigned long arg, cpumask_t *mask) goto out; } uptr += sizeof(struct s390_ctrset_cpudata) + cpuhw->used; - cond_resched(); } cpus = cpumask_weight(mask); if (put_user(cpus, &ctrset_read->no_cpus)) @@ -1616,7 +1687,7 @@ static long cfset_ioctl_start(unsigned long arg, struct file *file) if (!start.counter_sets) return -EINVAL; /* No counter set at all? */ - preq = kzalloc(sizeof(*preq), GFP_KERNEL); + preq = kzalloc_obj(*preq); if (!preq) return -ENOMEM; cpumask_clear(&preq->mask); diff --git a/arch/s390/kernel/perf_cpum_cf_events.c b/arch/s390/kernel/perf_cpum_cf_events.c index 7ace1f9e4ccf..ad8c32f0aaed 100644 --- a/arch/s390/kernel/perf_cpum_cf_events.c +++ b/arch/s390/kernel/perf_cpum_cf_events.c @@ -976,7 +976,7 @@ static __init struct attribute **merge_attr(struct attribute **a, j++; j++; - new = kmalloc_array(j, sizeof(struct attribute *), GFP_KERNEL); + new = kmalloc_objs(struct attribute *, j); if (!new) return NULL; j = 0; diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index 459af23a47a5..7bfeb5208177 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -841,7 +841,7 @@ static bool is_callchain_event(struct perf_event *event) u64 sample_type = event->attr.sample_type; return sample_type & (PERF_SAMPLE_CALLCHAIN | PERF_SAMPLE_REGS_USER | - PERF_SAMPLE_STACK_USER); + PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_STACK_USER); } static int cpumsf_pmu_event_init(struct perf_event *event) @@ -1168,6 +1168,7 @@ static void hw_collect_samples(struct perf_event *event, unsigned long *sdbt, static void hw_perf_event_update(struct perf_event *event, int flush_all) { unsigned long long event_overflow, sampl_overflow, num_sdb; + struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf); struct hw_perf_event *hwc = &event->hw; union hws_trailer_header prev, new; struct hws_trailer_entry *te; @@ -1247,8 +1248,11 @@ static void hw_perf_event_update(struct perf_event *event, int flush_all) * are dropped. * Slightly increase the interval to avoid hitting this limit. */ - if (event_overflow) + if (event_overflow) { SAMPL_RATE(hwc) += DIV_ROUND_UP(SAMPL_RATE(hwc), 10); + if (SAMPL_RATE(hwc) > cpuhw->qsi.max_sampl_rate) + SAMPL_RATE(hwc) = cpuhw->qsi.max_sampl_rate; + } } static inline unsigned long aux_sdb_index(struct aux_buffer *aux, @@ -1609,7 +1613,7 @@ static void *aux_buffer_setup(struct perf_event *event, void **pages, } /* Allocate aux_buffer struct for the event */ - aux = kzalloc(sizeof(struct aux_buffer), GFP_KERNEL); + aux = kzalloc_obj(struct aux_buffer); if (!aux) goto no_aux; sfb = &aux->sfb; diff --git a/arch/s390/kernel/perf_pai.c b/arch/s390/kernel/perf_pai.c index 810f5b6c5e01..013c3dae21ec 100644 --- a/arch/s390/kernel/perf_pai.c +++ b/arch/s390/kernel/perf_pai.c @@ -67,6 +67,7 @@ struct pai_mapptr { static struct pai_root { /* Anchor to per CPU data */ refcount_t refcnt; /* Overall active events */ + atomic_t tskctx; /* Overall per-task events */ struct pai_mapptr __percpu *mapptr; } pai_root[PAI_PMU_MAX]; @@ -93,14 +94,15 @@ struct pai_pmu { /* Define PAI PMU characteristics */ static struct pai_pmu pai_pmu[]; /* Forward declaration */ /* Free per CPU data when the last event is removed. */ -static void pai_root_free(int idx) +static void pai_root_free(int idx, int tasks) { - if (refcount_dec_and_test(&pai_root[idx].refcnt)) { + if (refcount_sub_and_test(tasks, &pai_root[idx].refcnt)) { free_percpu(pai_root[idx].mapptr); pai_root[idx].mapptr = NULL; } - debug_sprintf_event(paidbg, 5, "%s root[%d].refcount %d\n", __func__, - idx, refcount_read(&pai_root[idx].refcnt)); + debug_sprintf_event(paidbg, 5, "%s root[%d].refcount %d tskctx %d\n", + __func__, idx, refcount_read(&pai_root[idx].refcnt), + atomic_read(&pai_root[idx].tskctx)); } /* @@ -137,40 +139,54 @@ static void pai_free(struct pai_mapptr *mp) mp->mapptr = NULL; } -/* Adjust usage counters and remove allocated memory when all users are - * gone. - */ -static void pai_event_destroy_cpu(struct perf_event *event, int cpu) +/* Called under mutex_lock */ +static void pai_event_destroy_cpu(int idx, int cpu, bool hotplug) { - int idx = PAI_PMU_IDX(event); - struct pai_mapptr *mp = per_cpu_ptr(pai_root[idx].mapptr, cpu); - struct pai_map *cpump = mp->mapptr; + struct pai_mapptr *mp; + struct pai_map *cpump; + int tasks = 1; - mutex_lock(&pai_reserve_mutex); - debug_sprintf_event(paidbg, 5, "%s event %#llx idx %d cpu %d users %d " - "refcnt %u\n", __func__, event->attr.config, idx, - event->cpu, cpump->active_events, - refcount_read(&cpump->refcnt)); - if (refcount_dec_and_test(&cpump->refcnt)) + /* Check reference count and return when all gone. + * 1. An event is installed on online CPU X. + * 2. CPU x is offlined and the per-CPU data is removed. + * 3. Event is destroyed via close system call. + */ + if (!refcount_read(&pai_root[idx].refcnt)) + return; /* No events at all */ + mp = per_cpu_ptr(pai_root[idx].mapptr, cpu); + if (!mp || !mp->mapptr) /* No events on that CPU */ + return; + + /* When hotplug is true, invocation is from CPU hotplug callback. + * Delete per-CPU resource and adjust refcnt when per-task events + * are currently active. This can be more than one. + * In this case adjust counters. + */ + if (hotplug) + tasks = atomic_read(&pai_root[idx].tskctx); + + cpump = mp->mapptr; + if (refcount_sub_and_test(tasks, &cpump->refcnt)) pai_free(mp); - pai_root_free(idx); - mutex_unlock(&pai_reserve_mutex); + pai_root_free(idx, tasks); } static void pai_event_destroy(struct perf_event *event) { - int cpu; + int cpu = 0, idx = PAI_PMU_IDX(event); free_page(PAI_SAVE_AREA(event)); + cpus_read_lock(); + mutex_lock(&pai_reserve_mutex); if (event->cpu == -1) { - struct cpumask *mask = PAI_CPU_MASK(event); - - for_each_cpu(cpu, mask) - pai_event_destroy_cpu(event, cpu); - kfree(mask); + atomic_dec(&pai_root[idx].tskctx); + for_each_online_cpu(cpu) + pai_event_destroy_cpu(idx, cpu, false); } else { - pai_event_destroy_cpu(event, event->cpu); + pai_event_destroy_cpu(idx, event->cpu, false); } + mutex_unlock(&pai_reserve_mutex); + cpus_read_unlock(); } static void paicrypt_event_destroy(struct perf_event *event) @@ -186,6 +202,13 @@ static u64 pai_getctr(unsigned long *page, int nr, unsigned long offset) return page[nr]; } +static void pai_setctr(unsigned long *page, int nr, unsigned long offset, u64 v) +{ + if (offset) + nr += offset / sizeof(*page); + page[nr] = v; +} + /* Read the counter values. Return value from location in CMP. For base * event xxx_ALL sum up all events. Returns counter value. */ @@ -227,32 +250,37 @@ static u64 paicrypt_getall(struct perf_event *event) return sum; } -/* Check concurrent access of counting and sampling for crypto events. - * This function is called in process context and it is save to block. - * When the event initialization functions fails, no other call back will - * be invoked. - * - * Allocate the memory for the event. - */ -static int pai_alloc_cpu(struct perf_event *event, int cpu) +/* Called under mutex_lock */ +static int pai_alloc_cpu(int idx, int cpu, bool hotplug) { - int rc, idx = PAI_PMU_IDX(event); struct pai_map *cpump = NULL; bool need_paiext_cb = false; struct pai_mapptr *mp; + int tasks = 1, rc = 0; + + /* When hotplug is true, invocation is from CPU hotplug callback. + * Allocate per-CPU resource when per-task events are currently active. + * This can be more than one. In this case adjust all reference + * counters. Otherwise return, this ensures memory is only allocated + * when needed. + */ + if (hotplug) { + tasks = atomic_read(&pai_root[idx].tskctx); + if (!tasks) + goto out; + } - mutex_lock(&pai_reserve_mutex); /* Allocate root node */ rc = pai_root_alloc(idx); if (rc) - goto unlock; + goto out; /* Allocate node for this event */ mp = per_cpu_ptr(pai_root[idx].mapptr, cpu); cpump = mp->mapptr; if (!cpump) { /* Paicrypt_map allocated? */ rc = -ENOMEM; - cpump = kzalloc(sizeof(*cpump), GFP_KERNEL); + cpump = kzalloc_obj(*cpump); if (!cpump) goto undo; /* Allocate memory for counter page and counter extraction. @@ -281,63 +309,75 @@ static int pai_alloc_cpu(struct perf_event *event, int cpu) cpump->paiext_cb = kzalloc(PAIE1_CB_SZ, GFP_KERNEL); need_paiext_cb = true; } - cpump->save = kvmalloc_array(pai_pmu[idx].num_avail + 1, - sizeof(struct pai_userdata), - GFP_KERNEL); + cpump->save = kvmalloc_objs(struct pai_userdata, + pai_pmu[idx].num_avail + 1); if (!cpump->area || !cpump->save || (need_paiext_cb && !cpump->paiext_cb)) { pai_free(mp); goto undo; } INIT_LIST_HEAD(&cpump->syswide_list); - refcount_set(&cpump->refcnt, 1); + refcount_set(&cpump->refcnt, tasks); rc = 0; } else { - refcount_inc(&cpump->refcnt); + refcount_add(tasks, &cpump->refcnt); } + /* If tasks is greater than 1, we are called from CPU hotplug path + * and need to adjust the pai_root[idx].refcnt by the number of + * per-process events. Function pai_root_alloc(idx) already + * incremented by one. Adjust for the rest. + */ + if (tasks > 1) + refcount_add(tasks - 1, &pai_root[idx].refcnt); undo: if (rc) { /* Error in allocation of event, decrement anchor. Since * the event in not created, its destroy() function is never * invoked. Adjust the reference counter for the anchor. + * The failure happened in the case of variable + * cpump == NULL branch above. The pai_root[XXX].refcnt has + * been incremented by one. Then the per-CPU allocation + * failed, so decrement it by one, regardless of tasks. */ - pai_root_free(idx); + pai_root_free(idx, 1); } -unlock: - mutex_unlock(&pai_reserve_mutex); +out: /* If rc is non-zero, no increment of counter/sampler was done. */ return rc; } +/* Check concurrent access of counting and sampling for PAI events. + * This function is called in process context and it is safe to block. + * When the event initialization functions fails, no other call back will + * be invoked. + * Called under mutex_lock. + */ static int pai_alloc(struct perf_event *event) { + int idx = PAI_PMU_IDX(event); struct cpumask *maskptr; int cpu, rc = -ENOMEM; - maskptr = kzalloc(sizeof(*maskptr), GFP_KERNEL); + maskptr = kzalloc_obj(*maskptr); if (!maskptr) goto out; for_each_online_cpu(cpu) { - rc = pai_alloc_cpu(event, cpu); + rc = pai_alloc_cpu(idx, cpu, false); if (rc) { for_each_cpu(cpu, maskptr) - pai_event_destroy_cpu(event, cpu); - kfree(maskptr); - goto out; + pai_event_destroy_cpu(idx, cpu, false); + goto undo; } cpumask_set_cpu(cpu, maskptr); } - /* - * On error all cpumask are freed and all events have been destroyed. - * Save of which CPUs data structures have been allocated for. - * Release them in pai_event_destroy call back function - * for this event. - */ - PAI_CPU_MASK(event) = maskptr; rc = 0; + /* Trace per-task events for CPU hotplug. */ + atomic_inc(&pai_root[idx].tskctx); +undo: + kfree(maskptr); out: return rc; } @@ -385,10 +425,14 @@ static int pai_event_init(struct perf_event *event, int idx) } } + cpus_read_lock(); + mutex_lock(&pai_reserve_mutex); if (event->cpu >= 0) - rc = pai_alloc_cpu(event, event->cpu); + rc = pai_alloc_cpu(idx, event->cpu, false); else rc = pai_alloc(event); + mutex_unlock(&pai_reserve_mutex); + cpus_read_unlock(); if (rc) { free_page(PAI_SAVE_AREA(event)); goto out; @@ -458,6 +502,7 @@ static void pai_start(struct perf_event *event, int flags, cpump->event = event; } } + event->hw.state &= ~PERF_HES_STOPPED; } static void paicrypt_start(struct perf_event *event, int flags) @@ -504,6 +549,13 @@ static void pai_stop(struct perf_event *event, int flags) struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); struct pai_map *cpump = mp->mapptr; + /* Cope with multiple invocations: + * 1. perf_event_throttle() --> PMU->stop() + * 2. task schedules out --> PMU->stop() + * Check for event already stopped. + */ + if (event->hw.state & PERF_HES_STOPPED) + return; if (!event->attr.sample_period) { /* Counting */ pai_pmu[idx].pmu->read(event); } else { /* Sampling */ @@ -552,6 +604,8 @@ static void paicrypt_del(struct perf_event *event, int flags) /* Create raw data and save it in buffer. Calculate the delta for each * counter between this invocation and the last invocation. * Returns number of bytes copied. + * After reading from PAI counter page, save the read value to the old + * page to calculate PAI counter deltas. * Saves only entries with positive counter difference of the form * 2 bytes: Number of counter * 8 bytes: Value of counter @@ -563,16 +617,22 @@ static size_t pai_copy(struct pai_userdata *userdata, unsigned long *page, int i, outidx = 0; for (i = 1; i <= pp->num_avail; i++) { - u64 val = 0, val_old = 0; + u64 val = 0, val_old = 0, val_k = 0, val_old_k = 0; if (!exclude_kernel) { - val += pai_getctr(page, i, pp->kernel_offset); - val_old += pai_getctr(page_old, i, pp->kernel_offset); + val_k = pai_getctr(page, i, pp->kernel_offset); + val_old_k = pai_getctr(page_old, i, pp->kernel_offset); + if (val_k != val_old_k) + pai_setctr(page_old, i, pp->kernel_offset, val_k); } if (!exclude_user) { - val += pai_getctr(page, i, 0); - val_old += pai_getctr(page_old, i, 0); + val = pai_getctr(page, i, 0); + val_old = pai_getctr(page_old, i, 0); + if (val != val_old) + pai_setctr(page_old, i, 0, val); } + val += val_k; + val_old += val_old_k; if (val >= val_old) val -= val_old; else @@ -603,8 +663,6 @@ static size_t pai_copy(struct pai_userdata *userdata, unsigned long *page, static int pai_push_sample(size_t rawsize, struct pai_map *cpump, struct perf_event *event) { - int idx = PAI_PMU_IDX(event); - struct pai_pmu *pp = &pai_pmu[idx]; struct perf_sample_data data; struct perf_raw_record raw; struct pt_regs regs; @@ -635,8 +693,6 @@ static int pai_push_sample(size_t rawsize, struct pai_map *cpump, overflow = perf_event_overflow(event, &data, ®s); perf_event_update_userpage(event); - /* Save crypto counter lowcore page after reading event data. */ - memcpy((void *)PAI_SAVE_AREA(event), cpump->area, pp->area_size); return overflow; } @@ -652,7 +708,7 @@ static void pai_have_sample(struct perf_event *event, struct pai_map *cpump) rawsize = pai_copy(cpump->save, cpump->area, pp, (unsigned long *)PAI_SAVE_AREA(event), event->attr.exclude_user, - event->attr.exclude_kernel); + !pp->kernel_offset ? true : event->attr.exclude_kernel); if (rawsize) /* No incremented counters */ pai_push_sample(rawsize, cpump, event); } @@ -662,9 +718,9 @@ static void pai_have_samples(int idx) { struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); struct pai_map *cpump = mp->mapptr; - struct perf_event *event; + struct perf_event *event, *e2; - list_for_each_entry(event, &cpump->syswide_list, hw.tp_list) + list_for_each_entry_safe(event, e2, &cpump->syswide_list, hw.tp_list) pai_have_sample(event, cpump); } @@ -681,6 +737,17 @@ static void paicrypt_sched_task(struct perf_event_pmu_context *pmu_ctx, pai_have_samples(PAI_PMU_CRYPTO); } +/* Prevent ioctl(fd, PERF_EVENT_IOC_PERIOD, ...) call. + * It sets perf_event::event_limit to a positive value and causes + * perf_event_overflow() to invoke pai_stop() call back function when + * perf_event::event_limit hits zero. This is not supported because the + * sample events CRYPTO_ALL and NNPA_ALL are always taken at schedule out + * of a task. + */ +static int pai_check_period(struct perf_event *event, u64 value) +{ + return -EINVAL; +} /* ============================= paiext ====================================*/ static void paiext_event_destroy(struct perf_event *event) @@ -794,6 +861,7 @@ static struct pmu paicrypt = { .stop = paicrypt_stop, .read = paicrypt_read, .sched_task = paicrypt_sched_task, + .check_period = pai_check_period, .attr_groups = paicrypt_attr_groups }; @@ -1005,6 +1073,7 @@ static struct pmu paiext = { .stop = paiext_stop, .read = paiext_read, .sched_task = paiext_sched_task, + .check_period = pai_check_period, .attr_groups = paiext_attr_groups, }; @@ -1070,7 +1139,7 @@ static struct attribute * __init attr_event_init_one(int num, { struct perf_pmu_events_attr *pa; - pa = kzalloc(sizeof(*pa), GFP_KERNEL); + pa = kzalloc_obj(*pa); if (!pa) return NULL; @@ -1089,7 +1158,7 @@ static struct attribute ** __init attr_event_init(struct pai_pmu *p) struct attribute **attrs; unsigned int i; - attrs = kmalloc_array(min_attr + 1, sizeof(*attrs), GFP_KERNEL | __GFP_ZERO); + attrs = kmalloc_objs(*attrs, min_attr + 1, GFP_KERNEL | __GFP_ZERO); if (!attrs) goto out; for (i = 0; i < min_attr; i++) { @@ -1208,23 +1277,61 @@ static int __init paipmu_setup(void) return install_ok; } +static int pai_online_cpu(unsigned int cpu) +{ + int rc; + + mutex_lock(&pai_reserve_mutex); + rc = pai_alloc_cpu(PAI_PMU_CRYPTO, cpu, true); + if (rc) + goto out; + rc = pai_alloc_cpu(PAI_PMU_EXT, cpu, true); + if (rc) + pai_event_destroy_cpu(PAI_PMU_CRYPTO, cpu, true); +out: + mutex_unlock(&pai_reserve_mutex); + return rc; +} + +static int pai_offline_cpu(unsigned int cpu) +{ + mutex_lock(&pai_reserve_mutex); + pai_event_destroy_cpu(PAI_PMU_CRYPTO, cpu, true); + pai_event_destroy_cpu(PAI_PMU_EXT, cpu, true); + mutex_unlock(&pai_reserve_mutex); + return 0; +} + static int __init pai_init(void) { + int state, rc; + /* Setup s390dbf facility */ - paidbg = debug_register("pai", 32, 256, 128); + paidbg = debug_register("pai", 1, 1, 128); if (!paidbg) { pr_err("Registration of s390dbf pai failed\n"); return -ENOMEM; } debug_register_view(paidbg, &debug_sprintf_view); - if (!paipmu_setup()) { - /* No PMU registration, no need for debug buffer */ - debug_unregister_view(paidbg, &debug_sprintf_view); - debug_unregister(paidbg); - return -ENODEV; - } + /* CPUHP_BP_PREPARE_DYN --> before CPU is brought online */ + state = cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "perf/pai:prepare", + pai_online_cpu, pai_offline_cpu); + rc = state < 0 ? state : 0; + if (rc < 0) + goto out_debug; + + rc = -ENODEV; + if (!paipmu_setup()) + goto out_cpuhp; return 0; + +out_cpuhp: + cpuhp_remove_state(state); +out_debug: + debug_unregister_view(paidbg, &debug_sprintf_view); + debug_unregister(paidbg); + return rc; } device_initcall(pai_init); diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 0df95dcb2101..416650ae4871 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -50,7 +50,7 @@ void ret_from_fork(void) asm("ret_from_fork"); void __ret_from_fork(struct task_struct *prev, struct pt_regs *regs) { - void (*func)(void *arg); + int (*func)(void *arg); schedule_tail(prev); @@ -203,9 +203,6 @@ unsigned long __get_wchan(struct task_struct *p) struct unwind_state state; unsigned long ip = 0; - if (!task_stack_page(p)) - return 0; - if (!try_get_task_stack(p)) return 0; diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index ceaa1726e328..125ca4c4e30c 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -749,7 +749,7 @@ static int s390_gs_cb_set(struct task_struct *target, if (!cpu_has_gs()) return -ENODEV; if (!target->thread.gs_cb) { - data = kzalloc(sizeof(*data), GFP_KERNEL); + data = kzalloc_obj(*data); if (!data) return -ENOMEM; } @@ -800,7 +800,7 @@ static int s390_gs_bc_set(struct task_struct *target, if (!cpu_has_gs()) return -ENODEV; if (!data) { - data = kzalloc(sizeof(*data), GFP_KERNEL); + data = kzalloc_obj(*data); if (!data) return -ENOMEM; target->thread.gs_bc_cb = data; @@ -861,7 +861,7 @@ static int s390_runtime_instr_set(struct task_struct *target, return -ENODEV; if (!target->thread.ri_cb) { - data = kzalloc(sizeof(*data), GFP_KERNEL); + data = kzalloc_obj(*data); if (!data) return -ENOMEM; } diff --git a/arch/s390/kernel/runtime_instr.c b/arch/s390/kernel/runtime_instr.c index 1788a5454b6f..928bff8fcd4a 100644 --- a/arch/s390/kernel/runtime_instr.c +++ b/arch/s390/kernel/runtime_instr.c @@ -83,7 +83,7 @@ SYSCALL_DEFINE2(s390_runtime_instr, int, command, int, signum) return -EINVAL; if (!current->thread.ri_cb) { - cb = kzalloc(sizeof(*cb), GFP_KERNEL); + cb = kzalloc_obj(*cb); if (!cb) return -ENOMEM; } else { diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index c1fe0b53c5ac..b60284328fe3 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -963,8 +963,6 @@ void __init setup_arch(char **cmdline_p) setup_uv(); dma_contiguous_reserve(ident_map_size); vmcp_cma_reserve(); - if (cpu_has_edat2()) - hugetlb_cma_reserve(PUD_SHIFT - PAGE_SHIFT); reserve_crashkernel(); #ifdef CONFIG_CRASH_DUMP diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index b7429f30afc1..32499cad86f0 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -54,7 +54,6 @@ #include <asm/debug.h> #include <asm/os_info.h> #include <asm/sigp.h> -#include <asm/idle.h> #include <asm/nmi.h> #include <asm/stacktrace.h> #include <asm/topology.h> @@ -660,23 +659,13 @@ int smp_cpu_get_polarization(int cpu) return per_cpu(pcpu_devices, cpu).polarization; } -void smp_cpu_set_capacity(int cpu, unsigned long val) -{ - per_cpu(pcpu_devices, cpu).capacity = val; -} - -unsigned long smp_cpu_get_capacity(int cpu) -{ - return per_cpu(pcpu_devices, cpu).capacity; -} - void smp_set_core_capacity(int cpu, unsigned long val) { int i; cpu = smp_get_base_cpu(cpu); for (i = cpu; (i <= cpu + smp_cpu_mtid) && (i < nr_cpu_ids); i++) - smp_cpu_set_capacity(i, val); + topology_set_cpu_scale(i, val); } int smp_cpu_get_cpu_address(int cpu) @@ -728,7 +717,7 @@ static int smp_add_core(struct sclp_core_entry *core, cpumask_t *avail, else pcpu->state = CPU_STATE_STANDBY; smp_cpu_set_polarization(cpu, POLARIZATION_UNKNOWN); - smp_cpu_set_capacity(cpu, CPU_CAPACITY_HIGH); + topology_set_cpu_scale(cpu, CPU_CAPACITY_HIGH); set_cpu_present(cpu, true); if (!early && arch_register_cpu(cpu)) set_cpu_present(cpu, false); @@ -910,7 +899,6 @@ int __cpu_disable(void) cregs[6].val &= ~0xff000000UL; /* disable all I/O interrupts */ cregs[14].val &= ~0x1f000000UL; /* disable most machine checks */ __local_ctl_load(0, 15, cregs); - clear_cpu_flag(CIF_NOHZ_DELAY); return 0; } @@ -969,7 +957,7 @@ void __init smp_prepare_boot_cpu(void) ipl_pcpu->state = CPU_STATE_CONFIGURED; lc->pcpu = (unsigned long)ipl_pcpu; smp_cpu_set_polarization(0, POLARIZATION_UNKNOWN); - smp_cpu_set_capacity(0, CPU_CAPACITY_HIGH); + topology_set_cpu_scale(0, CPU_CAPACITY_HIGH); } void __init smp_setup_processor_id(void) @@ -1040,6 +1028,7 @@ static ssize_t cpu_configure_store(struct device *dev, per_cpu(pcpu_devices, cpu + i).state = CPU_STATE_STANDBY; smp_cpu_set_polarization(cpu + i, POLARIZATION_UNKNOWN); + set_cpu_enabled(cpu + i, false); } topology_expect_change(); break; @@ -1055,6 +1044,7 @@ static ssize_t cpu_configure_store(struct device *dev, per_cpu(pcpu_devices, cpu + i).state = CPU_STATE_CONFIGURED; smp_cpu_set_polarization(cpu + i, POLARIZATION_UNKNOWN); + set_cpu_enabled(cpu + i, true); } topology_expect_change(); break; @@ -1085,31 +1075,6 @@ static struct attribute_group cpu_common_attr_group = { .attrs = cpu_common_attrs, }; -static struct attribute *cpu_online_attrs[] = { - &dev_attr_idle_count.attr, - &dev_attr_idle_time_us.attr, - NULL, -}; - -static struct attribute_group cpu_online_attr_group = { - .attrs = cpu_online_attrs, -}; - -static int smp_cpu_online(unsigned int cpu) -{ - struct cpu *c = per_cpu_ptr(&cpu_devices, cpu); - - return sysfs_create_group(&c->dev.kobj, &cpu_online_attr_group); -} - -static int smp_cpu_pre_down(unsigned int cpu) -{ - struct cpu *c = per_cpu_ptr(&cpu_devices, cpu); - - sysfs_remove_group(&c->dev.kobj, &cpu_online_attr_group); - return 0; -} - bool arch_cpu_is_hotpluggable(int cpu) { return !!cpu; @@ -1117,6 +1082,7 @@ bool arch_cpu_is_hotpluggable(int cpu) int arch_register_cpu(int cpu) { + struct pcpu *pcpu = per_cpu_ptr(&pcpu_devices, cpu); struct cpu *c = per_cpu_ptr(&cpu_devices, cpu); int rc; @@ -1130,6 +1096,8 @@ int arch_register_cpu(int cpu) rc = topology_cpu_init(c); if (rc) goto out_topology; + if (pcpu->state != CPU_STATE_CONFIGURED) + set_cpu_enabled(cpu, false); return 0; out_topology: @@ -1145,13 +1113,13 @@ int __ref smp_rescan_cpus(bool early) struct sclp_core_info *info; int nr; - info = kzalloc(sizeof(*info), GFP_KERNEL); + info = kzalloc_obj(*info); if (!info) return -ENOMEM; smp_get_core_info(info, 0); nr = __smp_rescan_cpus(info, early); kfree(info); - if (nr) + if (nr && !early) topology_schedule_update(); return 0; } @@ -1175,18 +1143,13 @@ static DEVICE_ATTR_WO(rescan); static int __init s390_smp_init(void) { struct device *dev_root; - int rc; + int rc = 0; dev_root = bus_get_dev_root(&cpu_subsys); if (dev_root) { rc = device_create_file(dev_root, &dev_attr_rescan); put_device(dev_root); - if (rc) - return rc; } - rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "s390/smp:online", - smp_cpu_online, smp_cpu_pre_down); - rc = rc <= 0 ? rc : 0; return rc; } subsys_initcall(s390_smp_init); diff --git a/arch/s390/kernel/stackprotector.c b/arch/s390/kernel/stackprotector.c index d4e40483f008..8bd3ecf9200a 100644 --- a/arch/s390/kernel/stackprotector.c +++ b/arch/s390/kernel/stackprotector.c @@ -5,6 +5,7 @@ #endif #include <linux/export.h> +#include <linux/hex.h> #include <linux/uaccess.h> #include <linux/printk.h> #include <asm/abs_lowcore.h> diff --git a/arch/s390/kernel/syscall.c b/arch/s390/kernel/syscall.c index 795b6cca74c9..bcc0b76bd868 100644 --- a/arch/s390/kernel/syscall.c +++ b/arch/s390/kernel/syscall.c @@ -13,6 +13,7 @@ */ #include <linux/cpufeature.h> +#include <linux/nospec.h> #include <linux/errno.h> #include <linux/sched.h> #include <linux/mm.h> @@ -92,12 +93,13 @@ SYSCALL_DEFINE0(ni_syscall) return -ENOSYS; } -void noinstr __do_syscall(struct pt_regs *regs, int per_trap) +void noinstr __do_syscall(struct pt_regs *regs, unsigned long flags) { unsigned long nr; + bool permit; + + enter_from_user_mode_randomize_stack(regs); - add_random_kstack_offset(); - enter_from_user_mode(regs); regs->psw = get_lowcore()->svc_old_psw; regs->int_code = get_lowcore()->svc_int_code; update_timer_sys(); @@ -105,7 +107,7 @@ void noinstr __do_syscall(struct pt_regs *regs, int per_trap) current->thread.last_break = regs->last_break; local_irq_enable(); regs->orig_gpr2 = regs->gprs[2]; - if (unlikely(per_trap)) + if (unlikely(flags & SYSCALL_FLAG_PER_TRAP)) set_thread_flag(TIF_PER_TRAP); regs->flags = 0; set_pt_regs_flag(regs, PIF_SYSCALL); @@ -120,7 +122,9 @@ void noinstr __do_syscall(struct pt_regs *regs, int per_trap) regs->psw.addr = current->restart_block.arch_data; current->restart_block.arch_data = 1; } - nr = syscall_enter_from_user_mode_work(regs, nr); + + permit = syscall_enter_from_user_mode_work(regs, &nr); + /* * In the s390 ptrace ABI, both the syscall number and the return value * use gpr2. However, userspace puts the syscall number either in the @@ -128,11 +132,13 @@ void noinstr __do_syscall(struct pt_regs *regs, int per_trap) * work, the ptrace code sets PIF_SYSCALL_RET_SET, which is checked here * and if set, the syscall will be skipped. */ - if (unlikely(test_and_clear_pt_regs_flag(regs, PIF_SYSCALL_RET_SET))) + if (unlikely(test_and_clear_pt_regs_flag(regs, PIF_SYSCALL_RET_SET) || !permit)) goto out; regs->gprs[2] = -ENOSYS; - if (likely(nr < NR_syscalls)) + if (likely(nr < NR_syscalls)) { + nr = array_index_nospec(nr, NR_syscalls); regs->gprs[2] = sys_call_table[nr](regs); + } out: syscall_exit_to_user_mode(regs); } diff --git a/arch/s390/kernel/syscalls/syscall.tbl b/arch/s390/kernel/syscalls/syscall.tbl index 417ed16b3c63..1b45e68a217b 100644 --- a/arch/s390/kernel/syscalls/syscall.tbl +++ b/arch/s390/kernel/syscalls/syscall.tbl @@ -397,3 +397,5 @@ 468 common file_getattr sys_file_getattr 469 common file_setattr sys_file_setattr 470 common listns sys_listns +471 common rseq_slice_yield sys_rseq_slice_yield +472 common fchroot sys_fchroot diff --git a/arch/s390/kernel/sysinfo.c b/arch/s390/kernel/sysinfo.c index 33ca3e47a0e6..45b4f448fe3d 100644 --- a/arch/s390/kernel/sysinfo.c +++ b/arch/s390/kernel/sysinfo.c @@ -325,6 +325,7 @@ int unregister_service_level(struct service_level *slr) EXPORT_SYMBOL(unregister_service_level); static void *service_level_start(struct seq_file *m, loff_t *pos) +__acquires_shared(service_level_sem) { down_read(&service_level_sem); return seq_list_start(&service_level_list, *pos); @@ -336,6 +337,7 @@ static void *service_level_next(struct seq_file *m, void *p, loff_t *pos) } static void service_level_stop(struct seq_file *m, void *p) +__releases_shared(service_level_sem) { up_read(&service_level_sem); } diff --git a/arch/s390/kernel/text_amode31.S b/arch/s390/kernel/text_amode31.S index 26f2981aa09e..f007d892d0c5 100644 --- a/arch/s390/kernel/text_amode31.S +++ b/arch/s390/kernel/text_amode31.S @@ -5,6 +5,7 @@ * Copyright IBM Corp. 2019 */ +#include <linux/cfi_types.h> #include <linux/linkage.h> #include <asm/asm-extable.h> #include <asm/errno.h> @@ -26,7 +27,7 @@ /* * int _diag14_amode31(unsigned long rx, unsigned long ry1, unsigned long subcode) */ -SYM_FUNC_START(_diag14_amode31) +SYM_TYPED_FUNC_START(_diag14_amode31) lgr %r1,%r2 lgr %r2,%r3 lgr %r3,%r4 @@ -46,7 +47,7 @@ SYM_FUNC_END(_diag14_amode31) /* * int _diag210_amode31(struct diag210 *addr) */ -SYM_FUNC_START(_diag210_amode31) +SYM_TYPED_FUNC_START(_diag210_amode31) lgr %r1,%r2 lhi %r2,-1 sam31 @@ -64,7 +65,7 @@ SYM_FUNC_END(_diag210_amode31) /* * int diag8c(struct diag8c *addr, struct ccw_dev_id *devno, size_t len) */ -SYM_FUNC_START(_diag8c_amode31) +SYM_TYPED_FUNC_START(_diag8c_amode31) llgf %r3,0(%r3) sam31 diag %r2,%r4,0x8c @@ -77,7 +78,7 @@ SYM_FUNC_END(_diag8c_amode31) /* * int _diag26c_amode31(void *req, void *resp, enum diag26c_sc subcode) */ -SYM_FUNC_START(_diag26c_amode31) +SYM_TYPED_FUNC_START(_diag26c_amode31) lghi %r5,-EOPNOTSUPP sam31 diag %r2,%r4,0x26c @@ -91,7 +92,7 @@ SYM_FUNC_END(_diag26c_amode31) /* * void _diag0c_amode31(unsigned long rx) */ -SYM_FUNC_START(_diag0c_amode31) +SYM_TYPED_FUNC_START(_diag0c_amode31) sam31 diag %r2,%r2,0x0c sam64 @@ -103,7 +104,7 @@ SYM_FUNC_END(_diag0c_amode31) * * Calls diag 308 subcode 1 and continues execution */ -SYM_FUNC_START(_diag308_reset_amode31) +SYM_TYPED_FUNC_START(_diag308_reset_amode31) larl %r4,ctlregs # Save control registers stctg %c0,%c15,0(%r4) lg %r2,0(%r4) # Disable lowcore protection diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index bd0df61d1907..2b989bebd220 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -65,6 +65,7 @@ ATOMIC_NOTIFIER_HEAD(s390_epoch_delta_notifier); EXPORT_SYMBOL(s390_epoch_delta_notifier); unsigned char ptff_function_mask[16]; +EXPORT_SYMBOL(ptff_function_mask); static unsigned long lpar_offset; static unsigned long initial_leap_seconds; diff --git a/arch/s390/kernel/topology.c b/arch/s390/kernel/topology.c index 1913a5566ac2..42fc0294f543 100644 --- a/arch/s390/kernel/topology.c +++ b/arch/s390/kernel/topology.c @@ -147,7 +147,7 @@ static void add_cpus_to_mask(struct topology_core *tl_core, cpumask_set_cpu(cpu, &book->mask); cpumask_set_cpu(cpu, &socket->mask); smp_cpu_set_polarization(cpu, tl_core->pp); - smp_cpu_set_capacity(cpu, CPU_CAPACITY_HIGH); + topology_set_cpu_scale(cpu, CPU_CAPACITY_HIGH); } } } @@ -192,17 +192,21 @@ static void tl_to_masks(struct sysinfo_15_1_x *info) end = (union topology_entry *)((unsigned long)info + info->length); while (tle < end) { switch (tle->nl) { + /* + * Adjust drawer_id, book_id, and socked_id so they match the + * numbering scheme of e.g. the hardware management console. + */ case 3: drawer = drawer->next; - drawer->id = tle->container.id; + drawer->id = tle->container.id - 1; break; case 2: book = book->next; - book->id = tle->container.id; + book->id = tle->container.id - 1; break; case 1: socket = socket->next; - socket->id = tle->container.id; + socket->id = tle->container.id - 1; break; case 0: add_cpus_to_mask(&tle->cpu, drawer, book, socket); diff --git a/arch/s390/kernel/trace_clock.c b/arch/s390/kernel/trace_clock.c new file mode 100644 index 000000000000..9ca568058cbc --- /dev/null +++ b/arch/s390/kernel/trace_clock.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/trace_clock.h> +#include <linux/timex.h> +/* + * trace_clock_s390_tod(): trace clock based on the s390 TOD clock + * + * Unlike the other clocks, this is not in nanoseconds. + */ +u64 notrace trace_clock_s390_tod(void) +{ + return get_tod_clock(); +} diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index 19687dab32f7..b6ba4465f59d 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -9,7 +9,9 @@ * Copyright (C) 1991, 1992 Linus Torvalds */ +#include <linux/capability.h> #include <linux/cpufeature.h> +#include <linux/debugfs.h> #include <linux/kprobes.h> #include <linux/kdebug.h> #include <linux/randomize_kstack.h> @@ -23,6 +25,8 @@ #include <linux/cpu.h> #include <linux/entry-common.h> #include <linux/kmsan.h> +#include <linux/bug.h> +#include <asm/entry-percpu.h> #include <asm/asm-extable.h> #include <asm/irqflags.h> #include <asm/ptrace.h> @@ -31,6 +35,12 @@ #include <asm/fault.h> #include "entry.h" +struct pgm_stat { + unsigned int count[128]; +}; + +static DEFINE_PER_CPU_SHARED_ALIGNED(struct pgm_stat, pgm_stat); + static inline void __user *get_trap_ip(struct pt_regs *regs) { unsigned long address; @@ -220,11 +230,48 @@ static void space_switch_exception(struct pt_regs *regs) do_trap(regs, SIGILL, ILL_PRVOPC, "space switch event"); } +#if defined(CONFIG_BUG) && defined(CONFIG_CC_HAS_ASM_IMMEDIATE_STRINGS) + +void *__warn_args(struct arch_va_list *args, struct pt_regs *regs) +{ + struct stack_frame *stack_frame; + + /* + * Generate va_list from pt_regs. See ELF Application Binary Interface + * s390x Supplement documentation for details. + * + * - __overflow_arg_area needs to point to the parameter area, which + * is right above the standard stack frame (160 bytes) + * + * - __reg_save_area needs to point to a register save area where + * general registers (%r2 - %r6) can be found at offset 16. Which + * means that the gprs save area of pt_regs can be used + * + * - __gpr must be set to one, since the first parameter has been + * processed (pointer to bug_entry) + */ + stack_frame = (struct stack_frame *)regs->gprs[15]; + args->__overflow_arg_area = stack_frame + 1; + args->__reg_save_area = regs->gprs; + args->__gpr = 1; + return args; +} + +#endif /* CONFIG_BUG && CONFIG_CC_HAS_ASM_IMMEDIATE_STRINGS */ + static void monitor_event_exception(struct pt_regs *regs) { + enum bug_trap_type btt; + if (user_mode(regs)) return; - switch (report_bug(regs->psw.addr - (regs->int_code >> 16), regs)) { + if (regs->monitor_code == MONCODE_BUG_ARG) { + regs->psw.addr = regs->gprs[14]; + btt = report_bug_entry((struct bug_entry *)regs->gprs[2], regs); + } else { + btt = report_bug(regs->psw.addr - (regs->int_code >> 16), regs); + } + switch (btt) { case BUG_TRAP_TYPE_NONE: fixup_exception(regs); break; @@ -258,11 +305,12 @@ static void __init test_monitor_call(void) if (!IS_ENABLED(CONFIG_BUG)) return; asm_inline volatile( - " mc 0,0\n" + " mc %[monc](%%r0),0\n" "0: lhi %[val],0\n" "1:\n" EX_TABLE(0b, 1b) - : [val] "+d" (val)); + : [val] "+d" (val) + : [monc] "i" (MONCODE_BUG)); if (!val) panic("Monitor call doesn't work!\n"); } @@ -287,16 +335,23 @@ void __init trap_init(void) static void (*pgm_check_table[128])(struct pt_regs *regs); -void noinstr __do_pgm_check(struct pt_regs *regs) +void noinstr __do_pgm_check(struct pt_regs *regs, unsigned long flags) { struct lowcore *lc = get_lowcore(); + bool percpu_needs_fixup; irqentry_state_t state; + struct pgm_stat *stat; unsigned int trapnr; union teid teid; teid.val = lc->trans_exc_code; regs->int_code = lc->pgm_int_code; regs->int_parm_long = teid.val; + regs->monitor_code = lc->monitor_code; + + trapnr = regs->int_code & PGM_INT_CODE_MASK; + stat = this_cpu_ptr(&pgm_stat); + stat->count[trapnr]++; /* * In case of a guest fault, short-circuit the fault handler and return. * This way the sie64a() function will return 0; fault address and @@ -304,11 +359,12 @@ void noinstr __do_pgm_check(struct pt_regs *regs) * the fault number in current->thread.gmap_int_code. KVM will be * able to use this information to handle the fault. */ - if (test_pt_regs_flag(regs, PIF_GUEST_FAULT)) { + if (flags & PGM_FLAG_GUEST_FAULT) { current->thread.gmap_teid.val = regs->int_parm_long; current->thread.gmap_int_code = regs->int_code & 0xffff; return; } + percpu_entry(regs); state = irqentry_enter(regs); if (user_mode(regs)) { update_timer_sys(); @@ -340,13 +396,41 @@ void noinstr __do_pgm_check(struct pt_regs *regs) if (!irqs_disabled_flags(regs->psw.mask)) trace_hardirqs_on(); __arch_local_irq_ssm(regs->psw.mask & ~PSW_MASK_PER); - trapnr = regs->int_code & PGM_INT_CODE_MASK; if (trapnr) pgm_check_table[trapnr](regs); out: local_irq_disable(); + percpu_needs_fixup = percpu_code_check(regs); irqentry_exit(regs, state); + percpu_exit(regs, percpu_needs_fixup); +} + +static int pgm_check_stat_show(struct seq_file *p, void *v) +{ + int i, cpu; + + cpus_read_lock(); + seq_puts(p, " "); + for_each_online_cpu(cpu) + seq_printf(p, "CPU%-8d", cpu); + seq_putc(p, '\n'); + for (i = 0; i < 128; i++) { + seq_printf(p, "%02x: ", i); + for_each_online_cpu(cpu) + seq_printf(p, "%10u ", per_cpu(pgm_stat, cpu).count[i]); + seq_putc(p, '\n'); + } + cpus_read_unlock(); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(pgm_check_stat); + +static int __init debugfs_pgm_check_init(void) +{ + debugfs_create_file("exceptions", 0400, arch_debugfs_dir, NULL, &pgm_check_stat_fops); + return 0; } +late_initcall(debugfs_pgm_check_init); /* * The program check table contains exactly 128 (0x00-0x7f) entries. Each diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c index ed46950be86f..8ea9dd7704ff 100644 --- a/arch/s390/kernel/uv.c +++ b/arch/s390/kernel/uv.c @@ -16,6 +16,7 @@ #include <linux/swap.h> #include <linux/pagewalk.h> #include <linux/backing-dev.h> +#include <linux/vmalloc.h> #include <asm/facility.h> #include <asm/sections.h> #include <asm/uv.h> @@ -134,14 +135,15 @@ static int uv_destroy(unsigned long paddr) */ int uv_destroy_folio(struct folio *folio) { + unsigned long i; int rc; - /* Large folios cannot be secure */ - if (unlikely(folio_test_large(folio))) - return 0; - folio_get(folio); - rc = uv_destroy(folio_to_phys(folio)); + for (i = 0; i < (1 << folio_order(folio)); i++) { + rc = uv_destroy(folio_to_phys(folio) + i * PAGE_SIZE); + if (rc) + break; + } if (!rc) clear_bit(PG_arch_1, &folio->flags.f); folio_put(folio); @@ -183,14 +185,15 @@ EXPORT_SYMBOL_GPL(uv_convert_from_secure); */ int uv_convert_from_secure_folio(struct folio *folio) { + unsigned long i; int rc; - /* Large folios cannot be secure */ - if (unlikely(folio_test_large(folio))) - return 0; - folio_get(folio); - rc = uv_convert_from_secure(folio_to_phys(folio)); + for (i = 0; i < (1 << folio_order(folio)); i++) { + rc = uv_convert_from_secure(folio_to_phys(folio) + i * PAGE_SIZE); + if (rc) + break; + } if (!rc) clear_bit(PG_arch_1, &folio->flags.f); folio_put(folio); @@ -207,39 +210,70 @@ int uv_convert_from_secure_pte(pte_t pte) return uv_convert_from_secure_folio(pfn_folio(pte_pfn(pte))); } -/** - * should_export_before_import - Determine whether an export is needed - * before an import-like operation - * @uvcb: the Ultravisor control block of the UVC to be performed - * @mm: the mm of the process - * - * Returns whether an export is needed before every import-like operation. - * This is needed for shared pages, which don't trigger a secure storage - * exception when accessed from a different guest. - * - * Although considered as one, the Unpin Page UVC is not an actual import, - * so it is not affected. - * - * No export is needed also when there is only one protected VM, because the - * page cannot belong to the wrong VM in that case (there is no "other VM" - * it can belong to). - * - * Return: true if an export is needed before every import, otherwise false. - */ -static bool should_export_before_import(struct uv_cb_header *uvcb, struct mm_struct *mm) +static int uv_free_range_cb(pte_t *ptep, unsigned long addr, void *data) { + pte_t pte = ptep_get(ptep); + + if (!pte_present(pte)) + return 0; /* - * The misc feature indicates, among other things, that importing a - * shared page from a different protected VM will automatically also - * transfer its ownership. + * Note: do not update the pte here, since there is no code which + * accesses the memory range, besides bugs. The invalidation of ptes + * and TLB flushing is deferred like for regular vfree() calls. */ - if (uv_has_feature(BIT_UV_FEAT_MISC)) - return false; - if (uvcb->cmd == UVC_CMD_UNPIN_PAGE_SHARED) - return false; - return atomic_read(&mm->context.protected_count) > 1; + __free_page(pte_page(pte)); + return 0; +} + +void uv_free_stor_var(void *stor_var) +{ + unsigned long addr, size; + struct vm_struct *area; + + if (!stor_var) + return; + area = find_vm_area(stor_var); + if (WARN_ON_ONCE(!area || !(area->flags & VM_SPARSE))) + return; + size = get_vm_area_size(area); + addr = (unsigned long)area->addr; + apply_to_existing_page_range(&init_mm, addr, size, uv_free_range_cb, NULL); + free_vm_area(area); +} +EXPORT_SYMBOL_FOR_MODULES(uv_free_stor_var, "kvm"); + +static int uv_alloc_range_cb(pte_t *ptep, unsigned long addr, void *data) +{ + struct page *page; + pte_t pte; + + page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!page) + return -ENOMEM; + pte = __pte(page_to_phys(page) | pgprot_val(PAGE_KERNEL)); + set_pte(ptep, pte); + return 0; } +void *uv_alloc_stor_var(unsigned long size) +{ + struct vm_struct *area; + unsigned long addr; + + size = PAGE_ALIGN(size); + area = get_vm_area(size, VM_SPARSE); + if (!area) + return NULL; + addr = (unsigned long)area->addr; + if (apply_to_page_range(&init_mm, addr, size, uv_alloc_range_cb, NULL)) + goto out; + return area->addr; +out: + uv_free_stor_var(area->addr); + return NULL; +} +EXPORT_SYMBOL_FOR_MODULES(uv_alloc_stor_var, "kvm"); + /* * Calculate the expected ref_count for a folio that would otherwise have no * further pins. This was cribbed from similar functions in other places in @@ -279,7 +313,7 @@ static int expected_folio_refs(struct folio *folio) * (it's the same logic as split_folio()), and the folio must be * locked. */ -static int __make_folio_secure(struct folio *folio, struct uv_cb_header *uvcb) +int __make_folio_secure(struct folio *folio, struct uv_cb_header *uvcb) { int expected, cc = 0; @@ -309,20 +343,7 @@ static int __make_folio_secure(struct folio *folio, struct uv_cb_header *uvcb) return -EAGAIN; return uvcb->rc == 0x10a ? -ENXIO : -EINVAL; } - -static int make_folio_secure(struct mm_struct *mm, struct folio *folio, struct uv_cb_header *uvcb) -{ - int rc; - - if (!folio_trylock(folio)) - return -EAGAIN; - if (should_export_before_import(uvcb, mm)) - uv_convert_from_secure(folio_to_phys(folio)); - rc = __make_folio_secure(folio, uvcb); - folio_unlock(folio); - - return rc; -} +EXPORT_SYMBOL(__make_folio_secure); /** * s390_wiggle_split_folio() - try to drain extra references to a folio and @@ -337,7 +358,7 @@ static int make_folio_secure(struct mm_struct *mm, struct folio *folio, struct u * but another attempt can be made; * -EINVAL in case of other folio splitting errors. See split_folio(). */ -static int s390_wiggle_split_folio(struct mm_struct *mm, struct folio *folio) +int s390_wiggle_split_folio(struct mm_struct *mm, struct folio *folio) { int rc, tried_splits; @@ -409,56 +430,7 @@ static int s390_wiggle_split_folio(struct mm_struct *mm, struct folio *folio) } return -EAGAIN; } - -int make_hva_secure(struct mm_struct *mm, unsigned long hva, struct uv_cb_header *uvcb) -{ - struct vm_area_struct *vma; - struct folio_walk fw; - struct folio *folio; - int rc; - - mmap_read_lock(mm); - vma = vma_lookup(mm, hva); - if (!vma) { - mmap_read_unlock(mm); - return -EFAULT; - } - folio = folio_walk_start(&fw, vma, hva, 0); - if (!folio) { - mmap_read_unlock(mm); - return -ENXIO; - } - - folio_get(folio); - /* - * Secure pages cannot be huge and userspace should not combine both. - * In case userspace does it anyway this will result in an -EFAULT for - * the unpack. The guest is thus never reaching secure mode. - * If userspace plays dirty tricks and decides to map huge pages at a - * later point in time, it will receive a segmentation fault or - * KVM_RUN will return -EFAULT. - */ - if (folio_test_hugetlb(folio)) - rc = -EFAULT; - else if (folio_test_large(folio)) - rc = -E2BIG; - else if (!pte_write(fw.pte) || (pte_val(fw.pte) & _PAGE_INVALID)) - rc = -ENXIO; - else - rc = make_folio_secure(mm, folio, uvcb); - folio_walk_end(&fw, vma); - mmap_read_unlock(mm); - - if (rc == -E2BIG || rc == -EBUSY) { - rc = s390_wiggle_split_folio(mm, folio); - if (!rc) - rc = -EAGAIN; - } - folio_put(folio); - - return rc; -} -EXPORT_SYMBOL_GPL(make_hva_secure); +EXPORT_SYMBOL_GPL(s390_wiggle_split_folio); /* * To be called with the folio locked or with an extra reference! This will @@ -470,21 +442,18 @@ int arch_make_folio_accessible(struct folio *folio) { int rc = 0; - /* Large folios cannot be secure */ - if (unlikely(folio_test_large(folio))) - return 0; - /* - * PG_arch_1 is used in 2 places: - * 1. for storage keys of hugetlb folios and KVM - * 2. As an indication that this small folio might be secure. This can - * overindicate, e.g. we set the bit before calling - * convert_to_secure. - * As secure pages are never large folios, both variants can co-exists. + * PG_arch_1 is used as an indication that this small folio might be + * secure. This can overindicate, e.g. we set the bit before calling + * convert_to_secure. */ if (!test_bit(PG_arch_1, &folio->flags.f)) return 0; + /* Large folios cannot be secure. */ + if (WARN_ON_ONCE(folio_test_large(folio))) + return -EFAULT; + rc = uv_pin_shared(folio_to_phys(folio)); if (!rc) { clear_bit(PG_arch_1, &folio->flags.f); @@ -856,7 +825,7 @@ static int find_secret_in_page(const u8 secret_id[UV_SECRET_ID_LEN], { u16 i; - for (i = 0; i < list->total_num_secrets; i++) { + for (i = 0; i < list->num_secr_stored; i++) { if (memcmp(secret_id, list->secrets[i].id, UV_SECRET_ID_LEN) == 0) { *secret = list->secrets[i].hdr; return 0; @@ -877,11 +846,14 @@ int uv_find_secret(const u8 secret_id[UV_SECRET_ID_LEN], struct uv_secret_list *list, struct uv_secret_list_item_hdr *secret) { - u16 start_idx = 0; + u16 start_idx; u16 list_rc; int ret; + list->next_secret_idx = 0; + do { + start_idx = list->next_secret_idx; uv_list_secrets(list, start_idx, &list_rc, NULL); if (list_rc != UVC_RC_EXECUTED && list_rc != UVC_RC_MORE_DATA) { if (list_rc == UVC_RC_INV_CMD) @@ -892,7 +864,6 @@ int uv_find_secret(const u8 secret_id[UV_SECRET_ID_LEN], ret = find_secret_in_page(secret_id, list, secret); if (ret == 0) return ret; - start_idx = list->next_secret_idx; } while (list_rc == UVC_RC_MORE_DATA && start_idx < list->next_secret_idx); return -ENOENT; @@ -927,7 +898,7 @@ int uv_retrieve_secret(u16 secret_idx, u8 *buf, size_t buf_size) .buf_size = buf_size, }; - uv_call_sched(0, (u64)&uvcb); + uv_call(0, (u64)&uvcb); switch (uvcb.header.rc) { case UVC_RC_EXECUTED: diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c index a27a90a199be..0fe616c49fcc 100644 --- a/arch/s390/kernel/vdso.c +++ b/arch/s390/kernel/vdso.c @@ -132,7 +132,7 @@ static struct page ** __init vdso_setup_pages(void *start, void *end) struct page **pagelist; int i; - pagelist = kcalloc(pages + 1, sizeof(struct page *), GFP_KERNEL); + pagelist = kzalloc_objs(struct page *, pages + 1); if (!pagelist) panic("%s: Cannot allocate page list for VDSO", __func__); for (i = 0; i < pages; i++) diff --git a/arch/s390/kernel/vdso/Makefile b/arch/s390/kernel/vdso/Makefile index fece5d975eaf..35c834b895ec 100644 --- a/arch/s390/kernel/vdso/Makefile +++ b/arch/s390/kernel/vdso/Makefile @@ -30,7 +30,8 @@ KBUILD_CFLAGS_VDSO := $(filter-out -fno-asynchronous-unwind-tables,$(KBUILD_CFLA KBUILD_CFLAGS_VDSO += -fPIC -fno-common -fno-builtin -fasynchronous-unwind-tables KBUILD_CFLAGS_VDSO += -fno-stack-protector $(DISABLE_KSTACK_ERASE) ldflags-y := -shared -soname=linux-vdso.so.1 \ - --hash-style=both --build-id=sha1 -T + --hash-style=both --build-id=sha1 \ + $(call ld-option, --eh-frame-hdr) -T $(targets:%=$(obj)/%.dbg): KBUILD_CFLAGS = $(KBUILD_CFLAGS_VDSO) $(targets:%=$(obj)/%.dbg): KBUILD_AFLAGS = $(KBUILD_AFLAGS_VDSO) diff --git a/arch/s390/kernel/vdso/getcpu.c b/arch/s390/kernel/vdso/getcpu.c index 5c5d4a848b76..1e17665616c5 100644 --- a/arch/s390/kernel/vdso/getcpu.c +++ b/arch/s390/kernel/vdso/getcpu.c @@ -2,11 +2,10 @@ /* Copyright IBM Corp. 2020 */ #include <linux/compiler.h> -#include <linux/getcpu.h> #include <asm/timex.h> #include "vdso.h" -int __s390_vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused) +int __s390_vdso_getcpu(unsigned *cpu, unsigned *node, void *unused) { union tod_clock clk; diff --git a/arch/s390/kernel/vdso/note.S b/arch/s390/kernel/vdso/note.S index db19d0680a0a..d215781f26b5 100644 --- a/arch/s390/kernel/vdso/note.S +++ b/arch/s390/kernel/vdso/note.S @@ -4,6 +4,7 @@ * Here we can supply some information useful to userland. */ +#include <linux/build-salt.h> #include <linux/uts.h> #include <linux/version.h> #include <linux/elfnote.h> @@ -11,3 +12,5 @@ ELFNOTE_START(Linux, 0, "a") .long LINUX_VERSION_CODE ELFNOTE_END + +BUILD_SALT diff --git a/arch/s390/kernel/vdso/vdso.h b/arch/s390/kernel/vdso/vdso.h index 8cff033dd854..1fe52a6f5a56 100644 --- a/arch/s390/kernel/vdso/vdso.h +++ b/arch/s390/kernel/vdso/vdso.h @@ -4,9 +4,7 @@ #include <vdso/datapage.h> -struct getcpu_cache; - -int __s390_vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused); +int __s390_vdso_getcpu(unsigned *cpu, unsigned *node, void *unused); int __s390_vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz); int __s390_vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts); int __s390_vdso_clock_getres(clockid_t clock, struct __kernel_timespec *ts); diff --git a/arch/s390/kernel/vdso/vdso.lds.S b/arch/s390/kernel/vdso/vdso.lds.S index 7bec4de0e8e0..841daeec4be2 100644 --- a/arch/s390/kernel/vdso/vdso.lds.S +++ b/arch/s390/kernel/vdso/vdso.lds.S @@ -82,12 +82,15 @@ SECTIONS * We must supply the ELF program headers explicitly to get just one * PT_LOAD segment, and set the flags explicitly to make segments read-only. */ +#define PF_R FLAGS(4) +#define PF_RX FLAGS(5) + PHDRS { - text PT_LOAD FILEHDR PHDRS FLAGS(5); /* PF_R|PF_X */ - dynamic PT_DYNAMIC FLAGS(4); /* PF_R */ - note PT_NOTE FLAGS(4); /* PF_R */ - eh_frame_hdr PT_GNU_EH_FRAME; + text PT_LOAD PF_RX FILEHDR PHDRS; + dynamic PT_DYNAMIC PF_R; + note PT_NOTE PF_R; + eh_frame_hdr PT_GNU_EH_FRAME PF_R; } /* diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index 53bcbb91bb9b..1f0e71c58eb9 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -159,6 +159,13 @@ SECTIONS } #endif + . = ALIGN(8); + .s390dbf_info : { + __s390dbf_info = .; + *(.s390dbf_info) + __s390dbf_info_end = .; + } + /* * Table with the patch locations to undo expolines */ @@ -221,6 +228,7 @@ SECTIONS /* Debugging sections. */ STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS /* diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c index 234a0ba30510..efcbf406f03e 100644 --- a/arch/s390/kernel/vtime.c +++ b/arch/s390/kernel/vtime.c @@ -17,6 +17,7 @@ #include <asm/vtimer.h> #include <asm/vtime.h> #include <asm/cpu_mf.h> +#include <asm/idle.h> #include <asm/smp.h> #include "entry.h" @@ -31,7 +32,7 @@ static atomic64_t virt_timer_elapsed; DEFINE_PER_CPU(u64, mt_cycles[8]); static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 }; static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 }; -static DEFINE_PER_CPU(u64, mt_scaling_jiffies); +static DEFINE_PER_CPU(unsigned long, mt_scaling_jiffies); static inline void set_vtimer(u64 expires) { @@ -48,8 +49,7 @@ static inline void set_vtimer(u64 expires) static inline int virt_timer_forward(u64 elapsed) { - BUG_ON(!irqs_disabled()); - + lockdep_assert_irqs_disabled(); if (list_empty(&virt_timer_list)) return 0; elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed); @@ -81,7 +81,7 @@ static void update_mt_scaling(void) memcpy(cycles_old, cycles_new, sizeof(u64) * (smp_cpu_mtid + 1)); } - __this_cpu_write(mt_scaling_jiffies, jiffies_64); + __this_cpu_write(mt_scaling_jiffies, jiffies); } static inline u64 update_tsk_timer(unsigned long *tsk_vtime, u64 new) @@ -111,6 +111,16 @@ static void account_system_index_scaled(struct task_struct *p, u64 cputime, account_system_index_time(p, cputime_to_nsecs(cputime), index); } +static inline void vtime_reset_last_update(struct lowcore *lc) +{ + asm volatile( + " stpt %0\n" /* Store current cpu timer value */ + " stckf %1" /* Store current tod clock value */ + : "=Q" (lc->last_update_timer), + "=Q" (lc->last_update_clock) + : : "cc"); +} + /* * Update process times based on virtual cpu times stored by entry.S * to the lowcore fields user_timer, system_timer & steal_clock. @@ -122,12 +132,9 @@ static int do_account_vtime(struct task_struct *tsk) timer = lc->last_update_timer; clock = lc->last_update_clock; - asm volatile( - " stpt %0\n" /* Store current cpu timer value */ - " stckf %1" /* Store current tod clock value */ - : "=Q" (lc->last_update_timer), - "=Q" (lc->last_update_clock) - : : "cc"); + + vtime_reset_last_update(lc); + clock = lc->last_update_clock - clock; timer -= lc->last_update_timer; @@ -137,23 +144,16 @@ static int do_account_vtime(struct task_struct *tsk) lc->system_timer += timer; /* Update MT utilization calculation */ - if (smp_cpu_mtid && - time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies))) + if (smp_cpu_mtid && time_after(jiffies, __this_cpu_read(mt_scaling_jiffies))) update_mt_scaling(); /* Calculate cputime delta */ - user = update_tsk_timer(&tsk->thread.user_timer, - READ_ONCE(lc->user_timer)); - guest = update_tsk_timer(&tsk->thread.guest_timer, - READ_ONCE(lc->guest_timer)); - system = update_tsk_timer(&tsk->thread.system_timer, - READ_ONCE(lc->system_timer)); - hardirq = update_tsk_timer(&tsk->thread.hardirq_timer, - READ_ONCE(lc->hardirq_timer)); - softirq = update_tsk_timer(&tsk->thread.softirq_timer, - READ_ONCE(lc->softirq_timer)); - lc->steal_timer += - clock - user - guest - system - hardirq - softirq; + user = update_tsk_timer(&tsk->thread.user_timer, lc->user_timer); + guest = update_tsk_timer(&tsk->thread.guest_timer, lc->guest_timer); + system = update_tsk_timer(&tsk->thread.system_timer, lc->system_timer); + hardirq = update_tsk_timer(&tsk->thread.hardirq_timer, lc->hardirq_timer); + softirq = update_tsk_timer(&tsk->thread.softirq_timer, lc->softirq_timer); + lc->steal_timer += clock - user - guest - system - hardirq - softirq; /* Push account value */ if (user) { @@ -225,10 +225,6 @@ static u64 vtime_delta(void) return timer - lc->last_update_timer; } -/* - * Update process times based on virtual cpu times stored by entry.S - * to the lowcore fields user_timer, system_timer & steal_clock. - */ void vtime_account_kernel(struct task_struct *tsk) { struct lowcore *lc = get_lowcore(); @@ -238,27 +234,17 @@ void vtime_account_kernel(struct task_struct *tsk) lc->guest_timer += delta; else lc->system_timer += delta; - - virt_timer_forward(delta); } EXPORT_SYMBOL_GPL(vtime_account_kernel); void vtime_account_softirq(struct task_struct *tsk) { - u64 delta = vtime_delta(); - - get_lowcore()->softirq_timer += delta; - - virt_timer_forward(delta); + get_lowcore()->softirq_timer += vtime_delta(); } void vtime_account_hardirq(struct task_struct *tsk) { - u64 delta = vtime_delta(); - - get_lowcore()->hardirq_timer += delta; - - virt_timer_forward(delta); + get_lowcore()->hardirq_timer += vtime_delta(); } /* diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig index f4ec8c1ce214..8d3ee17a1bcb 100644 --- a/arch/s390/kvm/Kconfig +++ b/arch/s390/kvm/Kconfig @@ -28,8 +28,9 @@ config KVM select HAVE_KVM_INVALID_WAKEUPS select HAVE_KVM_NO_POLL select KVM_VFIO - select MMU_NOTIFIER select VIRT_XFER_TO_GUEST_WORK + select KVM_MMU_LOCKLESS_AGING + select KVM_GENERIC_PRE_FAULT_MEMORY help Support hosting paravirtualized guest machines using the SIE virtualization capability on the mainframe. This should work diff --git a/arch/s390/kvm/Makefile b/arch/s390/kvm/Makefile index 9a723c48b05a..a4e3875b5bdd 100644 --- a/arch/s390/kvm/Makefile +++ b/arch/s390/kvm/Makefile @@ -3,12 +3,4 @@ # # Copyright IBM Corp. 2008 -include $(srctree)/virt/kvm/Makefile.kvm - -ccflags-y := -Ivirt/kvm -Iarch/s390/kvm - -kvm-y += kvm-s390.o intercept.o interrupt.o priv.o sigp.o -kvm-y += diag.o gaccess.o guestdbg.o vsie.o pv.o gmap-vsie.o - -kvm-$(CONFIG_VFIO_PCI_ZDEV_KVM) += pci.o -obj-$(CONFIG_KVM) += kvm.o +obj-$(CONFIG_KVM) += s390/ diff --git a/arch/s390/kvm/gmap-vsie.c b/arch/s390/kvm/gmap-vsie.c deleted file mode 100644 index 56ef153eb8fe..000000000000 --- a/arch/s390/kvm/gmap-vsie.c +++ /dev/null @@ -1,141 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Guest memory management for KVM/s390 nested VMs. - * - * Copyright IBM Corp. 2008, 2020, 2024 - * - * Author(s): Claudio Imbrenda <imbrenda@linux.ibm.com> - * Martin Schwidefsky <schwidefsky@de.ibm.com> - * David Hildenbrand <david@redhat.com> - * Janosch Frank <frankja@linux.vnet.ibm.com> - */ - -#include <linux/compiler.h> -#include <linux/kvm.h> -#include <linux/kvm_host.h> -#include <linux/pgtable.h> -#include <linux/pagemap.h> -#include <linux/mman.h> - -#include <asm/lowcore.h> -#include <asm/gmap.h> -#include <asm/uv.h> - -#include "kvm-s390.h" - -/** - * gmap_find_shadow - find a specific asce in the list of shadow tables - * @parent: pointer to the parent gmap - * @asce: ASCE for which the shadow table is created - * @edat_level: edat level to be used for the shadow translation - * - * Returns the pointer to a gmap if a shadow table with the given asce is - * already available, ERR_PTR(-EAGAIN) if another one is just being created, - * otherwise NULL - * - * Context: Called with parent->shadow_lock held - */ -static struct gmap *gmap_find_shadow(struct gmap *parent, unsigned long asce, int edat_level) -{ - struct gmap *sg; - - lockdep_assert_held(&parent->shadow_lock); - list_for_each_entry(sg, &parent->children, list) { - if (!gmap_shadow_valid(sg, asce, edat_level)) - continue; - if (!sg->initialized) - return ERR_PTR(-EAGAIN); - refcount_inc(&sg->ref_count); - return sg; - } - return NULL; -} - -/** - * gmap_shadow - create/find a shadow guest address space - * @parent: pointer to the parent gmap - * @asce: ASCE for which the shadow table is created - * @edat_level: edat level to be used for the shadow translation - * - * The pages of the top level page table referred by the asce parameter - * will be set to read-only and marked in the PGSTEs of the kvm process. - * The shadow table will be removed automatically on any change to the - * PTE mapping for the source table. - * - * Returns a guest address space structure, ERR_PTR(-ENOMEM) if out of memory, - * ERR_PTR(-EAGAIN) if the caller has to retry and ERR_PTR(-EFAULT) if the - * parent gmap table could not be protected. - */ -struct gmap *gmap_shadow(struct gmap *parent, unsigned long asce, int edat_level) -{ - struct gmap *sg, *new; - unsigned long limit; - int rc; - - if (KVM_BUG_ON(parent->mm->context.allow_gmap_hpage_1m, (struct kvm *)parent->private) || - KVM_BUG_ON(gmap_is_shadow(parent), (struct kvm *)parent->private)) - return ERR_PTR(-EFAULT); - spin_lock(&parent->shadow_lock); - sg = gmap_find_shadow(parent, asce, edat_level); - spin_unlock(&parent->shadow_lock); - if (sg) - return sg; - /* Create a new shadow gmap */ - limit = -1UL >> (33 - (((asce & _ASCE_TYPE_MASK) >> 2) * 11)); - if (asce & _ASCE_REAL_SPACE) - limit = -1UL; - new = gmap_alloc(limit); - if (!new) - return ERR_PTR(-ENOMEM); - new->mm = parent->mm; - new->parent = gmap_get(parent); - new->private = parent->private; - new->orig_asce = asce; - new->edat_level = edat_level; - new->initialized = false; - spin_lock(&parent->shadow_lock); - /* Recheck if another CPU created the same shadow */ - sg = gmap_find_shadow(parent, asce, edat_level); - if (sg) { - spin_unlock(&parent->shadow_lock); - gmap_free(new); - return sg; - } - if (asce & _ASCE_REAL_SPACE) { - /* only allow one real-space gmap shadow */ - list_for_each_entry(sg, &parent->children, list) { - if (sg->orig_asce & _ASCE_REAL_SPACE) { - spin_lock(&sg->guest_table_lock); - gmap_unshadow(sg); - spin_unlock(&sg->guest_table_lock); - list_del(&sg->list); - gmap_put(sg); - break; - } - } - } - refcount_set(&new->ref_count, 2); - list_add(&new->list, &parent->children); - if (asce & _ASCE_REAL_SPACE) { - /* nothing to protect, return right away */ - new->initialized = true; - spin_unlock(&parent->shadow_lock); - return new; - } - spin_unlock(&parent->shadow_lock); - /* protect after insertion, so it will get properly invalidated */ - mmap_read_lock(parent->mm); - rc = __kvm_s390_mprotect_many(parent, asce & _ASCE_ORIGIN, - ((asce & _ASCE_TABLE_LENGTH) + 1), - PROT_READ, GMAP_NOTIFY_SHADOW); - mmap_read_unlock(parent->mm); - spin_lock(&parent->shadow_lock); - new->initialized = true; - if (rc) { - list_del(&new->list); - gmap_free(new); - new = ERR_PTR(rc); - } - spin_unlock(&parent->shadow_lock); - return new; -} diff --git a/arch/s390/kvm/gmap/Makefile b/arch/s390/kvm/gmap/Makefile new file mode 100644 index 000000000000..dd4ef062c536 --- /dev/null +++ b/arch/s390/kvm/gmap/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 + +GMAP ?= ../gmap + +gmap-y += $(GMAP)/dat.o $(GMAP)/gmap.o $(GMAP)/faultin.o $(GMAP)/kvm_mmu.o diff --git a/arch/s390/kvm/gmap/dat.c b/arch/s390/kvm/gmap/dat.c new file mode 100644 index 000000000000..ff80d02c9f56 --- /dev/null +++ b/arch/s390/kvm/gmap/dat.c @@ -0,0 +1,1350 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KVM guest address space mapping code + * + * Copyright IBM Corp. 2007, 2020, 2024 + * Author(s): Claudio Imbrenda <imbrenda@linux.ibm.com> + * Martin Schwidefsky <schwidefsky@de.ibm.com> + * David Hildenbrand <david@redhat.com> + * Janosch Frank <frankja@linux.ibm.com> + */ + +#include <linux/kernel.h> +#include <linux/pagewalk.h> +#include <linux/swap.h> +#include <linux/smp.h> +#include <linux/spinlock.h> +#include <linux/slab.h> +#include <linux/swapops.h> +#include <linux/ksm.h> +#include <linux/mm.h> +#include <linux/mman.h> +#include <linux/pgtable.h> +#include <linux/kvm_types.h> +#include <linux/kvm_host.h> +#include <linux/pgalloc.h> + +#include <asm/page-states.h> +#include <asm/tlb.h> +#include "dat.h" + +int kvm_s390_mmu_cache_topup(struct kvm_s390_mmu_cache *mc) +{ + void *o; + + for ( ; mc->n_crsts < KVM_S390_MMU_CACHE_N_CRSTS; mc->n_crsts++) { + o = (void *)__get_free_pages(GFP_KERNEL_ACCOUNT | __GFP_COMP, CRST_ALLOC_ORDER); + if (!o) + return -ENOMEM; + mc->crsts[mc->n_crsts] = o; + } + for ( ; mc->n_pts < KVM_S390_MMU_CACHE_N_PTS; mc->n_pts++) { + o = (void *)__get_free_page(GFP_KERNEL_ACCOUNT); + if (!o) + return -ENOMEM; + mc->pts[mc->n_pts] = o; + } + for ( ; mc->n_rmaps < KVM_S390_MMU_CACHE_N_RMAPS; mc->n_rmaps++) { + o = kzalloc_obj(struct vsie_rmap, GFP_KERNEL_ACCOUNT); + if (!o) + return -ENOMEM; + mc->rmaps[mc->n_rmaps] = o; + } + return 0; +} + +static inline struct page_table *dat_alloc_pt_noinit(struct kvm_s390_mmu_cache *mc) +{ + struct page_table *res; + + res = kvm_s390_mmu_cache_alloc_pt(mc); + if (res) + __arch_set_page_dat(res, 1); + return res; +} + +static inline struct crst_table *dat_alloc_crst_noinit(struct kvm_s390_mmu_cache *mc) +{ + struct crst_table *res; + + res = kvm_s390_mmu_cache_alloc_crst(mc); + if (res) + __arch_set_page_dat(res, 1UL << CRST_ALLOC_ORDER); + return res; +} + +struct crst_table *dat_alloc_crst_sleepable(unsigned long init) +{ + struct page *page; + void *virt; + + page = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_COMP, CRST_ALLOC_ORDER); + if (!page) + return NULL; + virt = page_to_virt(page); + __arch_set_page_dat(virt, 1UL << CRST_ALLOC_ORDER); + crst_table_init(virt, init); + return virt; +} + +void dat_free_level(struct crst_table *table, bool owns_ptes) +{ + unsigned int i; + + for (i = 0; i < _CRST_ENTRIES; i++) { + if (table->crstes[i].h.fc || table->crstes[i].h.i) + continue; + if (!is_pmd(table->crstes[i])) + dat_free_level(dereference_crste(table->crstes[i]), owns_ptes); + else if (owns_ptes) + dat_free_pt(dereference_pmd(table->crstes[i].pmd)); + } + dat_free_crst(table); +} + +int dat_set_asce_limit(struct kvm_s390_mmu_cache *mc, union asce *asce, int newtype) +{ + struct crst_table *table; + union crste crste; + + while (asce->dt > newtype) { + table = dereference_asce(*asce); + crste = table->crstes[0]; + if (crste.h.fc) + return 0; + if (!crste.h.i) { + asce->rsto = crste.h.fc0.to; + dat_free_crst(table); + } else { + crste.h.tt--; + crst_table_init((void *)table, crste.val); + } + asce->dt--; + } + while (asce->dt < newtype) { + crste = _crste_fc0(asce->rsto, asce->dt + 1); + table = dat_alloc_crst_noinit(mc); + if (!table) + return -ENOMEM; + crst_table_init((void *)table, _CRSTE_HOLE(crste.h.tt).val); + table->crstes[0] = crste; + asce->rsto = __pa(table) >> PAGE_SHIFT; + asce->dt++; + } + return 0; +} + +/** + * dat_crstep_xchg_atomic() - Atomically exchange a gmap CRSTE with another. + * @crstep: Pointer to the CRST entry. + * @old: Expected old value. + * @new: Replacement entry. + * @gfn: The affected guest address. + * @asce: The asce of the address space. + * + * This function is needed to atomically exchange a CRSTE that potentially + * maps a prefix area, without having to invalidate it inbetween. + * + * Context: This function is assumed to be called with kvm->mmu_lock held. + * + * Return: %true if the exchange was successful. + */ +bool __must_check dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, + gfn_t gfn, union asce asce) +{ + if (old.h.i) + return arch_try_cmpxchg((long *)crstep, &old.val, new.val); + if (cpu_has_edat2()) + return crdte_crste(crstep, old, new, gfn, asce); + return cspg_crste(crstep, old, new); +} + +static void dat_set_storage_key_from_pgste(union pte pte, union pgste pgste) +{ + union skey nkey = { .acc = pgste.acc, .fp = pgste.fp }; + + page_set_storage_key(pte_origin(pte), nkey.skey, 0); +} + +static void dat_move_storage_key(union pte old, union pte new) +{ + page_set_storage_key(pte_origin(new), page_get_storage_key(pte_origin(old)), 1); +} + +static union pgste dat_save_storage_key_into_pgste(union pte pte, union pgste pgste) +{ + union skey skey; + + skey.skey = page_get_storage_key(pte_origin(pte)); + + pgste.acc = skey.acc; + pgste.fp = skey.fp; + pgste.gr |= skey.r; + pgste.gc |= skey.c; + + return pgste; +} + +union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, gfn_t gfn, + union asce asce, bool uses_skeys) +{ + union pte old = READ_ONCE(*ptep); + + /* Updating only the software bits while holding the pgste lock. */ + if (!((ptep->val ^ new.val) & ~_PAGE_SW_BITS)) { + WRITE_ONCE(ptep->swbyte, new.swbyte); + return pgste; + } + + if (!old.h.i) { + unsigned long opts = IPTE_GUEST_ASCE | (pgste.nodat ? IPTE_NODAT : 0); + + if (machine_has_tlb_guest()) + __ptep_ipte(gfn_to_gpa(gfn), (void *)ptep, opts, asce.val, IPTE_GLOBAL); + else + __ptep_ipte(gfn_to_gpa(gfn), (void *)ptep, 0, 0, IPTE_GLOBAL); + } + + if (uses_skeys) { + if (old.h.i && !new.h.i) + /* Invalid to valid: restore storage keys from PGSTE. */ + dat_set_storage_key_from_pgste(new, pgste); + else if (!old.h.i && new.h.i) + /* Valid to invalid: save storage keys to PGSTE. */ + pgste = dat_save_storage_key_into_pgste(old, pgste); + else if (!old.h.i && !new.h.i) + /* Valid to valid: move storage keys. */ + if (old.h.pfra != new.h.pfra) + dat_move_storage_key(old, new); + /* Invalid to invalid: nothing to do. */ + } + + WRITE_ONCE(*ptep, new); + return pgste; +} + +/* + * dat_split_ste() - Split a segment table entry into page table entries. + * + * Context: This function is assumed to be called with kvm->mmu_lock held. + * + * Return: 0 in case of success, -ENOMEM if running out of memory. + */ +static int dat_split_ste(struct kvm_s390_mmu_cache *mc, union pmd *pmdp, gfn_t gfn, + union asce asce, bool uses_skeys) +{ + union pgste pgste_init; + struct page_table *pt; + union pmd new, old; + union pte init; + int i; + + BUG_ON(!mc); + old = READ_ONCE(*pmdp); + + /* Already split, nothing to do. */ + if (!old.h.i && !old.h.fc) + return 0; + + pt = dat_alloc_pt_noinit(mc); + if (!pt) + return -ENOMEM; + new.val = virt_to_phys(pt); + + while (old.h.i || old.h.fc) { + init.val = pmd_origin_large(old); + init.h.p = old.h.p; + init.h.i = old.h.i; + init.s.d = old.s.fc1.d; + init.s.w = old.s.fc1.w; + init.s.y = old.s.fc1.y; + init.s.sd = old.s.fc1.sd; + init.s.pr = old.s.fc1.pr; + pgste_init.val = 0; + if (old.h.fc) { + for (i = 0; i < _PAGE_ENTRIES; i++) + pt->ptes[i].val = init.val | i * PAGE_SIZE; + /* No need to take locks as the page table is not installed yet. */ + pgste_init.prefix_notif = old.s.fc1.prefix_notif; + pgste_init.vsie_notif = old.s.fc1.vsie_notif; + pgste_init.vsie_gmem = old.s.fc1.vsie_notif; + pgste_init.pcl = uses_skeys && init.h.i; + dat_init_pgstes(pt, pgste_init.val); + } else { + dat_init_page_table(pt, init.val, 0); + } + + if (dat_pmdp_xchg_atomic(pmdp, old, new, gfn, asce)) { + if (!pgste_init.pcl) + return 0; + for (i = 0; i < _PAGE_ENTRIES; i++) { + union pgste pgste = pt->pgstes[i]; + + pgste = dat_save_storage_key_into_pgste(pt->ptes[i], pgste); + pgste_set_unlock(pt->ptes + i, pgste); + } + return 0; + } + old = READ_ONCE(*pmdp); + } + + dat_free_pt(pt); + return 0; +} + +/* + * dat_split_crste() - Split a crste into smaller crstes. + * + * Context: This function is assumed to be called with kvm->mmu_lock held. + * + * Return: %0 in case of success, %-ENOMEM if running out of memory. + */ +static int dat_split_crste(struct kvm_s390_mmu_cache *mc, union crste *crstep, + gfn_t gfn, union asce asce, bool uses_skeys) +{ + struct crst_table *table; + union crste old, new, init; + int i; + + old = READ_ONCE(*crstep); + if (is_pmd(old)) + return dat_split_ste(mc, &crstep->pmd, gfn, asce, uses_skeys); + + BUG_ON(!mc); + + /* Already split, nothing to do. */ + if (!old.h.i && !old.h.fc) + return 0; + + table = dat_alloc_crst_noinit(mc); + if (!table) + return -ENOMEM; + + new.val = virt_to_phys(table); + new.h.tt = old.h.tt; + new.h.fc0.tl = _REGION_ENTRY_LENGTH; + + while (old.h.i || old.h.fc) { + init = old; + init.h.tt--; + if (old.h.fc) { + for (i = 0; i < _CRST_ENTRIES; i++) + table->crstes[i].val = init.val | i * HPAGE_SIZE; + } else { + crst_table_init((void *)table, init.val); + } + if (dat_crstep_xchg_atomic(crstep, old, new, gfn, asce)) + return 0; + old = READ_ONCE(*crstep); + } + + dat_free_crst(table); + return 0; +} + +/** + * dat_entry_walk() - Walk the gmap page tables. + * @mc: Cache to use to allocate dat tables, if needed; can be NULL if neither + * %DAT_WALK_SPLIT or %DAT_WALK_ALLOC is specified in @flags. + * @gfn: Guest frame. + * @asce: The ASCE of the address space. + * @flags: Flags from WALK_* macros. + * @walk_level: Level to walk to, from LEVEL_* macros. + * @last: Will be filled the last visited non-pte DAT entry. + * @ptepp: Will be filled the last visited pte entry, if any, otherwise NULL. + * + * Returns a table entry pointer for the given guest address and @walk_level. + * + * The @flags have the following meanings: + * * %DAT_WALK_IGN_HOLES: consider holes as normal table entries + * * %DAT_WALK_ALLOC: allocate new tables to reach the requested level, if needed + * * %DAT_WALK_SPLIT: split existing large pages to reach the requested level, if needed + * * %DAT_WALK_LEAF: return successfully whenever a large page is encountered + * * %DAT_WALK_ANY: return successfully even if the requested level could not be reached + * * %DAT_WALK_CONTINUE: walk to the requested level with the specified flags, and then try to + * continue walking to ptes with only DAT_WALK_ANY + * * %DAT_WALK_USES_SKEYS: storage keys are in use + * + * Context: called with kvm->mmu_lock held. + * + * Return: + * * %PGM_ADDRESSING if the requested address lies outside memory + * * a PIC number if the requested address lies in a memory hole of type _DAT_TOKEN_PIC + * * %-EFAULT if the requested address lies inside a memory hole of a different type + * * %-EINVAL if the given ASCE is not compatible with the requested level + * * %-EFBIG if the requested level could not be reached because a larger frame was found + * * %-ENOENT if the requested level could not be reached for other reasons + * * %-ENOMEM if running out of memory while allocating or splitting a table + */ +int dat_entry_walk(struct kvm_s390_mmu_cache *mc, gfn_t gfn, union asce asce, int flags, + int walk_level, union crste **last, union pte **ptepp) +{ + union vaddress vaddr = { .addr = gfn_to_gpa(gfn) }; + bool continue_anyway = flags & DAT_WALK_CONTINUE; + bool uses_skeys = flags & DAT_WALK_USES_SKEYS; + bool ign_holes = flags & DAT_WALK_IGN_HOLES; + bool allocate = flags & DAT_WALK_ALLOC; + bool split = flags & DAT_WALK_SPLIT; + bool leaf = flags & DAT_WALK_LEAF; + bool any = flags & DAT_WALK_ANY; + struct page_table *pgtable; + struct crst_table *table; + union crste entry; + int rc; + + *last = NULL; + *ptepp = NULL; + if (WARN_ON_ONCE(unlikely(!asce.val))) + return -EINVAL; + if (WARN_ON_ONCE(unlikely(walk_level > asce.dt))) + return -EINVAL; + if (!asce_contains_gfn(asce, gfn)) + return PGM_ADDRESSING; + + table = dereference_asce(asce); + if (asce.dt >= ASCE_TYPE_REGION1) { + *last = table->crstes + vaddr.rfx; + entry = READ_ONCE(**last); + if (WARN_ON_ONCE(entry.h.tt != TABLE_TYPE_REGION1)) + return -EINVAL; + if (crste_hole(entry) && !ign_holes) + return entry.tok.type == _DAT_TOKEN_PIC ? entry.tok.par : -EFAULT; + if (walk_level == TABLE_TYPE_REGION1) + return 0; + if (entry.pgd.h.i) { + if (!allocate) + return any ? 0 : -ENOENT; + rc = dat_split_crste(mc, *last, gfn, asce, uses_skeys); + if (rc) + return rc; + entry = READ_ONCE(**last); + } + table = dereference_crste(entry.pgd); + } + + if (asce.dt >= ASCE_TYPE_REGION2) { + *last = table->crstes + vaddr.rsx; + entry = READ_ONCE(**last); + if (WARN_ON_ONCE(entry.h.tt != TABLE_TYPE_REGION2)) + return -EINVAL; + if (crste_hole(entry) && !ign_holes) + return entry.tok.type == _DAT_TOKEN_PIC ? entry.tok.par : -EFAULT; + if (walk_level == TABLE_TYPE_REGION2) + return 0; + if (entry.p4d.h.i) { + if (!allocate) + return any ? 0 : -ENOENT; + rc = dat_split_crste(mc, *last, gfn, asce, uses_skeys); + if (rc) + return rc; + entry = READ_ONCE(**last); + } + table = dereference_crste(entry.p4d); + } + + if (asce.dt >= ASCE_TYPE_REGION3) { + *last = table->crstes + vaddr.rtx; + entry = READ_ONCE(**last); + if (WARN_ON_ONCE(entry.h.tt != TABLE_TYPE_REGION3)) + return -EINVAL; + if (crste_hole(entry) && !ign_holes) + return entry.tok.type == _DAT_TOKEN_PIC ? entry.tok.par : -EFAULT; + if (walk_level == TABLE_TYPE_REGION3 && + continue_anyway && !entry.pud.h.fc && !entry.h.i) { + walk_level = TABLE_TYPE_PAGE_TABLE; + allocate = false; + } + if (walk_level == TABLE_TYPE_REGION3 || ((leaf || any) && entry.pud.h.fc)) + return 0; + if (entry.pud.h.i && !entry.pud.h.fc) { + if (!allocate) + return any ? 0 : -ENOENT; + rc = dat_split_crste(mc, *last, gfn, asce, uses_skeys); + if (rc) + return rc; + entry = READ_ONCE(**last); + } + if (walk_level <= TABLE_TYPE_SEGMENT && entry.pud.h.fc) { + if (!split) + return -EFBIG; + rc = dat_split_crste(mc, *last, gfn, asce, uses_skeys); + if (rc) + return rc; + entry = READ_ONCE(**last); + } + table = dereference_crste(entry.pud); + } + + *last = table->crstes + vaddr.sx; + entry = READ_ONCE(**last); + if (WARN_ON_ONCE(entry.h.tt != TABLE_TYPE_SEGMENT)) + return -EINVAL; + if (crste_hole(entry) && !ign_holes) + return entry.tok.type == _DAT_TOKEN_PIC ? entry.tok.par : -EFAULT; + if (continue_anyway && !entry.pmd.h.fc && !entry.h.i) { + walk_level = TABLE_TYPE_PAGE_TABLE; + allocate = false; + } + if (walk_level == TABLE_TYPE_SEGMENT || ((leaf || any) && entry.pmd.h.fc)) + return 0; + + if (entry.pmd.h.i && !entry.pmd.h.fc) { + if (!allocate) + return any ? 0 : -ENOENT; + rc = dat_split_ste(mc, &(*last)->pmd, gfn, asce, uses_skeys); + if (rc) + return rc; + entry = READ_ONCE(**last); + } + if (walk_level <= TABLE_TYPE_PAGE_TABLE && entry.pmd.h.fc) { + if (!split) + return -EFBIG; + rc = dat_split_ste(mc, &(*last)->pmd, gfn, asce, uses_skeys); + if (rc) + return rc; + entry = READ_ONCE(**last); + } + pgtable = dereference_pmd(entry.pmd); + *ptepp = pgtable->ptes + vaddr.px; + if (pte_hole(**ptepp) && !ign_holes) + return (*ptepp)->tok.type == _DAT_TOKEN_PIC ? (*ptepp)->tok.par : -EFAULT; + return 0; +} + +static long dat_pte_walk_range(gfn_t gfn, gfn_t end, struct page_table *table, struct dat_walk *w) +{ + unsigned int idx = gfn & (_PAGE_ENTRIES - 1); + long rc = 0; + + for ( ; gfn < end; idx++, gfn++) { + if (pte_hole(READ_ONCE(table->ptes[idx]))) { + if (!(w->flags & DAT_WALK_IGN_HOLES)) + return -EFAULT; + if (!(w->flags & DAT_WALK_ANY)) + continue; + } + + rc = w->ops->pte_entry(table->ptes + idx, gfn, gfn + 1, w); + if (rc) + break; + } + return rc; +} + +static long dat_crste_walk_range(gfn_t start, gfn_t end, struct crst_table *table, + struct dat_walk *walk) +{ + unsigned long idx, cur_shift, cur_size; + dat_walk_op the_op; + union crste crste; + gfn_t cur, next; + long rc = 0; + + cur_shift = 8 + table->crstes[0].h.tt * 11; + idx = (start >> cur_shift) & (_CRST_ENTRIES - 1); + cur_size = 1UL << cur_shift; + + for (cur = ALIGN_DOWN(start, cur_size); cur < end; idx++, cur = next) { + next = cur + cur_size; + walk->last = table->crstes + idx; + crste = READ_ONCE(*walk->last); + + if (crste_hole(crste)) { + if (!(walk->flags & DAT_WALK_IGN_HOLES)) + return -EFAULT; + if (!(walk->flags & DAT_WALK_ANY)) + continue; + } + + the_op = walk->ops->crste_ops[crste.h.tt]; + if (the_op) { + rc = the_op(walk->last, cur, next, walk); + crste = READ_ONCE(*walk->last); + } + if (rc) + break; + if (!crste.h.i && !crste.h.fc) { + if (!is_pmd(crste)) + rc = dat_crste_walk_range(max(start, cur), min(end, next), + _dereference_crste(crste), walk); + else if (walk->ops->pte_entry) + rc = dat_pte_walk_range(max(start, cur), min(end, next), + dereference_pmd(crste.pmd), walk); + if (rc) + break; + } + } + return rc; +} + +/** + * _dat_walk_gfn_range() - Walk DAT tables. + * @start: The first guest page frame to walk. + * @end: The guest page frame immediately after the last one to walk. + * @asce: The ASCE of the guest mapping. + * @ops: The gmap_walk_ops that will be used to perform the walk. + * @flags: Flags from WALK_* (currently only WALK_IGN_HOLES is supported). + * @priv: Will be passed as-is to the callbacks. + * + * Any callback returning non-zero causes the walk to stop immediately. + * + * Return: %-EINVAL in case of error, %-EFAULT if @start is too high for the + * given ASCE unless the DAT_WALK_IGN_HOLES flag is specified, + * otherwise it returns whatever the callbacks return. + */ +long _dat_walk_gfn_range(gfn_t start, gfn_t end, union asce asce, + const struct dat_walk_ops *ops, int flags, void *priv) +{ + struct crst_table *table = dereference_asce(asce); + struct dat_walk walk = { + .ops = ops, + .asce = asce, + .priv = priv, + .flags = flags, + .start = start, + .end = end, + }; + + if (WARN_ON_ONCE(unlikely(!asce.val))) + return -EINVAL; + if (!asce_contains_gfn(asce, start)) + return (flags & DAT_WALK_IGN_HOLES) ? 0 : -EFAULT; + + return dat_crste_walk_range(start, min(end, asce_end(asce)), table, &walk); +} + +#if KVM_S390_MANAGES_S390_GUEST +int dat_get_storage_key(union asce asce, gfn_t gfn, union skey *skey) +{ + union crste *crstep; + union pgste pgste; + union pte *ptep; + int rc; + +again: + skey->skey = 0; + rc = dat_entry_walk(NULL, gfn, asce, DAT_WALK_ANY, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); + if (rc) + return rc; + + if (!ptep) { + union crste crste = READ_ONCE(*crstep); + + if (!crste_leaf(crste) && !crste.h.i) + goto again; + if (!crste.s.fc1.pr) + return 0; + + skey->skey = page_get_storage_key(large_crste_to_phys(crste, gfn)); + return 0; + } + pgste = pgste_get_lock(ptep); + if (ptep->h.i) { + skey->acc = pgste.acc; + skey->fp = pgste.fp; + } else { + skey->skey = page_get_storage_key(pte_origin(*ptep)); + } + skey->r |= pgste.gr; + skey->c |= pgste.gc; + pgste_set_unlock(ptep, pgste); + return 0; +} + +static void dat_update_ptep_sd(union pgste old, union pgste pgste, union pte *ptep) +{ + if (pgste.acc != old.acc || pgste.fp != old.fp || pgste.gr != old.gr || pgste.gc != old.gc) + __atomic64_or(_PAGE_SD, &ptep->val); +} + +int dat_set_storage_key(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn, + union skey skey, bool nq) +{ + union pgste pgste, old; + union crste *crstep; + union pte *ptep; + int rc; + +again: + rc = dat_entry_walk(mc, gfn, asce, DAT_WALK_LEAF_ALLOC, TABLE_TYPE_PAGE_TABLE, + &crstep, &ptep); + if (rc) + return rc; + + if (!ptep) { + union crste crste = READ_ONCE(*crstep); + + /* A large page has been split concurrently, try again */ + if (!crste_leaf(crste)) + goto again; + + page_set_storage_key(large_crste_to_phys(crste, gfn), skey.skey, !nq); + return 0; + } + + old = pgste_get_lock(ptep); + pgste = old; + + pgste.acc = skey.acc; + pgste.fp = skey.fp; + pgste.gc = skey.c; + pgste.gr = skey.r; + + if (!ptep->h.i) { + union skey old_skey; + + old_skey.skey = page_get_storage_key(pte_origin(*ptep)); + pgste.hc |= old_skey.c; + pgste.hr |= old_skey.r; + old_skey.c = old.gc; + old_skey.r = old.gr; + skey.r = 0; + skey.c = 0; + page_set_storage_key(pte_origin(*ptep), skey.skey, !nq); + } + + dat_update_ptep_sd(old, pgste, ptep); + pgste_set_unlock(ptep, pgste); + return 0; +} + +static bool page_cond_set_storage_key(phys_addr_t paddr, union skey skey, union skey *oldkey, + bool nq, bool mr, bool mc) +{ + oldkey->skey = page_get_storage_key(paddr); + if (oldkey->acc == skey.acc && oldkey->fp == skey.fp && + (oldkey->r == skey.r || mr) && (oldkey->c == skey.c || mc)) + return false; + page_set_storage_key(paddr, skey.skey, !nq); + return true; +} + +int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gfn_t gfn, + union skey skey, union skey *oldkey, bool nq, bool mr, bool mc) +{ + union pgste pgste, old; + union crste *crstep; + union skey prev; + union pte *ptep; + int rc; + +again: + rc = dat_entry_walk(mmc, gfn, asce, DAT_WALK_LEAF_ALLOC, TABLE_TYPE_PAGE_TABLE, + &crstep, &ptep); + if (rc) + return rc; + + if (!ptep) { + union crste crste = READ_ONCE(*crstep); + + /* A large page has been split concurrently, try again */ + if (!crste_leaf(crste)) + goto again; + if (!oldkey) + oldkey = &prev; + + return page_cond_set_storage_key(large_crste_to_phys(crste, gfn), skey, oldkey, + nq, mr, mc); + } + + old = pgste_get_lock(ptep); + pgste = old; + + rc = 1; + pgste.acc = skey.acc; + pgste.fp = skey.fp; + pgste.gc = skey.c; + pgste.gr = skey.r; + prev.skey = 0; + + if (!ptep->h.i) { + rc = page_cond_set_storage_key(pte_origin(*ptep), skey, &prev, nq, mr, mc); + pgste.hc |= prev.c; + pgste.hr |= prev.r; + prev.c |= old.gc; + prev.r |= old.gr; + } else { + prev.acc = old.acc; + prev.fp = old.fp; + prev.c = old.gc; + prev.r = old.gr; + } + if (oldkey) + *oldkey = prev; + + dat_update_ptep_sd(old, pgste, ptep); + pgste_set_unlock(ptep, pgste); + return rc; +} + +int dat_reset_reference_bit(union asce asce, gfn_t gfn, union skey *skey) +{ + union pgste pgste, old; + union crste *crstep; + union pte *ptep; + int rc; + + skey->skey = 0; +again: + rc = dat_entry_walk(NULL, gfn, asce, DAT_WALK_ANY, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); + if (rc) + return rc; + + if (!ptep) { + union crste crste = READ_ONCE(*crstep); + + /* A large page has been split concurrently, try again */ + if (!crste_leaf(crste) && !crste.h.i) + goto again; + if (!crste.s.fc1.pr) + return 0; + skey->skey = page_reset_referenced(large_crste_to_phys(crste, gfn)) << 1; + return 0; + } + old = pgste_get_lock(ptep); + pgste = old; + + if (!ptep->h.i) { + skey->skey = page_reset_referenced(pte_origin(*ptep)) << 1; + pgste.hr = skey->r; + } + skey->r |= pgste.gr; + skey->c |= pgste.gc; + pgste.gr = 0; + + dat_update_ptep_sd(old, pgste, ptep); + pgste_set_unlock(ptep, pgste); + return 0; +} + +static long dat_reset_skeys_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) +{ + union pgste pgste; + + pgste = pgste_get_lock(ptep); + pgste.acc = 0; + pgste.fp = 0; + pgste.gr = 0; + pgste.gc = 0; + if (ptep->s.pr) + page_set_storage_key(pte_origin(*ptep), PAGE_DEFAULT_KEY, 1); + pgste_set_unlock(ptep, pgste); + + if (need_resched()) + return next; + return 0; +} + +static long dat_reset_skeys_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) +{ + phys_addr_t addr, end, origin = crste_origin_large(*crstep); + + if (!crstep->h.fc || !crstep->s.fc1.pr) + return 0; + + addr = ((max(gfn, walk->start) - gfn) << PAGE_SHIFT) + origin; + end = ((min(next, walk->end) - gfn) << PAGE_SHIFT) + origin; + while (ALIGN(addr + 1, _SEGMENT_SIZE) <= end) + addr = sske_frame(addr, PAGE_DEFAULT_KEY); + for ( ; addr < end; addr += PAGE_SIZE) + page_set_storage_key(addr, PAGE_DEFAULT_KEY, 1); + + if (need_resched()) + return next; + return 0; +} + +long dat_reset_skeys(union asce asce, gfn_t start) +{ + const struct dat_walk_ops ops = { + .pte_entry = dat_reset_skeys_pte, + .pmd_entry = dat_reset_skeys_crste, + .pud_entry = dat_reset_skeys_crste, + }; + + return _dat_walk_gfn_range(start, asce_end(asce), asce, &ops, DAT_WALK_IGN_HOLES, NULL); +} +#endif /* KVM_S390_MANAGES_S390_GUEST */ + +static long _dat_slot_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) +{ + union pte new_pte, pte = READ_ONCE(*ptep); + union pgste pgste; + + new_pte = walk->priv ? _PTE_EMPTY : _PTE_TOK(_DAT_TOKEN_PIC, PGM_ADDRESSING); + + /* Table entry already in the desired state. */ + if (pte.val == new_pte.val) + return 0; + + pgste = pgste_get_lock(ptep); + pgste = __dat_ptep_xchg(ptep, pgste, new_pte, gfn, walk->asce, false); + pgste.cmma_d = 0; + pgste_set_unlock(ptep, pgste); + + return 0; +} + +static long _dat_slot_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) +{ + union crste new_crste, crste = READ_ONCE(*crstep); + struct kvm_s390_mmu_cache *mc = walk->priv; + + new_crste = mc ? _CRSTE_EMPTY(crste.h.tt) : _CRSTE_HOLE(crste.h.tt); + + /* Table entry already in the desired state. */ + if (crste.val == new_crste.val) + return 0; + + /* This table entry needs to be updated. */ + if (walk->start <= gfn && walk->end >= next) { + if (!dat_crstep_xchg_atomic(crstep, crste, new_crste, gfn, walk->asce)) + return -EINVAL; + /* A lower level table was present, needs to be freed. */ + if (!crste.h.fc && !crste.h.i) { + if (is_pmd(crste)) + dat_free_pt(dereference_pmd(crste.pmd)); + else + dat_free_level(dereference_crste(crste), true); + } + return 0; + } + + /* A lower level table is present, things will handled there. */ + if (!crste.h.fc && !crste.h.i) + return 0; + /* Split (install a lower level table), and handle things there. */ + if (mc) + return dat_split_crste(mc, crstep, gfn, walk->asce, false); + /* A large page should never cross memslots boundaries */ + return -EINVAL; +} + +static const struct dat_walk_ops dat_slot_ops = { + .pte_entry = _dat_slot_pte, + .crste_ops = { _dat_slot_crste, _dat_slot_crste, _dat_slot_crste, _dat_slot_crste, }, +}; + +int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end) +{ + return _dat_walk_gfn_range(start, end, asce, &dat_slot_ops, + DAT_WALK_IGN_HOLES | DAT_WALK_ANY, mc); +} + +static void pgste_set_unlock_multiple(union pte *first, int n, union pgste *pgstes) +{ + int i; + + for (i = 0; i < n; i++) + pgste_set_unlock(first + i, pgstes[i]); +} + +static bool pgste_get_trylock_multiple(union pte *first, int n, union pgste *pgstes) +{ + int i; + + for (i = 0; i < n; i++) { + if (!pgste_get_trylock(first + i, pgstes + i)) + break; + } + if (i == n) + return true; + pgste_set_unlock_multiple(first, i, pgstes); + return false; +} + +unsigned long dat_get_ptval(struct page_table *table, struct ptval_param param) +{ + union pgste pgstes[4] = {}; + unsigned long res = 0; + int i, n; + + n = param.len + 1; + + while (!pgste_get_trylock_multiple(table->ptes + param.offset, n, pgstes)) + cpu_relax(); + + for (i = 0; i < n; i++) + res = res << 16 | pgstes[i].val16; + + pgste_set_unlock_multiple(table->ptes + param.offset, n, pgstes); + return res; +} + +void dat_set_ptval(struct page_table *table, struct ptval_param param, unsigned long val) +{ + union pgste pgstes[4] = {}; + int i, n; + + n = param.len + 1; + + while (!pgste_get_trylock_multiple(table->ptes + param.offset, n, pgstes)) + cpu_relax(); + + for (i = param.len; i >= 0; i--) { + pgstes[i].val16 = val; + val = val >> 16; + } + + pgste_set_unlock_multiple(table->ptes + param.offset, n, pgstes); +} + +static long _dat_test_young_pte(union pte *ptep, gfn_t start, gfn_t end, struct dat_walk *walk) +{ + return ptep->s.y; +} + +static long _dat_test_young_crste(union crste *crstep, gfn_t start, gfn_t end, + struct dat_walk *walk) +{ + return crstep->h.fc && crstep->s.fc1.y; +} + +static const struct dat_walk_ops test_age_ops = { + .pte_entry = _dat_test_young_pte, + .pmd_entry = _dat_test_young_crste, + .pud_entry = _dat_test_young_crste, +}; + +/** + * dat_test_age_gfn() - Test young. + * @asce: The ASCE whose address range is to be tested. + * @start: The first guest frame of the range to check. + * @end: The guest frame after the last in the range. + * + * Context: called by KVM common code with the kvm mmu write lock held. + * + * Return: %true if any page in the given range is young, otherwise %false. + */ +bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end) +{ + return _dat_walk_gfn_range(start, end, asce, &test_age_ops, 0, NULL) > 0; +} + +#if KVM_S390_MANAGES_S390_GUEST +static long dat_set_pn_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) +{ + union crste newcrste, oldcrste; + int *n = walk->priv; + + do { + oldcrste = READ_ONCE(*crstep); + if (!oldcrste.h.fc || oldcrste.h.i || oldcrste.h.p) + return 0; + if (oldcrste.s.fc1.prefix_notif) + break; + newcrste = oldcrste; + newcrste.s.fc1.prefix_notif = 1; + } while (!dat_crstep_xchg_atomic(crstep, oldcrste, newcrste, gfn, walk->asce)); + *n = 2; + return 0; +} + +static long dat_set_pn_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) +{ + int *n = walk->priv; + union pgste pgste; + + pgste = pgste_get_lock(ptep); + if (!ptep->h.i && !ptep->h.p) { + pgste.prefix_notif = 1; + *n += 1; + } + pgste_set_unlock(ptep, pgste); + return 0; +} + +int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn) +{ + static const struct dat_walk_ops ops = { + .pte_entry = dat_set_pn_pte, + .pmd_entry = dat_set_pn_crste, + .pud_entry = dat_set_pn_crste, + }; + + int n = 0; + + _dat_walk_gfn_range(gfn, gfn + 2, asce, &ops, DAT_WALK_IGN_HOLES, &n); + if (n != 2) + return -EAGAIN; + return 0; +} + +/** + * dat_perform_essa() - Perform ESSA actions on the PGSTE. + * @asce: The asce to operate on. + * @gfn: The guest page frame to operate on. + * @orc: The specific action to perform, see the ESSA_SET_* macros. + * @state: The storage attributes to be returned to the guest. + * @dirty: Returns whether the function dirtied a previously clean entry. + * + * Context: Called with kvm->mmu_lock held. + * + * Return: + * * %1 if the page state has been altered and the page is to be added to the CBRL + * * %0 if the page state has been altered, but the page is not to be added to the CBRL + * * %-1 if the page state has not been altered and the page is not to be added to the CBRL + */ +int dat_perform_essa(union asce asce, gfn_t gfn, int orc, union essa_state *state, bool *dirty) +{ + union crste *crstep; + union pgste pgste; + union pte *ptep; + int res = 0; + + if (dat_entry_walk(NULL, gfn, asce, 0, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep)) { + *state = (union essa_state) { .exception = 1 }; + return -1; + } + + pgste = pgste_get_lock(ptep); + + *state = (union essa_state) { + .content = (ptep->h.i << 1) + (ptep->h.i && pgste.zero), + .nodat = pgste.nodat, + .usage = pgste.usage, + }; + + switch (orc) { + case ESSA_GET_STATE: + res = -1; + break; + case ESSA_SET_STABLE: + pgste.usage = PGSTE_GPS_USAGE_STABLE; + pgste.nodat = 0; + break; + case ESSA_SET_UNUSED: + pgste.usage = PGSTE_GPS_USAGE_UNUSED; + if (ptep->h.i) + res = 1; + break; + case ESSA_SET_VOLATILE: + pgste.usage = PGSTE_GPS_USAGE_VOLATILE; + if (ptep->h.i) + res = 1; + break; + case ESSA_SET_POT_VOLATILE: + if (!ptep->h.i) { + pgste.usage = PGSTE_GPS_USAGE_POT_VOLATILE; + } else if (pgste.zero) { + pgste.usage = PGSTE_GPS_USAGE_VOLATILE; + } else if (!pgste.gc) { + pgste.usage = PGSTE_GPS_USAGE_VOLATILE; + res = 1; + } + break; + case ESSA_SET_STABLE_RESIDENT: + pgste.usage = PGSTE_GPS_USAGE_STABLE; + /* + * Since the resident state can go away any time after this + * call, we will not make this page resident. We can revisit + * this decision if a guest will ever start using this. + */ + break; + case ESSA_SET_STABLE_IF_RESIDENT: + if (!ptep->h.i) + pgste.usage = PGSTE_GPS_USAGE_STABLE; + break; + case ESSA_SET_STABLE_NODAT: + pgste.usage = PGSTE_GPS_USAGE_STABLE; + pgste.nodat = 1; + break; + default: + WARN_ONCE(1, "Invalid ORC!"); + res = -1; + break; + } + /* If we are discarding a page, set it to logical zero. */ + pgste.zero = res == 1; + if (orc > 0) { + *dirty = !pgste.cmma_d; + pgste.cmma_d = 1; + } + + pgste_set_unlock(ptep, pgste); + + return res; +} + +static long dat_reset_cmma_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) +{ + union pgste pgste; + + pgste = pgste_get_lock(ptep); + pgste.usage = 0; + pgste.nodat = 0; + pgste.cmma_d = 0; + pgste_set_unlock(ptep, pgste); + if (need_resched()) + return next; + return 0; +} + +long dat_reset_cmma(union asce asce, gfn_t start) +{ + const struct dat_walk_ops dat_reset_cmma_ops = { + .pte_entry = dat_reset_cmma_pte, + }; + + return _dat_walk_gfn_range(start, asce_end(asce), asce, &dat_reset_cmma_ops, + DAT_WALK_IGN_HOLES, NULL); +} + +struct dat_get_cmma_state { + gfn_t start; + gfn_t end; + unsigned int count; + u8 *values; + atomic64_t *remaining; +}; + +static long __dat_peek_cmma_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) +{ + struct dat_get_cmma_state *state = walk->priv; + union pgste pgste; + + pgste = pgste_get_lock(ptep); + state->values[gfn - walk->start] = pgste.usage | (pgste.nodat << 6); + pgste_set_unlock(ptep, pgste); + state->end = next; + + return 0; +} + +static long __dat_peek_cmma_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) +{ + struct dat_get_cmma_state *state = walk->priv; + + if (crstep->h.i) + state->end = min(walk->end, next); + return 0; +} + +int dat_peek_cmma(gfn_t start, union asce asce, unsigned int *count, u8 *values) +{ + const struct dat_walk_ops ops = { + .pte_entry = __dat_peek_cmma_pte, + .pmd_entry = __dat_peek_cmma_crste, + .pud_entry = __dat_peek_cmma_crste, + .p4d_entry = __dat_peek_cmma_crste, + .pgd_entry = __dat_peek_cmma_crste, + }; + struct dat_get_cmma_state state = { .values = values, }; + int rc; + + rc = _dat_walk_gfn_range(start, start + *count, asce, &ops, DAT_WALK_DEFAULT, &state); + *count = state.end >= start ? state.end - start : 0; + /* Return success if at least one value was saved, otherwise an error. */ + return (rc == -EFAULT && *count > 0) ? 0 : rc; +} + +static long __dat_get_cmma_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) +{ + struct dat_get_cmma_state *state = walk->priv; + union pgste pgste; + + if (state->start != -1) { + if ((gfn - state->end) > KVM_S390_MAX_BIT_DISTANCE) + return 1; + if (gfn - state->start >= state->count) + return 1; + } + + if (!READ_ONCE(*pgste_of(ptep)).cmma_d) + return 0; + + pgste = pgste_get_lock(ptep); + if (pgste.cmma_d) { + if (state->start == -1) + state->start = gfn; + pgste.cmma_d = 0; + atomic64_dec(state->remaining); + state->values[gfn - state->start] = pgste.usage | pgste.nodat << 6; + state->end = next; + } + pgste_set_unlock(ptep, pgste); + return 0; +} + +int dat_get_cmma(union asce asce, gfn_t *start, unsigned int *count, u8 *values, atomic64_t *rem) +{ + const struct dat_walk_ops ops = { .pte_entry = __dat_get_cmma_pte, }; + struct dat_get_cmma_state state = { + .remaining = rem, + .values = values, + .count = *count, + .start = -1, + }; + + _dat_walk_gfn_range(*start, asce_end(asce), asce, &ops, DAT_WALK_IGN_HOLES, &state); + /* If no dirty pages were found, wrap around and continue searching */ + if (*start && state.start == -1) + _dat_walk_gfn_range(0, *start, asce, &ops, DAT_WALK_IGN_HOLES, &state); + + if (state.start == -1) { + *count = 0; + } else { + *count = state.end - state.start; + *start = state.start; + } + + return 0; +} + +struct dat_set_cmma_state { + unsigned long mask; + const u8 *bits; +}; + +static long __dat_set_cmma_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) +{ + struct dat_set_cmma_state *state = walk->priv; + union pgste pgste, tmp; + + tmp.val = (state->bits[gfn - walk->start] << 24) & state->mask; + + pgste = pgste_get_lock(ptep); + pgste.usage = tmp.usage; + pgste.nodat = tmp.nodat; + pgste_set_unlock(ptep, pgste); + + return 0; +} + +/** + * dat_set_cmma_bits() - Set CMMA bits for a range of guest pages. + * @mc: Cache used for allocations. + * @asce: The ASCE of the guest. + * @gfn: The guest frame of the fist page whose CMMA bits are to set. + * @count: How many pages need to be processed. + * @mask: Which PGSTE bits should be set. + * @bits: Points to an array with the CMMA attributes. + * + * This function sets the CMMA attributes for the given pages. If the input + * buffer has zero length, no action is taken, otherwise the attributes are + * set and the mm->context.uses_cmm flag is set. + * + * Each byte in @bits contains new values for bits 32-39 of the PGSTE. + * Currently, only the fields NT and US are applied. + * + * Return: %0 in case of success, a negative error value otherwise. + */ +int dat_set_cmma_bits(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn, + unsigned long count, unsigned long mask, const uint8_t *bits) +{ + const struct dat_walk_ops ops = { .pte_entry = __dat_set_cmma_pte, }; + struct dat_set_cmma_state state = { .mask = mask, .bits = bits, }; + union crste *crstep; + union pte *ptep; + gfn_t cur; + int rc; + + for (cur = ALIGN_DOWN(gfn, _PAGE_ENTRIES); cur < gfn + count; cur += _PAGE_ENTRIES) { + rc = dat_entry_walk(mc, cur, asce, DAT_WALK_ALLOC, TABLE_TYPE_PAGE_TABLE, + &crstep, &ptep); + if (rc) + return rc; + } + return _dat_walk_gfn_range(gfn, gfn + count, asce, &ops, DAT_WALK_IGN_HOLES, &state); +} +#endif /* KVM_S390_MANAGES_S390_GUEST */ diff --git a/arch/s390/kvm/gmap/dat.h b/arch/s390/kvm/gmap/dat.h new file mode 100644 index 000000000000..90389d47ba4e --- /dev/null +++ b/arch/s390/kvm/gmap/dat.h @@ -0,0 +1,991 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * KVM guest address space mapping code + * + * Copyright IBM Corp. 2024, 2025 + * Author(s): Claudio Imbrenda <imbrenda@linux.ibm.com> + */ + +#ifndef ARCH_KVM_GMAP_DAT_H +#define ARCH_KVM_GMAP_DAT_H + +#include <linux/kvm_host.h> +#include <linux/radix-tree.h> +#include <linux/refcount.h> +#include <linux/io.h> +#include <linux/kvm_types.h> +#include <linux/pgalloc.h> +#include <asm/tlbflush.h> +#include <asm/dat-bits.h> + +/* + * Base address and length must be sent at the start of each block, therefore + * it's cheaper to send some clean data, as long as it's less than the size of + * two longs. + */ +#define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *)) +/* For consistency */ +#define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX) + +#define _ASCE(x) ((union asce) { .val = (x), }) +#define NULL_ASCE _ASCE(0) + +enum { + _DAT_TOKEN_NONE = 0, + _DAT_TOKEN_PIC, +}; + +#define _CRSTE_TOK(l, t, p) ((union crste) { \ + .tok.i = 1, \ + .tok.tt = (l), \ + .tok.type = (t), \ + .tok.par = (p) \ + }) +#define _CRSTE_PIC(l, p) _CRSTE_TOK(l, _DAT_TOKEN_PIC, p) + +#define _CRSTE_HOLE(l) _CRSTE_PIC(l, PGM_ADDRESSING) +#define _CRSTE_EMPTY(l) _CRSTE_TOK(l, _DAT_TOKEN_NONE, 0) + +#define _PMD_EMPTY _CRSTE_EMPTY(TABLE_TYPE_SEGMENT) + +#define _PTE_TOK(t, p) ((union pte) { .tok.i = 1, .tok.type = (t), .tok.par = (p) }) +#define _PTE_EMPTY _PTE_TOK(_DAT_TOKEN_NONE, 0) + +/* This fake table type is used for page table walks (both for normal page tables and vSIE) */ +#define TABLE_TYPE_PAGE_TABLE -1 + +enum dat_walk_flags { + DAT_WALK_USES_SKEYS = 0x40, + DAT_WALK_CONTINUE = 0x20, + DAT_WALK_IGN_HOLES = 0x10, + DAT_WALK_SPLIT = 0x08, + DAT_WALK_ALLOC = 0x04, + DAT_WALK_ANY = 0x02, + DAT_WALK_LEAF = 0x01, + DAT_WALK_DEFAULT = 0 +}; + +#define DAT_WALK_SPLIT_ALLOC (DAT_WALK_SPLIT | DAT_WALK_ALLOC) +#define DAT_WALK_ALLOC_CONTINUE (DAT_WALK_CONTINUE | DAT_WALK_ALLOC) +#define DAT_WALK_LEAF_ALLOC (DAT_WALK_LEAF | DAT_WALK_ALLOC) + +union pte { + unsigned long val; + union page_table_entry h; + struct { + unsigned long :56; /* Hardware bits */ + unsigned long u : 1; /* Page unused */ + unsigned long s : 1; /* Special */ + unsigned long w : 1; /* Writable */ + unsigned long r : 1; /* Readable */ + unsigned long d : 1; /* Dirty */ + unsigned long y : 1; /* Young */ + unsigned long sd: 1; /* Soft dirty */ + unsigned long pr: 1; /* Present */ + } s; + struct { + unsigned char hwbytes[7]; + unsigned char swbyte; + }; + union { + struct { + unsigned long type :16; /* Token type */ + unsigned long par :16; /* Token parameter */ + unsigned long :20; + unsigned long : 1; /* Must be 0 */ + unsigned long i : 1; /* Must be 1 */ + unsigned long : 2; + unsigned long : 7; + unsigned long pr : 1; /* Must be 0 */ + }; + struct { + unsigned long token:32; /* Token and parameter */ + unsigned long :32; + }; + } tok; +}; + +#define _SEGMENT_FR_MASK (_SEGMENT_MASK >> PAGE_SHIFT) +#define _REGION3_FR_MASK (_REGION3_MASK >> PAGE_SHIFT) +#define _PAGES_PER_SEGMENT _PAGE_ENTRIES +#define _PAGES_PER_REGION3 (_PAGES_PER_SEGMENT * _CRST_ENTRIES) + +/* Soft dirty, needed as macro for atomic operations on ptes */ +#define _PAGE_SD 0x002 + +/* Needed as macro to perform atomic operations */ +#define PGSTE_PCL_BIT 0x0080000000000000UL /* PCL lock, HW bit */ +#define PGSTE_CMMA_D_BIT 0x0000000000008000UL /* CMMA dirty soft-bit */ + +enum pgste_gps_usage { + PGSTE_GPS_USAGE_STABLE = 0, + PGSTE_GPS_USAGE_UNUSED, + PGSTE_GPS_USAGE_POT_VOLATILE, + PGSTE_GPS_USAGE_VOLATILE, +}; + +union pgste { + unsigned long val; + struct { + unsigned long acc : 4; + unsigned long fp : 1; + unsigned long : 3; + unsigned long pcl : 1; + unsigned long hr : 1; + unsigned long hc : 1; + unsigned long : 2; + unsigned long gr : 1; + unsigned long gc : 1; + unsigned long : 1; + unsigned long :16; /* val16 */ + unsigned long zero : 1; + unsigned long nodat : 1; + unsigned long : 4; + unsigned long usage : 2; + unsigned long : 8; + unsigned long cmma_d : 1; /* Dirty flag for CMMA bits */ + unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ + unsigned long vsie_notif : 1; /* Referenced in a shadow table */ + unsigned long vsie_gmem : 1; /* Contains nested guest memory */ + unsigned long : 4; + unsigned long : 8; + }; + struct { + unsigned short hwbytes0; + unsigned short val16; /* Used to store chunked values, see dat_{s,g}et_ptval() */ + unsigned short hwbytes4; + unsigned char flags; /* Maps to the software bits */ + unsigned char hwbyte7; + } __packed; +}; + +union pmd { + unsigned long val; + union segment_table_entry h; + struct { + struct { + unsigned long :44; /* HW */ + unsigned long : 3; /* Unused */ + unsigned long : 1; /* HW */ + unsigned long s : 1; /* Special */ + unsigned long w : 1; /* Writable soft-bit */ + unsigned long r : 1; /* Readable soft-bit */ + unsigned long d : 1; /* Dirty */ + unsigned long y : 1; /* Young */ + unsigned long : 3; /* HW */ + unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ + unsigned long vsie_notif : 1; /* Referenced in a shadow table */ + unsigned long : 4; /* HW */ + unsigned long sd : 1; /* Soft-Dirty */ + unsigned long pr : 1; /* Present */ + } fc1; + } s; +}; + +union pud { + unsigned long val; + union region3_table_entry h; + struct { + struct { + unsigned long :33; /* HW */ + unsigned long :14; /* Unused */ + unsigned long : 1; /* HW */ + unsigned long s : 1; /* Special */ + unsigned long w : 1; /* Writable soft-bit */ + unsigned long r : 1; /* Readable soft-bit */ + unsigned long d : 1; /* Dirty */ + unsigned long y : 1; /* Young */ + unsigned long : 3; /* HW */ + unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ + unsigned long vsie_notif : 1; /* Referenced in a shadow table */ + unsigned long : 4; /* HW */ + unsigned long sd : 1; /* Soft-Dirty */ + unsigned long pr : 1; /* Present */ + } fc1; + } s; +}; + +union p4d { + unsigned long val; + union region2_table_entry h; +}; + +union pgd { + unsigned long val; + union region1_table_entry h; +}; + +union crste { + unsigned long val; + union { + struct { + unsigned long :52; + unsigned long : 1; + unsigned long fc: 1; + unsigned long p : 1; + unsigned long : 1; + unsigned long : 2; + unsigned long i : 1; + unsigned long : 1; + unsigned long tt: 2; + unsigned long : 2; + }; + struct { + unsigned long to:52; + unsigned long : 1; + unsigned long fc: 1; + unsigned long p : 1; + unsigned long : 1; + unsigned long tf: 2; + unsigned long i : 1; + unsigned long : 1; + unsigned long tt: 2; + unsigned long tl: 2; + } fc0; + struct { + unsigned long :47; + unsigned long av : 1; /* ACCF-Validity Control */ + unsigned long acc: 4; /* Access-Control Bits */ + unsigned long f : 1; /* Fetch-Protection Bit */ + unsigned long fc : 1; /* Format-Control */ + unsigned long p : 1; /* DAT-Protection Bit */ + unsigned long iep: 1; /* Instruction-Execution-Protection */ + unsigned long : 2; + unsigned long i : 1; /* Segment-Invalid Bit */ + unsigned long cs : 1; /* Common-Segment Bit */ + unsigned long tt : 2; /* Table-Type Bits */ + unsigned long : 2; + } fc1; + } h; + struct { + struct { + unsigned long :47; + unsigned long : 1; /* HW (should be 0) */ + unsigned long s : 1; /* Special */ + unsigned long w : 1; /* Writable */ + unsigned long r : 1; /* Readable */ + unsigned long d : 1; /* Dirty */ + unsigned long y : 1; /* Young */ + unsigned long : 3; /* HW */ + unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ + unsigned long vsie_notif : 1; /* Referenced in a shadow table */ + unsigned long : 4; /* HW */ + unsigned long sd : 1; /* Soft-Dirty */ + unsigned long pr : 1; /* Present */ + } fc1; + } s; + union { + struct { + unsigned long type :16; /* Token type */ + unsigned long par :16; /* Token parameter */ + unsigned long :26; + unsigned long i : 1; /* Must be 1 */ + unsigned long : 1; + unsigned long tt : 2; + unsigned long : 1; + unsigned long pr : 1; /* Must be 0 */ + }; + struct { + unsigned long token:32; /* Token and parameter */ + unsigned long :32; + }; + } tok; + union pmd pmd; + union pud pud; + union p4d p4d; + union pgd pgd; +}; + +union skey { + unsigned char skey; + struct { + unsigned char acc :4; + unsigned char fp :1; + unsigned char r :1; + unsigned char c :1; + unsigned char zero:1; + }; +}; + +static_assert(sizeof(union pgste) == sizeof(unsigned long)); +static_assert(sizeof(union pte) == sizeof(unsigned long)); +static_assert(sizeof(union pmd) == sizeof(unsigned long)); +static_assert(sizeof(union pud) == sizeof(unsigned long)); +static_assert(sizeof(union p4d) == sizeof(unsigned long)); +static_assert(sizeof(union pgd) == sizeof(unsigned long)); +static_assert(sizeof(union crste) == sizeof(unsigned long)); +static_assert(sizeof(union skey) == sizeof(char)); + +struct segment_table { + union pmd pmds[_CRST_ENTRIES]; +}; + +struct region3_table { + union pud puds[_CRST_ENTRIES]; +}; + +struct region2_table { + union p4d p4ds[_CRST_ENTRIES]; +}; + +struct region1_table { + union pgd pgds[_CRST_ENTRIES]; +}; + +struct crst_table { + union { + union crste crstes[_CRST_ENTRIES]; + struct segment_table segment; + struct region3_table region3; + struct region2_table region2; + struct region1_table region1; + }; +}; + +struct page_table { + union pte ptes[_PAGE_ENTRIES]; + union pgste pgstes[_PAGE_ENTRIES]; +}; + +static_assert(sizeof(struct crst_table) == _CRST_TABLE_SIZE); +static_assert(sizeof(struct page_table) == PAGE_SIZE); + +struct dat_walk; + +typedef long (*dat_walk_op)(union crste *crste, gfn_t gfn, gfn_t next, struct dat_walk *w); + +struct dat_walk_ops { + union { + dat_walk_op crste_ops[4]; + struct { + dat_walk_op pmd_entry; + dat_walk_op pud_entry; + dat_walk_op p4d_entry; + dat_walk_op pgd_entry; + }; + }; + long (*pte_entry)(union pte *pte, gfn_t gfn, gfn_t next, struct dat_walk *w); +}; + +struct dat_walk { + const struct dat_walk_ops *ops; + union crste *last; + union pte *last_pte; + union asce asce; + gfn_t start; + gfn_t end; + int flags; + void *priv; +}; + +struct ptval_param { + unsigned char offset : 6; + unsigned char len : 2; +}; + +/** + * _pte() - Useful constructor for union pte + * @pfn: the pfn this pte should point to. + * @writable: whether the pte should be writable. + * @dirty: whether the pte should be dirty. + * @special: whether the pte should be marked as special + * + * The pte is also marked as young and present. If the pte is marked as dirty, + * it gets marked as soft-dirty too. If the pte is not dirty, the hardware + * protect bit is set (independently of the write softbit); this way proper + * dirty tracking can be performed. + * + * Return: a union pte value. + */ +static inline union pte _pte(kvm_pfn_t pfn, bool writable, bool dirty, bool special) +{ + union pte res = { .val = PFN_PHYS(pfn) }; + + res.h.p = !dirty; + res.s.y = 1; + res.s.pr = 1; + res.s.w = writable; + res.s.d = dirty; + res.s.sd = dirty; + res.s.s = special; + return res; +} + +static inline union crste _crste_fc0(kvm_pfn_t pfn, int tt) +{ + union crste res = { .val = PFN_PHYS(pfn) }; + + res.h.tt = tt; + res.h.fc0.tl = _REGION_ENTRY_LENGTH; + res.h.fc0.tf = 0; + return res; +} + +/** + * _crste() - Useful constructor for union crste with FC=1 + * @pfn: the pfn this pte should point to. + * @tt: the table type + * @writable: whether the pte should be writable. + * @dirty: whether the pte should be dirty. + * + * The crste is also marked as young and present. If the crste is marked as + * dirty, it gets marked as soft-dirty too. If the crste is not dirty, the + * hardware protect bit is set (independently of the write softbit); this way + * proper dirty tracking can be performed. + * + * Return: a union crste value. + */ +static inline union crste _crste_fc1(kvm_pfn_t pfn, int tt, bool writable, bool dirty) +{ + union crste res = { .val = PFN_PHYS(pfn) & _SEGMENT_MASK }; + + res.h.tt = tt; + res.h.p = !dirty; + res.h.fc = 1; + res.s.fc1.y = 1; + res.s.fc1.pr = 1; + res.s.fc1.w = writable; + res.s.fc1.d = dirty; + res.s.fc1.sd = dirty; + return res; +} + +union essa_state { + unsigned char val; + struct { + unsigned char : 2; + unsigned char nodat : 1; + unsigned char exception : 1; + unsigned char usage : 2; + unsigned char content : 2; + }; +}; + +/** + * struct vsie_rmap - reverse mapping for shadow page table entries + * @next: pointer to next rmap in the list + * @r_gfn: virtual rmap address in the shadow guest address space + */ +struct vsie_rmap { + struct vsie_rmap *next; + union { + unsigned long val; + struct { + long level: 8; + unsigned long : 4; + unsigned long r_gfn:52; + }; + }; +}; + +static_assert(sizeof(struct vsie_rmap) == 2 * sizeof(long)); + +#define KVM_S390_MMU_CACHE_N_CRSTS 6 +#define KVM_S390_MMU_CACHE_N_PTS 2 +#define KVM_S390_MMU_CACHE_N_RMAPS 16 +struct kvm_s390_mmu_cache { + void *crsts[KVM_S390_MMU_CACHE_N_CRSTS]; + void *pts[KVM_S390_MMU_CACHE_N_PTS]; + void *rmaps[KVM_S390_MMU_CACHE_N_RMAPS]; + short int n_crsts; + short int n_pts; + short int n_rmaps; +}; + +struct guest_fault { + gfn_t gfn; /* Guest frame */ + kvm_pfn_t pfn; /* Host PFN */ + struct page *page; /* Host page */ + union pte *ptep; /* Used to resolve the fault, or NULL */ + union crste *crstep; /* Used to resolve the fault, or NULL */ + bool writable; /* Mapping is writable */ + bool write_attempt; /* Write access attempted */ + bool attempt_pfault; /* Attempt a pfault first */ + bool valid; /* This entry contains valid data */ + bool crste_region3; /* Whether crstep refers to a region3 entry */ + void (*callback)(struct guest_fault *f); + void *priv; +}; + +/* + * 0 1 2 3 4 5 6 7 + * +-------+-------+-------+-------+-------+-------+-------+-------+ + * 0 | | PGT_ADDR | + * 8 | VMADDR | | + * 16 | | + * 24 | | + */ +#define MKPTVAL(o, l) ((struct ptval_param) { .offset = (o), .len = ((l) + 1) / 2 - 1}) +#define PTVAL_PGT_ADDR MKPTVAL(4, 8) +#define PTVAL_VMADDR MKPTVAL(8, 6) + +union pgste __must_check __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, + gfn_t gfn, union asce asce, bool uses_skeys); +bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, gfn_t gfn, + union asce asce); +void dat_crstep_xchg(union crste *crstep, union crste new, gfn_t gfn, union asce asce); + +long _dat_walk_gfn_range(gfn_t start, gfn_t end, union asce asce, + const struct dat_walk_ops *ops, int flags, void *priv); + +int dat_entry_walk(struct kvm_s390_mmu_cache *mc, gfn_t gfn, union asce asce, int flags, + int walk_level, union crste **last, union pte **ptepp); +void dat_free_level(struct crst_table *table, bool owns_ptes); +struct crst_table *dat_alloc_crst_sleepable(unsigned long init); +int dat_set_asce_limit(struct kvm_s390_mmu_cache *mc, union asce *asce, int newtype); + +#if KVM_S390_MANAGES_S390_GUEST +int dat_get_storage_key(union asce asce, gfn_t gfn, union skey *skey); +int dat_set_storage_key(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn, + union skey skey, bool nq); +int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gfn_t gfn, + union skey skey, union skey *oldkey, bool nq, bool mr, bool mc); +int dat_reset_reference_bit(union asce asce, gfn_t gfn, union skey *skey); +long dat_reset_skeys(union asce asce, gfn_t start); +#endif /* KVM_S390_MANAGES_S390_GUEST */ + +unsigned long dat_get_ptval(struct page_table *table, struct ptval_param param); +void dat_set_ptval(struct page_table *table, struct ptval_param param, unsigned long val); + +int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end); + +#if KVM_S390_MANAGES_S390_GUEST +int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn); +#else +static inline int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn) +{ + return 0; +} +#endif /* KVM_S390_MANAGES_S390_GUEST */ + +bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end); + +#if KVM_S390_MANAGES_S390_GUEST +int dat_perform_essa(union asce asce, gfn_t gfn, int orc, union essa_state *state, bool *dirty); +long dat_reset_cmma(union asce asce, gfn_t start_gfn); +int dat_peek_cmma(gfn_t start, union asce asce, unsigned int *count, u8 *values); +int dat_get_cmma(union asce asce, gfn_t *start, unsigned int *count, u8 *values, atomic64_t *rem); +int dat_set_cmma_bits(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn, + unsigned long count, unsigned long mask, const uint8_t *bits); +#endif /* KVM_S390_MANAGES_S390_GUEST */ + +int kvm_s390_mmu_cache_topup(struct kvm_s390_mmu_cache *mc); + +#define GFP_KVM_S390_MMU_CACHE (GFP_ATOMIC | __GFP_ACCOUNT | __GFP_NOWARN) + +static inline struct page_table *kvm_s390_mmu_cache_alloc_pt(struct kvm_s390_mmu_cache *mc) +{ + if (mc->n_pts) + return mc->pts[--mc->n_pts]; + return (void *)__get_free_page(GFP_KVM_S390_MMU_CACHE); +} + +static inline struct crst_table *kvm_s390_mmu_cache_alloc_crst(struct kvm_s390_mmu_cache *mc) +{ + if (mc->n_crsts) + return mc->crsts[--mc->n_crsts]; + return (void *)__get_free_pages(GFP_KVM_S390_MMU_CACHE | __GFP_COMP, CRST_ALLOC_ORDER); +} + +static inline struct vsie_rmap *kvm_s390_mmu_cache_alloc_rmap(struct kvm_s390_mmu_cache *mc) +{ + if (mc->n_rmaps) + return mc->rmaps[--mc->n_rmaps]; + return kzalloc_obj(struct vsie_rmap, GFP_KVM_S390_MMU_CACHE); +} + +static inline struct crst_table *crste_table_start(union crste *crstep) +{ + return (struct crst_table *)ALIGN_DOWN((unsigned long)crstep, _CRST_TABLE_SIZE); +} + +static inline struct page_table *pte_table_start(union pte *ptep) +{ + return (struct page_table *)ALIGN_DOWN((unsigned long)ptep, _PAGE_TABLE_SIZE); +} + +static inline bool crdte_crste(union crste *crstep, union crste old, union crste new, gfn_t gfn, + union asce asce) +{ + unsigned long dtt = 0x10 | new.h.tt << 2; + void *table = crste_table_start(crstep); + + return crdte(old.val, new.val, table, dtt, gfn_to_gpa(gfn), asce.val); +} + +/** + * idte_crste() - invalidate a crste entry using idte + * @crstep: pointer to the crste to be invalidated + * @gfn: a gfn mapped by the crste + * @opt: options for the idte instruction + * @asce: the asce + * @local: whether the operation is cpu-local + */ +static __always_inline void idte_crste(union crste *crstep, gfn_t gfn, unsigned long opt, + union asce asce, int local) +{ + unsigned long table_origin = __pa(crste_table_start(crstep)); + unsigned long gaddr = gfn_to_gpa(gfn) & HPAGE_MASK; + + if (__builtin_constant_p(opt) && opt == 0) { + /* flush without guest asce */ + asm volatile("idte %[table_origin],0,%[gaddr],%[local]" + : "+m" (*crstep) + : [table_origin] "a" (table_origin), [gaddr] "a" (gaddr), + [local] "i" (local) + : "cc"); + } else { + /* flush with guest asce */ + asm volatile("idte %[table_origin],%[asce],%[gaddr_opt],%[local]" + : "+m" (*crstep) + : [table_origin] "a" (table_origin), [gaddr_opt] "a" (gaddr | opt), + [asce] "a" (asce.val), [local] "i" (local) + : "cc"); + } +} + +static inline void dat_init_pgstes(struct page_table *pt, unsigned long val) +{ + memset64((void *)pt->pgstes, val, PTRS_PER_PTE); +} + +static inline void dat_init_page_table(struct page_table *pt, unsigned long ptes, + unsigned long pgstes) +{ + memset64((void *)pt->ptes, ptes, PTRS_PER_PTE); + dat_init_pgstes(pt, pgstes); +} + +static inline gfn_t asce_end(union asce asce) +{ + return 1ULL << ((asce.dt + 1) * 11 + _SEGMENT_SHIFT - PAGE_SHIFT); +} + +#define _CRSTE(x) ((union crste) { .val = _Generic((x), \ + union pgd : (x).val, \ + union p4d : (x).val, \ + union pud : (x).val, \ + union pmd : (x).val, \ + union crste : (x).val)}) + +#define _CRSTEP(x) ((union crste *)_Generic((*(x)), \ + union pgd : (x), \ + union p4d : (x), \ + union pud : (x), \ + union pmd : (x), \ + union crste : (x))) + +#define _CRSTP(x) ((struct crst_table *)_Generic((*(x)), \ + struct crst_table : (x), \ + struct segment_table : (x), \ + struct region3_table : (x), \ + struct region2_table : (x), \ + struct region1_table : (x))) + +static inline bool asce_contains_gfn(union asce asce, gfn_t gfn) +{ + return gfn < asce_end(asce); +} + +static inline bool is_pmd(union crste crste) +{ + return crste.h.tt == TABLE_TYPE_SEGMENT; +} + +static inline bool is_pud(union crste crste) +{ + return crste.h.tt == TABLE_TYPE_REGION3; +} + +static inline bool is_p4d(union crste crste) +{ + return crste.h.tt == TABLE_TYPE_REGION2; +} + +static inline bool is_pgd(union crste crste) +{ + return crste.h.tt == TABLE_TYPE_REGION1; +} + +static inline phys_addr_t pmd_origin_large(union pmd pmd) +{ + return pmd.val & _SEGMENT_ENTRY_ORIGIN_LARGE; +} + +static inline phys_addr_t pud_origin_large(union pud pud) +{ + return pud.val & _REGION3_ENTRY_ORIGIN_LARGE; +} + +/** + * crste_origin_large() - Return the large frame origin of a large crste + * @crste: The crste whose origin is to be returned. Should be either a + * region-3 table entry or a segment table entry, in both cases with + * FC set to 1 (large pages). + * + * Return: The origin of the large frame pointed to by @crste, or -1 if the + * crste was not large (wrong table type, or FC==0) + */ +static inline phys_addr_t crste_origin_large(union crste crste) +{ + if (unlikely(!crste.h.fc || crste.h.tt > TABLE_TYPE_REGION3)) + return -1; + if (is_pmd(crste)) + return pmd_origin_large(crste.pmd); + return pud_origin_large(crste.pud); +} + +#define crste_origin(x) (_Generic((x), \ + union pmd : (x).val & _SEGMENT_ENTRY_ORIGIN, \ + union pud : (x).val & _REGION_ENTRY_ORIGIN, \ + union p4d : (x).val & _REGION_ENTRY_ORIGIN, \ + union pgd : (x).val & _REGION_ENTRY_ORIGIN)) + +static inline unsigned long pte_origin(union pte pte) +{ + return pte.val & PAGE_MASK; +} + +static inline bool pmd_prefix(union pmd pmd) +{ + return pmd.h.fc && pmd.s.fc1.prefix_notif; +} + +static inline bool pud_prefix(union pud pud) +{ + return pud.h.fc && pud.s.fc1.prefix_notif; +} + +static inline bool crste_leaf(union crste crste) +{ + return (crste.h.tt <= TABLE_TYPE_REGION3) && crste.h.fc; +} + +static inline bool crste_prefix(union crste crste) +{ + return crste_leaf(crste) && crste.s.fc1.prefix_notif; +} + +static inline bool crste_dirty(union crste crste) +{ + return crste_leaf(crste) && crste.s.fc1.d; +} + +static inline union pgste *pgste_of(union pte *pte) +{ + return (union pgste *)(pte + _PAGE_ENTRIES); +} + +static inline bool pte_hole(union pte pte) +{ + return pte.h.i && !pte.tok.pr && pte.tok.type != _DAT_TOKEN_NONE; +} + +static inline bool _crste_hole(union crste crste) +{ + return crste.h.i && !crste.tok.pr && crste.tok.type != _DAT_TOKEN_NONE; +} + +#define crste_hole(x) _crste_hole(_CRSTE(x)) + +static inline bool _crste_none(union crste crste) +{ + return crste.h.i && !crste.tok.pr && crste.tok.type == _DAT_TOKEN_NONE; +} + +#define crste_none(x) _crste_none(_CRSTE(x)) + +static inline phys_addr_t large_pud_to_phys(union pud pud, gfn_t gfn) +{ + return pud_origin_large(pud) | (gfn_to_gpa(gfn) & ~_REGION3_MASK); +} + +static inline phys_addr_t large_pmd_to_phys(union pmd pmd, gfn_t gfn) +{ + return pmd_origin_large(pmd) | (gfn_to_gpa(gfn) & ~_SEGMENT_MASK); +} + +static inline phys_addr_t large_crste_to_phys(union crste crste, gfn_t gfn) +{ + if (unlikely(!crste.h.fc || crste.h.tt > TABLE_TYPE_REGION3)) + return -1; + if (is_pmd(crste)) + return large_pmd_to_phys(crste.pmd, gfn); + return large_pud_to_phys(crste.pud, gfn); +} + +static inline bool cspg_crste(union crste *crstep, union crste old, union crste new) +{ + return cspg(&crstep->val, old.val, new.val); +} + +static inline struct page_table *dereference_pmd(union pmd pmd) +{ + return phys_to_virt(crste_origin(pmd)); +} + +static inline struct segment_table *dereference_pud(union pud pud) +{ + return phys_to_virt(crste_origin(pud)); +} + +static inline struct region3_table *dereference_p4d(union p4d p4d) +{ + return phys_to_virt(crste_origin(p4d)); +} + +static inline struct region2_table *dereference_pgd(union pgd pgd) +{ + return phys_to_virt(crste_origin(pgd)); +} + +static inline struct crst_table *_dereference_crste(union crste crste) +{ + if (unlikely(is_pmd(crste))) + return NULL; + return phys_to_virt(crste_origin(crste.pud)); +} + +#define dereference_crste(x) (_Generic((x), \ + union pud : _dereference_crste(_CRSTE(x)), \ + union p4d : _dereference_crste(_CRSTE(x)), \ + union pgd : _dereference_crste(_CRSTE(x)), \ + union crste : _dereference_crste(_CRSTE(x)))) + +static inline struct crst_table *dereference_asce(union asce asce) +{ + return phys_to_virt(asce.val & _ASCE_ORIGIN); +} + +static inline void asce_flush_tlb(union asce asce) +{ + __tlb_flush_idte(asce.val); +} + +static inline bool pgste_get_trylock(union pte *ptep, union pgste *res) +{ + union pgste *pgstep = pgste_of(ptep); + union pgste old_pgste; + + if (READ_ONCE(pgstep->val) & PGSTE_PCL_BIT) + return false; + old_pgste.val = __atomic64_or_barrier(PGSTE_PCL_BIT, &pgstep->val); + if (old_pgste.pcl) + return false; + old_pgste.pcl = 1; + *res = old_pgste; + return true; +} + +static inline union pgste pgste_get_lock(union pte *ptep) +{ + union pgste res; + + while (!pgste_get_trylock(ptep, &res)) + cpu_relax(); + return res; +} + +static inline void pgste_set_unlock(union pte *ptep, union pgste pgste) +{ + pgste.pcl = 0; + barrier(); + WRITE_ONCE(*pgste_of(ptep), pgste); +} + +static inline void dat_ptep_xchg(union pte *ptep, union pte new, gfn_t gfn, union asce asce, + bool has_skeys) +{ + union pgste pgste; + + pgste = pgste_get_lock(ptep); + pgste = __dat_ptep_xchg(ptep, pgste, new, gfn, asce, has_skeys); + pgste_set_unlock(ptep, pgste); +} + +static inline void dat_ptep_clear(union pte *ptep, gfn_t gfn, union asce asce, bool has_skeys) +{ + dat_ptep_xchg(ptep, _PTE_EMPTY, gfn, asce, has_skeys); +} + +static inline void dat_free_pt(struct page_table *pt) +{ + free_page((unsigned long)pt); +} + +static inline void _dat_free_crst(struct crst_table *table) +{ + free_pages((unsigned long)table, CRST_ALLOC_ORDER); +} + +#define dat_free_crst(x) _dat_free_crst(_CRSTP(x)) + +static inline void kvm_s390_free_mmu_cache(struct kvm_s390_mmu_cache *mc) +{ + if (!mc) + return; + while (mc->n_pts) + dat_free_pt(mc->pts[--mc->n_pts]); + while (mc->n_crsts) + _dat_free_crst(mc->crsts[--mc->n_crsts]); + while (mc->n_rmaps) + kfree(mc->rmaps[--mc->n_rmaps]); + kfree(mc); +} + +DEFINE_FREE(kvm_s390_mmu_cache, struct kvm_s390_mmu_cache *, if (_T) kvm_s390_free_mmu_cache(_T)) + +static inline struct kvm_s390_mmu_cache *kvm_s390_new_mmu_cache(void) +{ + struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL; + + mc = kzalloc_obj(*mc, GFP_KERNEL_ACCOUNT); + if (mc && !kvm_s390_mmu_cache_topup(mc)) + return_ptr(mc); + return NULL; +} + +static inline bool dat_pmdp_xchg_atomic(union pmd *pmdp, union pmd old, union pmd new, + gfn_t gfn, union asce asce) +{ + return dat_crstep_xchg_atomic(_CRSTEP(pmdp), _CRSTE(old), _CRSTE(new), gfn, asce); +} + +static inline bool dat_pudp_xchg_atomic(union pud *pudp, union pud old, union pud new, + gfn_t gfn, union asce asce) +{ + return dat_crstep_xchg_atomic(_CRSTEP(pudp), _CRSTE(old), _CRSTE(new), gfn, asce); +} + +static inline union crste dat_crstep_clear_atomic(union crste *crstep, gfn_t gfn, union asce asce) +{ + union crste oldcrste, empty = _CRSTE_EMPTY(crstep->h.tt); + + do { + oldcrste = READ_ONCE(*crstep); + } while (!dat_crstep_xchg_atomic(crstep, oldcrste, empty, gfn, asce)); + return oldcrste; +} + +static inline int get_level(union crste *crstep, union pte *ptep) +{ + return ptep ? TABLE_TYPE_PAGE_TABLE : crstep->h.tt; +} + +static inline int dat_delete_slot(union asce asce, gfn_t start, unsigned long npages) +{ + return dat_set_slot(NULL, asce, start, start + npages); +} + +static inline int dat_create_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, + unsigned long npages) +{ + return dat_set_slot(mc, asce, start, start + npages); +} + +static inline bool crste_is_ucas(union crste crste) +{ + return is_pmd(crste) && crste.h.i && crste.h.fc0.tl == 1 && crste.h.fc == 0; +} + +#endif /* ARCH_KVM_GMAP_DAT_H */ diff --git a/arch/s390/kvm/gmap/faultin.c b/arch/s390/kvm/gmap/faultin.c new file mode 100644 index 000000000000..1dc79807012c --- /dev/null +++ b/arch/s390/kvm/gmap/faultin.c @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KVM guest fault handling. + * + * Copyright IBM Corp. 2025 + * Author(s): Claudio Imbrenda <imbrenda@linux.ibm.com> + */ +#include <linux/kvm_types.h> +#include <linux/kvm_host.h> + +#include "gmap.h" +#include "faultin.h" + +bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu); +#define CREATE_TRACE_POINTS +#include "trace_gmap.h" + +/* + * kvm_s390_faultin_gfn() - handle a dat fault. + * @vcpu: The vCPU whose gmap is to be fixed up, or NULL if operating on the VM. + * @kvm: The VM whose gmap is to be fixed up, or NULL if operating on a vCPU. + * @f: The guest fault that needs to be resolved. + * + * Return: + * * 0 on success + * * < 0 in case of error + * * > 0 in case of guest exceptions + * + * Context: + * * The mm lock must not be held before calling + * * kvm->srcu must be held + * * may sleep + */ +int kvm_s390_faultin_gfn(struct kvm_vcpu *vcpu, struct kvm *kvm, struct guest_fault *f) +{ + struct kvm_s390_mmu_cache *local_mc __free(kvm_s390_mmu_cache) = NULL; + struct kvm_s390_mmu_cache *mc = NULL; + struct kvm_memory_slot *slot; + unsigned long inv_seq; + int rc = -EAGAIN; + int foll; + + foll = f->write_attempt ? FOLL_WRITE : 0; + foll |= f->attempt_pfault ? FOLL_NOWAIT : 0; + + if (vcpu) { + kvm = vcpu->kvm; + mc = vcpu->arch.mc; + } + + lockdep_assert_held(&kvm->srcu); + + scoped_guard(read_lock, &kvm->mmu_lock) { + if (gmap_try_fixup_minor(kvm->arch.gmap, f) == 0) + return 0; + } + + if (!mc) { + local_mc = kvm_s390_new_mmu_cache(); + if (!local_mc) + return -ENOMEM; + mc = local_mc; + } + + while (rc == -EAGAIN) { + f->valid = false; + inv_seq = kvm->mmu_invalidate_seq; + /* Pairs with the smp_wmb() in kvm_mmu_invalidate_end(). */ + smp_rmb(); + + if (vcpu) + slot = kvm_vcpu_gfn_to_memslot(vcpu, f->gfn); + else + slot = gfn_to_memslot(kvm, f->gfn); + f->pfn = __kvm_faultin_pfn(slot, f->gfn, foll, &f->writable, &f->page); + + /* Needs I/O, try to setup async pfault (only possible with FOLL_NOWAIT). */ + if (f->pfn == KVM_PFN_ERR_NEEDS_IO) { + if (unlikely(!f->attempt_pfault)) + return -EAGAIN; + if (unlikely(!vcpu)) + return -EINVAL; + trace_kvm_s390_major_guest_pfault(vcpu); + if (kvm_arch_setup_async_pf(vcpu)) + return 0; + vcpu->stat.pfault_sync++; + /* Could not setup async pfault, try again synchronously. */ + foll &= ~FOLL_NOWAIT; + f->pfn = __kvm_faultin_pfn(slot, f->gfn, foll, &f->writable, &f->page); + } + + /* Access outside memory, addressing exception. */ + if (is_noslot_pfn(f->pfn)) + return PGM_ADDRESSING; + /* Fatal signal pending: bail out. */ + if (is_sigpending_pfn(f->pfn)) + return -EINTR; + /* Check if it's read-only memory; don't try to actually handle that case. */ + if (f->pfn == KVM_PFN_ERR_RO_FAULT) + return -EOPNOTSUPP; + /* Any other error. */ + if (is_error_pfn(f->pfn)) + return -EFAULT; + + /* Loop, release the faulted page. */ + if (mmu_invalidate_retry_gfn_unsafe(kvm, inv_seq, f->gfn)) { + kvm_release_faultin_page(kvm, f->page, true, false); + continue; + } + + scoped_guard(read_lock, &kvm->mmu_lock) { + if (!mmu_invalidate_retry_gfn(kvm, inv_seq, f->gfn)) { + f->valid = true; + rc = gmap_link(mc, kvm->arch.gmap, f, slot); + } + kvm_release_faultin_page(kvm, f->page, !!rc, f->write_attempt); + } + + if (rc == -ENOMEM) { + rc = kvm_s390_mmu_cache_topup(mc); + if (rc) + return rc; + rc = -EAGAIN; + } + } + + return rc; +} + +int kvm_s390_get_guest_page(struct kvm *kvm, struct guest_fault *f, gfn_t gfn, bool w) +{ + struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); + int foll = w ? FOLL_WRITE : 0; + + f->write_attempt = w; + f->gfn = gfn; + f->pfn = __kvm_faultin_pfn(slot, gfn, foll, &f->writable, &f->page); + if (is_noslot_pfn(f->pfn)) + return PGM_ADDRESSING; + if (is_sigpending_pfn(f->pfn)) + return -EINTR; + if (f->pfn == KVM_PFN_ERR_NEEDS_IO) + return -EAGAIN; + if (is_error_pfn(f->pfn)) + return -EFAULT; + + f->valid = true; + return 0; +} diff --git a/arch/s390/kvm/gmap/faultin.h b/arch/s390/kvm/gmap/faultin.h new file mode 100644 index 000000000000..f343b6fb6f16 --- /dev/null +++ b/arch/s390/kvm/gmap/faultin.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * KVM guest fault handling. + * + * Copyright IBM Corp. 2025 + * Author(s): Claudio Imbrenda <imbrenda@linux.ibm.com> + */ + +#ifndef ARCH_KVM_GMAP_FAULTIN_H +#define ARCH_KVM_GMAP_FAULTIN_H + +#include <linux/kvm_host.h> + +#include "dat.h" + +int kvm_s390_faultin_gfn(struct kvm_vcpu *vcpu, struct kvm *kvm, struct guest_fault *f); +int kvm_s390_get_guest_page(struct kvm *kvm, struct guest_fault *f, gfn_t gfn, bool w); + +static inline int kvm_s390_faultin_gfn_simple(struct kvm_vcpu *vcpu, struct kvm *kvm, + gfn_t gfn, bool wr) +{ + struct guest_fault f = { .gfn = gfn, .write_attempt = wr, }; + + return kvm_s390_faultin_gfn(vcpu, kvm, &f); +} + +static inline int kvm_s390_get_guest_page_and_read_gpa(struct kvm *kvm, struct guest_fault *f, + gpa_t gaddr, unsigned long *val) +{ + int rc; + + rc = kvm_s390_get_guest_page(kvm, f, gpa_to_gfn(gaddr), false); + if (rc) + return rc; + + *val = *(unsigned long *)phys_to_virt(pfn_to_phys(f->pfn) | offset_in_page(gaddr)); + + return 0; +} + +static inline void kvm_s390_release_multiple(struct kvm *kvm, struct guest_fault *guest_faults, + int n, bool ignore) +{ + int i; + + for (i = 0; i < n; i++) { + kvm_release_faultin_page(kvm, guest_faults[i].page, ignore, + guest_faults[i].write_attempt); + guest_faults[i].page = NULL; + } +} + +static inline bool kvm_s390_multiple_faults_need_retry(struct kvm *kvm, unsigned long seq, + struct guest_fault *guest_faults, int n, + bool unsafe) +{ + int i; + + for (i = 0; i < n; i++) { + if (!guest_faults[i].valid) + continue; + if (unsafe && mmu_invalidate_retry_gfn_unsafe(kvm, seq, guest_faults[i].gfn)) + return true; + if (!unsafe && mmu_invalidate_retry_gfn(kvm, seq, guest_faults[i].gfn)) + return true; + } + return false; +} + +static inline int kvm_s390_get_guest_pages(struct kvm *kvm, struct guest_fault *guest_faults, + gfn_t start, int n_pages, bool write_attempt) +{ + int i, rc; + + for (i = 0; i < n_pages; i++) { + rc = kvm_s390_get_guest_page(kvm, guest_faults + i, start + i, write_attempt); + if (rc) + break; + } + return rc; +} + +#define kvm_s390_release_faultin_array(kvm, array, ignore) \ + kvm_s390_release_multiple(kvm, array, ARRAY_SIZE(array), ignore) + +#define kvm_s390_array_needs_retry_unsafe(kvm, seq, array) \ + kvm_s390_multiple_faults_need_retry(kvm, seq, array, ARRAY_SIZE(array), true) + +#define kvm_s390_array_needs_retry_safe(kvm, seq, array) \ + kvm_s390_multiple_faults_need_retry(kvm, seq, array, ARRAY_SIZE(array), false) + +#endif /* ARCH_KVM_GMAP_FAULTIN_H */ diff --git a/arch/s390/kvm/gmap/gmap.c b/arch/s390/kvm/gmap/gmap.c new file mode 100644 index 000000000000..3f3fa864cc36 --- /dev/null +++ b/arch/s390/kvm/gmap/gmap.c @@ -0,0 +1,1410 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Guest memory management for KVM/s390 + * + * Copyright IBM Corp. 2008, 2020, 2024 + * + * Author(s): Claudio Imbrenda <imbrenda@linux.ibm.com> + * Martin Schwidefsky <schwidefsky@de.ibm.com> + * David Hildenbrand <david@redhat.com> + * Janosch Frank <frankja@linux.ibm.com> + */ + +#include <linux/compiler.h> +#include <linux/kvm.h> +#include <linux/kvm_host.h> +#include <linux/pgtable.h> +#include <linux/pagemap.h> +#include <asm/lowcore.h> +#include <asm/uv.h> +#include <asm/gmap_helpers.h> + +#include "dat.h" +#include "gmap.h" +#include "s390.h" +#include "faultin.h" + +static int gmap_limit_to_type(gfn_t limit) +{ + if (!limit) + return TABLE_TYPE_REGION1; + if (limit <= _REGION3_SIZE >> PAGE_SHIFT) + return TABLE_TYPE_SEGMENT; + if (limit <= _REGION2_SIZE >> PAGE_SHIFT) + return TABLE_TYPE_REGION3; + if (limit <= _REGION1_SIZE >> PAGE_SHIFT) + return TABLE_TYPE_REGION2; + return TABLE_TYPE_REGION1; +} + +/** + * gmap_new() - Allocate and initialize a guest address space. + * @kvm: The kvm owning the guest. + * @limit: Maximum address of the gmap address space. + * + * Return: A guest address space structure. + */ +struct gmap *gmap_new(struct kvm *kvm, gfn_t limit) +{ + struct crst_table *table; + struct gmap *gmap; + int type; + + type = gmap_limit_to_type(limit); + + gmap = kzalloc_obj(*gmap, GFP_KERNEL_ACCOUNT); + if (!gmap) + return NULL; + INIT_LIST_HEAD(&gmap->children); + INIT_LIST_HEAD(&gmap->list); + INIT_LIST_HEAD(&gmap->scb_users); + INIT_RADIX_TREE(&gmap->host_to_rmap, GFP_KVM_S390_MMU_CACHE); + spin_lock_init(&gmap->children_lock); + spin_lock_init(&gmap->host_to_rmap_lock); + refcount_set(&gmap->refcount, 1); + + table = dat_alloc_crst_sleepable(_CRSTE_EMPTY(type).val); + if (!table) { + kfree(gmap); + return NULL; + } + + gmap->asce.val = __pa(table); + gmap->asce.dt = type; + gmap->asce.tl = _ASCE_TABLE_LENGTH; + gmap->asce.x = 1; + gmap->asce.p = 1; + gmap->asce.s = 1; + gmap->kvm = kvm; + set_bit(GMAP_FLAG_OWNS_PAGETABLES, &gmap->flags); + + return gmap; +} + +static void gmap_add_child(struct gmap *parent, struct gmap *child) +{ + KVM_BUG_ON(is_ucontrol(parent) && parent->parent, parent->kvm); + KVM_BUG_ON(is_ucontrol(parent) && !owns_page_tables(parent), parent->kvm); + KVM_BUG_ON(!refcount_read(&child->refcount), parent->kvm); + lockdep_assert_held(&parent->children_lock); + + child->parent = parent; + + if (is_ucontrol(parent)) + set_bit(GMAP_FLAG_IS_UCONTROL, &child->flags); + else + clear_bit(GMAP_FLAG_IS_UCONTROL, &child->flags); + + if (test_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &parent->flags)) + set_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &child->flags); + else + clear_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &child->flags); + + if (test_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &parent->flags)) + set_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &child->flags); + else + clear_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &child->flags); + + if (kvm_is_ucontrol(parent->kvm)) + clear_bit(GMAP_FLAG_OWNS_PAGETABLES, &child->flags); + list_add(&child->list, &parent->children); +} + +struct gmap *gmap_new_child(struct gmap *parent, gfn_t limit) +{ + struct gmap *res; + + lockdep_assert_not_held(&parent->children_lock); + res = gmap_new(parent->kvm, limit); + if (res) { + scoped_guard(spinlock, &parent->children_lock) + gmap_add_child(parent, res); + } + return res; +} + +int gmap_set_limit(struct gmap *gmap, gfn_t limit) +{ + struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL; + int rc, type; + + type = gmap_limit_to_type(limit); + + mc = kvm_s390_new_mmu_cache(); + if (!mc) + return -ENOMEM; + + do { + rc = kvm_s390_mmu_cache_topup(mc); + if (rc) + return rc; + scoped_guard(write_lock, &gmap->kvm->mmu_lock) + rc = dat_set_asce_limit(mc, &gmap->asce, type); + } while (rc == -ENOMEM); + + return 0; +} + +static void gmap_rmap_radix_tree_free(struct radix_tree_root *root) +{ + struct vsie_rmap *rmap, *rnext, *head; + struct radix_tree_iter iter; + unsigned long indices[16]; + unsigned long index; + void __rcu **slot; + int i, nr; + + /* A radix tree is freed by deleting all of its entries */ + index = 0; + do { + nr = 0; + radix_tree_for_each_slot(slot, root, &iter, index) { + indices[nr] = iter.index; + if (++nr == 16) + break; + } + for (i = 0; i < nr; i++) { + index = indices[i]; + head = radix_tree_delete(root, index); + gmap_for_each_rmap_safe(rmap, rnext, head) + kfree(rmap); + } + } while (nr > 0); +} + +void gmap_remove_child(struct gmap *child) +{ + if (KVM_BUG_ON(!child->parent, child->kvm)) + return; + lockdep_assert_held(&child->parent->children_lock); + + list_del(&child->list); + child->parent = NULL; + child->invalidated = true; +} + +/** + * gmap_dispose() - Remove and free a guest address space and its children. + * @gmap: Pointer to the guest address space structure. + */ +void gmap_dispose(struct gmap *gmap) +{ + /* The gmap must have been removed from the parent beforehands */ + KVM_BUG_ON(gmap->parent, gmap->kvm); + /* All children of this gmap must have been removed beforehands */ + KVM_BUG_ON(!list_empty(&gmap->children), gmap->kvm); + /* No VSIE shadow block is allowed to use this gmap */ + KVM_BUG_ON(!list_empty(&gmap->scb_users), gmap->kvm); + /* The ASCE must be valid */ + KVM_BUG_ON(!gmap->asce.val, gmap->kvm); + /* The refcount must be 0 */ + KVM_BUG_ON(refcount_read(&gmap->refcount), gmap->kvm); + + /* Flush tlb of all gmaps */ + asce_flush_tlb(gmap->asce); + + /* Free all DAT tables. */ + dat_free_level(dereference_asce(gmap->asce), owns_page_tables(gmap)); + + /* Free additional data for a shadow gmap */ + if (is_shadow(gmap)) + gmap_rmap_radix_tree_free(&gmap->host_to_rmap); + + kfree(gmap); +} + +/** + * s390_replace_asce() - Try to replace the current ASCE of a gmap with a copy. + * @gmap: The gmap whose ASCE needs to be replaced. + * + * If the ASCE is a SEGMENT type then this function will return -EINVAL, + * otherwise the pointers in the host_to_guest radix tree will keep pointing + * to the wrong pages, causing use-after-free and memory corruption. + * If the allocation of the new top level page table fails, the ASCE is not + * replaced. + * In any case, the old ASCE is always removed from the gmap CRST list. + * Therefore the caller has to make sure to save a pointer to it + * beforehand, unless a leak is actually intended. + * + * Return: 0 in case of success, -EINVAL if the ASCE is segment type ASCE, + * -ENOMEM if runinng out of memory. + */ +int s390_replace_asce(struct gmap *gmap) +{ + struct crst_table *table; + union asce asce; + + /* Replacing segment type ASCEs would cause serious issues */ + if (gmap->asce.dt == ASCE_TYPE_SEGMENT) + return -EINVAL; + + table = dat_alloc_crst_sleepable(0); + if (!table) + return -ENOMEM; + memcpy(table, dereference_asce(gmap->asce), sizeof(*table)); + + /* Set new table origin while preserving existing ASCE control bits */ + asce = gmap->asce; + asce.rsto = virt_to_pfn(table); + WRITE_ONCE(gmap->asce, asce); + + return 0; +} + +#if KVM_S390_MANAGES_S390_GUEST +static inline bool kvm_s390_is_in_sie(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.sie_block->prog0c & PROG_IN_SIE; +} + +bool _gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end, bool hint) +{ + struct kvm *kvm = gmap->kvm; + struct kvm_vcpu *vcpu; + gfn_t prefix_gfn; + unsigned long i; + + if (is_shadow(gmap)) + return false; + kvm_for_each_vcpu(i, vcpu, kvm) { + /* Match against both prefix pages */ + prefix_gfn = gpa_to_gfn(kvm_s390_get_prefix(vcpu)); + if (prefix_gfn < end && gfn <= prefix_gfn + 1) { + if (hint && kvm_s390_is_in_sie(vcpu)) + return false; + VCPU_EVENT(vcpu, 2, "gmap notifier for %llx-%llx", + gfn_to_gpa(gfn), gfn_to_gpa(end)); + kvm_s390_sync_request(KVM_REQ_REFRESH_GUEST_PREFIX, vcpu); + } + } + return true; +} +#endif /* KVM_S390_MANAGES_S390_GUEST */ + +struct clear_young_pte_priv { + struct gmap *gmap; + bool young; +}; + +static long gmap_clear_young_pte(union pte *ptep, gfn_t gfn, gfn_t end, struct dat_walk *walk) +{ + struct clear_young_pte_priv *p = walk->priv; + union pgste pgste; + union pte pte, new; + + pte = READ_ONCE(*ptep); + + if (!pte.s.pr || (!pte.s.y && pte.h.i)) + return 0; + + pgste = pgste_get_lock(ptep); + if (!pgste.prefix_notif || gmap_mkold_prefix(p->gmap, gfn, end)) { + new = pte; + new.h.i = 1; + new.s.y = 0; + if ((new.s.d || !new.h.p) && !new.s.s) + folio_set_dirty(pfn_folio(pte.h.pfra)); + new.s.d = 0; + new.h.p = 1; + + pgste.prefix_notif = 0; + pgste = __dat_ptep_xchg(ptep, pgste, new, gfn, walk->asce, uses_skeys(p->gmap)); + } + p->young = 1; + pgste_set_unlock(ptep, pgste); + return 0; +} + +static long gmap_clear_young_crste(union crste *crstep, gfn_t gfn, gfn_t end, struct dat_walk *walk) +{ + struct clear_young_pte_priv *priv = walk->priv; + union crste crste, new; + + do { + crste = READ_ONCE(*crstep); + + if (!crste.h.fc) + return 0; + if (!crste.s.fc1.y && crste.h.i) + return 0; + if (crste_prefix(crste) && !gmap_mkold_prefix(priv->gmap, gfn, end)) + break; + + new = crste; + new.h.i = 1; + new.s.fc1.y = 0; + new.s.fc1.prefix_notif = 0; + if ((new.s.fc1.d || !new.h.p) && !new.s.fc1.s) + folio_set_dirty(phys_to_folio(crste_origin_large(crste))); + new.s.fc1.d = 0; + new.h.p = 1; + } while (!dat_crstep_xchg_atomic(crstep, crste, new, gfn, walk->asce)); + + priv->young = 1; + return 0; +} + +/** + * gmap_age_gfn() - Clear young. + * @gmap: The guest gmap. + * @start: The first gfn to test. + * @end: The gfn after the last one to test. + * + * Context: Called with the kvm mmu write lock held. + * Return: 1 if any page in the given range was young, otherwise 0. + */ +bool gmap_age_gfn(struct gmap *gmap, gfn_t start, gfn_t end) +{ + const struct dat_walk_ops ops = { + .pte_entry = gmap_clear_young_pte, + .pmd_entry = gmap_clear_young_crste, + .pud_entry = gmap_clear_young_crste, + }; + struct clear_young_pte_priv priv = { + .gmap = gmap, + .young = false, + }; + + _dat_walk_gfn_range(start, end, gmap->asce, &ops, 0, &priv); + + return priv.young; +} + +struct gmap_unmap_priv { + struct gmap *gmap; + struct kvm_memory_slot *slot; +}; + +static long _gmap_unmap_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *w) +{ + struct gmap_unmap_priv *priv = w->priv; + struct folio *folio = NULL; + unsigned long vmaddr; + union pgste pgste; + + pgste = pgste_get_lock(ptep); + if (ptep->s.pr && pgste.usage == PGSTE_GPS_USAGE_UNUSED) { + vmaddr = __gfn_to_hva_memslot(priv->slot, gfn); + gmap_helper_try_set_pte_unused(priv->gmap->kvm->mm, vmaddr); + } + if (ptep->s.pr && test_bit(GMAP_FLAG_EXPORT_ON_UNMAP, &priv->gmap->flags)) + folio = pfn_folio(ptep->h.pfra); + pgste = gmap_ptep_xchg(priv->gmap, ptep, _PTE_EMPTY, pgste, gfn); + pgste_set_unlock(ptep, pgste); + if (folio) + uv_convert_from_secure_folio(folio); + + return 0; +} + +static long _gmap_unmap_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) +{ + struct gmap_unmap_priv *priv = walk->priv; + struct folio *folio = NULL; + union crste old = *crstep; + bool ok; + + if (!old.h.fc) + return 0; + + if (old.s.fc1.pr && test_bit(GMAP_FLAG_EXPORT_ON_UNMAP, &priv->gmap->flags)) + folio = phys_to_folio(crste_origin_large(old)); + /* + * No races should happen because kvm->mmu_lock is held in write mode, + * but the unmap operation could have triggered an unshadow, which + * causes gmap_crstep_xchg_atomic() to return false and clear the + * vsie_notif bit. Allow the operation to fail once, if the old crste + * had the vsie_notif bit set. A second failure is not allowed, for + * the reasons above. + */ + ok = gmap_crstep_xchg_atomic(priv->gmap, crstep, old, _CRSTE_EMPTY(old.h.tt), gfn); + if (!ok) { + KVM_BUG_ON(!old.s.fc1.vsie_notif, priv->gmap->kvm); + old.s.fc1.vsie_notif = 0; + ok = gmap_crstep_xchg_atomic(priv->gmap, crstep, old, _CRSTE_EMPTY(old.h.tt), gfn); + KVM_BUG_ON(!ok, priv->gmap->kvm); + } + if (folio) + uv_convert_from_secure_folio(folio); + + return 0; +} + +/** + * gmap_unmap_gfn_range() - Unmap a range of guest addresses. + * @gmap: The gmap to act on. + * @slot: The memslot in which the range is located. + * @start: The first gfn to unmap. + * @end: The gfn after the last one to unmap. + * + * Context: Called with the kvm mmu write lock held. + * Return: false + */ +bool gmap_unmap_gfn_range(struct gmap *gmap, struct kvm_memory_slot *slot, gfn_t start, gfn_t end) +{ + const struct dat_walk_ops ops = { + .pte_entry = _gmap_unmap_pte, + .pmd_entry = _gmap_unmap_crste, + .pud_entry = _gmap_unmap_crste, + }; + struct gmap_unmap_priv priv = { + .gmap = gmap, + .slot = slot, + }; + + lockdep_assert_held_write(&gmap->kvm->mmu_lock); + + _dat_walk_gfn_range(start, end, gmap->asce, &ops, 0, &priv); + return false; +} + +static union pgste __pte_test_and_clear_softdirty(union pte *ptep, union pgste pgste, gfn_t gfn, + struct gmap *gmap) +{ + union pte pte = READ_ONCE(*ptep); + + if (!pte.s.pr || (pte.h.p && !pte.s.sd)) + return pgste; + + /* + * If this page contains one or more prefixes of vCPUS that are currently + * running, do not reset the protection, leave it marked as dirty. + */ + if (!pgste.prefix_notif || gmap_mkold_prefix(gmap, gfn, gfn + 1)) { + pte.h.p = 1; + pte.s.sd = 0; + pgste = gmap_ptep_xchg(gmap, ptep, pte, pgste, gfn); + } + + mark_page_dirty(gmap->kvm, gfn); + + return pgste; +} + +static long _pte_test_and_clear_softdirty(union pte *ptep, gfn_t gfn, gfn_t end, + struct dat_walk *walk) +{ + struct gmap *gmap = walk->priv; + union pgste pgste; + + pgste = pgste_get_lock(ptep); + pgste = __pte_test_and_clear_softdirty(ptep, pgste, gfn, gmap); + pgste_set_unlock(ptep, pgste); + return 0; +} + +static long _crste_test_and_clear_softdirty(union crste *table, gfn_t gfn, gfn_t end, + struct dat_walk *walk) +{ + struct gmap *gmap = walk->priv; + union crste crste, new; + + if (fatal_signal_pending(current)) + return 1; + do { + crste = READ_ONCE(*table); + if (!crste.h.fc) + return 0; + if (crste.h.p && !crste.s.fc1.sd) + return 0; + + /* + * If this large page contains one or more prefixes of vCPUs that are + * currently running, do not reset the protection, leave it marked as + * dirty. + */ + if (crste.s.fc1.prefix_notif && !gmap_mkold_prefix(gmap, gfn, end)) + break; + new = crste; + new.h.p = 1; + new.s.fc1.sd = 0; + } while (!gmap_crstep_xchg_atomic(gmap, table, crste, new, gfn)); + + for ( ; gfn < end; gfn++) + mark_page_dirty(gmap->kvm, gfn); + + return 0; +} + +void gmap_sync_dirty_log(struct gmap *gmap, gfn_t start, gfn_t end) +{ + const struct dat_walk_ops walk_ops = { + .pte_entry = _pte_test_and_clear_softdirty, + .pmd_entry = _crste_test_and_clear_softdirty, + .pud_entry = _crste_test_and_clear_softdirty, + }; + + lockdep_assert_held(&gmap->kvm->mmu_lock); + + _dat_walk_gfn_range(start, end, gmap->asce, &walk_ops, 0, gmap); +} + +static int gmap_handle_minor_crste_fault(struct gmap *gmap, struct guest_fault *f) +{ + union crste newcrste, oldcrste = READ_ONCE(*f->crstep); + + /* Somehow the crste is not large anymore, let the slow path deal with it. */ + if (!oldcrste.h.fc) + return 1; + + f->pfn = PHYS_PFN(large_crste_to_phys(oldcrste, f->gfn)); + f->writable = oldcrste.s.fc1.w; + + f->crste_region3 = is_pud(oldcrste); + /* Appropriate permissions already (race with another handler), nothing to do. */ + if (!oldcrste.h.i && !(f->write_attempt && oldcrste.h.p)) + return 0; + + if (!f->write_attempt || oldcrste.s.fc1.w) { + f->write_attempt |= oldcrste.s.fc1.w && oldcrste.s.fc1.d; + newcrste = oldcrste; + newcrste.h.i = 0; + newcrste.s.fc1.y = 1; + if (f->write_attempt) { + newcrste.h.p = 0; + newcrste.s.fc1.d = 1; + newcrste.s.fc1.sd = 1; + } + /* In case of races, let the slow path deal with it. */ + return !gmap_crstep_xchg_atomic(gmap, f->crstep, oldcrste, newcrste, f->gfn); + } + /* Trying to write on a read-only page, let the slow path deal with it. */ + return 1; +} + +static int _gmap_handle_minor_pte_fault(struct gmap *gmap, union pgste *pgste, + struct guest_fault *f) +{ + union pte newpte, oldpte = READ_ONCE(*f->ptep); + + f->pfn = oldpte.h.pfra; + f->writable = oldpte.s.w; + + /* Appropriate permissions already (race with another handler), nothing to do. */ + if (!oldpte.h.i && !(f->write_attempt && oldpte.h.p)) + return 0; + /* Trying to write on a read-only page, let the slow path deal with it. */ + if (!oldpte.s.pr || (f->write_attempt && !oldpte.s.w)) + return 1; + + newpte = oldpte; + newpte.h.i = 0; + newpte.s.y = 1; + if (f->write_attempt) { + newpte.h.p = 0; + newpte.s.d = 1; + newpte.s.sd = 1; + } + *pgste = gmap_ptep_xchg(gmap, f->ptep, newpte, *pgste, f->gfn); + + return 0; +} + +/** + * gmap_try_fixup_minor() -- Try to fixup a minor gmap fault. + * @gmap: The gmap whose fault needs to be resolved. + * @fault: Describes the fault that is being resolved. + * + * A minor fault is a fault that can be resolved quickly within gmap. + * The page is already mapped, the fault is only due to dirty/young tracking. + * + * Return: 0 in case of success, < 0 in case of error, > 0 if the fault could + * not be resolved and needs to go through the slow path. + */ +int gmap_try_fixup_minor(struct gmap *gmap, struct guest_fault *fault) +{ + union pgste pgste; + int rc; + + lockdep_assert_held(&gmap->kvm->mmu_lock); + + rc = dat_entry_walk(NULL, fault->gfn, gmap->asce, DAT_WALK_LEAF, TABLE_TYPE_PAGE_TABLE, + &fault->crstep, &fault->ptep); + /* If a PTE or a leaf CRSTE could not be reached, slow path. */ + if (rc) + return 1; + + if (fault->ptep) { + pgste = pgste_get_lock(fault->ptep); + rc = _gmap_handle_minor_pte_fault(gmap, &pgste, fault); + if (!rc && fault->callback) + fault->callback(fault); + pgste_set_unlock(fault->ptep, pgste); + } else { + rc = gmap_handle_minor_crste_fault(gmap, fault); + if (!rc && fault->callback) + fault->callback(fault); + } + return rc; +} + +/** + * gmap_2g_allowed() - Check whether a 2G hugepage is allowed. + * @gmap: The gmap of the guest. + * @f: Describes the fault that is being resolved. + * @slot: The memslot the faulting address belongs to. + * + * The function checks whether the GMAP_FLAG_ALLOW_HPAGE_2G flag is set for + * @gmap, whether the offset of the address in the 2G virtual frame is the + * same as the offset in the physical 2G frame, and finally whether the whole + * 2G page would fit in the given memslot. + * + * Return: true if a 2G hugepage is allowed to back the faulting address, false + * otherwise. + */ +static inline bool gmap_2g_allowed(struct gmap *gmap, struct guest_fault *f, + struct kvm_memory_slot *slot) +{ + return test_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &gmap->flags) && + !((f->gfn ^ f->pfn) & ~_REGION3_FR_MASK) && + slot->base_gfn <= ALIGN_DOWN(f->gfn, _PAGES_PER_REGION3) && + slot->base_gfn + slot->npages >= ALIGN(f->gfn + 1, _PAGES_PER_REGION3); +} + +/** + * gmap_1m_allowed() - Check whether a 1M hugepage is allowed. + * @gmap: The gmap of the guest. + * @f: Describes the fault that is being resolved. + * @slot: The memslot the faulting address belongs to. + * + * The function checks whether the GMAP_FLAG_ALLOW_HPAGE_1M flag is set for + * @gmap, whether the offset of the address in the 1M virtual frame is the + * same as the offset in the physical 1M frame, and finally whether the whole + * 1M page would fit in the given memslot. + * + * Return: true if a 1M hugepage is allowed to back the faulting address, false + * otherwise. + */ +static inline bool gmap_1m_allowed(struct gmap *gmap, struct guest_fault *f, + struct kvm_memory_slot *slot) +{ + return test_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &gmap->flags) && + !((f->gfn ^ f->pfn) & ~_SEGMENT_FR_MASK) && + slot->base_gfn <= ALIGN_DOWN(f->gfn, _PAGES_PER_SEGMENT) && + slot->base_gfn + slot->npages >= ALIGN(f->gfn + 1, _PAGES_PER_SEGMENT); +} + +static int _gmap_link(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, int level, + struct guest_fault *f) +{ + union crste oldval, newval; + union pte newpte, oldpte; + union pgste pgste; + int rc = 0; + + rc = dat_entry_walk(mc, f->gfn, gmap->asce, DAT_WALK_ALLOC_CONTINUE, level, + &f->crstep, &f->ptep); + if (rc == -ENOMEM) + return rc; + if (KVM_BUG_ON(rc == -EINVAL, gmap->kvm)) + return rc; + if (rc) + return -EAGAIN; + if (KVM_BUG_ON(get_level(f->crstep, f->ptep) > level, gmap->kvm)) + return -EINVAL; + + if (f->ptep) { + pgste = pgste_get_lock(f->ptep); + oldpte = *f->ptep; + newpte = _pte(f->pfn, f->writable, f->write_attempt | oldpte.s.d, !f->page); + newpte.s.sd = oldpte.s.sd; + oldpte.s.sd = 0; + if (oldpte.val == _PTE_EMPTY.val || oldpte.h.pfra == f->pfn) { + pgste = gmap_ptep_xchg(gmap, f->ptep, newpte, pgste, f->gfn); + if (f->callback) + f->callback(f); + } else { + rc = -EAGAIN; + } + pgste_set_unlock(f->ptep, pgste); + } else { + do { + oldval = READ_ONCE(*f->crstep); + newval = _crste_fc1(f->pfn, oldval.h.tt, f->writable, + f->write_attempt | oldval.s.fc1.d); + newval.s.fc1.s = !f->page; + newval.s.fc1.sd = oldval.s.fc1.sd; + if (oldval.val != _CRSTE_EMPTY(oldval.h.tt).val && + crste_origin_large(oldval) != crste_origin_large(newval)) + return -EAGAIN; + f->crste_region3 = is_pud(newval); + } while (!gmap_crstep_xchg_atomic(gmap, f->crstep, oldval, newval, f->gfn)); + if (f->callback) + f->callback(f); + } + + return rc; +} + +int gmap_link(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, struct guest_fault *f, + struct kvm_memory_slot *slot) +{ + unsigned int order; + int level; + + lockdep_assert_held(&gmap->kvm->mmu_lock); + + level = TABLE_TYPE_PAGE_TABLE; + if (f->page) { + order = folio_order(page_folio(f->page)); + if (order >= get_order(_REGION3_SIZE) && gmap_2g_allowed(gmap, f, slot)) + level = TABLE_TYPE_REGION3; + else if (order >= get_order(_SEGMENT_SIZE) && gmap_1m_allowed(gmap, f, slot)) + level = TABLE_TYPE_SEGMENT; + } + return _gmap_link(mc, gmap, level, f); +} + +static int gmap_ucas_map_one(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, + gfn_t p_gfn, gfn_t c_gfn, bool force_alloc) +{ + union crste newcrste, oldcrste; + struct page_table *pt; + union crste *crstep; + union pte *ptep; + int rc; + + if (force_alloc) + rc = dat_entry_walk(mc, p_gfn, gmap->parent->asce, DAT_WALK_ALLOC, + TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); + else + rc = dat_entry_walk(mc, p_gfn, gmap->parent->asce, DAT_WALK_ALLOC_CONTINUE, + TABLE_TYPE_SEGMENT, &crstep, &ptep); + if (rc) + return rc; + if (!ptep) { + newcrste = _crste_fc0(p_gfn, TABLE_TYPE_SEGMENT); + newcrste.h.i = 1; + newcrste.h.fc0.tl = 1; + } else { + pt = pte_table_start(ptep); + dat_set_ptval(pt, PTVAL_VMADDR, p_gfn >> (_SEGMENT_SHIFT - PAGE_SHIFT)); + newcrste = _crste_fc0(virt_to_pfn(pt), TABLE_TYPE_SEGMENT); + } + rc = dat_entry_walk(mc, c_gfn, gmap->asce, DAT_WALK_ALLOC, TABLE_TYPE_SEGMENT, + &crstep, &ptep); + if (rc) + return rc; + do { + oldcrste = READ_ONCE(*crstep); + if (oldcrste.val == newcrste.val) + break; + } while (!dat_crstep_xchg_atomic(crstep, oldcrste, newcrste, c_gfn, gmap->asce)); + return 0; +} + +static int gmap_ucas_translate_simple(struct gmap *gmap, gpa_t *gaddr, union crste **crstepp) +{ + union pte *ptep; + int rc; + + rc = dat_entry_walk(NULL, gpa_to_gfn(*gaddr), gmap->asce, DAT_WALK_CONTINUE, + TABLE_TYPE_SEGMENT, crstepp, &ptep); + if (rc || (!ptep && !crste_is_ucas(**crstepp))) + return -EREMOTE; + if (!ptep) + return 1; + *gaddr &= ~_SEGMENT_MASK; + *gaddr |= dat_get_ptval(pte_table_start(ptep), PTVAL_VMADDR) << _SEGMENT_SHIFT; + return 0; +} + +/** + * gmap_ucas_translate() - Translate a vcpu address into a host gmap address + * @mc: The memory cache to be used for allocations. + * @gmap: The per-cpu gmap. + * @gaddr: Pointer to the address to be translated, will get overwritten with + * the translated address in case of success. + * Translates the per-vCPU guest address into a fake guest address, which can + * then be used with the fake memslots that are identity mapping userspace. + * This allows ucontrol VMs to use the normal fault resolution path, like + * normal VMs. + * + * Return: %0 in case of success, otherwise %-EREMOTE. + */ +int gmap_ucas_translate(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, gpa_t *gaddr) +{ + gpa_t translated_address; + union crste *crstep; + gfn_t gfn; + int rc; + + gfn = gpa_to_gfn(*gaddr); + + scoped_guard(read_lock, &gmap->kvm->mmu_lock) { + rc = gmap_ucas_translate_simple(gmap, gaddr, &crstep); + if (rc <= 0) + return rc; + } + do { + scoped_guard(write_lock, &gmap->kvm->mmu_lock) { + rc = gmap_ucas_translate_simple(gmap, gaddr, &crstep); + if (rc <= 0) + return rc; + translated_address = (*gaddr & ~_SEGMENT_MASK) | + (crstep->val & _SEGMENT_MASK); + rc = gmap_ucas_map_one(mc, gmap, gpa_to_gfn(translated_address), gfn, true); + } + if (!rc) { + *gaddr = translated_address; + return 0; + } + if (rc != -ENOMEM) + return -EREMOTE; + rc = kvm_s390_mmu_cache_topup(mc); + if (rc) + return rc; + } while (1); + return 0; +} + +int gmap_ucas_map(struct gmap *gmap, gfn_t p_gfn, gfn_t c_gfn, unsigned long count) +{ + struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL; + int rc = 0; + + mc = kvm_s390_new_mmu_cache(); + if (!mc) + return -ENOMEM; + + while (count) { + scoped_guard(write_lock, &gmap->kvm->mmu_lock) + rc = gmap_ucas_map_one(mc, gmap, p_gfn, c_gfn, false); + if (rc == -ENOMEM) { + rc = kvm_s390_mmu_cache_topup(mc); + if (rc) + return rc; + continue; + } + if (rc) + return rc; + + count--; + c_gfn += _PAGE_ENTRIES; + p_gfn += _PAGE_ENTRIES; + } + return rc; +} + +static void gmap_ucas_unmap_one(struct gmap *gmap, gfn_t c_gfn) +{ + union crste *crstep; + union pte *ptep; + int rc; + + rc = dat_entry_walk(NULL, c_gfn, gmap->asce, 0, TABLE_TYPE_SEGMENT, &crstep, &ptep); + if (rc) + return; + while (!dat_crstep_xchg_atomic(crstep, READ_ONCE(*crstep), _PMD_EMPTY, c_gfn, gmap->asce)) + ; +} + +void gmap_ucas_unmap(struct gmap *gmap, gfn_t c_gfn, unsigned long count) +{ + guard(read_lock)(&gmap->kvm->mmu_lock); + + for ( ; count; count--, c_gfn += _PAGE_ENTRIES) + gmap_ucas_unmap_one(gmap, c_gfn); +} + +static long _gmap_split_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) +{ + struct gmap *gmap = walk->priv; + union crste crste, newcrste; + + crste = READ_ONCE(*crstep); + newcrste = _CRSTE_EMPTY(crste.h.tt); + + while (crste_leaf(crste)) { + if (crste_prefix(crste)) + gmap_unmap_prefix(gmap, gfn, next); + if (crste.s.fc1.vsie_notif) + gmap_handle_vsie_unshadow_event(gmap, gfn); + if (dat_crstep_xchg_atomic(crstep, crste, newcrste, gfn, walk->asce)) + break; + crste = READ_ONCE(*crstep); + } + + if (need_resched()) + return next; + + return 0; +} + +void gmap_split_huge_pages(struct gmap *gmap) +{ + const struct dat_walk_ops ops = { + .pmd_entry = _gmap_split_crste, + .pud_entry = _gmap_split_crste, + }; + gfn_t start = 0; + + do { + scoped_guard(read_lock, &gmap->kvm->mmu_lock) + start = _dat_walk_gfn_range(start, asce_end(gmap->asce), gmap->asce, + &ops, DAT_WALK_IGN_HOLES, gmap); + } while (start); +} + +#if KVM_S390_MANAGES_S390_GUEST + +static int _gmap_enable_skeys(struct gmap *gmap) +{ + gfn_t start = 0; + int rc; + + if (uses_skeys(gmap)) + return 0; + + set_bit(GMAP_FLAG_USES_SKEYS, &gmap->flags); + rc = gmap_helper_disable_cow_sharing(); + if (rc) { + clear_bit(GMAP_FLAG_USES_SKEYS, &gmap->flags); + return rc; + } + + do { + scoped_guard(write_lock, &gmap->kvm->mmu_lock) + start = dat_reset_skeys(gmap->asce, start); + } while (start); + return 0; +} + +int gmap_enable_skeys(struct gmap *gmap) +{ + int rc; + + mmap_write_lock(gmap->kvm->mm); + rc = _gmap_enable_skeys(gmap); + mmap_write_unlock(gmap->kvm->mm); + return rc; +} +#endif /* KVM_S390_MANAGES_S390_GUEST */ + +static long _destroy_pages_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) +{ + if (!ptep->s.pr) + return 0; + __kvm_s390_pv_destroy_page(phys_to_page(pte_origin(*ptep))); + if (need_resched()) + return next; + return 0; +} + +static long _destroy_pages_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) +{ + phys_addr_t origin, cur, end; + union crste crste; + + crste = READ_ONCE(*crstep); + if (!crste.h.fc || !crste.s.fc1.pr) + return 0; + + origin = crste_origin_large(crste); + cur = ((max(gfn, walk->start) - gfn) << PAGE_SHIFT) + origin; + end = ((min(next, walk->end) - gfn) << PAGE_SHIFT) + origin; + for ( ; cur < end; cur += PAGE_SIZE) + __kvm_s390_pv_destroy_page(phys_to_page(cur)); + if (need_resched()) + return next; + return 0; +} + +int gmap_pv_destroy_range(struct gmap *gmap, gfn_t start, gfn_t end, bool interruptible) +{ + const struct dat_walk_ops ops = { + .pte_entry = _destroy_pages_pte, + .pmd_entry = _destroy_pages_crste, + .pud_entry = _destroy_pages_crste, + }; + + do { + scoped_guard(read_lock, &gmap->kvm->mmu_lock) + start = _dat_walk_gfn_range(start, end, gmap->asce, &ops, + DAT_WALK_IGN_HOLES, NULL); + if (interruptible && fatal_signal_pending(current)) + return -EINTR; + } while (start && start < end); + return 0; +} + +int gmap_insert_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn, + gfn_t r_gfn, int level) +{ + struct vsie_rmap *rmap __free(kvfree) = NULL; + struct vsie_rmap *temp; + void __rcu **slot; + int rc = 0; + + KVM_BUG_ON(!is_shadow(sg), sg->kvm); + lockdep_assert_held(&sg->host_to_rmap_lock); + + rmap = kvm_s390_mmu_cache_alloc_rmap(mc); + if (!rmap) + return -ENOMEM; + + rmap->r_gfn = r_gfn; + rmap->level = level; + slot = radix_tree_lookup_slot(&sg->host_to_rmap, p_gfn); + if (slot) { + rmap->next = radix_tree_deref_slot_protected(slot, &sg->host_to_rmap_lock); + for (temp = rmap->next; temp; temp = temp->next) { + if (temp->val == rmap->val) + return 0; + } + radix_tree_replace_slot(&sg->host_to_rmap, slot, rmap); + } else { + rmap->next = NULL; + rc = radix_tree_insert(&sg->host_to_rmap, p_gfn, rmap); + if (rc) + return rc; + } + rmap = NULL; + + return 0; +} + +int gmap_protect_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn, + kvm_pfn_t pfn, int level, bool wr) +{ + unsigned long bitmask; + union crste *crstep; + union pgste pgste; + union pte *ptep; + union pte pte; + int flags, rc; + + if (KVM_BUG_ON(!is_shadow(sg) || level <= TABLE_TYPE_PAGE_TABLE, sg->kvm)) + return -EINVAL; + lockdep_assert_held(&sg->parent->children_lock); + + flags = DAT_WALK_SPLIT_ALLOC | (uses_skeys(sg->parent) ? DAT_WALK_USES_SKEYS : 0); + rc = dat_entry_walk(mc, p_gfn, sg->parent->asce, flags, + TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); + if (rc) + return rc; + if (level <= TABLE_TYPE_REGION1) { + bitmask = -1UL << (8 + 11 * level); + scoped_guard(spinlock, &sg->host_to_rmap_lock) + rc = gmap_insert_rmap(mc, sg, p_gfn, r_gfn & bitmask, level); + } + if (rc) + return rc; + + if (!pgste_get_trylock(ptep, &pgste)) + return -EAGAIN; + pte = ptep->s.pr ? *ptep : _pte(pfn, wr, false, false); + pte.h.p = 1; + pgste = _gmap_ptep_xchg(sg->parent, ptep, pte, pgste, p_gfn, false); + pgste.vsie_notif = 1; + pgste_set_unlock(ptep, pgste); + + return 0; +} + +#if KVM_S390_MANAGES_S390_GUEST +static long __set_cmma_clean_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) +{ + union pgste pgste; + + pgste = pgste_get_lock(ptep); + pgste.cmma_d = 0; + pgste_set_unlock(ptep, pgste); + + if (need_resched()) + return next; + return 0; +} + +static long __set_cmma_dirty_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) +{ + union pgste pgste; + + pgste = pgste_get_lock(ptep); + if (!pgste.cmma_d) + atomic64_inc(walk->priv); + pgste.cmma_d = 1; + pgste_set_unlock(ptep, pgste); + + if (need_resched()) + return next; + return 0; +} + +void _gmap_set_cmma_all(struct gmap *gmap, bool dirty) +{ + const struct dat_walk_ops ops = { + .pte_entry = dirty ? __set_cmma_dirty_pte : __set_cmma_clean_pte, + }; + gfn_t gfn = 0; + + do { + scoped_guard(read_lock, &gmap->kvm->mmu_lock) + gfn = _dat_walk_gfn_range(gfn, asce_end(gmap->asce), gmap->asce, &ops, + DAT_WALK_IGN_HOLES, + &gmap->kvm->arch.cmma_dirty_pages); + } while (gfn); +} +#endif /* KVM_S390_MANAGES_S390_GUEST */ + +static void gmap_unshadow_level(struct gmap *sg, gfn_t r_gfn, int level) +{ + unsigned long align = PAGE_SIZE; + gpa_t gaddr = gfn_to_gpa(r_gfn); + union crste *crstep; + union crste crste; + union pte *ptep; + + if (level > TABLE_TYPE_PAGE_TABLE) + align = 1UL << (11 * level + _SEGMENT_SHIFT); + kvm_s390_vsie_gmap_notifier(sg, ALIGN_DOWN(gaddr, align), ALIGN(gaddr + 1, align)); + sg->invalidated = true; + if (dat_entry_walk(NULL, r_gfn, sg->asce, 0, level, &crstep, &ptep)) + return; + if (ptep) { + if (READ_ONCE(*ptep).val != _PTE_EMPTY.val) + dat_ptep_xchg(ptep, _PTE_EMPTY, r_gfn, sg->asce, uses_skeys(sg)); + return; + } + + crste = dat_crstep_clear_atomic(crstep, r_gfn, sg->asce); + if (crste_leaf(crste) || crste.h.i) + return; + if (is_pmd(crste)) + dat_free_pt(dereference_pmd(crste.pmd)); + else + dat_free_level(dereference_crste(crste), true); +} + +static void gmap_unshadow(struct gmap *sg) +{ + struct gmap_cache *gmap_cache, *next; + + KVM_BUG_ON(!is_shadow(sg), sg->kvm); + KVM_BUG_ON(!sg->parent, sg->kvm); + + lockdep_assert_held(&sg->parent->children_lock); + + gmap_remove_child(sg); + kvm_s390_vsie_gmap_notifier(sg, 0, -1UL); + + list_for_each_entry_safe(gmap_cache, next, &sg->scb_users, list) { + gmap_cache->gmap = NULL; + list_del(&gmap_cache->list); + } + + gmap_put(sg); +} + +void _gmap_handle_vsie_unshadow_event(struct gmap *parent, gfn_t gfn) +{ + struct vsie_rmap *rmap, *rnext, *head; + struct gmap *sg, *next; + gfn_t start, end; + + list_for_each_entry_safe(sg, next, &parent->children, list) { + start = sg->guest_asce.rsto; + end = start + sg->guest_asce.tl + 1; + if (!sg->guest_asce.r && gfn >= start && gfn < end) { + gmap_unshadow(sg); + continue; + } + scoped_guard(spinlock, &sg->host_to_rmap_lock) + head = radix_tree_delete(&sg->host_to_rmap, gfn); + gmap_for_each_rmap_safe(rmap, rnext, head) { + gmap_unshadow_level(sg, rmap->r_gfn, rmap->level); + kfree(rmap); + } + } +} + +/** + * gmap_find_shadow() - Find a specific ASCE in the list of shadow tables. + * @parent: Pointer to the parent gmap. + * @asce: ASCE for which the shadow table is created. + * @edat_level: Edat level to be used for the shadow translation. + * + * Context: Called with parent->children_lock held. + * + * Return: The pointer to a gmap if a shadow table with the given asce is + * already available, ERR_PTR(-EAGAIN) if another one is just being created, + * otherwise NULL. + */ +static struct gmap *gmap_find_shadow(struct gmap *parent, union asce asce, int edat_level) +{ + struct gmap *sg; + + lockdep_assert_held(&parent->children_lock); + list_for_each_entry(sg, &parent->children, list) { + if (!gmap_is_shadow_valid(sg, asce, edat_level)) + continue; + return sg; + } + return NULL; +} + +#define CRST_TABLE_PAGES (_CRST_TABLE_SIZE / PAGE_SIZE) +struct gmap_protect_asce_top_level { + unsigned long seq; + struct guest_fault f[CRST_TABLE_PAGES]; +}; + +static inline int __gmap_protect_asce_top_level(struct kvm_s390_mmu_cache *mc, struct gmap *sg, + struct gmap_protect_asce_top_level *context) +{ + struct gmap *parent; + int rc, i; + + guard(write_lock)(&sg->kvm->mmu_lock); + + if (kvm_s390_array_needs_retry_safe(sg->kvm, context->seq, context->f)) + return -EAGAIN; + + parent = READ_ONCE(sg->parent); + if (!parent) + return -EAGAIN; + scoped_guard(spinlock, &parent->children_lock) { + if (READ_ONCE(sg->parent) != parent) + return -EAGAIN; + sg->invalidated = false; + for (i = 0; i < CRST_TABLE_PAGES; i++) { + if (!context->f[i].valid) + continue; + rc = gmap_protect_rmap(mc, sg, context->f[i].gfn, 0, context->f[i].pfn, + TABLE_TYPE_REGION1 + 1, context->f[i].writable); + if (rc) + return rc; + } + gmap_add_child(sg->parent, sg); + } + + kvm_s390_release_faultin_array(sg->kvm, context->f, false); + return 0; +} + +static inline int _gmap_protect_asce_top_level(struct kvm_s390_mmu_cache *mc, struct gmap *sg, + struct gmap_protect_asce_top_level *context) +{ + int rc; + + if (kvm_s390_array_needs_retry_unsafe(sg->kvm, context->seq, context->f)) + return -EAGAIN; + do { + rc = kvm_s390_mmu_cache_topup(mc); + if (rc) + return rc; + rc = radix_tree_preload(GFP_KERNEL); + if (rc) + return rc; + rc = __gmap_protect_asce_top_level(mc, sg, context); + radix_tree_preload_end(); + } while (rc == -ENOMEM); + + return rc; +} + +static int gmap_protect_asce_top_level(struct kvm_s390_mmu_cache *mc, struct gmap *sg) +{ + struct gmap_protect_asce_top_level context = {}; + union asce asce = sg->guest_asce; + int rc; + + KVM_BUG_ON(!is_shadow(sg), sg->kvm); + + context.seq = sg->kvm->mmu_invalidate_seq; + /* Pairs with the smp_wmb() in kvm_mmu_invalidate_end(). */ + smp_rmb(); + + rc = kvm_s390_get_guest_pages(sg->kvm, context.f, asce.rsto, asce.tl + 1, false); + if (rc > 0) + rc = -EFAULT; + if (!rc) + rc = _gmap_protect_asce_top_level(mc, sg, &context); + if (rc) + kvm_s390_release_faultin_array(sg->kvm, context.f, true); + return rc; +} + +/** + * gmap_create_shadow() - Create/find a shadow guest address space. + * @mc: The cache to use to allocate dat tables. + * @parent: Pointer to the parent gmap. + * @asce: ASCE for which the shadow table is created. + * @edat_level: Edat level to be used for the shadow translation. + * + * The pages of the top level page table referred by the asce parameter + * will be set to read-only and marked in the PGSTEs of the kvm process. + * The shadow table will be removed automatically on any change to the + * PTE mapping for the source table. + * + * The returned shadow gmap will be returned with one extra reference. + * + * Return: A guest address space structure, ERR_PTR(-ENOMEM) if out of memory, + * ERR_PTR(-EAGAIN) if the caller has to retry and ERR_PTR(-EFAULT) if the + * parent gmap table could not be protected. + */ +struct gmap *gmap_create_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *parent, + union asce asce, int edat_level) +{ + struct gmap *sg, *new; + int rc; + + if (WARN_ON(!parent)) + return ERR_PTR(-EINVAL); + + scoped_guard(spinlock, &parent->children_lock) { + sg = gmap_find_shadow(parent, asce, edat_level); + if (sg) { + gmap_get(sg); + return sg; + } + } + /* Create a new shadow gmap. */ + new = gmap_new(parent->kvm, asce.r ? 1UL << (64 - PAGE_SHIFT) : asce_end(asce)); + if (!new) + return ERR_PTR(-ENOMEM); + new->guest_asce = asce; + new->edat_level = edat_level; + set_bit(GMAP_FLAG_SHADOW, &new->flags); + + scoped_guard(spinlock, &parent->children_lock) { + /* Recheck if another CPU created the same shadow. */ + sg = gmap_find_shadow(parent, asce, edat_level); + if (sg) { + gmap_put(new); + gmap_get(sg); + return sg; + } + if (asce.r) { + /* Only allow one real-space gmap shadow. */ + list_for_each_entry(sg, &parent->children, list) { + if (sg->guest_asce.r) { + if (write_trylock(&parent->kvm->mmu_lock)) { + gmap_unshadow(sg); + write_unlock(&parent->kvm->mmu_lock); + } else { + gmap_put(new); + return ERR_PTR(-EAGAIN); + } + break; + } + } + gmap_add_child(parent, new); + /* Nothing to protect, return right away. */ + gmap_get(new); + return new; + } + } + + gmap_get(new); + new->parent = parent; + /* Protect while inserting, protects against invalidation races. */ + rc = gmap_protect_asce_top_level(mc, new); + if (rc) { + new->parent = NULL; + gmap_put(new); + gmap_put(new); + return ERR_PTR(rc); + } + return new; +} diff --git a/arch/s390/kvm/gmap/gmap.h b/arch/s390/kvm/gmap/gmap.h new file mode 100644 index 000000000000..8f47f29c89fd --- /dev/null +++ b/arch/s390/kvm/gmap/gmap.h @@ -0,0 +1,348 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * KVM guest address space mapping code + * + * Copyright IBM Corp. 2007, 2016, 2025 + * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> + * Claudio Imbrenda <imbrenda@linux.ibm.com> + */ + +#ifndef ARCH_KVM_GMAP_GMAP_H +#define ARCH_KVM_GMAP_GMAP_H + +#include <linux/kvm_host.h> + +#include "dat.h" + +/** + * enum gmap_flags - Flags of a gmap. + * + * @GMAP_FLAG_SHADOW: The gmap is a vsie shadow gmap. + * @GMAP_FLAG_OWNS_PAGETABLES: The gmap owns all dat levels; normally 1, is 0 + * only for ucontrol per-cpu gmaps, since they + * share the page tables with the main gmap. + * @GMAP_FLAG_IS_UCONTROL: The gmap is ucontrol (main gmap or per-cpu gmap). + * @GMAP_FLAG_ALLOW_HPAGE_1M: 1M hugepages are allowed for this gmap, + * independently of the page size used by userspace. + * @GMAP_FLAG_ALLOW_HPAGE_2G: 2G hugepages are allowed for this gmap, + * independently of the page size used by userspace. + * @GMAP_FLAG_PFAULT_ENABLED: Pfault is enabled for the gmap. + * @GMAP_FLAG_USES_SKEYS: If the guest uses storage keys. + * @GMAP_FLAG_USES_CMM: Whether the guest uses CMMA. + * @GMAP_FLAG_EXPORT_ON_UNMAP: Whether to export guest pages when unmapping. + */ +enum gmap_flags { + GMAP_FLAG_SHADOW = 0, + GMAP_FLAG_OWNS_PAGETABLES, + GMAP_FLAG_IS_UCONTROL, + GMAP_FLAG_ALLOW_HPAGE_1M, + GMAP_FLAG_ALLOW_HPAGE_2G, + GMAP_FLAG_PFAULT_ENABLED, + GMAP_FLAG_USES_SKEYS, + GMAP_FLAG_USES_CMM, + GMAP_FLAG_EXPORT_ON_UNMAP, +}; + +/** + * struct gmap_struct - Guest address space. + * + * @flags: GMAP_FLAG_* flags. + * @edat_level: The edat level of this shadow gmap. + * @kvm: The vm. + * @asce: The ASCE used by this gmap. + * @list: List head used in children gmaps for the children gmap list. + * @children_lock: Protects children and scb_users. + * @children: List of child gmaps of this gmap. + * @scb_users: List of vsie_scb that use this shadow gmap. + * @parent: Parent gmap of a child gmap. + * @guest_asce: Original ASCE of this shadow gmap. + * @host_to_rmap_lock: Protects host_to_rmap. + * @host_to_rmap: Radix tree mapping host addresses to guest addresses. + */ +struct gmap { + unsigned long flags; + unsigned char edat_level; + bool invalidated; + struct kvm *kvm; + union asce asce; + struct list_head list; + spinlock_t children_lock; /* Protects: children, scb_users */ + struct list_head children; + struct list_head scb_users; + struct gmap *parent; + union asce guest_asce; + spinlock_t host_to_rmap_lock; /* Protects host_to_rmap */ + struct radix_tree_root host_to_rmap; + refcount_t refcount; +}; + +struct gmap_cache { + struct list_head list; + struct gmap *gmap; +}; + +#define gmap_for_each_rmap_safe(pos, n, head) \ + for (pos = (head); n = pos ? pos->next : NULL, pos; pos = n) + +int s390_replace_asce(struct gmap *gmap); +bool gmap_age_gfn(struct gmap *gmap, gfn_t start, gfn_t end); +bool gmap_unmap_gfn_range(struct gmap *gmap, struct kvm_memory_slot *slot, gfn_t start, gfn_t end); +int gmap_try_fixup_minor(struct gmap *gmap, struct guest_fault *fault); +struct gmap *gmap_new(struct kvm *kvm, gfn_t limit); +struct gmap *gmap_new_child(struct gmap *parent, gfn_t limit); +void gmap_remove_child(struct gmap *child); +void gmap_dispose(struct gmap *gmap); +int gmap_link(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, struct guest_fault *fault, + struct kvm_memory_slot *slot); +void gmap_sync_dirty_log(struct gmap *gmap, gfn_t start, gfn_t end); +int gmap_set_limit(struct gmap *gmap, gfn_t limit); +int gmap_ucas_translate(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, gpa_t *gaddr); +int gmap_ucas_map(struct gmap *gmap, gfn_t p_gfn, gfn_t c_gfn, unsigned long count); +void gmap_ucas_unmap(struct gmap *gmap, gfn_t c_gfn, unsigned long count); + +#if KVM_S390_MANAGES_S390_GUEST +int gmap_enable_skeys(struct gmap *gmap); +#endif /* KVM_S390_MANAGES_S390_GUEST */ + +int gmap_pv_destroy_range(struct gmap *gmap, gfn_t start, gfn_t end, bool interruptible); +int gmap_insert_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn, + gfn_t r_gfn, int level); +int gmap_protect_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn, + kvm_pfn_t pfn, int level, bool wr); +void _gmap_handle_vsie_unshadow_event(struct gmap *parent, gfn_t gfn); +struct gmap *gmap_create_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, + union asce asce, int edat_level); +void gmap_split_huge_pages(struct gmap *gmap); + +static inline bool uses_skeys(struct gmap *gmap) +{ + return test_bit(GMAP_FLAG_USES_SKEYS, &gmap->flags); +} + +static inline bool uses_cmm(struct gmap *gmap) +{ + return test_bit(GMAP_FLAG_USES_CMM, &gmap->flags); +} + +static inline bool pfault_enabled(struct gmap *gmap) +{ + return test_bit(GMAP_FLAG_PFAULT_ENABLED, &gmap->flags); +} + +static inline bool is_ucontrol(struct gmap *gmap) +{ + return test_bit(GMAP_FLAG_IS_UCONTROL, &gmap->flags); +} + +static inline bool is_shadow(struct gmap *gmap) +{ + return test_bit(GMAP_FLAG_SHADOW, &gmap->flags); +} + +static inline bool owns_page_tables(struct gmap *gmap) +{ + return test_bit(GMAP_FLAG_OWNS_PAGETABLES, &gmap->flags); +} + +static inline struct gmap *gmap_put(struct gmap *gmap) +{ + if (refcount_dec_and_test(&gmap->refcount)) + gmap_dispose(gmap); + return NULL; +} + +static inline void gmap_get(struct gmap *gmap) +{ + WARN_ON_ONCE(unlikely(!refcount_inc_not_zero(&gmap->refcount))); +} + +static inline void gmap_handle_vsie_unshadow_event(struct gmap *parent, gfn_t gfn) +{ + scoped_guard(spinlock, &parent->children_lock) + _gmap_handle_vsie_unshadow_event(parent, gfn); +} + +#if KVM_S390_MANAGES_S390_GUEST +bool _gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end, bool hint); +#else +static inline bool _gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end, bool hint) +{ + return true; +} +#endif /* KVM_S390_MANAGES_S390_GUEST */ +static inline bool gmap_mkold_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end) +{ + return _gmap_unmap_prefix(gmap, gfn, end, true); +} + +static inline bool gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end) +{ + return _gmap_unmap_prefix(gmap, gfn, end, false); +} + +/** + * pte_needs_unshadow() -- Check if the pte operations triggers unshadowing. + * @oldpte: the previous value for the guest pte. + * @newpte: the new pte being set. + * @pgste: the pgste for the pte entry. + * + * If the pgste.vsie_notif bit is not set, return false: the page is not + * involved in vsie and thus should not trigger an unshadow operation. + * + * If the pgste.vsie_gmem bit is set, this pte represents shadowed guest + * memory. The access rights on g3's memory should be synchronized with g1's + * and g2's. Therefore unshadowing is triggered if the new and old pte + * differ in protection, or if the new pte is invalid. + * + * If the pgste.vsie_gmem bit is not set, this pte maps the g2 dat tables + * for g3. If the entry becomes writable or absent, it becomes impossible to + * guarantee that the shadow mapping will match g2's mapping. In that case, + * trigger an unshadow event. + * + * Return: true if an unshadow event should be triggered, otherwise false. + */ +static inline bool pte_needs_unshadow(union pte oldpte, union pte newpte, union pgste pgste) +{ + if (!pgste.vsie_notif) + return false; + if (pgste.vsie_gmem) + return (oldpte.h.p != newpte.h.p) || newpte.h.i; + return !newpte.h.p || !newpte.s.pr; +} + +#if KVM_S390_MANAGES_S390_GUEST +void _gmap_set_cmma_all(struct gmap *gmap, bool dirty); +static inline void gmap_set_cmma_all_dirty(struct gmap *gmap) +{ + _gmap_set_cmma_all(gmap, true); +} + +static inline void gmap_set_cmma_all_clean(struct gmap *gmap) +{ + _gmap_set_cmma_all(gmap, false); +} +#endif /* KVM_S390_MANAGES_S390_GUEST */ + +static inline union pgste _gmap_ptep_xchg(struct gmap *gmap, union pte *ptep, union pte newpte, + union pgste pgste, gfn_t gfn, bool needs_lock) +{ + lockdep_assert_held(&gmap->kvm->mmu_lock); + if (!needs_lock) + lockdep_assert_held(&gmap->children_lock); + else + lockdep_assert_not_held(&gmap->children_lock); + + if (pgste.prefix_notif && (newpte.h.p || newpte.h.i)) { + pgste.prefix_notif = 0; + gmap_unmap_prefix(gmap, gfn, gfn + 1); + } + if (pte_needs_unshadow(*ptep, newpte, pgste)) { + pgste.vsie_notif = 0; + pgste.vsie_gmem = 0; + if (needs_lock) + gmap_handle_vsie_unshadow_event(gmap, gfn); + else + _gmap_handle_vsie_unshadow_event(gmap, gfn); + } + if (!ptep->s.d && newpte.s.d && !newpte.s.s) + SetPageDirty(pfn_to_page(newpte.h.pfra)); + pgste.zero = 0; + return __dat_ptep_xchg(ptep, pgste, newpte, gfn, gmap->asce, uses_skeys(gmap)); +} + +static inline union pgste gmap_ptep_xchg(struct gmap *gmap, union pte *ptep, union pte newpte, + union pgste pgste, gfn_t gfn) +{ + return _gmap_ptep_xchg(gmap, ptep, newpte, pgste, gfn, true); +} + +/** + * crste_needs_unshadow() -- Check if the crste operations triggers unshadowing. + * @oldcrste: the previous value for the crste. + * @newcrste: the new value for the crste. + * + * If the old crste did not have the vsie_notif bit set, return false: the + * page is not involved in vsie and thus should not trigger an unshadow + * operation. Conversely, if the bit is set, it can only be g3 memory, since + * dat tables are never mapped using large pages. + * + * Similar to the pgste.vsie_gmem case of pte_needs_unshadow(), if the + * protection bit is changing or the new page is invalid, trigger an + * unshadow event. Also trigger an unshadow event if the new crste does not + * have the vsie_notif bit set. + * + * Return: true if an unshadow event should be triggered, otherwise false. + */ +static inline bool crste_needs_unshadow(union crste oldcrste, union crste newcrste) +{ + if (!oldcrste.s.fc1.vsie_notif) + return false; + return (newcrste.h.p != oldcrste.h.p) || newcrste.h.i || !newcrste.s.fc1.vsie_notif; +} + +static inline bool __must_check _gmap_crstep_xchg_atomic(struct gmap *gmap, union crste *crstep, + union crste oldcrste, union crste newcrste, + gfn_t gfn, bool needs_lock) +{ + unsigned long align = is_pmd(newcrste) ? _PAGE_ENTRIES : _PAGE_ENTRIES * _CRST_ENTRIES; + + if (KVM_BUG_ON(crstep->h.tt != oldcrste.h.tt || newcrste.h.tt != oldcrste.h.tt, gmap->kvm)) + return true; + + lockdep_assert_held(&gmap->kvm->mmu_lock); + if (!needs_lock) + lockdep_assert_held(&gmap->children_lock); + + gfn = ALIGN_DOWN(gfn, align); + if (crste_prefix(oldcrste) && (newcrste.h.p || newcrste.h.i || !crste_prefix(newcrste))) { + newcrste.s.fc1.prefix_notif = 0; + gmap_unmap_prefix(gmap, gfn, gfn + align); + } + if (crste_leaf(oldcrste) && crste_needs_unshadow(oldcrste, newcrste)) { + newcrste = oldcrste; + newcrste.s.fc1.vsie_notif = 0; + if (needs_lock) + gmap_handle_vsie_unshadow_event(gmap, gfn); + else + _gmap_handle_vsie_unshadow_event(gmap, gfn); + if (!dat_crstep_xchg_atomic(crstep, oldcrste, newcrste, gfn, gmap->asce)) + return false; + /* + * Return false even if the swap was successful, as it only + * indicates that the best effort clearing of the vsie_notif + * bit was successful. The caller will have to try again + * regardless, since the desired value has not been set. + * This pointless check is needed to silence a potential + * __must_check warning. + */ + return false; + } + if (!oldcrste.s.fc1.d && newcrste.s.fc1.d && !newcrste.s.fc1.s) + SetPageDirty(phys_to_page(crste_origin_large(newcrste))); + return dat_crstep_xchg_atomic(crstep, oldcrste, newcrste, gfn, gmap->asce); +} + +static inline bool __must_check gmap_crstep_xchg_atomic(struct gmap *gmap, union crste *crstep, + union crste oldcrste, union crste newcrste, + gfn_t gfn) +{ + return _gmap_crstep_xchg_atomic(gmap, crstep, oldcrste, newcrste, gfn, true); +} + +/** + * gmap_is_shadow_valid() - check if a shadow guest address space matches the + * given properties and is still valid. + * @sg: Pointer to the shadow guest address space structure. + * @asce: ASCE for which the shadow table is requested. + * @edat_level: Edat level to be used for the shadow translation. + * + * Return: true if the gmap shadow is still valid and matches the given + * properties and the caller can continue using it; false otherwise, the + * caller has to request a new shadow gmap in this case. + */ +static inline bool gmap_is_shadow_valid(struct gmap *sg, union asce asce, int edat_level) +{ + return sg->guest_asce.val == asce.val && sg->edat_level == edat_level; +} + +#endif /* ARCH_KVM_GMAP_GMAP_H */ diff --git a/arch/s390/kvm/gmap/kvm_mmu.c b/arch/s390/kvm/gmap/kvm_mmu.c new file mode 100644 index 000000000000..c2ffb5e59ec6 --- /dev/null +++ b/arch/s390/kvm/gmap/kvm_mmu.c @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/kvm_types.h> +#include <linux/kvm_host.h> + +#include "s390.h" +#include "gmap.h" +#include "dat.h" +#include "kvm_mmu.h" + +/* + * Get (and clear) the dirty memory log for a memory slot. + */ +int s390_kvm_mmu_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log) +{ + int r; + unsigned long n; + struct kvm_memory_slot *memslot; + int is_dirty; + + if (kvm_is_ucontrol(kvm)) + return -EINVAL; + + mutex_lock(&kvm->slots_lock); + + r = -EINVAL; + if (log->slot >= KVM_USER_MEM_SLOTS) + goto out; + + r = kvm_get_dirty_log(kvm, log, &is_dirty, &memslot); + if (r) + goto out; + + /* Clear the dirty log */ + if (is_dirty) { + n = kvm_dirty_bitmap_bytes(memslot); + memset(memslot->dirty_bitmap, 0, n); + } + r = 0; +out: + mutex_unlock(&kvm->slots_lock); + return r; +} + +int s390_kvm_mmu_prepare_memory_region(struct kvm *kvm, + const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ + struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL; + int rc = 0; + + if (kvm_is_ucontrol(kvm) && new && new->id < KVM_USER_MEM_SLOTS) + return -EINVAL; + + /* When we are protected, we should not change the memory slots */ + if (kvm_s390_pv_get_handle(kvm)) + return -EINVAL; + + if (change != KVM_MR_DELETE && change != KVM_MR_FLAGS_ONLY) { + /* + * A few sanity checks. The memory in userland is ok to be + * fragmented into various different vmas. It is okay to mmap() + * and munmap() stuff in this slot after doing this call at any + * time. + */ + if (change != KVM_MR_MOVE && change != KVM_MR_CREATE) { + WARN(1, "Unknown KVM MR CHANGE: %d\n", change); + return -EINVAL; + } + if (new->userspace_addr & ~PAGE_MASK) + return -EINVAL; + if ((new->base_gfn + new->npages) * PAGE_SIZE > kvm->arch.mem_limit) + return -EINVAL; + if (!asce_contains_gfn(kvm->arch.gmap->asce, new->base_gfn + new->npages - 1)) + return -EINVAL; + } + + if (kvm->arch.migration_mode) { + /* + * Turn off migration mode when: + * - userspace creates a new memslot with dirty logging off, + * - userspace modifies an existing memslot (MOVE or FLAGS_ONLY) + * and dirty logging is turned off. + * Migration mode expects dirty page logging being enabled to + * store its dirty bitmap. + */ + if (change != KVM_MR_DELETE && + !(new->flags & KVM_MEM_LOG_DIRTY_PAGES)) + WARN(kvm_s390_vm_stop_migration(kvm), + "Failed to stop migration mode"); + } + + if (change == KVM_MR_FLAGS_ONLY) + return 0; + if (change != KVM_MR_DELETE) { + /* Enough capacity to add a new memslot */ + mc = kvm_s390_new_mmu_cache(); + if (!mc) + return -ENOMEM; + } + scoped_guard(write_lock, &kvm->mmu_lock) { + kvm_s390_update_cmma_dirty(kvm, old); + if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) + rc = dat_delete_slot(kvm->arch.gmap->asce, old->base_gfn, old->npages); + if (!rc && (change == KVM_MR_MOVE || change == KVM_MR_CREATE)) + rc = dat_create_slot(mc, kvm->arch.gmap->asce, new->base_gfn, new->npages); + } + /* + * Can only be triggered if dat_{create,delete}_slot() found an + * internal inconsistency or if the mmu cache ran out of memory; + * both should be impossible. + */ + KVM_BUG_ON(rc, kvm); + return rc; +} diff --git a/arch/s390/kvm/gmap/kvm_mmu.h b/arch/s390/kvm/gmap/kvm_mmu.h new file mode 100644 index 000000000000..43cde61bae03 --- /dev/null +++ b/arch/s390/kvm/gmap/kvm_mmu.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef ARCH_KVM_GMAP_KVM_MMU_H +#define ARCH_KVM_GMAP_KVM_MMU_H + +#include <linux/kvm_host.h> + +int s390_kvm_mmu_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log); +int s390_kvm_mmu_prepare_memory_region(struct kvm *kvm, + const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, + enum kvm_mr_change change); + +#endif /* ARCH_KVM_GMAP_KVM_MMU_H */ diff --git a/arch/s390/kvm/gmap/trace_gmap.h b/arch/s390/kvm/gmap/trace_gmap.h new file mode 100644 index 000000000000..15ec88bdce55 --- /dev/null +++ b/arch/s390/kvm/gmap/trace_gmap.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#if !defined(GMAP_TRACE_KVM_H) || defined(TRACE_HEADER_MULTI_READ) +#define GMAP_TRACE_KVM_H + +#include <linux/tracepoint.h> + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM kvm +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH ../gmap +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace_gmap + +#define __KVM_FIELDS \ + __field(unsigned long, pswmask) \ + __field(unsigned long, pswaddr) +#define __KVM_ASSIGN ({\ + __entry->pswmask = vcpu->arch.sie_block->gpsw.mask; \ + __entry->pswaddr = vcpu->arch.sie_block->gpsw.addr; \ + }) +#define __KVM_PRINT \ + __entry->pswmask,\ + __entry->pswaddr + +TRACE_EVENT(kvm_s390_major_guest_pfault, + TP_PROTO(struct kvm_vcpu *vcpu), + TP_ARGS(vcpu), + + TP_STRUCT__entry( + __field(int, id) + __KVM_FIELDS + ), + + TP_fast_assign( + __entry->id = vcpu->vcpu_id; + __KVM_ASSIGN + ), + TP_printk("%02d[%016lx-%016lx]: major fault, maybe applicable for pfault", + __entry->id, + __KVM_PRINT + ) + ); + +#endif /* GMAP_TRACE_KVM_H */ + +/* This part must be outside protection */ +#include <trace/define_trace.h> diff --git a/arch/s390/kvm/s390/Makefile b/arch/s390/kvm/s390/Makefile new file mode 100644 index 000000000000..762a63826423 --- /dev/null +++ b/arch/s390/kvm/s390/Makefile @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0 + +KVM := ../../../../virt/kvm +include $(srctree)/virt/kvm/Makefile.kvm +include $(srctree)/arch/s390/kvm/gmap/Makefile + +ccflags-y := -I$(src) -I$(srctree)/arch/s390/kvm/gmap + +kvm-y += s390.o intercept.o interrupt.o priv.o sigp.o +kvm-y += diag.o gaccess.o guestdbg.o vsie.o pv.o +kvm-y += $(gmap-y) + +kvm-$(CONFIG_VFIO_PCI_ZDEV_KVM) += pci.o +obj-$(CONFIG_KVM) += kvm.o diff --git a/arch/s390/kvm/diag.c b/arch/s390/kvm/s390/diag.c index 53233dec8cad..09033c8ff1bb 100644 --- a/arch/s390/kvm/diag.c +++ b/arch/s390/kvm/s390/diag.c @@ -10,13 +10,13 @@ #include <linux/kvm.h> #include <linux/kvm_host.h> -#include <asm/gmap.h> #include <asm/gmap_helpers.h> #include <asm/virtio-ccw.h> -#include "kvm-s390.h" +#include "s390.h" #include "trace.h" #include "trace-s390.h" #include "gaccess.h" +#include "gmap.h" static void do_discard_gfn_range(struct kvm_vcpu *vcpu, gfn_t gfn_start, gfn_t gfn_end) { @@ -186,7 +186,8 @@ static int diag9c_forwarding_overrun(void) static int __diag_time_slice_end_directed(struct kvm_vcpu *vcpu) { struct kvm_vcpu *tcpu; - int tcpu_cpu; + const char *result; + int tcpu_cpu = -1; int tid; tid = vcpu->run->s.regs.gprs[(vcpu->arch.sie_block->ipa & 0xf0) >> 4]; @@ -211,21 +212,23 @@ static int __diag_time_slice_end_directed(struct kvm_vcpu *vcpu) if (!vcpu_is_preempted(tcpu_cpu)) goto no_yield; smp_yield_cpu(tcpu_cpu); - VCPU_EVENT(vcpu, 5, - "diag time slice end directed to %d: yield forwarded", - tid); vcpu->stat.diag_9c_forward++; - return 0; + result = "yield forwarded"; + goto out; } if (kvm_vcpu_yield_to(tcpu) <= 0) goto no_yield; - VCPU_EVENT(vcpu, 5, "diag time slice end directed to %d: done", tid); - return 0; + result = "done"; + goto out; no_yield: - VCPU_EVENT(vcpu, 5, "diag time slice end directed to %d: ignored", tid); vcpu->stat.diag_9c_ignored++; + result = "ignored"; +out: + VCPU_EVENT(vcpu, 5, "diag time slice end directed to %d: %s", tid, + result); + trace_kvm_s390_diag_9c(vcpu, tid, tcpu_cpu, result); return 0; } diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/s390/gaccess.c index 41ca6b0ee7a9..405345ccc4f1 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/s390/gaccess.c @@ -11,41 +11,43 @@ #include <linux/err.h> #include <linux/pgtable.h> #include <linux/bitfield.h> +#include <linux/kvm_host.h> +#include <linux/kvm_types.h> +#include <asm/diag.h> #include <asm/access-regs.h> #include <asm/fault.h> -#include <asm/gmap.h> #include <asm/dat-bits.h> -#include "kvm-s390.h" +#include "s390.h" +#include "dat.h" +#include "gmap.h" #include "gaccess.h" +#include "faultin.h" #define GMAP_SHADOW_FAKE_TABLE 1ULL -/* - * vaddress union in order to easily decode a virtual address into its - * region first index, region second index etc. parts. - */ -union vaddress { - unsigned long addr; - struct { - unsigned long rfx : 11; - unsigned long rsx : 11; - unsigned long rtx : 11; - unsigned long sx : 11; - unsigned long px : 8; - unsigned long bx : 12; - }; - struct { - unsigned long rfx01 : 2; - unsigned long : 9; - unsigned long rsx01 : 2; - unsigned long : 9; - unsigned long rtx01 : 2; - unsigned long : 9; - unsigned long sx01 : 2; - unsigned long : 29; - }; +union dat_table_entry { + unsigned long val; + union region1_table_entry pgd; + union region2_table_entry p4d; + union region3_table_entry pud; + union segment_table_entry pmd; + union page_table_entry pte; +}; + +#define WALK_N_ENTRIES 7 +#define LEVEL_MEM -2 +struct pgtwalk { + struct guest_fault raw_entries[WALK_N_ENTRIES]; + gpa_t last_addr; + int level; + bool p; }; +static inline struct guest_fault *get_entries(struct pgtwalk *w) +{ + return w->raw_entries - LEVEL_MEM; +} + /* * raddress union which will contain the result (real or absolute address) * after a page table walk. The rfaa, sfaa and pfra members are used to @@ -107,6 +109,28 @@ struct aste { /* .. more fields there */ }; +union oac { + unsigned int val; + struct { + struct { + unsigned short key : 4; + unsigned short : 4; + unsigned short as : 2; + unsigned short : 4; + unsigned short k : 1; + unsigned short a : 1; + } oac1; + struct { + unsigned short key : 4; + unsigned short : 4; + unsigned short as : 2; + unsigned short : 4; + unsigned short k : 1; + unsigned short a : 1; + } oac2; + }; +}; + int ipte_lock_held(struct kvm *kvm) { if (sclp.has_siif) @@ -127,10 +151,8 @@ retry: ic = &kvm->arch.sca->ipte_control; old = READ_ONCE(*ic); do { - if (old.k) { - cond_resched(); + if (old.k) goto retry; - } new = old; new.k = 1; } while (!try_cmpxchg(&ic->val, &old.val, new.val)); @@ -165,10 +187,8 @@ retry: ic = &kvm->arch.sca->ipte_control; old = READ_ONCE(*ic); do { - if (old.kg) { - cond_resched(); + if (old.kg) goto retry; - } new = old; new.k = 1; new.kh++; @@ -423,7 +443,7 @@ static int deref_table(struct kvm *kvm, unsigned long gpa, unsigned long *val) } /** - * guest_translate - translate a guest virtual into a guest absolute address + * guest_translate_gva() - translate a guest virtual into a guest absolute address * @vcpu: virtual cpu * @gva: guest virtual address * @gpa: points to where guest physical (absolute) address should be stored @@ -443,9 +463,9 @@ static int deref_table(struct kvm *kvm, unsigned long gpa, unsigned long *val) * the returned value is the program interruption code as defined * by the architecture */ -static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva, - unsigned long *gpa, const union asce asce, - enum gacc_mode mode, enum prot_type *prot) +static unsigned long guest_translate_gva(struct kvm_vcpu *vcpu, unsigned long gva, + unsigned long *gpa, const union asce asce, + enum gacc_mode mode, enum prot_type *prot) { union vaddress vaddr = {.addr = gva}; union raddress raddr = {.addr = gva}; @@ -626,31 +646,19 @@ static int low_address_protection_enabled(struct kvm_vcpu *vcpu, return 1; } -static int vm_check_access_key(struct kvm *kvm, u8 access_key, - enum gacc_mode mode, gpa_t gpa) +static int vm_check_access_key_gpa(struct kvm *kvm, u8 access_key, + enum gacc_mode mode, gpa_t gpa) { - u8 storage_key, access_control; - bool fetch_protected; - unsigned long hva; + union skey storage_key; int r; - if (access_key == 0) - return 0; - - hva = gfn_to_hva(kvm, gpa_to_gfn(gpa)); - if (kvm_is_error_hva(hva)) - return PGM_ADDRESSING; - - mmap_read_lock(current->mm); - r = get_guest_storage_key(current->mm, hva, &storage_key); - mmap_read_unlock(current->mm); + scoped_guard(read_lock, &kvm->mmu_lock) + r = dat_get_storage_key(kvm->arch.gmap->asce, gpa_to_gfn(gpa), &storage_key); if (r) return r; - access_control = FIELD_GET(_PAGE_ACC_BITS, storage_key); - if (access_control == access_key) + if (access_key == 0 || storage_key.acc == access_key) return 0; - fetch_protected = storage_key & _PAGE_FP_BIT; - if ((mode == GACC_FETCH || mode == GACC_IFETCH) && !fetch_protected) + if ((mode == GACC_FETCH || mode == GACC_IFETCH) && !storage_key.fp) return 0; return PGM_PROTECTION; } @@ -689,12 +697,11 @@ static bool storage_prot_override_applies(u8 access_control) return access_control == PAGE_SPO_ACC; } -static int vcpu_check_access_key(struct kvm_vcpu *vcpu, u8 access_key, - enum gacc_mode mode, union asce asce, gpa_t gpa, - unsigned long ga, unsigned int len) +static int vcpu_check_access_key_gpa(struct kvm_vcpu *vcpu, u8 access_key, + enum gacc_mode mode, union asce asce, gpa_t gpa, + unsigned long ga, unsigned int len) { - u8 storage_key, access_control; - unsigned long hva; + union skey storage_key; int r; /* access key 0 matches any storage key -> allow */ @@ -704,26 +711,23 @@ static int vcpu_check_access_key(struct kvm_vcpu *vcpu, u8 access_key, * caller needs to ensure that gfn is accessible, so we can * assume that this cannot fail */ - hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(gpa)); - mmap_read_lock(current->mm); - r = get_guest_storage_key(current->mm, hva, &storage_key); - mmap_read_unlock(current->mm); + scoped_guard(read_lock, &vcpu->kvm->mmu_lock) + r = dat_get_storage_key(vcpu->arch.gmap->asce, gpa_to_gfn(gpa), &storage_key); if (r) return r; - access_control = FIELD_GET(_PAGE_ACC_BITS, storage_key); /* access key matches storage key -> allow */ - if (access_control == access_key) + if (storage_key.acc == access_key) return 0; if (mode == GACC_FETCH || mode == GACC_IFETCH) { /* it is a fetch and fetch protection is off -> allow */ - if (!(storage_key & _PAGE_FP_BIT)) + if (!storage_key.fp) return 0; if (fetch_prot_override_applicable(vcpu, mode, asce) && fetch_prot_override_applies(ga, len)) return 0; } if (storage_prot_override_applicable(vcpu) && - storage_prot_override_applies(access_control)) + storage_prot_override_applies(storage_key.acc)) return 0; return PGM_PROTECTION; } @@ -783,7 +787,7 @@ static int guest_range_to_gpas(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar, return trans_exc(vcpu, PGM_PROTECTION, ga, ar, mode, PROT_TYPE_LA); if (psw_bits(*psw).dat) { - rc = guest_translate(vcpu, ga, &gpa, asce, mode, &prot); + rc = guest_translate_gva(vcpu, ga, &gpa, asce, mode, &prot); if (rc < 0) return rc; } else { @@ -795,8 +799,7 @@ static int guest_range_to_gpas(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar, } if (rc) return trans_exc(vcpu, rc, ga, ar, mode, prot); - rc = vcpu_check_access_key(vcpu, access_key, mode, asce, gpa, ga, - fragment_len); + rc = vcpu_check_access_key_gpa(vcpu, access_key, mode, asce, gpa, ga, fragment_len); if (rc) return trans_exc(vcpu, rc, ga, ar, mode, PROT_TYPE_KEYC); if (gpas) @@ -808,8 +811,8 @@ static int guest_range_to_gpas(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar, return 0; } -static int access_guest_page(struct kvm *kvm, enum gacc_mode mode, gpa_t gpa, - void *data, unsigned int len) +static int access_guest_page_gpa(struct kvm *kvm, enum gacc_mode mode, gpa_t gpa, + void *data, unsigned int len) { const unsigned int offset = offset_in_page(gpa); const gfn_t gfn = gpa_to_gfn(gpa); @@ -824,38 +827,79 @@ static int access_guest_page(struct kvm *kvm, enum gacc_mode mode, gpa_t gpa, return rc; } -static int -access_guest_page_with_key(struct kvm *kvm, enum gacc_mode mode, gpa_t gpa, - void *data, unsigned int len, u8 access_key) +static int mvcos_key(void *to, const void *from, unsigned long size, u8 dst_key, u8 src_key) { - struct kvm_memory_slot *slot; - bool writable; - gfn_t gfn; - hva_t hva; - int rc; + union oac spec = { + .oac1.key = dst_key, + .oac1.k = !!dst_key, + .oac2.key = src_key, + .oac2.k = !!src_key, + }; + int exception = PGM_PROTECTION; + + asm_inline volatile( + " lr %%r0,%[spec]\n" + "0: mvcos %[to],%[from],%[size]\n" + "1: lhi %[exc],0\n" + "2:\n" + EX_TABLE(0b, 2b) + EX_TABLE(1b, 2b) + : [size] "+d" (size), [to] "=Q" (*(char *)to), [exc] "+d" (exception) + : [spec] "d" (spec.val), [from] "Q" (*(const char *)from) + : "memory", "cc", "0"); + return exception; +} - gfn = gpa >> PAGE_SHIFT; - slot = gfn_to_memslot(kvm, gfn); - hva = gfn_to_hva_memslot_prot(slot, gfn, &writable); +struct acc_page_key_context { + void *data; + int exception; + unsigned short offset; + unsigned short len; + bool store; + u8 access_key; +}; - if (kvm_is_error_hva(hva)) - return PGM_ADDRESSING; - /* - * Check if it's a ro memslot, even tho that can't occur (they're unsupported). - * Don't try to actually handle that case. - */ - if (!writable && mode == GACC_STORE) - return -EOPNOTSUPP; - hva += offset_in_page(gpa); - if (mode == GACC_STORE) - rc = copy_to_user_key((void __user *)hva, data, len, access_key); +static void _access_guest_page_with_key_gpa(struct guest_fault *f) +{ + struct acc_page_key_context *context = f->priv; + void *ptr; + int r; + + ptr = __va(PFN_PHYS(f->pfn) | context->offset); + + if (context->store) + r = mvcos_key(ptr, context->data, context->len, context->access_key, 0); else - rc = copy_from_user_key(data, (void __user *)hva, len, access_key); + r = mvcos_key(context->data, ptr, context->len, 0, context->access_key); + + context->exception = r; +} + +static int access_guest_page_with_key_gpa(struct kvm *kvm, enum gacc_mode mode, gpa_t gpa, + void *data, unsigned int len, u8 acc) +{ + struct acc_page_key_context context = { + .offset = offset_in_page(gpa), + .len = len, + .data = data, + .access_key = acc, + .store = mode == GACC_STORE, + }; + struct guest_fault fault = { + .gfn = gpa_to_gfn(gpa), + .priv = &context, + .write_attempt = mode == GACC_STORE, + .callback = _access_guest_page_with_key_gpa, + }; + int rc; + + if (KVM_BUG_ON((len + context.offset) > PAGE_SIZE, kvm)) + return -EINVAL; + + rc = kvm_s390_faultin_gfn(NULL, kvm, &fault); if (rc) - return PGM_PROTECTION; - if (mode == GACC_STORE) - mark_page_dirty_in_slot(kvm, slot, gfn); - return 0; + return rc; + return context.exception; } int access_guest_abs_with_key(struct kvm *kvm, gpa_t gpa, void *data, @@ -867,7 +911,7 @@ int access_guest_abs_with_key(struct kvm *kvm, gpa_t gpa, void *data, while (min(PAGE_SIZE - offset, len) > 0) { fragment_len = min(PAGE_SIZE - offset, len); - rc = access_guest_page_with_key(kvm, mode, gpa, data, fragment_len, access_key); + rc = access_guest_page_with_key_gpa(kvm, mode, gpa, data, fragment_len, access_key); if (rc) return rc; offset = 0; @@ -927,15 +971,14 @@ int access_guest_with_key(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar, for (idx = 0; idx < nr_pages; idx++) { fragment_len = min(PAGE_SIZE - offset_in_page(gpas[idx]), len); if (try_fetch_prot_override && fetch_prot_override_applies(ga, fragment_len)) { - rc = access_guest_page(vcpu->kvm, mode, gpas[idx], - data, fragment_len); + rc = access_guest_page_gpa(vcpu->kvm, mode, gpas[idx], data, fragment_len); } else { - rc = access_guest_page_with_key(vcpu->kvm, mode, gpas[idx], - data, fragment_len, access_key); + rc = access_guest_page_with_key_gpa(vcpu->kvm, mode, gpas[idx], + data, fragment_len, access_key); } if (rc == PGM_PROTECTION && try_storage_prot_override) - rc = access_guest_page_with_key(vcpu->kvm, mode, gpas[idx], - data, fragment_len, PAGE_SPO_ACC); + rc = access_guest_page_with_key_gpa(vcpu->kvm, mode, gpas[idx], + data, fragment_len, PAGE_SPO_ACC); if (rc) break; len -= fragment_len; @@ -969,7 +1012,7 @@ int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra, while (len && !rc) { gpa = kvm_s390_real_to_abs(vcpu, gra); fragment_len = min(PAGE_SIZE - offset_in_page(gpa), len); - rc = access_guest_page(vcpu->kvm, mode, gpa, data, fragment_len); + rc = access_guest_page_gpa(vcpu->kvm, mode, gpa, data, fragment_len); len -= fragment_len; gra += fragment_len; data += fragment_len; @@ -980,17 +1023,101 @@ int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra, } /** + * __cmpxchg_with_key() - Perform cmpxchg, honoring storage keys. + * @ptr: Address of value to compare to *@old and exchange with + * @new. Must be aligned to @size. + * @old: Old value. Compared to the content pointed to by @ptr in order to + * determine if the exchange occurs. The old value read from *@ptr is + * written here. + * @new: New value to place at *@ptr. + * @size: Size of the operation in bytes, may only be a power of two up to 16. + * @access_key: Access key to use for checking storage key protection. + * + * Perform a cmpxchg on guest memory, honoring storage key protection. + * @access_key alone determines how key checking is performed, neither + * storage-protection-override nor fetch-protection-override apply. + * In case of an exception *@uval is set to zero. + * + * Return: + * * %0: cmpxchg executed successfully + * * %1: cmpxchg executed unsuccessfully + * * %PGM_PROTECTION: an exception happened when trying to access *@ptr + * * %-EAGAIN: maxed out number of retries (byte and short only) + * * %-EINVAL: invalid value for @size + */ +static int __cmpxchg_with_key(union kvm_s390_quad *ptr, union kvm_s390_quad *old, + union kvm_s390_quad new, int size, u8 access_key) +{ + union kvm_s390_quad tmp = { .sixteen = 0 }; + int rc; + + /* + * The cmpxchg_key macro depends on the type of "old", so we need + * a case for each valid length and get some code duplication as long + * as we don't introduce a new macro. + */ + switch (size) { + case 1: + rc = __cmpxchg_key1(&ptr->one, &tmp.one, old->one, new.one, access_key); + break; + case 2: + rc = __cmpxchg_key2(&ptr->two, &tmp.two, old->two, new.two, access_key); + break; + case 4: + rc = __cmpxchg_key4(&ptr->four, &tmp.four, old->four, new.four, access_key); + break; + case 8: + rc = __cmpxchg_key8(&ptr->eight, &tmp.eight, old->eight, new.eight, access_key); + break; + case 16: + rc = __cmpxchg_key16(&ptr->sixteen, &tmp.sixteen, old->sixteen, new.sixteen, + access_key); + break; + default: + return -EINVAL; + } + if (!rc && memcmp(&tmp, old, size)) + rc = 1; + *old = tmp; + /* + * Assume that the fault is caused by protection, either key protection + * or user page write protection. + */ + if (rc == -EFAULT) + rc = PGM_PROTECTION; + return rc; +} + +struct cmpxchg_key_context { + union kvm_s390_quad new; + union kvm_s390_quad *old; + int exception; + unsigned short offset; + u8 access_key; + u8 len; +}; + +static void _cmpxchg_guest_abs_with_key(struct guest_fault *f) +{ + struct cmpxchg_key_context *context = f->priv; + + context->exception = __cmpxchg_with_key(__va(PFN_PHYS(f->pfn) | context->offset), + context->old, context->new, context->len, + context->access_key); +} + +/** * cmpxchg_guest_abs_with_key() - Perform cmpxchg on guest absolute address. * @kvm: Virtual machine instance. * @gpa: Absolute guest address of the location to be changed. * @len: Operand length of the cmpxchg, required: 1 <= len <= 16. Providing a * non power of two will result in failure. - * @old_addr: Pointer to old value. If the location at @gpa contains this value, - * the exchange will succeed. After calling cmpxchg_guest_abs_with_key() - * *@old_addr contains the value at @gpa before the attempt to - * exchange the value. + * @old: Pointer to old value. If the location at @gpa contains this value, + * the exchange will succeed. After calling cmpxchg_guest_abs_with_key() + * *@old contains the value at @gpa before the attempt to + * exchange the value. * @new: The value to place at @gpa. - * @access_key: The access key to use for the guest access. + * @acc: The access key to use for the guest access. * @success: output value indicating if an exchange occurred. * * Atomically exchange the value at @gpa by @new, if it contains *@old. @@ -1003,89 +1130,36 @@ int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra, * * -EAGAIN: transient failure (len 1 or 2) * * -EOPNOTSUPP: read-only memslot (should never occur) */ -int cmpxchg_guest_abs_with_key(struct kvm *kvm, gpa_t gpa, int len, - __uint128_t *old_addr, __uint128_t new, - u8 access_key, bool *success) +int cmpxchg_guest_abs_with_key(struct kvm *kvm, gpa_t gpa, int len, union kvm_s390_quad *old, + union kvm_s390_quad new, u8 acc, bool *success) { - gfn_t gfn = gpa_to_gfn(gpa); - struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); - bool writable; - hva_t hva; - int ret; - - if (!IS_ALIGNED(gpa, len)) - return -EINVAL; - - hva = gfn_to_hva_memslot_prot(slot, gfn, &writable); - if (kvm_is_error_hva(hva)) - return PGM_ADDRESSING; - /* - * Check if it's a read-only memslot, even though that cannot occur - * since those are unsupported. - * Don't try to actually handle that case. - */ - if (!writable) - return -EOPNOTSUPP; - - hva += offset_in_page(gpa); - /* - * The cmpxchg_user_key macro depends on the type of "old", so we need - * a case for each valid length and get some code duplication as long - * as we don't introduce a new macro. - */ - switch (len) { - case 1: { - u8 old; - - ret = cmpxchg_user_key((u8 __user *)hva, &old, *old_addr, new, access_key); - *success = !ret && old == *old_addr; - *old_addr = old; - break; - } - case 2: { - u16 old; - - ret = cmpxchg_user_key((u16 __user *)hva, &old, *old_addr, new, access_key); - *success = !ret && old == *old_addr; - *old_addr = old; - break; - } - case 4: { - u32 old; - - ret = cmpxchg_user_key((u32 __user *)hva, &old, *old_addr, new, access_key); - *success = !ret && old == *old_addr; - *old_addr = old; - break; - } - case 8: { - u64 old; + struct cmpxchg_key_context context = { + .old = old, + .new = new, + .offset = offset_in_page(gpa), + .len = len, + .access_key = acc, + }; + struct guest_fault fault = { + .gfn = gpa_to_gfn(gpa), + .priv = &context, + .write_attempt = true, + .callback = _cmpxchg_guest_abs_with_key, + }; + int rc; - ret = cmpxchg_user_key((u64 __user *)hva, &old, *old_addr, new, access_key); - *success = !ret && old == *old_addr; - *old_addr = old; - break; - } - case 16: { - __uint128_t old; + lockdep_assert_held(&kvm->srcu); - ret = cmpxchg_user_key((__uint128_t __user *)hva, &old, *old_addr, new, access_key); - *success = !ret && old == *old_addr; - *old_addr = old; - break; - } - default: + if (len > 16 || !IS_ALIGNED(gpa, len)) return -EINVAL; - } - if (*success) - mark_page_dirty_in_slot(kvm, slot, gfn); - /* - * Assume that the fault is caused by protection, either key protection - * or user page write protection. - */ - if (ret == -EFAULT) - ret = PGM_PROTECTION; - return ret; + + rc = kvm_s390_faultin_gfn(NULL, kvm, &fault); + if (rc) + return rc; + *success = !context.exception; + if (context.exception == 1) + return 0; + return context.exception; } /** @@ -1160,7 +1234,7 @@ int check_gpa_range(struct kvm *kvm, unsigned long gpa, unsigned long length, while (length && !rc) { fragment_len = min(PAGE_SIZE - offset_in_page(gpa), length); - rc = vm_check_access_key(kvm, access_key, mode, gpa); + rc = vm_check_access_key_gpa(kvm, access_key, mode, gpa); length -= fragment_len; gpa += fragment_len; } @@ -1187,304 +1261,437 @@ int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra) } /** - * kvm_s390_shadow_tables - walk the guest page table and create shadow tables - * @sg: pointer to the shadow guest address space structure - * @saddr: faulting address in the shadow gmap - * @pgt: pointer to the beginning of the page table for the given address if - * successful (return value 0), or to the first invalid DAT entry in - * case of exceptions (return value > 0) - * @dat_protection: referenced memory is write protected - * @fake: pgt references contiguous guest memory block, not a pgtable + * walk_guest_tables() - Walk the guest page table and pin the dat tables. + * @sg: Pointer to the shadow guest address space structure. + * @saddr: Faulting address in the shadow gmap. + * @w: Will be filled with information on the pinned pages. + * @wr: Wndicates a write access if true. + * + * Return: + * * %0 in case of success, + * * a PIC code > 0 in case the address translation fails + * * an error code < 0 if other errors happen in the host */ -static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr, - unsigned long *pgt, int *dat_protection, - int *fake) +static int walk_guest_tables(struct gmap *sg, unsigned long saddr, struct pgtwalk *w, bool wr) { - struct kvm *kvm; - struct gmap *parent; - union asce asce; + struct gmap *parent = sg->parent; + struct guest_fault *entries; + union dat_table_entry table; union vaddress vaddr; unsigned long ptr; + struct kvm *kvm; + union asce asce; int rc; - *fake = 0; - *dat_protection = 0; - kvm = sg->private; - parent = sg->parent; + if (!parent) + return -EAGAIN; + kvm = parent->kvm; + WARN_ON(!kvm); + asce = sg->guest_asce; + entries = get_entries(w); + + w->level = LEVEL_MEM; + w->last_addr = saddr; + if (asce.r) + return kvm_s390_get_guest_page(kvm, entries + LEVEL_MEM, gpa_to_gfn(saddr), false); + vaddr.addr = saddr; - asce.val = sg->orig_asce; ptr = asce.rsto * PAGE_SIZE; - if (asce.r) { - *fake = 1; - ptr = 0; - asce.dt = ASCE_TYPE_REGION1; - } + + if (!asce_contains_gfn(asce, gpa_to_gfn(saddr))) + return PGM_ASCE_TYPE; switch (asce.dt) { case ASCE_TYPE_REGION1: - if (vaddr.rfx01 > asce.tl && !*fake) + if (vaddr.rfx01 > asce.tl) return PGM_REGION_FIRST_TRANS; break; case ASCE_TYPE_REGION2: - if (vaddr.rfx) - return PGM_ASCE_TYPE; if (vaddr.rsx01 > asce.tl) return PGM_REGION_SECOND_TRANS; break; case ASCE_TYPE_REGION3: - if (vaddr.rfx || vaddr.rsx) - return PGM_ASCE_TYPE; if (vaddr.rtx01 > asce.tl) return PGM_REGION_THIRD_TRANS; break; case ASCE_TYPE_SEGMENT: - if (vaddr.rfx || vaddr.rsx || vaddr.rtx) - return PGM_ASCE_TYPE; if (vaddr.sx01 > asce.tl) return PGM_SEGMENT_TRANSLATION; break; } + w->level = asce.dt; switch (asce.dt) { - case ASCE_TYPE_REGION1: { - union region1_table_entry rfte; - - if (*fake) { - ptr += vaddr.rfx * _REGION1_SIZE; - rfte.val = ptr; - goto shadow_r2t; - } - *pgt = ptr + vaddr.rfx * 8; - rc = gmap_read_table(parent, ptr + vaddr.rfx * 8, &rfte.val); + case ASCE_TYPE_REGION1: + w->last_addr = ptr + vaddr.rfx * 8; + rc = kvm_s390_get_guest_page_and_read_gpa(kvm, entries + w->level, + w->last_addr, &table.val); if (rc) return rc; - if (rfte.i) + if (table.pgd.i) return PGM_REGION_FIRST_TRANS; - if (rfte.tt != TABLE_TYPE_REGION1) + if (table.pgd.tt != TABLE_TYPE_REGION1) return PGM_TRANSLATION_SPEC; - if (vaddr.rsx01 < rfte.tf || vaddr.rsx01 > rfte.tl) + if (vaddr.rsx01 < table.pgd.tf || vaddr.rsx01 > table.pgd.tl) return PGM_REGION_SECOND_TRANS; if (sg->edat_level >= 1) - *dat_protection |= rfte.p; - ptr = rfte.rto * PAGE_SIZE; -shadow_r2t: - rc = gmap_shadow_r2t(sg, saddr, rfte.val, *fake); - if (rc) - return rc; - kvm->stat.gmap_shadow_r1_entry++; - } + w->p |= table.pgd.p; + ptr = table.pgd.rto * PAGE_SIZE; + w->level--; fallthrough; - case ASCE_TYPE_REGION2: { - union region2_table_entry rste; - - if (*fake) { - ptr += vaddr.rsx * _REGION2_SIZE; - rste.val = ptr; - goto shadow_r3t; - } - *pgt = ptr + vaddr.rsx * 8; - rc = gmap_read_table(parent, ptr + vaddr.rsx * 8, &rste.val); + case ASCE_TYPE_REGION2: + w->last_addr = ptr + vaddr.rsx * 8; + rc = kvm_s390_get_guest_page_and_read_gpa(kvm, entries + w->level, + w->last_addr, &table.val); if (rc) return rc; - if (rste.i) + if (table.p4d.i) return PGM_REGION_SECOND_TRANS; - if (rste.tt != TABLE_TYPE_REGION2) + if (table.p4d.tt != TABLE_TYPE_REGION2) return PGM_TRANSLATION_SPEC; - if (vaddr.rtx01 < rste.tf || vaddr.rtx01 > rste.tl) + if (vaddr.rtx01 < table.p4d.tf || vaddr.rtx01 > table.p4d.tl) return PGM_REGION_THIRD_TRANS; if (sg->edat_level >= 1) - *dat_protection |= rste.p; - ptr = rste.rto * PAGE_SIZE; -shadow_r3t: - rste.p |= *dat_protection; - rc = gmap_shadow_r3t(sg, saddr, rste.val, *fake); - if (rc) - return rc; - kvm->stat.gmap_shadow_r2_entry++; - } + w->p |= table.p4d.p; + ptr = table.p4d.rto * PAGE_SIZE; + w->level--; fallthrough; - case ASCE_TYPE_REGION3: { - union region3_table_entry rtte; - - if (*fake) { - ptr += vaddr.rtx * _REGION3_SIZE; - rtte.val = ptr; - goto shadow_sgt; - } - *pgt = ptr + vaddr.rtx * 8; - rc = gmap_read_table(parent, ptr + vaddr.rtx * 8, &rtte.val); + case ASCE_TYPE_REGION3: + w->last_addr = ptr + vaddr.rtx * 8; + rc = kvm_s390_get_guest_page_and_read_gpa(kvm, entries + w->level, + w->last_addr, &table.val); if (rc) return rc; - if (rtte.i) + if (table.pud.i) return PGM_REGION_THIRD_TRANS; - if (rtte.tt != TABLE_TYPE_REGION3) + if (table.pud.tt != TABLE_TYPE_REGION3) return PGM_TRANSLATION_SPEC; - if (rtte.cr && asce.p && sg->edat_level >= 2) + if (table.pud.cr && asce.p && sg->edat_level >= 2) return PGM_TRANSLATION_SPEC; - if (rtte.fc && sg->edat_level >= 2) { - *dat_protection |= rtte.fc0.p; - *fake = 1; - ptr = rtte.fc1.rfaa * _REGION3_SIZE; - rtte.val = ptr; - goto shadow_sgt; + if (sg->edat_level >= 1) + w->p |= table.pud.p; + if (table.pud.fc && sg->edat_level >= 2) { + table.val = u64_replace_bits(table.val, saddr, ~_REGION3_MASK); + goto edat_applies; } - if (vaddr.sx01 < rtte.fc0.tf || vaddr.sx01 > rtte.fc0.tl) + if (vaddr.sx01 < table.pud.fc0.tf || vaddr.sx01 > table.pud.fc0.tl) return PGM_SEGMENT_TRANSLATION; - if (sg->edat_level >= 1) - *dat_protection |= rtte.fc0.p; - ptr = rtte.fc0.sto * PAGE_SIZE; -shadow_sgt: - rtte.fc0.p |= *dat_protection; - rc = gmap_shadow_sgt(sg, saddr, rtte.val, *fake); - if (rc) - return rc; - kvm->stat.gmap_shadow_r3_entry++; - } + ptr = table.pud.fc0.sto * PAGE_SIZE; + w->level--; fallthrough; - case ASCE_TYPE_SEGMENT: { - union segment_table_entry ste; - - if (*fake) { - ptr += vaddr.sx * _SEGMENT_SIZE; - ste.val = ptr; - goto shadow_pgt; - } - *pgt = ptr + vaddr.sx * 8; - rc = gmap_read_table(parent, ptr + vaddr.sx * 8, &ste.val); + case ASCE_TYPE_SEGMENT: + w->last_addr = ptr + vaddr.sx * 8; + rc = kvm_s390_get_guest_page_and_read_gpa(kvm, entries + w->level, + w->last_addr, &table.val); if (rc) return rc; - if (ste.i) + if (table.pmd.i) return PGM_SEGMENT_TRANSLATION; - if (ste.tt != TABLE_TYPE_SEGMENT) + if (table.pmd.tt != TABLE_TYPE_SEGMENT) return PGM_TRANSLATION_SPEC; - if (ste.cs && asce.p) + if (table.pmd.cs && asce.p) return PGM_TRANSLATION_SPEC; - *dat_protection |= ste.fc0.p; - if (ste.fc && sg->edat_level >= 1) { - *fake = 1; - ptr = ste.fc1.sfaa * _SEGMENT_SIZE; - ste.val = ptr; - goto shadow_pgt; + w->p |= table.pmd.p; + if (table.pmd.fc && sg->edat_level >= 1) { + table.val = u64_replace_bits(table.val, saddr, ~_SEGMENT_MASK); + goto edat_applies; } - ptr = ste.fc0.pto * (PAGE_SIZE / 2); -shadow_pgt: - ste.fc0.p |= *dat_protection; - rc = gmap_shadow_pgt(sg, saddr, ste.val, *fake); + ptr = table.pmd.fc0.pto * (PAGE_SIZE / 2); + w->level--; + } + w->last_addr = ptr + vaddr.px * 8; + rc = kvm_s390_get_guest_page_and_read_gpa(kvm, entries + w->level, + w->last_addr, &table.val); + if (rc) + return rc; + if (table.pte.i) + return PGM_PAGE_TRANSLATION; + if (table.pte.z) + return PGM_TRANSLATION_SPEC; + w->p |= table.pte.p; +edat_applies: + if (wr && w->p) + return PGM_PROTECTION; + + return kvm_s390_get_guest_page(kvm, entries + LEVEL_MEM, table.pte.pfra, wr); +} + +static int _do_shadow_pte(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gpa_t raddr, + union pte *ptep_h, union pte *ptep, struct guest_fault *f, bool p) +{ + union pgste pgste; + union pte newpte; + int rc; + + lockdep_assert_held(&sg->kvm->mmu_lock); + lockdep_assert_held(&sg->parent->children_lock); + + scoped_guard(spinlock, &sg->host_to_rmap_lock) + rc = gmap_insert_rmap(mc, sg, f->gfn, gpa_to_gfn(raddr), TABLE_TYPE_PAGE_TABLE); + if (rc) + return rc; + + if (!pgste_get_trylock(ptep_h, &pgste)) + return -EAGAIN; + newpte = _pte(f->pfn, f->writable, !p, ptep_h->s.s); + newpte.s.d |= ptep_h->s.d; + newpte.s.sd |= ptep_h->s.sd; + newpte.h.p &= ptep_h->h.p; + if (!newpte.h.p && !f->writable) { + rc = -EOPNOTSUPP; + } else { + pgste = _gmap_ptep_xchg(sg->parent, ptep_h, newpte, pgste, f->gfn, false); + pgste.vsie_notif = 1; + pgste.vsie_gmem = 1; + } + pgste_set_unlock(ptep_h, pgste); + if (rc) + return rc; + if (sg->invalidated) + return -EAGAIN; + + newpte = _pte(f->pfn, 0, !p, 0); + if (!pgste_get_trylock(ptep, &pgste)) + return -EAGAIN; + pgste = __dat_ptep_xchg(ptep, pgste, newpte, gpa_to_gfn(raddr), sg->asce, uses_skeys(sg)); + pgste_set_unlock(ptep, pgste); + + return 0; +} + +static int _do_shadow_crste(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gpa_t raddr, + union crste *host, union crste *table, struct guest_fault *f, bool p) +{ + union crste newcrste, oldcrste; + unsigned long mask; + gfn_t r_gfn; + int rc; + + lockdep_assert_held(&sg->kvm->mmu_lock); + lockdep_assert_held(&sg->parent->children_lock); + + mask = is_pmd(*table) ? _SEGMENT_FR_MASK : _REGION3_FR_MASK; + r_gfn = gpa_to_gfn(raddr) & mask; + scoped_guard(spinlock, &sg->host_to_rmap_lock) + rc = gmap_insert_rmap(mc, sg, f->gfn & mask, r_gfn, host->h.tt); + if (rc) + return rc; + + do { + /* _gmap_crstep_xchg_atomic() could have unshadowed this shadow gmap */ + if (sg->invalidated) + return -EAGAIN; + oldcrste = READ_ONCE(*host); + newcrste = _crste_fc1(f->pfn, oldcrste.h.tt, f->writable, !p); + newcrste.s.fc1.d |= oldcrste.s.fc1.d; + newcrste.s.fc1.sd |= oldcrste.s.fc1.sd; + newcrste.h.p &= oldcrste.h.p; + newcrste.s.fc1.vsie_notif = 1; + newcrste.s.fc1.prefix_notif = oldcrste.s.fc1.prefix_notif; + newcrste.s.fc1.s = oldcrste.s.fc1.s; + if (!newcrste.h.p && !f->writable) + return -EOPNOTSUPP; + } while (!_gmap_crstep_xchg_atomic(sg->parent, host, oldcrste, newcrste, f->gfn, false)); + if (sg->invalidated) + return -EAGAIN; + + newcrste = _crste_fc1(f->pfn, oldcrste.h.tt, 0, !p); + while (!dat_crstep_xchg_atomic(table, READ_ONCE(*table), newcrste, r_gfn, sg->asce)) + ; + return 0; +} + +static int _gaccess_do_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *sg, + unsigned long saddr, struct pgtwalk *w) +{ + struct guest_fault *entries; + int flags, i, hl, gl, l, rc; + union crste *table, *host; + union pte *ptep, *ptep_h; + + lockdep_assert_held(&sg->kvm->mmu_lock); + lockdep_assert_held(&sg->parent->children_lock); + + entries = get_entries(w); + ptep_h = NULL; + ptep = NULL; + + rc = dat_entry_walk(NULL, gpa_to_gfn(saddr), sg->asce, DAT_WALK_ANY, TABLE_TYPE_PAGE_TABLE, + &table, &ptep); + if (rc) + return rc; + + /* A race occurred. The shadow mapping is already valid, nothing to do */ + if ((ptep && !ptep->h.i && ptep->h.p == w->p) || + (!ptep && crste_leaf(*table) && !table->h.i && table->h.p == w->p)) + return 0; + + gl = get_level(table, ptep); + + /* In case of a real address space */ + if (w->level <= LEVEL_MEM) { + l = TABLE_TYPE_PAGE_TABLE; + hl = TABLE_TYPE_REGION1; + goto real_address_space; + } + + /* + * Skip levels that are already protected. For each level, protect + * only the page containing the entry, not the whole table. + */ + for (i = gl ; i >= w->level; i--) { + rc = gmap_protect_rmap(mc, sg, entries[i].gfn, gpa_to_gfn(saddr), + entries[i].pfn, i + 1, entries[i].writable); if (rc) return rc; - kvm->stat.gmap_shadow_sg_entry++; + if (sg->invalidated) + return -EAGAIN; } + + rc = dat_entry_walk(NULL, entries[LEVEL_MEM].gfn, sg->parent->asce, DAT_WALK_LEAF, + TABLE_TYPE_PAGE_TABLE, &host, &ptep_h); + if (rc) + return rc; + + hl = get_level(host, ptep_h); + /* Get the smallest granularity */ + l = min3(gl, hl, w->level); + +real_address_space: + flags = DAT_WALK_SPLIT_ALLOC | (uses_skeys(sg->parent) ? DAT_WALK_USES_SKEYS : 0); + /* If necessary, create the shadow mapping */ + if (l < gl) { + rc = dat_entry_walk(mc, gpa_to_gfn(saddr), sg->asce, flags, l, &table, &ptep); + if (rc) + return rc; } - /* Return the parent address of the page table */ - *pgt = ptr; - return 0; + if (l < hl) { + rc = dat_entry_walk(mc, entries[LEVEL_MEM].gfn, sg->parent->asce, + flags, l, &host, &ptep_h); + if (rc) + return rc; + } + + if (KVM_BUG_ON(l > TABLE_TYPE_REGION3, sg->kvm)) + return -EFAULT; + if (l == TABLE_TYPE_PAGE_TABLE) + return _do_shadow_pte(mc, sg, saddr, ptep_h, ptep, entries + LEVEL_MEM, w->p); + return _do_shadow_crste(mc, sg, saddr, host, table, entries + LEVEL_MEM, w->p); } -/** - * shadow_pgt_lookup() - find a shadow page table - * @sg: pointer to the shadow guest address space structure - * @saddr: the address in the shadow aguest address space - * @pgt: parent gmap address of the page table to get shadowed - * @dat_protection: if the pgtable is marked as protected by dat - * @fake: pgt references contiguous guest memory block, not a pgtable - * - * Returns 0 if the shadow page table was found and -EAGAIN if the page - * table was not found. - * - * Called with sg->mm->mmap_lock in read. - */ -static int shadow_pgt_lookup(struct gmap *sg, unsigned long saddr, unsigned long *pgt, - int *dat_protection, int *fake) +static inline int ___gaccess_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg, gpa_t saddr, + unsigned long seq, struct pgtwalk *walk) { - unsigned long pt_index; - unsigned long *table; - struct page *page; + struct gmap *parent; int rc; - spin_lock(&sg->guest_table_lock); - table = gmap_table_walk(sg, saddr, 1); /* get segment pointer */ - if (table && !(*table & _SEGMENT_ENTRY_INVALID)) { - /* Shadow page tables are full pages (pte+pgste) */ - page = pfn_to_page(*table >> PAGE_SHIFT); - pt_index = gmap_pgste_get_pgt_addr(page_to_virt(page)); - *pgt = pt_index & ~GMAP_SHADOW_FAKE_TABLE; - *dat_protection = !!(*table & _SEGMENT_ENTRY_PROTECT); - *fake = !!(pt_index & GMAP_SHADOW_FAKE_TABLE); - rc = 0; - } else { - rc = -EAGAIN; + if (kvm_s390_array_needs_retry_safe(vcpu->kvm, seq, walk->raw_entries)) + return -EAGAIN; + parent = READ_ONCE(sg->parent); + if (!parent) + return -EAGAIN; +retry: + scoped_guard(spinlock, &parent->children_lock) { + if (READ_ONCE(sg->parent) != parent) + return -EAGAIN; + sg->invalidated = false; + rc = _gaccess_do_shadow(vcpu->arch.mc, sg, saddr, walk); + } + if (rc == -ENOENT) { + struct kvm_memory_slot *slot; + struct guest_fault *entries; + + entries = get_entries(walk); + slot = kvm_vcpu_gfn_to_memslot(vcpu, entries[LEVEL_MEM].gfn); + if (!slot) + return PGM_ADDRESSING; + rc = gmap_link(vcpu->arch.mc, parent, entries + LEVEL_MEM, slot); + if (!rc) + goto retry; } - spin_unlock(&sg->guest_table_lock); + if (!rc) + kvm_s390_release_faultin_array(vcpu->kvm, walk->raw_entries, false); + return rc; +} + +static inline int _gaccess_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg, gpa_t saddr, + unsigned long seq, struct pgtwalk *walk) +{ + int rc; + + if (kvm_s390_array_needs_retry_unsafe(vcpu->kvm, seq, walk->raw_entries)) + return -EAGAIN; + + do { + rc = kvm_s390_mmu_cache_topup(vcpu->arch.mc); + if (rc) + return rc; + rc = radix_tree_preload(GFP_KERNEL); + if (rc) + return rc; + scoped_guard(read_lock, &vcpu->kvm->mmu_lock) + rc = ___gaccess_shadow_fault(vcpu, sg, saddr, seq, walk); + radix_tree_preload_end(); + } while (rc == -ENOMEM); + return rc; } /** - * kvm_s390_shadow_fault - handle fault on a shadow page table - * @vcpu: virtual cpu - * @sg: pointer to the shadow guest address space structure - * @saddr: faulting address in the shadow gmap - * @datptr: will contain the address of the faulting DAT table entry, or of - * the valid leaf, plus some flags + * __gaccess_shadow_fault() - Handle fault on a shadow page table. + * @vcpu: Virtual cpu that triggered the action. + * @sg: The shadow guest address space structure. + * @saddr: Faulting address in the shadow gmap. + * @datptr: Will contain the address of the faulting DAT table entry, or of + * the valid leaf, plus some flags. + * @wr: Whether this is a write access. * - * Returns: - 0 if the shadow fault was successfully resolved - * - > 0 (pgm exception code) on exceptions while faulting - * - -EAGAIN if the caller can retry immediately - * - -EFAULT when accessing invalid guest addresses - * - -ENOMEM if out of memory + * Return: + * * %0 if the shadow fault was successfully resolved + * * > 0 (pgm exception code) on exceptions while faulting + * * %-EAGAIN if the caller can retry immediately + * * %-EFAULT when accessing invalid guest addresses + * * %-ENOMEM if out of memory */ -int kvm_s390_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg, - unsigned long saddr, unsigned long *datptr) +static int __gaccess_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg, gpa_t saddr, + union mvpg_pei *datptr, bool wr) { - union vaddress vaddr; - union page_table_entry pte; - unsigned long pgt = 0; - int dat_protection, fake; + struct pgtwalk walk = { .p = false, }; + unsigned long seq; int rc; - if (KVM_BUG_ON(!gmap_is_shadow(sg), vcpu->kvm)) - return -EFAULT; - - mmap_read_lock(sg->mm); - /* - * We don't want any guest-2 tables to change - so the parent - * tables/pointers we read stay valid - unshadowing is however - * always possible - only guest_table_lock protects us. - */ - ipte_lock(vcpu->kvm); + seq = vcpu->kvm->mmu_invalidate_seq; + /* Pairs with the smp_wmb() in kvm_mmu_invalidate_end(). */ + smp_rmb(); - rc = shadow_pgt_lookup(sg, saddr, &pgt, &dat_protection, &fake); + rc = walk_guest_tables(sg, saddr, &walk, wr); + if (datptr) { + datptr->val = walk.last_addr; + datptr->dat_prot = wr && walk.p; + datptr->not_pte = walk.level > TABLE_TYPE_PAGE_TABLE; + datptr->real = sg->guest_asce.r; + } + if (!rc) + rc = _gaccess_shadow_fault(vcpu, sg, saddr, seq, &walk); if (rc) - rc = kvm_s390_shadow_tables(sg, saddr, &pgt, &dat_protection, - &fake); + kvm_s390_release_faultin_array(vcpu->kvm, walk.raw_entries, true); + return rc; +} - vaddr.addr = saddr; - if (fake) { - pte.val = pgt + vaddr.px * PAGE_SIZE; - goto shadow_page; - } +int gaccess_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg, gpa_t saddr, + union mvpg_pei *datptr, bool wr) +{ + int rc; - switch (rc) { - case PGM_SEGMENT_TRANSLATION: - case PGM_REGION_THIRD_TRANS: - case PGM_REGION_SECOND_TRANS: - case PGM_REGION_FIRST_TRANS: - pgt |= PEI_NOT_PTE; - break; - case 0: - pgt += vaddr.px * 8; - rc = gmap_read_table(sg->parent, pgt, &pte.val); - } - if (datptr) - *datptr = pgt | dat_protection * PEI_DAT_PROT; - if (!rc && pte.i) - rc = PGM_PAGE_TRANSLATION; - if (!rc && pte.z) - rc = PGM_TRANSLATION_SPEC; -shadow_page: - pte.p |= dat_protection; - if (!rc) - rc = gmap_shadow_page(sg, saddr, __pte(pte.val)); - vcpu->kvm->stat.gmap_shadow_pg_entry++; + if (KVM_BUG_ON(!test_bit(GMAP_FLAG_SHADOW, &sg->flags), vcpu->kvm)) + return -EFAULT; + + rc = kvm_s390_mmu_cache_topup(vcpu->arch.mc); + if (rc) + return rc; + + ipte_lock(vcpu->kvm); + rc = __gaccess_shadow_fault(vcpu, sg, saddr, datptr, wr || sg->guest_asce.r); ipte_unlock(vcpu->kvm); - mmap_read_unlock(sg->mm); + return rc; } diff --git a/arch/s390/kvm/gaccess.h b/arch/s390/kvm/s390/gaccess.h index 3fde45a151f2..ef922b3b4990 100644 --- a/arch/s390/kvm/gaccess.h +++ b/arch/s390/kvm/s390/gaccess.h @@ -14,7 +14,7 @@ #include <linux/kvm_host.h> #include <linux/uaccess.h> #include <linux/ptrace.h> -#include "kvm-s390.h" +#include "s390.h" /** * kvm_s390_real_to_abs - convert guest real address to guest absolute address @@ -206,8 +206,8 @@ int access_guest_with_key(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar, int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra, void *data, unsigned long len, enum gacc_mode mode); -int cmpxchg_guest_abs_with_key(struct kvm *kvm, gpa_t gpa, int len, __uint128_t *old, - __uint128_t new, u8 access_key, bool *success); +int cmpxchg_guest_abs_with_key(struct kvm *kvm, gpa_t gpa, int len, union kvm_s390_quad *old, + union kvm_s390_quad new, u8 access_key, bool *success); /** * write_guest_with_key - copy data from kernel space to guest space @@ -450,11 +450,17 @@ void ipte_unlock(struct kvm *kvm); int ipte_lock_held(struct kvm *kvm); int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra); -/* MVPG PEI indication bits */ -#define PEI_DAT_PROT 2 -#define PEI_NOT_PTE 4 +union mvpg_pei { + unsigned long val; + struct { + unsigned long addr : 61; + unsigned long not_pte : 1; + unsigned long dat_prot: 1; + unsigned long real : 1; + }; +}; -int kvm_s390_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *shadow, - unsigned long saddr, unsigned long *datptr); +int gaccess_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg, gpa_t saddr, + union mvpg_pei *datptr, bool wr); #endif /* __KVM_S390_GACCESS_H */ diff --git a/arch/s390/kvm/guestdbg.c b/arch/s390/kvm/s390/guestdbg.c index 80879fc73c90..1bf7e91b7e61 100644 --- a/arch/s390/kvm/guestdbg.c +++ b/arch/s390/kvm/s390/guestdbg.c @@ -8,7 +8,7 @@ */ #include <linux/kvm_host.h> #include <linux/errno.h> -#include "kvm-s390.h" +#include "s390.h" #include "gaccess.h" /* @@ -184,7 +184,7 @@ static int __import_wp_info(struct kvm_vcpu *vcpu, if (wp_info->len < 0 || wp_info->len > MAX_WP_SIZE) return -EINVAL; - wp_info->old_data = kmalloc(bp_data->len, GFP_KERNEL_ACCOUNT); + wp_info->old_data = kmalloc(wp_info->len, GFP_KERNEL_ACCOUNT); if (!wp_info->old_data) return -ENOMEM; /* try to backup the original value */ @@ -232,18 +232,14 @@ int kvm_s390_import_bp_data(struct kvm_vcpu *vcpu, } if (nr_wp > 0) { - wp_info = kmalloc_array(nr_wp, - sizeof(*wp_info), - GFP_KERNEL_ACCOUNT); + wp_info = kmalloc_objs(*wp_info, nr_wp, GFP_KERNEL_ACCOUNT); if (!wp_info) { ret = -ENOMEM; goto error; } } if (nr_bp > 0) { - bp_info = kmalloc_array(nr_bp, - sizeof(*bp_info), - GFP_KERNEL_ACCOUNT); + bp_info = kmalloc_objs(*bp_info, nr_bp, GFP_KERNEL_ACCOUNT); if (!bp_info) { ret = -ENOMEM; goto error; @@ -256,7 +252,7 @@ int kvm_s390_import_bp_data(struct kvm_vcpu *vcpu, ret = __import_wp_info(vcpu, &bp_data[i], &wp_info[nr_wp]); if (ret) - goto error; + goto error_wp; nr_wp++; break; case KVM_HW_BP: @@ -271,7 +267,12 @@ int kvm_s390_import_bp_data(struct kvm_vcpu *vcpu, vcpu->arch.guestdbg.hw_bp_info = bp_info; vcpu->arch.guestdbg.nr_hw_wp = nr_wp; vcpu->arch.guestdbg.hw_wp_info = wp_info; + kfree(bp_data); return 0; + +error_wp: + while (nr_wp--) + kfree(wp_info[nr_wp].old_data); error: kfree(bp_data); kfree(wp_info); diff --git a/arch/s390/kvm/intercept.c b/arch/s390/kvm/s390/intercept.c index 420ae62977e2..ca1205dfac8b 100644 --- a/arch/s390/kvm/intercept.c +++ b/arch/s390/kvm/s390/intercept.c @@ -17,10 +17,11 @@ #include <asm/sysinfo.h> #include <asm/uv.h> -#include "kvm-s390.h" +#include "s390.h" #include "gaccess.h" #include "trace.h" #include "trace-s390.h" +#include "faultin.h" u8 kvm_s390_get_ilen(struct kvm_vcpu *vcpu) { @@ -367,8 +368,11 @@ static int handle_mvpg_pei(struct kvm_vcpu *vcpu) reg2, &srcaddr, GACC_FETCH, 0); if (rc) return kvm_s390_inject_prog_cond(vcpu, rc); - rc = kvm_s390_handle_dat_fault(vcpu, srcaddr, 0); - if (rc != 0) + + do { + rc = kvm_s390_faultin_gfn_simple(vcpu, NULL, gpa_to_gfn(srcaddr), false); + } while (rc == -EAGAIN); + if (rc) return rc; /* Ensure that the source is paged-in, no actual access -> no key checking */ @@ -376,8 +380,11 @@ static int handle_mvpg_pei(struct kvm_vcpu *vcpu) reg1, &dstaddr, GACC_STORE, 0); if (rc) return kvm_s390_inject_prog_cond(vcpu, rc); - rc = kvm_s390_handle_dat_fault(vcpu, dstaddr, FOLL_WRITE); - if (rc != 0) + + do { + rc = kvm_s390_faultin_gfn_simple(vcpu, NULL, gpa_to_gfn(dstaddr), true); + } while (rc == -EAGAIN); + if (rc) return rc; kvm_s390_retry_instr(vcpu); @@ -510,8 +517,9 @@ static int handle_pv_spx(struct kvm_vcpu *vcpu) static int handle_pv_sclp(struct kvm_vcpu *vcpu) { struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); /* * 2 cases: * a: an sccb answering interrupt was already pending or in flight. @@ -527,7 +535,7 @@ static int handle_pv_sclp(struct kvm_vcpu *vcpu) fi->srv_signal.ext_params |= 0x43000; set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); clear_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/s390/interrupt.c index 249cdc822ec5..f892f4307883 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/s390/interrupt.c @@ -26,14 +26,14 @@ #include <linux/uaccess.h> #include <asm/sclp.h> #include <asm/isc.h> -#include <asm/gmap.h> #include <asm/nmi.h> #include <asm/airq.h> #include <asm/tpi.h> -#include "kvm-s390.h" +#include "s390.h" #include "gaccess.h" #include "trace-s390.h" #include "pci.h" +#include "gmap.h" #define PFAULT_INIT 0x0600 #define PFAULT_DONE 0x0680 @@ -45,13 +45,16 @@ static struct kvm_s390_gib *gib; static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id) { struct esca_block *sca = vcpu->kvm->arch.sca; - union esca_sigp_ctrl sigp_ctrl = sca->cpu[vcpu->vcpu_id].sigp_ctrl; + union esca_sigp_ctrl sigp_ctrl; if (!kvm_s390_test_cpuflags(vcpu, CPUSTAT_ECALL_PEND)) return 0; + if (kvm_is_ucontrol(vcpu->kvm)) + return 0; BUG_ON(!kvm_s390_use_sca_entries()); + sigp_ctrl = sca->cpu[vcpu->vcpu_id].sigp_ctrl; if (src_id) *src_id = sigp_ctrl.scn; @@ -60,13 +63,16 @@ static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id) static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id) { - struct esca_block *sca = vcpu->kvm->arch.sca; - union esca_sigp_ctrl *sigp_ctrl = &sca->cpu[vcpu->vcpu_id].sigp_ctrl; union esca_sigp_ctrl old_val, new_val = {.scn = src_id, .c = 1}; + struct esca_block *sca = vcpu->kvm->arch.sca; + union esca_sigp_ctrl *sigp_ctrl; int expect, rc; BUG_ON(!kvm_s390_use_sca_entries()); + if (kvm_is_ucontrol(vcpu->kvm)) + return -EINVAL; + sigp_ctrl = &sca->cpu[vcpu->vcpu_id].sigp_ctrl; old_val = READ_ONCE(*sigp_ctrl); old_val.c = 0; @@ -84,10 +90,13 @@ static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id) static void sca_clear_ext_call(struct kvm_vcpu *vcpu) { struct esca_block *sca = vcpu->kvm->arch.sca; - union esca_sigp_ctrl *sigp_ctrl = &sca->cpu[vcpu->vcpu_id].sigp_ctrl; + union esca_sigp_ctrl *sigp_ctrl; - if (!kvm_s390_use_sca_entries()) + if (!kvm_s390_use_sca_entries() || !vcpu->arch.initialized || kvm_is_ucontrol(vcpu->kvm)) return; + + /* Initialize after the above check, to prevent going out of bounds */ + sigp_ctrl = &sca->cpu[vcpu->vcpu_id].sigp_ctrl; kvm_s390_clear_cpuflags(vcpu, CPUSTAT_ECALL_PEND); WRITE_ONCE(sigp_ctrl->value, 0); @@ -264,6 +273,11 @@ static inline int gisa_tac_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc) return test_and_clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa); } +static inline int gisa_test_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc) +{ + return test_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *)gisa); +} + static inline unsigned long pending_irqs_no_gisa(struct kvm_vcpu *vcpu) { unsigned long pending = vcpu->kvm->arch.float_int.pending_irqs | @@ -624,8 +638,9 @@ static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu) struct kvm_s390_mchk_info mchk = {}; int deliver = 0; int rc = 0; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); spin_lock(&li->lock); if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) || test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) { @@ -654,7 +669,7 @@ static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu) deliver = 1; } spin_unlock(&li->lock); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); if (deliver) { VCPU_EVENT(vcpu, 3, "deliver: machine check mcic 0x%llx", @@ -941,11 +956,12 @@ static int __must_check __deliver_service(struct kvm_vcpu *vcpu) { struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; struct kvm_s390_ext_info ext; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs) || !(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) { - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } ext = fi->srv_signal; @@ -954,7 +970,10 @@ static int __must_check __deliver_service(struct kvm_vcpu *vcpu) clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs); if (kvm_s390_pv_cpu_is_protected(vcpu)) set_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); + + if (!ext.ext_params) + return 0; VCPU_EVENT(vcpu, 4, "deliver: sclp parameter 0x%x", ext.ext_params); @@ -969,17 +988,18 @@ static int __must_check __deliver_service_ev(struct kvm_vcpu *vcpu) { struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; struct kvm_s390_ext_info ext; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); if (!(test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs))) { - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } ext = fi->srv_signal; /* only clear the event bits */ fi->srv_signal.ext_params &= ~SCCB_EVENT_PENDING; clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); VCPU_EVENT(vcpu, 4, "%s", "deliver: sclp parameter event"); vcpu->stat.deliver_service_signal++; @@ -994,8 +1014,9 @@ static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu) struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; struct kvm_s390_interrupt_info *inti; int rc = 0; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT], struct kvm_s390_interrupt_info, list); @@ -1005,7 +1026,7 @@ static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu) } if (list_empty(&fi->lists[FIRQ_LIST_PFAULT])) clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); if (inti) { trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, @@ -1036,8 +1057,9 @@ static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu) struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; struct kvm_s390_interrupt_info *inti; int rc = 0; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO], struct kvm_s390_interrupt_info, list); @@ -1055,7 +1077,7 @@ static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu) } if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO])) clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); if (inti) { rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, @@ -1113,10 +1135,11 @@ static int __must_check __deliver_io(struct kvm_vcpu *vcpu, struct kvm_s390_io_info io; u32 isc; int rc = 0; + unsigned long flags; fi = &vcpu->kvm->arch.float_int; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); isc = irq_type_to_isc(irq_type); isc_list = &fi->lists[isc]; inti = list_first_entry_or_null(isc_list, @@ -1143,7 +1166,7 @@ static int __must_check __deliver_io(struct kvm_vcpu *vcpu, } if (list_empty(isc_list)) clear_bit(irq_type, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); if (inti) { rc = __do_deliver_io(vcpu, &(inti->io)); @@ -1532,23 +1555,21 @@ static int __inject_set_prefix(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) } #define KVM_S390_STOP_SUPP_FLAGS (KVM_S390_STOP_FLAG_STORE_STATUS) -static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) +static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq, bool *storestatus) { struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; struct kvm_s390_stop_info *stop = &li->irq.stop; - int rc = 0; vcpu->stat.inject_stop_signal++; trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_STOP, 0, 0); if (irq->u.stop.flags & ~KVM_S390_STOP_SUPP_FLAGS) return -EINVAL; - if (is_vcpu_stopped(vcpu)) { - if (irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS) - rc = kvm_s390_store_status_unloaded(vcpu, - KVM_S390_STORE_STATUS_NOADDR); - return rc; + if (!(irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS)) + return 0; + *storestatus = true; + return -EWOULDBLOCK; } if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs)) @@ -1659,8 +1680,9 @@ static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm, struct kvm_s390_interrupt_info *iter; u16 id = (schid & 0xffff0000U) >> 16; u16 nr = schid & 0x0000ffffU; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); list_for_each_entry(iter, isc_list, list) { if (schid && (id != iter->io.subchannel_id || nr != iter->io.subchannel_nr)) @@ -1670,10 +1692,10 @@ static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm, fi->counters[FIRQ_CNTR_IO] -= 1; if (list_empty(isc_list)) clear_bit(isc_to_irq_type(isc), &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return iter; } - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return NULL; } @@ -1749,7 +1771,7 @@ struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm, goto out; } gisa_out: - tmp_inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT); + tmp_inti = kzalloc_obj(*inti, GFP_KERNEL_ACCOUNT); if (tmp_inti) { tmp_inti->type = KVM_S390_INT_IO(1, 0, 0, 0); tmp_inti->io.io_int_word = isc_to_int_word(isc); @@ -1766,9 +1788,10 @@ static int __inject_service(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + unsigned long flags; kvm->stat.inject_service_signal++; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING; /* We always allow events, track them separately from the sccb ints */ @@ -1788,7 +1811,7 @@ static int __inject_service(struct kvm *kvm, fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK; set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); out: - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); kfree(inti); return 0; } @@ -1797,17 +1820,18 @@ static int __inject_virtio(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + unsigned long flags; kvm->stat.inject_virtio++; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) { - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return -EBUSY; } fi->counters[FIRQ_CNTR_VIRTIO] += 1; list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]); set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } @@ -1815,18 +1839,19 @@ static int __inject_pfault_done(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + unsigned long flags; kvm->stat.inject_pfault_done++; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); if (fi->counters[FIRQ_CNTR_PFAULT] >= (ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) { - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return -EBUSY; } fi->counters[FIRQ_CNTR_PFAULT] += 1; list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]); set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } @@ -1835,13 +1860,14 @@ static int __inject_float_mchk(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + unsigned long flags; kvm->stat.inject_float_mchk++; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS); fi->mchk.mcic |= inti->mchk.mcic; set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); kfree(inti); return 0; } @@ -1852,6 +1878,7 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) struct kvm_s390_float_interrupt *fi; struct list_head *list; int isc; + unsigned long flags; kvm->stat.inject_io++; isc = int_word_to_isc(inti->io.io_int_word); @@ -1870,9 +1897,9 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) } fi = &kvm->arch.float_int; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) { - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return -EBUSY; } fi->counters[FIRQ_CNTR_IO] += 1; @@ -1887,7 +1914,7 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc]; list_add_tail(&inti->list, list); set_bit(isc_to_irq_type(isc), &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } @@ -1963,15 +1990,10 @@ static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) } int kvm_s390_inject_vm(struct kvm *kvm, - struct kvm_s390_interrupt *s390int) + struct kvm_s390_interrupt *s390int, struct kvm_s390_interrupt_info *inti) { - struct kvm_s390_interrupt_info *inti; int rc; - inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT); - if (!inti) - return -ENOMEM; - inti->type = s390int->type; switch (inti->type) { case KVM_S390_INT_VIRTIO: @@ -2000,15 +2022,13 @@ int kvm_s390_inject_vm(struct kvm *kvm, inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull; break; default: - kfree(inti); return -EINVAL; } trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64, 2); rc = __inject_vm(kvm, inti); - if (rc) - kfree(inti); + return rc; } @@ -2085,7 +2105,7 @@ void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu) spin_unlock(&li->lock); } -static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) +static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq, bool *storestatus) { int rc; @@ -2097,7 +2117,7 @@ static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) rc = __inject_set_prefix(vcpu, irq); break; case KVM_S390_SIGP_STOP: - rc = __inject_sigp_stop(vcpu, irq); + rc = __inject_sigp_stop(vcpu, irq, storestatus); break; case KVM_S390_RESTART: rc = __inject_sigp_restart(vcpu); @@ -2133,11 +2153,16 @@ static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) { struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; + bool storestatus = false; int rc; spin_lock(&li->lock); - rc = do_inject_vcpu(vcpu, irq); + rc = do_inject_vcpu(vcpu, irq, &storestatus); spin_unlock(&li->lock); + + if (rc == -EWOULDBLOCK && storestatus) + rc = kvm_s390_store_status_unloaded(vcpu, KVM_S390_STORE_STATUS_NOADDR); + if (!rc) kvm_s390_vcpu_wakeup(vcpu); return rc; @@ -2173,12 +2198,13 @@ void kvm_s390_clear_float_irqs(struct kvm *kvm) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; int i; + unsigned long flags; mutex_lock(&kvm->lock); if (!kvm_s390_pv_is_protected(kvm)) fi->masked_irqs = 0; mutex_unlock(&kvm->lock); - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); fi->pending_irqs = 0; memset(&fi->srv_signal, 0, sizeof(fi->srv_signal)); memset(&fi->mchk, 0, sizeof(fi->mchk)); @@ -2186,7 +2212,7 @@ void kvm_s390_clear_float_irqs(struct kvm *kvm) clear_irq_list(&fi->lists[i]); for (i = 0; i < FIRQ_MAX_COUNT; i++) fi->counters[i] = 0; - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); kvm_s390_gisa_clear(kvm); }; @@ -2201,6 +2227,7 @@ static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) int ret = 0; int n = 0; int i; + unsigned long flags; if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0) return -EINVAL; @@ -2223,7 +2250,7 @@ static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) ret = -ENOMEM; goto out_nolock; } - if (gisa_tac_ipm_gisc(gi->origin, i)) { + if (gisa_test_ipm_gisc(gi->origin, i)) { irq = (struct kvm_s390_irq *) &buf[n]; irq->type = KVM_S390_INT_IO(1, 0, 0, 0); irq->u.io.io_int_word = isc_to_int_word(i); @@ -2232,7 +2259,7 @@ static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) } } fi = &kvm->arch.float_int; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); for (i = 0; i < FIRQ_LIST_COUNT; i++) { list_for_each_entry(inti, &fi->lists[i], list) { if (n == max_irqs) { @@ -2269,7 +2296,7 @@ static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) } out: - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); out_nolock: if (!ret && n > 0) { if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n)) @@ -2284,6 +2311,7 @@ static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; struct kvm_s390_ais_all ais; + unsigned long flags; if (attr->attr < sizeof(ais)) return -EINVAL; @@ -2291,10 +2319,10 @@ static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr) if (!test_kvm_facility(kvm, 72)) return -EOPNOTSUPP; - mutex_lock(&fi->ais_lock); + spin_lock_irqsave(&fi->ais_lock, flags); ais.simm = fi->simm; ais.nimm = fi->nimm; - mutex_unlock(&fi->ais_lock); + spin_unlock_irqrestore(&fi->ais_lock, flags); if (copy_to_user((void __user *)attr->addr, &ais, sizeof(ais))) return -EFAULT; @@ -2374,7 +2402,7 @@ static int enqueue_floating_irq(struct kvm_device *dev, return -EINVAL; while (len >= sizeof(struct kvm_s390_irq)) { - inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT); + inti = kzalloc_obj(*inti, GFP_KERNEL_ACCOUNT); if (!inti) return -ENOMEM; @@ -2408,34 +2436,46 @@ static int register_io_adapter(struct kvm_device *dev, { struct s390_io_adapter *adapter; struct kvm_s390_io_adapter adapter_info; + int rc = 0; + mutex_lock(&dev->kvm->lock); if (copy_from_user(&adapter_info, - (void __user *)attr->addr, sizeof(adapter_info))) - return -EFAULT; - - if (adapter_info.id >= MAX_S390_IO_ADAPTERS) - return -EINVAL; - + (void __user *)attr->addr, sizeof(adapter_info))) { + rc = -EFAULT; + goto out; + } + if (adapter_info.id >= MAX_S390_IO_ADAPTERS) { + rc = -EINVAL; + goto out; + } adapter_info.id = array_index_nospec(adapter_info.id, MAX_S390_IO_ADAPTERS); - if (dev->kvm->arch.adapters[adapter_info.id] != NULL) - return -EINVAL; - - adapter = kzalloc(sizeof(*adapter), GFP_KERNEL_ACCOUNT); - if (!adapter) - return -ENOMEM; + if (dev->kvm->arch.adapters[adapter_info.id] != NULL) { + rc = -EINVAL; + goto out; + } + adapter = kzalloc_obj(*adapter, GFP_KERNEL_ACCOUNT); + if (!adapter) { + rc = -ENOMEM; + goto out; + } + INIT_LIST_HEAD(&adapter->maps); + spin_lock_init(&adapter->maps_lock); + adapter->nr_maps = 0; adapter->id = adapter_info.id; adapter->isc = adapter_info.isc; adapter->maskable = adapter_info.maskable; adapter->masked = false; adapter->swap = adapter_info.swap; - adapter->suppressible = (adapter_info.flags) & + adapter->suppressible = adapter_info.flags & KVM_S390_ADAPTER_SUPPRESSIBLE; dev->kvm->arch.adapters[adapter->id] = adapter; - return 0; +out: + mutex_unlock(&dev->kvm->lock); + return rc; } int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked) @@ -2450,12 +2490,176 @@ int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked) return ret; } +static struct page *pin_map_page(struct kvm *kvm, u64 uaddr, + unsigned int gup_flags) +{ + struct mm_struct *mm = kvm->mm; + struct page *page = NULL; + int locked = 1; + + if (mmget_not_zero(mm)) { + mmap_read_lock(mm); + pin_user_pages_remote(mm, uaddr, 1, FOLL_WRITE | gup_flags, + &page, &locked); + if (locked) + mmap_read_unlock(mm); + mmput(mm); + } + + return page; +} + +static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr) +{ + struct s390_io_adapter *adapter = get_io_adapter(kvm, id); + struct s390_map_info *map; + unsigned long flags; + __u64 host_addr; + int ret, idx; + + if (!adapter || !addr) + return -EINVAL; + + map = kzalloc_obj(*map, GFP_KERNEL_ACCOUNT); + if (!map) + return -ENOMEM; + + INIT_LIST_HEAD(&map->list); + idx = srcu_read_lock(&kvm->srcu); + host_addr = gpa_to_hva(kvm, addr); + if (kvm_is_error_hva(host_addr)) { + srcu_read_unlock(&kvm->srcu, idx); + ret = -EFAULT; + goto out; + } + srcu_read_unlock(&kvm->srcu, idx); + map->guest_addr = addr; + map->addr = host_addr; + map->page = pin_map_page(kvm, host_addr, FOLL_LONGTERM); + if (!map->page) { + /* + * Long-term pinning may fail for memory types such as file-backed + * memory. Verify that short-term pinning succeeds so that the + * non-atomic irqfd path can handle interrupt injection. + */ + map->page = pin_map_page(kvm, host_addr, 0); + if (!map->page) { + ret = -EINVAL; + goto out; + } + unpin_user_page(map->page); + map->page = NULL; + map->pinned = false; + /* Add an entry to preserve MAP/UNMAP symmetry. */ + } else { + map->pinned = true; + } + spin_lock_irqsave(&adapter->maps_lock, flags); + if (adapter->nr_maps < MAX_S390_ADAPTER_MAPS) { + list_add_tail(&map->list, &adapter->maps); + adapter->nr_maps++; + ret = 0; + } else { + ret = -EINVAL; + } + spin_unlock_irqrestore(&adapter->maps_lock, flags); + if (ret && map->page) + unpin_user_page(map->page); +out: + if (ret) + kfree(map); + return ret; +} + +static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr) +{ + struct s390_io_adapter *adapter = get_io_adapter(kvm, id); + struct s390_map_info *map, *tmp, *map_to_free; + struct page *map_page_to_put = NULL; + u64 map_addr_to_mark = 0; + bool map_pinned = false; + unsigned long flags; + int found = 0, idx; + + if (!adapter || !addr) + return -EINVAL; + + spin_lock_irqsave(&adapter->maps_lock, flags); + list_for_each_entry_safe(map, tmp, &adapter->maps, list) { + if (map->guest_addr == addr) { + found = 1; + adapter->nr_maps--; + list_del(&map->list); + map_page_to_put = map->page; + map_addr_to_mark = map->guest_addr; + map_pinned = map->pinned; + map_to_free = map; + break; + } + } + spin_unlock_irqrestore(&adapter->maps_lock, flags); + + if (found) { + kfree(map_to_free); + if (map_pinned) { + /* + * Only long-term pinned pages need to be marked dirty + * and released. Fallback entries exist only for + * MAP/UNMAP symmetry. + */ + idx = srcu_read_lock(&kvm->srcu); + mark_page_dirty(kvm, map_addr_to_mark >> PAGE_SHIFT); + set_page_dirty_lock(map_page_to_put); + srcu_read_unlock(&kvm->srcu, idx); + unpin_user_page(map_page_to_put); + } + } + + return found ? 0 : -ENOENT; +} + +void kvm_s390_unmap_all_adapters(struct kvm *kvm) +{ + struct s390_map_info *map, *tmp; + unsigned long flags; + int i, idx; + + for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) { + struct s390_io_adapter *adapter = kvm->arch.adapters[i]; + LIST_HEAD(local_list); + + if (!adapter) + continue; + + spin_lock_irqsave(&adapter->maps_lock, flags); + list_splice_init(&adapter->maps, &local_list); + adapter->nr_maps = 0; + spin_unlock_irqrestore(&adapter->maps_lock, flags); + + list_for_each_entry_safe(map, tmp, &local_list, list) { + list_del(&map->list); + if (map->pinned) { + idx = srcu_read_lock(&kvm->srcu); + mark_page_dirty(kvm, map->guest_addr >> PAGE_SHIFT); + set_page_dirty_lock(map->page); + srcu_read_unlock(&kvm->srcu, idx); + unpin_user_page(map->page); + } + kfree(map); + } + } +} + void kvm_s390_destroy_adapters(struct kvm *kvm) { int i; - for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) + kvm_s390_unmap_all_adapters(kvm); + + for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) { kfree(kvm->arch.adapters[i]); + kvm->arch.adapters[i] = NULL; + } } static int modify_io_adapter(struct kvm_device *dev, @@ -2477,14 +2681,22 @@ static int modify_io_adapter(struct kvm_device *dev, if (ret > 0) ret = 0; break; - /* - * The following operations are no longer needed and therefore no-ops. - * The gpa to hva translation is done when an IRQ route is set up. The - * set_irq code uses get_user_pages_remote() to do the actual write. - */ case KVM_S390_IO_ADAPTER_MAP: case KVM_S390_IO_ADAPTER_UNMAP: - ret = 0; + /* If in Secure Execution mode do not long term pin. */ + mutex_lock(&dev->kvm->lock); + if (kvm_s390_pv_is_protected(dev->kvm)) { + mutex_unlock(&dev->kvm->lock); + return 0; + } + if (req.type == KVM_S390_IO_ADAPTER_MAP) { + dev->kvm->stat.io_390_adapter_map++; + ret = kvm_s390_adapter_map(dev->kvm, req.id, req.addr); + } else { + dev->kvm->stat.io_390_adapter_unmap++; + ret = kvm_s390_adapter_unmap(dev->kvm, req.id, req.addr); + } + mutex_unlock(&dev->kvm->lock); break; default: ret = -EINVAL; @@ -2521,6 +2733,7 @@ static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr) struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; struct kvm_s390_ais_req req; int ret = 0; + unsigned long flags; if (!test_kvm_facility(kvm, 72)) return -EOPNOTSUPP; @@ -2537,7 +2750,7 @@ static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr) 2 : KVM_S390_AIS_MODE_SINGLE : KVM_S390_AIS_MODE_ALL, req.mode); - mutex_lock(&fi->ais_lock); + spin_lock_irqsave(&fi->ais_lock, flags); switch (req.mode) { case KVM_S390_AIS_MODE_ALL: fi->simm &= ~AIS_MODE_MASK(req.isc); @@ -2550,7 +2763,7 @@ static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr) default: ret = -EINVAL; } - mutex_unlock(&fi->ais_lock); + spin_unlock_irqrestore(&fi->ais_lock, flags); return ret; } @@ -2564,25 +2777,41 @@ static int kvm_s390_inject_airq(struct kvm *kvm, .parm = 0, .parm64 = isc_to_int_word(adapter->isc), }; + struct kvm_s390_interrupt_info *inti; + unsigned long flags; + int ret = 0; - if (!test_kvm_facility(kvm, 72) || !adapter->suppressible) - return kvm_s390_inject_vm(kvm, &s390int); + inti = kzalloc_obj(*inti, GFP_KERNEL_ACCOUNT); + if (!inti) + return -ENOMEM; + + if (!test_kvm_facility(kvm, 72) || !adapter->suppressible) { + ret = kvm_s390_inject_vm(kvm, &s390int, inti); + if (ret) + kfree(inti); + return ret; + } - mutex_lock(&fi->ais_lock); + spin_lock_irqsave(&fi->ais_lock, flags); if (fi->nimm & AIS_MODE_MASK(adapter->isc)) { trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc); - goto out; + spin_unlock_irqrestore(&fi->ais_lock, flags); + kfree(inti); + return ret; } - ret = kvm_s390_inject_vm(kvm, &s390int); + ret = kvm_s390_inject_vm(kvm, &s390int, inti); + if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) { fi->nimm |= AIS_MODE_MASK(adapter->isc); trace_kvm_s390_modify_ais_mode(adapter->isc, KVM_S390_AIS_MODE_SINGLE, 2); } -out: - mutex_unlock(&fi->ais_lock); + + spin_unlock_irqrestore(&fi->ais_lock, flags); + if (ret) + kfree(inti); return ret; } @@ -2591,6 +2820,8 @@ static int flic_inject_airq(struct kvm *kvm, struct kvm_device_attr *attr) unsigned int id = attr->attr; struct s390_io_adapter *adapter = get_io_adapter(kvm, id); + kvm->stat.io_flic_inject_airq++; + if (!adapter) return -EINVAL; @@ -2601,6 +2832,7 @@ static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; struct kvm_s390_ais_all ais; + unsigned long flags; if (!test_kvm_facility(kvm, 72)) return -EOPNOTSUPP; @@ -2608,10 +2840,10 @@ static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr) if (copy_from_user(&ais, (void __user *)attr->addr, sizeof(ais))) return -EFAULT; - mutex_lock(&fi->ais_lock); + spin_lock_irqsave(&fi->ais_lock, flags); fi->simm = ais.simm; fi->nimm = ais.nimm; - mutex_unlock(&fi->ais_lock); + spin_unlock_irqrestore(&fi->ais_lock, flags); return 0; } @@ -2632,12 +2864,12 @@ static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) case KVM_DEV_FLIC_APF_ENABLE: if (kvm_is_ucontrol(dev->kvm)) return -EINVAL; - dev->kvm->arch.gmap->pfault_enabled = 1; + set_bit(GMAP_FLAG_PFAULT_ENABLED, &dev->kvm->arch.gmap->flags); break; case KVM_DEV_FLIC_APF_DISABLE_WAIT: if (kvm_is_ucontrol(dev->kvm)) return -EINVAL; - dev->kvm->arch.gmap->pfault_enabled = 0; + clear_bit(GMAP_FLAG_PFAULT_ENABLED, &dev->kvm->arch.gmap->flags); /* * Make sure no async faults are in transition when * clearing the queues. So we don't need to worry @@ -2724,62 +2956,129 @@ static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap) bit = bit_nr + (addr % PAGE_SIZE) * 8; + /* kvm_set_routing_entry() should never allow this to happen */ + WARN_ON_ONCE(bit > (PAGE_SIZE * BITS_PER_BYTE - 1)); + return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit; } -static struct page *get_map_page(struct kvm *kvm, u64 uaddr) +static struct s390_map_info *get_map_info(struct s390_io_adapter *adapter, + u64 addr) { - struct mm_struct *mm = kvm->mm; - struct page *page = NULL; - int locked = 1; + struct s390_map_info *map; - if (mmget_not_zero(mm)) { - mmap_read_lock(mm); - get_user_pages_remote(mm, uaddr, 1, FOLL_WRITE, - &page, &locked); - if (locked) - mmap_read_unlock(mm); - mmput(mm); - } + if (!adapter) + return NULL; - return page; + list_for_each_entry(map, &adapter->maps, list) { + if (map->addr == addr) { + if (!map->pinned) + return NULL; + return map; + } + } + return NULL; } static int adapter_indicators_set(struct kvm *kvm, struct s390_io_adapter *adapter, struct kvm_s390_adapter_int *adapter_int) { - unsigned long bit; - int summary_set, idx; + struct s390_map_info *ind_info, *summary_info; struct page *ind_page, *summary_page; + unsigned long bit; + int summary_set; void *map; - ind_page = get_map_page(kvm, adapter_int->ind_addr); - if (!ind_page) - return -1; - summary_page = get_map_page(kvm, adapter_int->summary_addr); - if (!summary_page) { - put_page(ind_page); - return -1; + ind_page = NULL; + + scoped_guard(spinlock_irqsave, &adapter->maps_lock) { + ind_info = get_map_info(adapter, adapter_int->ind_addr); + if (ind_info) { + map = page_address(ind_info->page); + bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap); + set_bit(bit, map); + } } + if (!ind_info) { + ind_page = pin_map_page(kvm, adapter_int->ind_addr, 0); + if (!ind_page) + return -1; + map = page_address(ind_page); + bit = get_ind_bit(adapter_int->ind_addr, + adapter_int->ind_offset, adapter->swap); + set_bit(bit, map); + set_page_dirty_lock(ind_page); + unpin_user_page(ind_page); + } + scoped_guard(srcu, &kvm->srcu) + mark_page_dirty(kvm, gpa_to_gfn(adapter_int->ind_gaddr)); + + scoped_guard(spinlock_irqsave, &adapter->maps_lock) { + summary_info = get_map_info(adapter, adapter_int->summary_addr); + if (summary_info) { + map = page_address(summary_info->page); + bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset, + adapter->swap); + summary_set = test_and_set_bit(bit, map); + } + } + if (!summary_info) { + summary_page = pin_map_page(kvm, adapter_int->summary_addr, 0); + if (!summary_page) + return -1; + map = page_address(summary_page); + bit = get_ind_bit(adapter_int->summary_addr, + adapter_int->summary_offset, adapter->swap); + summary_set = test_and_set_bit(bit, map); + set_page_dirty_lock(summary_page); + unpin_user_page(summary_page); + } + scoped_guard(srcu, &kvm->srcu) + mark_page_dirty(kvm, gpa_to_gfn(adapter_int->summary_gaddr)); - idx = srcu_read_lock(&kvm->srcu); - map = page_address(ind_page); - bit = get_ind_bit(adapter_int->ind_addr, - adapter_int->ind_offset, adapter->swap); - set_bit(bit, map); - mark_page_dirty(kvm, adapter_int->ind_addr >> PAGE_SHIFT); - set_page_dirty_lock(ind_page); - map = page_address(summary_page); - bit = get_ind_bit(adapter_int->summary_addr, - adapter_int->summary_offset, adapter->swap); - summary_set = test_and_set_bit(bit, map); - mark_page_dirty(kvm, adapter_int->summary_addr >> PAGE_SHIFT); - set_page_dirty_lock(summary_page); - srcu_read_unlock(&kvm->srcu, idx); + return summary_set ? 0 : 1; +} + +static int adapter_indicators_set_fast(struct kvm *kvm, + struct s390_io_adapter *adapter, + struct kvm_s390_adapter_int *adapter_int, + int setbit) +{ + struct s390_map_info *ind_info, *summary_info; + unsigned long bit; + int summary_set; + void *map; + + guard(srcu)(&kvm->srcu); + guard(spinlock)(&adapter->maps_lock); + + ind_info = get_map_info(adapter, adapter_int->ind_addr); + if (!ind_info) + return -EWOULDBLOCK; + + map = page_address(ind_info->page); + bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap); + if (setbit) { + set_bit(bit, map); + mark_page_dirty(kvm, gpa_to_gfn(adapter_int->ind_gaddr)); + } + + summary_info = get_map_info(adapter, adapter_int->summary_addr); + if (!summary_info) + return -EWOULDBLOCK; + + map = page_address(summary_info->page); + bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset, + adapter->swap); + /* If setbit then set summary bit. Else if falling back to the slow path */ + /* with setbit==0 then clear the summary bit so the slow path re-injects */ + if (setbit) + summary_set = test_and_set_bit(bit, map); + else + summary_set = test_and_clear_bit(bit, map); + mark_page_dirty(kvm, gpa_to_gfn(adapter_int->summary_gaddr)); - put_page(ind_page); - put_page(summary_page); return summary_set ? 0 : 1; } @@ -2795,6 +3094,8 @@ static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e, int ret; struct s390_io_adapter *adapter; + kvm->stat.io_set_adapter_int++; + /* We're only interested in the 0->1 transition. */ if (!level) return 0; @@ -2816,14 +3117,18 @@ static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e, void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu, struct mcck_volatile_info *mcck_info) { - struct kvm_s390_interrupt_info inti; - struct kvm_s390_irq irq; - struct kvm_s390_mchk_info *mchk; + struct kvm_s390_irq irq = {}; union mci mci; __u64 cr14 = 0; /* upper bits are not used */ int rc; mci.val = mcck_info->mcic; + + /* log machine checks being reinjected on all debugs */ + VCPU_EVENT(vcpu, 2, "guest machine check %lx", mci.val); + KVM_EVENT(2, "guest machine check %lx", mci.val); + pr_info("guest machine check pid %d: %lx", current->pid, mci.val); + if (mci.sr) cr14 |= CR14_RECOVERY_SUBMASK; if (mci.dg) @@ -2831,20 +3136,14 @@ void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu, if (mci.w) cr14 |= CR14_WARNING_SUBMASK; - mchk = mci.ck ? &inti.mchk : &irq.u.mchk; - mchk->cr14 = cr14; - mchk->mcic = mcck_info->mcic; - mchk->ext_damage_code = mcck_info->ext_damage_code; - mchk->failing_storage_address = mcck_info->failing_storage_address; - if (mci.ck) { - /* Inject the floating machine check */ - inti.type = KVM_S390_MCHK; - rc = __inject_vm(vcpu->kvm, &inti); - } else { - /* Inject the machine check to specified vcpu */ - irq.type = KVM_S390_MCHK; - rc = kvm_s390_inject_vcpu(vcpu, &irq); - } + irq.u.mchk.cr14 = cr14; + irq.u.mchk.mcic = mcck_info->mcic; + irq.u.mchk.ext_damage_code = mcck_info->ext_damage_code; + irq.u.mchk.failing_storage_address = mcck_info->failing_storage_address; + + /* Inject the machine check to specified vcpu */ + irq.type = KVM_S390_MCHK; + rc = kvm_s390_inject_vcpu(vcpu, &irq); WARN_ON_ONCE(rc); } @@ -2852,16 +3151,24 @@ int kvm_set_routing_entry(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, const struct kvm_irq_routing_entry *ue) { + const struct kvm_irq_routing_s390_adapter *adapter; u64 uaddr_s, uaddr_i; int idx; switch (ue->type) { - /* we store the userspace addresses instead of the guest addresses */ case KVM_IRQ_ROUTING_S390_ADAPTER: if (kvm_is_ucontrol(kvm)) return -EINVAL; e->set = set_adapter_int; + adapter = &ue->u.adapter; + if (adapter->summary_addr + (adapter->summary_offset / 8) >= + (adapter->summary_addr & PAGE_MASK) + PAGE_SIZE) + return -EINVAL; + if (adapter->ind_addr + (adapter->ind_offset / 8) >= + (adapter->ind_addr & PAGE_MASK) + PAGE_SIZE) + return -EINVAL; + idx = srcu_read_lock(&kvm->srcu); uaddr_s = gpa_to_hva(kvm, ue->u.adapter.summary_addr); uaddr_i = gpa_to_hva(kvm, ue->u.adapter.ind_addr); @@ -2870,7 +3177,9 @@ int kvm_set_routing_entry(struct kvm *kvm, if (kvm_is_error_hva(uaddr_s) || kvm_is_error_hva(uaddr_i)) return -EFAULT; e->adapter.summary_addr = uaddr_s; + e->adapter.summary_gaddr = ue->u.adapter.summary_addr; e->adapter.ind_addr = uaddr_i; + e->adapter.ind_gaddr = ue->u.adapter.ind_addr; e->adapter.summary_offset = ue->u.adapter.summary_offset; e->adapter.ind_offset = ue->u.adapter.ind_offset; e->adapter.adapter_id = ue->u.adapter.adapter_id; @@ -2889,7 +3198,8 @@ int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len) { struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; - struct kvm_s390_irq *buf; + struct kvm_s390_irq *buf __free(kvfree) = NULL; + bool tmp, storestatus = false; int r = 0; int n; @@ -2897,31 +3207,33 @@ int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len if (!buf) return -ENOMEM; - if (copy_from_user((void *) buf, irqstate, len)) { - r = -EFAULT; - goto out_free; - } + if (copy_from_user((void *)buf, irqstate, len)) + return -EFAULT; - /* - * Don't allow setting the interrupt state - * when there are already interrupts pending - */ - spin_lock(&li->lock); - if (li->pending_irqs) { - r = -EBUSY; - goto out_unlock; - } + scoped_guard(spinlock, &li->lock) { + /* + * Don't allow setting the interrupt state + * when there are already interrupts pending + */ + if (li->pending_irqs) + return -EBUSY; - for (n = 0; n < len / sizeof(*buf); n++) { - r = do_inject_vcpu(vcpu, &buf[n]); - if (r) - break; + for (n = 0; n < len / sizeof(*buf); n++) { + tmp = false; + r = do_inject_vcpu(vcpu, &buf[n], &tmp); + if (r == -EWOULDBLOCK && tmp) { + storestatus = true; + r = 0; + } + if (r) + break; + } + } + if (storestatus) { + scoped_guard(srcu, &vcpu->kvm->srcu) + n = kvm_s390_store_status_unloaded(vcpu, KVM_S390_STORE_STATUS_NOADDR); + return r ? r : n; } - -out_unlock: - spin_unlock(&li->lock); -out_free: - vfree(buf); return r; } @@ -3287,8 +3599,7 @@ static void aen_host_forward(unsigned long si) struct zpci_gaite *gaite; struct kvm *kvm; - gaite = (struct zpci_gaite *)aift->gait + - (si * sizeof(struct zpci_gaite)); + gaite = aift->gait + si; if (gaite->count == 0) return; if (gaite->aisb != 0) @@ -3437,3 +3748,86 @@ out_free_gib: out: return rc; } + +/* + * kvm_arch_set_irq_inatomic: fast-path for irqfd injection + */ +int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, + struct kvm *kvm, int irq_source_id, int level, + bool line_status) +{ + int ret, setbit; + struct s390_io_adapter *adapter; + struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + struct kvm_s390_interrupt_info *inti; + struct kvm_s390_interrupt s390int = { + .type = KVM_S390_INT_IO(1, 0, 0, 0), + .parm = 0, + }; + + kvm->stat.io_390_inatomic++; + + /* We're only interested in the 0->1 transition. */ + if (!level) + return 0; + if (e->type != KVM_IRQ_ROUTING_S390_ADAPTER) + return -EWOULDBLOCK; + + adapter = get_io_adapter(kvm, e->adapter.adapter_id); + if (!adapter) + return -EWOULDBLOCK; + + s390int.parm64 = isc_to_int_word(adapter->isc); + setbit = 1; + ret = adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit); + if (ret < 0) + return -EWOULDBLOCK; + if (!ret || adapter->masked) { + kvm->stat.io_390_inatomic_no_inject++; + return 0; + } + + inti = kzalloc_obj(*inti, GFP_ATOMIC); + if (!inti) { + setbit = 0; + adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit); + return -EWOULDBLOCK; + } + + if (!test_kvm_facility(kvm, 72) || !adapter->suppressible) { + ret = kvm_s390_inject_vm(kvm, &s390int, inti); + if (ret == 0) { + return ret; + } else { + setbit = 0; + adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit); + kfree(inti); + return -EWOULDBLOCK; + } + } + + spin_lock(&fi->ais_lock); + if (fi->nimm & AIS_MODE_MASK(adapter->isc)) { + trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc); + spin_unlock(&fi->ais_lock); + kfree(inti); + kvm->stat.io_390_inatomic_no_inject++; + return 0; + } + + ret = kvm_s390_inject_vm(kvm, &s390int, inti); + if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) { + fi->nimm |= AIS_MODE_MASK(adapter->isc); + trace_kvm_s390_modify_ais_mode(adapter->isc, + KVM_S390_AIS_MODE_SINGLE, 2); + } else if (ret) { + spin_unlock(&fi->ais_lock); + setbit = 0; + adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit); + kfree(inti); + return -EWOULDBLOCK; + } + + spin_unlock(&fi->ais_lock); + return 0; +} diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/s390/pci.c index 8c40154ff50f..82892e1e03d9 100644 --- a/arch/s390/kvm/pci.c +++ b/arch/s390/kvm/s390/pci.c @@ -14,7 +14,7 @@ #include <asm/pci_io.h> #include <asm/sclp.h> #include "pci.h" -#include "kvm-s390.h" +#include "s390.h" struct zpci_aift *aift; @@ -54,7 +54,7 @@ static int zpci_setup_aipb(u8 nisc) struct page *page; int size, rc; - zpci_aipb = kzalloc(sizeof(union zpci_sic_iib), GFP_KERNEL); + zpci_aipb = kzalloc_obj(union zpci_sic_iib); if (!zpci_aipb) return -ENOMEM; @@ -126,8 +126,7 @@ int kvm_s390_pci_aen_init(u8 nisc) return -EPERM; mutex_lock(&aift->aift_lock); - aift->kzdev = kcalloc(ZPCI_NR_DEVICES, sizeof(struct kvm_zdev *), - GFP_KERNEL); + aift->kzdev = kzalloc_objs(struct kvm_zdev *, ZPCI_NR_DEVICES); if (!aift->kzdev) { rc = -ENOMEM; goto unlock; @@ -167,7 +166,7 @@ static int kvm_zpci_set_airq(struct zpci_dev *zdev) fib.fmt0.noi = airq_iv_end(zdev->aibv); fib.fmt0.aibv = virt_to_phys(zdev->aibv->vector); fib.fmt0.aibvo = 0; - fib.fmt0.aisb = virt_to_phys(aift->sbv->vector + (zdev->aisb / 64) * 8); + fib.fmt0.aisb = virt_to_phys(aift->sbv->vector) + (zdev->aisb / 64) * 8; fib.fmt0.aisbo = zdev->aisb & 63; fib.gd = zdev->gisa; @@ -191,40 +190,61 @@ static int kvm_zpci_clear_airq(struct zpci_dev *zdev) return cc ? -EIO : 0; } -static inline void unaccount_mem(unsigned long nr_pages) +static inline void unaccount_mem(struct kvm_zdev *kzdev, unsigned long nr_pages) { - struct user_struct *user = get_uid(current_user()); + struct user_struct *user = kzdev->user_account; + struct mm_struct *mm_account = kzdev->mm_account; - if (user) + if (user) { atomic_long_sub(nr_pages, &user->locked_vm); - if (current->mm) - atomic64_sub(nr_pages, ¤t->mm->pinned_vm); + free_uid(user); + kzdev->user_account = NULL; + } + + if (mm_account) { + atomic64_sub(nr_pages, &mm_account->pinned_vm); + mmdrop(mm_account); + kzdev->mm_account = NULL; + } } -static inline int account_mem(unsigned long nr_pages) +static inline int account_mem(struct kvm_zdev *kzdev, unsigned long nr_pages) { struct user_struct *user = get_uid(current_user()); unsigned long page_limit, cur_pages, new_pages; + int rc = 0; page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; cur_pages = atomic_long_read(&user->locked_vm); do { new_pages = cur_pages + nr_pages; - if (new_pages > page_limit) - return -ENOMEM; + if (new_pages > page_limit) { + rc = -ENOMEM; + goto out; + } } while (!atomic_long_try_cmpxchg(&user->locked_vm, &cur_pages, new_pages)); - atomic64_add(nr_pages, ¤t->mm->pinned_vm); + if (current->mm) { + mmgrab(current->mm); + atomic64_add(nr_pages, ¤t->mm->pinned_vm); + } + + kzdev->user_account = user; + kzdev->mm_account = current->mm; return 0; + +out: + free_uid(user); + return rc; } static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, bool assist) { struct page *pages[1], *aibv_page, *aisb_page = NULL; - unsigned int msi_vecs, idx; + unsigned int msi_vecs, idx, size; struct zpci_gaite *gaite; unsigned long hva, bit; struct kvm *kvm; @@ -238,6 +258,10 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, if (zdev->gisa == 0) return -EINVAL; + /* AIF already enabled for the device */ + if (zdev->kzdev->fib.fmt0.aibv != 0) + return -EINVAL; + kvm = zdev->kzdev->kvm; msi_vecs = min_t(unsigned int, fib->fmt0.noi, zdev->max_msi); @@ -247,6 +271,14 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, return gisc; /* Replace AIBV address */ + size = BITS_TO_LONGS(msi_vecs + fib->fmt0.aibvo) * sizeof(unsigned long); + npages = DIV_ROUND_UP((fib->fmt0.aibv & ~PAGE_MASK) + size, PAGE_SIZE); + /* AIBV cannot span more than 1 page */ + if (npages > 1) { + rc = -EINVAL; + goto out; + } + idx = srcu_read_lock(&kvm->srcu); hva = gfn_to_hva(kvm, gpa_to_gfn((gpa_t)fib->fmt0.aibv)); npages = pin_user_pages_fast(hva, 1, FOLL_WRITE | FOLL_LONGTERM, pages); @@ -262,6 +294,12 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, /* Pin the guest AISB if one was specified */ if (fib->fmt0.sum == 1) { + /* AISB must be dword aligned */ + if (fib->fmt0.aisb & 0x7) { + rc = -EINVAL; + goto unpin1; + } + idx = srcu_read_lock(&kvm->srcu); hva = gfn_to_hva(kvm, gpa_to_gfn((gpa_t)fib->fmt0.aisb)); npages = pin_user_pages_fast(hva, 1, FOLL_WRITE | FOLL_LONGTERM, @@ -276,23 +314,30 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, } /* Account for pinned pages, roll back on failure */ - if (account_mem(pcount)) + rc = account_mem(zdev->kzdev, pcount); + if (rc) goto unpin2; /* AISB must be allocated before we can fill in GAITE */ mutex_lock(&aift->aift_lock); bit = airq_iv_alloc_bit(aift->sbv); - if (bit == -1UL) + if (bit == -1UL) { + rc = -ENOMEM; goto unlock; + } zdev->aisb = bit; /* store the summary bit number */ zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA | AIRQ_IV_BITLOCK | AIRQ_IV_GUESTVEC, phys_to_virt(fib->fmt0.aibv)); + if (!zdev->aibv) { + rc = -ENOMEM; + goto free_aisb; + } + spin_lock_irq(&aift->gait_lock); - gaite = (struct zpci_gaite *)aift->gait + (zdev->aisb * - sizeof(struct zpci_gaite)); + gaite = aift->gait + zdev->aisb; /* If assist not requested, host will get all alerts */ if (assist) @@ -302,27 +347,50 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, gaite->gisc = fib->fmt0.isc; gaite->count++; - gaite->aisbo = fib->fmt0.aisbo; - gaite->aisb = virt_to_phys(page_address(aisb_page) + (fib->fmt0.aisb & - ~PAGE_MASK)); + if (fib->fmt0.sum == 1) { + gaite->aisbo = fib->fmt0.aisbo; + gaite->aisb = virt_to_phys(page_address(aisb_page) + + (fib->fmt0.aisb & ~PAGE_MASK)); + } else { + gaite->aisbo = 0; + gaite->aisb = 0; + } aift->kzdev[zdev->aisb] = zdev->kzdev; spin_unlock_irq(&aift->gait_lock); - /* Update guest FIB for re-issue */ - fib->fmt0.aisbo = zdev->aisb & 63; - fib->fmt0.aisb = virt_to_phys(aift->sbv->vector + (zdev->aisb / 64) * 8); - fib->fmt0.isc = gisc; - /* Save some guest fib values in the host for later use */ - zdev->kzdev->fib.fmt0.isc = fib->fmt0.isc; + zdev->kzdev->fib.fmt0.isc = gisc; zdev->kzdev->fib.fmt0.aibv = fib->fmt0.aibv; - mutex_unlock(&aift->aift_lock); /* Issue the clp to setup the irq now */ rc = kvm_zpci_set_airq(zdev); - return rc; + if (!rc) { + mutex_unlock(&aift->aift_lock); + return rc; + } + + /* Start cleanup */ + zdev->kzdev->fib.fmt0.isc = 0; + zdev->kzdev->fib.fmt0.aibv = 0; + + spin_lock_irq(&aift->gait_lock); + gaite->count--; + gaite->aisb = 0; + gaite->gisc = 0; + gaite->aisbo = 0; + gaite->gisa = 0; + aift->kzdev[zdev->aisb] = NULL; + spin_unlock_irq(&aift->gait_lock); + airq_iv_release(zdev->aibv); + zdev->aibv = NULL; + +free_aisb: + airq_iv_free_bit(aift->sbv, zdev->aisb); + zdev->aisb = 0; unlock: + if (pcount > 0) + unaccount_mem(zdev->kzdev, pcount); mutex_unlock(&aift->aift_lock); unpin2: if (fib->fmt0.sum == 1) @@ -330,6 +398,7 @@ unpin2: unpin1: unpin_user_page(aibv_page); out: + kvm_s390_gisc_unregister(kvm, fib->fmt0.isc); return rc; } @@ -358,8 +427,7 @@ static int kvm_s390_pci_aif_disable(struct zpci_dev *zdev, bool force) if (zdev->kzdev->fib.fmt0.aibv == 0) goto out; spin_lock_irq(&aift->gait_lock); - gaite = (struct zpci_gaite *)aift->gait + (zdev->aisb * - sizeof(struct zpci_gaite)); + gaite = aift->gait + zdev->aisb; isc = gaite->gisc; gaite->count--; if (gaite->count == 0) { @@ -393,7 +461,7 @@ static int kvm_s390_pci_aif_disable(struct zpci_dev *zdev, bool force) pcount++; } if (pcount > 0) - unaccount_mem(pcount); + unaccount_mem(kzdev, pcount); out: mutex_unlock(&aift->aift_lock); @@ -404,7 +472,7 @@ static int kvm_s390_pci_dev_open(struct zpci_dev *zdev) { struct kvm_zdev *kzdev; - kzdev = kzalloc(sizeof(struct kvm_zdev), GFP_KERNEL); + kzdev = kzalloc_obj(struct kvm_zdev); if (!kzdev) return -ENOMEM; @@ -666,7 +734,7 @@ int __init kvm_s390_pci_init(void) if (!kvm_s390_pci_interp_allowed()) return 0; - aift = kzalloc(sizeof(struct zpci_aift), GFP_KERNEL); + aift = kzalloc_obj(struct zpci_aift); if (!aift) return -ENOMEM; diff --git a/arch/s390/kvm/pci.h b/arch/s390/kvm/s390/pci.h index ff0972dd5e71..fdf8c7bf4ed0 100644 --- a/arch/s390/kvm/pci.h +++ b/arch/s390/kvm/s390/pci.h @@ -22,6 +22,8 @@ struct kvm_zdev { struct kvm *kvm; struct zpci_fib fib; struct list_head entry; + struct user_struct *user_account; + struct mm_struct *mm_account; }; struct zpci_gaite { diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/s390/priv.c index 0b14d894f38a..b3cb2c2c3aa7 100644 --- a/arch/s390/kvm/priv.c +++ b/arch/s390/kvm/s390/priv.c @@ -21,13 +21,14 @@ #include <asm/ebcdic.h> #include <asm/sysinfo.h> #include <asm/page-states.h> -#include <asm/gmap.h> #include <asm/ptrace.h> #include <asm/sclp.h> #include <asm/ap.h> +#include <asm/gmap_helpers.h> #include "gaccess.h" -#include "kvm-s390.h" +#include "s390.h" #include "trace.h" +#include "gmap.h" static int handle_ri(struct kvm_vcpu *vcpu) { @@ -222,7 +223,7 @@ int kvm_s390_skey_check_enable(struct kvm_vcpu *vcpu) if (vcpu->arch.skey_enabled) return 0; - rc = s390_enable_skey(); + rc = gmap_enable_skeys(vcpu->arch.gmap); VCPU_EVENT(vcpu, 3, "enabling storage keys for guest: %d", rc); if (rc) return rc; @@ -255,10 +256,9 @@ static int try_handle_skey(struct kvm_vcpu *vcpu) static int handle_iske(struct kvm_vcpu *vcpu) { - unsigned long gaddr, vmaddr; - unsigned char key; + unsigned long gaddr; int reg1, reg2; - bool unlocked; + union skey key; int rc; vcpu->stat.instruction_iske++; @@ -275,37 +275,22 @@ static int handle_iske(struct kvm_vcpu *vcpu) gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; gaddr = kvm_s390_logical_to_effective(vcpu, gaddr); gaddr = kvm_s390_real_to_abs(vcpu, gaddr); - vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(gaddr)); - if (kvm_is_error_hva(vmaddr)) - return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); -retry: - unlocked = false; - mmap_read_lock(current->mm); - rc = get_guest_storage_key(current->mm, vmaddr, &key); - - if (rc) { - rc = fixup_user_fault(current->mm, vmaddr, - FAULT_FLAG_WRITE, &unlocked); - if (!rc) { - mmap_read_unlock(current->mm); - goto retry; - } - } - mmap_read_unlock(current->mm); - if (rc == -EFAULT) - return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); + scoped_guard(read_lock, &vcpu->kvm->mmu_lock) + rc = dat_get_storage_key(vcpu->arch.gmap->asce, gpa_to_gfn(gaddr), &key); + if (rc > 0) + return kvm_s390_inject_program_int(vcpu, rc); if (rc < 0) return rc; vcpu->run->s.regs.gprs[reg1] &= ~0xff; - vcpu->run->s.regs.gprs[reg1] |= key; + vcpu->run->s.regs.gprs[reg1] |= key.skey; return 0; } static int handle_rrbe(struct kvm_vcpu *vcpu) { - unsigned long vmaddr, gaddr; + unsigned long gaddr; + union skey skey; int reg1, reg2; - bool unlocked; int rc; vcpu->stat.instruction_rrbe++; @@ -322,27 +307,13 @@ static int handle_rrbe(struct kvm_vcpu *vcpu) gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; gaddr = kvm_s390_logical_to_effective(vcpu, gaddr); gaddr = kvm_s390_real_to_abs(vcpu, gaddr); - vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(gaddr)); - if (kvm_is_error_hva(vmaddr)) - return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); -retry: - unlocked = false; - mmap_read_lock(current->mm); - rc = reset_guest_reference_bit(current->mm, vmaddr); - if (rc < 0) { - rc = fixup_user_fault(current->mm, vmaddr, - FAULT_FLAG_WRITE, &unlocked); - if (!rc) { - mmap_read_unlock(current->mm); - goto retry; - } - } - mmap_read_unlock(current->mm); - if (rc == -EFAULT) - return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); + scoped_guard(read_lock, &vcpu->kvm->mmu_lock) + rc = dat_reset_reference_bit(vcpu->arch.gmap->asce, gpa_to_gfn(gaddr), &skey); + if (rc > 0) + return kvm_s390_inject_program_int(vcpu, rc); if (rc < 0) return rc; - kvm_s390_set_psw_cc(vcpu, rc); + kvm_s390_set_psw_cc(vcpu, (skey.skey >> 1) & 3); return 0; } @@ -354,9 +325,8 @@ static int handle_sske(struct kvm_vcpu *vcpu) { unsigned char m3 = vcpu->arch.sie_block->ipb >> 28; unsigned long start, end; - unsigned char key, oldkey; + union skey key, oldkey; int reg1, reg2; - bool unlocked; int rc; vcpu->stat.instruction_sske++; @@ -377,7 +347,7 @@ static int handle_sske(struct kvm_vcpu *vcpu) kvm_s390_get_regs_rre(vcpu, ®1, ®2); - key = vcpu->run->s.regs.gprs[reg1] & 0xfe; + key.skey = vcpu->run->s.regs.gprs[reg1] & 0xfe; start = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; start = kvm_s390_logical_to_effective(vcpu, start); if (m3 & SSKE_MB) { @@ -389,27 +359,19 @@ static int handle_sske(struct kvm_vcpu *vcpu) } while (start != end) { - unsigned long vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start)); - unlocked = false; - - if (kvm_is_error_hva(vmaddr)) - return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); - - mmap_read_lock(current->mm); - rc = cond_set_guest_storage_key(current->mm, vmaddr, key, &oldkey, - m3 & SSKE_NQ, m3 & SSKE_MR, - m3 & SSKE_MC); - - if (rc < 0) { - rc = fixup_user_fault(current->mm, vmaddr, - FAULT_FLAG_WRITE, &unlocked); - rc = !rc ? -EAGAIN : rc; + scoped_guard(read_lock, &vcpu->kvm->mmu_lock) { + rc = dat_cond_set_storage_key(vcpu->arch.mc, vcpu->arch.gmap->asce, + gpa_to_gfn(start), key, &oldkey, + m3 & SSKE_NQ, m3 & SSKE_MR, m3 & SSKE_MC); } - mmap_read_unlock(current->mm); - if (rc == -EFAULT) + if (rc > 1) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); - if (rc == -EAGAIN) + if (rc == -ENOMEM) { + rc = kvm_s390_mmu_cache_topup(vcpu->arch.mc); + if (rc) + return rc; continue; + } if (rc < 0) return rc; start += PAGE_SIZE; @@ -422,7 +384,7 @@ static int handle_sske(struct kvm_vcpu *vcpu) } else { kvm_s390_set_psw_cc(vcpu, rc); vcpu->run->s.regs.gprs[reg1] &= ~0xff00UL; - vcpu->run->s.regs.gprs[reg1] |= (u64) oldkey << 8; + vcpu->run->s.regs.gprs[reg1] |= (u64)oldkey.skey << 8; } } if (m3 & SSKE_MB) { @@ -755,12 +717,13 @@ int kvm_s390_handle_lpsw(struct kvm_vcpu *vcpu) { psw_t *gpsw = &vcpu->arch.sie_block->gpsw; psw32_t new_psw; - u64 addr; + u64 addr, iaddr; int rc; u8 ar; vcpu->stat.instruction_lpsw++; + iaddr = gpsw->addr - kvm_s390_get_ilen(vcpu); if (gpsw->mask & PSW_MASK_PSTATE) return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); @@ -778,18 +741,20 @@ int kvm_s390_handle_lpsw(struct kvm_vcpu *vcpu) gpsw->addr = new_psw.addr & ~PSW32_ADDR_AMODE; if (!is_valid_psw(gpsw)) return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); + vcpu->arch.sie_block->gbea = iaddr; return 0; } static int handle_lpswe(struct kvm_vcpu *vcpu) { psw_t new_psw; - u64 addr; + u64 addr, iaddr; int rc; u8 ar; vcpu->stat.instruction_lpswe++; + iaddr = vcpu->arch.sie_block->gpsw.addr - kvm_s390_get_ilen(vcpu); if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); @@ -802,6 +767,7 @@ static int handle_lpswe(struct kvm_vcpu *vcpu) vcpu->arch.sie_block->gpsw = new_psw; if (!is_valid_psw(&vcpu->arch.sie_block->gpsw)) return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); + vcpu->arch.sie_block->gbea = iaddr; return 0; } @@ -1082,7 +1048,7 @@ static int handle_pfmf(struct kvm_vcpu *vcpu) bool mr = false, mc = false, nq; int reg1, reg2; unsigned long start, end; - unsigned char key; + union skey key; vcpu->stat.instruction_pfmf++; @@ -1110,7 +1076,7 @@ static int handle_pfmf(struct kvm_vcpu *vcpu) } nq = vcpu->run->s.regs.gprs[reg1] & PFMF_NQ; - key = vcpu->run->s.regs.gprs[reg1] & PFMF_KEY; + key.skey = vcpu->run->s.regs.gprs[reg1] & PFMF_KEY; start = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; start = kvm_s390_logical_to_effective(vcpu, start); @@ -1141,14 +1107,6 @@ static int handle_pfmf(struct kvm_vcpu *vcpu) } while (start != end) { - unsigned long vmaddr; - bool unlocked = false; - - /* Translate guest address to host address */ - vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start)); - if (kvm_is_error_hva(vmaddr)) - return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); - if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) { if (kvm_clear_guest(vcpu->kvm, start, PAGE_SIZE)) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); @@ -1159,19 +1117,19 @@ static int handle_pfmf(struct kvm_vcpu *vcpu) if (rc) return rc; - mmap_read_lock(current->mm); - rc = cond_set_guest_storage_key(current->mm, vmaddr, - key, NULL, nq, mr, mc); - if (rc < 0) { - rc = fixup_user_fault(current->mm, vmaddr, - FAULT_FLAG_WRITE, &unlocked); - rc = !rc ? -EAGAIN : rc; + scoped_guard(read_lock, &vcpu->kvm->mmu_lock) { + rc = dat_cond_set_storage_key(vcpu->arch.mc, vcpu->arch.gmap->asce, + gpa_to_gfn(start), key, + NULL, nq, mr, mc); } - mmap_read_unlock(current->mm); - if (rc == -EFAULT) - return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); - if (rc == -EAGAIN) + if (rc > 1) + return kvm_s390_inject_program_int(vcpu, rc); + if (rc == -ENOMEM) { + rc = kvm_s390_mmu_cache_topup(vcpu->arch.mc); + if (rc) + return rc; continue; + } if (rc < 0) return rc; } @@ -1195,8 +1153,10 @@ static int handle_pfmf(struct kvm_vcpu *vcpu) static inline int __do_essa(struct kvm_vcpu *vcpu, const int orc) { int r1, r2, nappended, entries; - unsigned long gfn, hva, res, pgstev, ptev; + union essa_state state; unsigned long *cbrlo; + unsigned long gfn; + bool dirtied; /* * We don't need to set SD.FPF.SK to 1 here, because if we have a @@ -1205,33 +1165,12 @@ static inline int __do_essa(struct kvm_vcpu *vcpu, const int orc) kvm_s390_get_regs_rre(vcpu, &r1, &r2); gfn = vcpu->run->s.regs.gprs[r2] >> PAGE_SHIFT; - hva = gfn_to_hva(vcpu->kvm, gfn); entries = (vcpu->arch.sie_block->cbrlo & ~PAGE_MASK) >> 3; - if (kvm_is_error_hva(hva)) - return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); - - nappended = pgste_perform_essa(vcpu->kvm->mm, hva, orc, &ptev, &pgstev); - if (nappended < 0) { - res = orc ? 0x10 : 0; - vcpu->run->s.regs.gprs[r1] = res; /* Exception Indication */ + nappended = dat_perform_essa(vcpu->arch.gmap->asce, gfn, orc, &state, &dirtied); + vcpu->run->s.regs.gprs[r1] = state.val; + if (nappended < 0) return 0; - } - res = (pgstev & _PGSTE_GPS_USAGE_MASK) >> 22; - /* - * Set the block-content state part of the result. 0 means resident, so - * nothing to do if the page is valid. 2 is for preserved pages - * (non-present and non-zero), and 3 for zero pages (non-present and - * zero). - */ - if (ptev & _PAGE_INVALID) { - res |= 2; - if (pgstev & _PGSTE_GPS_ZERO) - res |= 1; - } - if (pgstev & _PGSTE_GPS_NODAT) - res |= 0x20; - vcpu->run->s.regs.gprs[r1] = res; /* * It is possible that all the normal 511 slots were full, in which case * we will now write in the 512th slot, which is reserved for host use. @@ -1243,17 +1182,38 @@ static inline int __do_essa(struct kvm_vcpu *vcpu, const int orc) cbrlo[entries] = gfn << PAGE_SHIFT; } - if (orc) { - struct kvm_memory_slot *ms = gfn_to_memslot(vcpu->kvm, gfn); - - /* Increment only if we are really flipping the bit */ - if (ms && !test_and_set_bit(gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms))) - atomic64_inc(&vcpu->kvm->arch.cmma_dirty_pages); - } + if (dirtied) + atomic64_inc(&vcpu->kvm->arch.cmma_dirty_pages); return nappended; } +static void _essa_clear_cbrl(struct kvm_vcpu *vcpu, unsigned long *cbrl, int len) +{ + union crste *crstep; + union pgste pgste; + union pte *ptep; + hva_t hva; + int i; + + lockdep_assert_held(&vcpu->kvm->mmu_lock); + + for (i = 0; i < len; i++) { + if (dat_entry_walk(NULL, gpa_to_gfn(cbrl[i]), vcpu->arch.gmap->asce, + 0, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep)) + continue; + if (!ptep || ptep->s.pr) + continue; + pgste = pgste_get_lock(ptep); + if (pgste.usage == PGSTE_GPS_USAGE_UNUSED || pgste.zero) { + hva = gpa_to_hva(vcpu->kvm, cbrl[i]); + if (!kvm_is_error_hva(hva)) + gmap_helper_zap_one_page(vcpu->kvm->mm, hva); + } + pgste_set_unlock(ptep, pgste); + } +} + static int handle_essa(struct kvm_vcpu *vcpu) { lockdep_assert_held(&vcpu->kvm->srcu); @@ -1261,11 +1221,9 @@ static int handle_essa(struct kvm_vcpu *vcpu) /* entries expected to be 1FF */ int entries = (vcpu->arch.sie_block->cbrlo & ~PAGE_MASK) >> 3; unsigned long *cbrlo; - struct gmap *gmap; int i, orc; VCPU_EVENT(vcpu, 4, "ESSA: release %d pages", entries); - gmap = vcpu->arch.gmap; vcpu->stat.instruction_essa++; if (!vcpu->kvm->arch.use_cmma) return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); @@ -1279,7 +1237,7 @@ static int handle_essa(struct kvm_vcpu *vcpu) : ESSA_SET_STABLE_IF_RESIDENT)) return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); - if (!vcpu->kvm->arch.migration_mode) { + if (!READ_ONCE(vcpu->kvm->arch.migration_mode)) { /* * CMMA is enabled in the KVM settings, but is disabled in * the SIE block and in the mm_context, and we are not doing @@ -1289,11 +1247,7 @@ static int handle_essa(struct kvm_vcpu *vcpu) * value really needs to be written to; if the value is * already correct, we do nothing and avoid the lock. */ - if (vcpu->kvm->mm->context.uses_cmm == 0) { - mmap_write_lock(vcpu->kvm->mm); - vcpu->kvm->mm->context.uses_cmm = 1; - mmap_write_unlock(vcpu->kvm->mm); - } + set_bit(GMAP_FLAG_USES_CMM, &vcpu->arch.gmap->flags); /* * If we are here, we are supposed to have CMMA enabled in * the SIE block. Enabling CMMA works on a per-CPU basis, @@ -1307,20 +1261,23 @@ static int handle_essa(struct kvm_vcpu *vcpu) /* Retry the ESSA instruction */ kvm_s390_retry_instr(vcpu); } else { - mmap_read_lock(vcpu->kvm->mm); - i = __do_essa(vcpu, orc); - mmap_read_unlock(vcpu->kvm->mm); + scoped_guard(mutex, &vcpu->kvm->slots_arch_lock) + scoped_guard(read_lock, &vcpu->kvm->mmu_lock) + i = __do_essa(vcpu, orc); if (i < 0) return i; /* Account for the possible extra cbrl entry */ entries += i; } - vcpu->arch.sie_block->cbrlo &= PAGE_MASK; /* reset nceo */ + /* reset nceo */ + vcpu->arch.sie_block->cbrlo &= PAGE_MASK; cbrlo = phys_to_virt(vcpu->arch.sie_block->cbrlo); - mmap_read_lock(gmap->mm); - for (i = 0; i < entries; ++i) - __gmap_zap(gmap, cbrlo[i]); - mmap_read_unlock(gmap->mm); + + mmap_read_lock(vcpu->kvm->mm); + scoped_guard(read_lock, &vcpu->kvm->mmu_lock) + _essa_clear_cbrl(vcpu, cbrlo, entries); + mmap_read_unlock(vcpu->kvm->mm); + return 0; } diff --git a/arch/s390/kvm/pv.c b/arch/s390/kvm/s390/pv.c index 6ba5a0305e25..b18abd0e29ef 100644 --- a/arch/s390/kvm/pv.c +++ b/arch/s390/kvm/s390/pv.c @@ -12,13 +12,17 @@ #include <linux/minmax.h> #include <linux/pagemap.h> #include <linux/sched/signal.h> -#include <asm/gmap.h> #include <asm/uv.h> #include <asm/mman.h> #include <linux/pagewalk.h> #include <linux/sched/mm.h> #include <linux/mmu_notifier.h> -#include "kvm-s390.h" +#include <asm/gmap_helpers.h> +#include "s390.h" +#include "dat.h" +#include "gaccess.h" +#include "gmap.h" +#include "faultin.h" bool kvm_s390_pv_is_protected(struct kvm *kvm) { @@ -35,6 +39,103 @@ bool kvm_s390_pv_cpu_is_protected(struct kvm_vcpu *vcpu) EXPORT_SYMBOL_GPL(kvm_s390_pv_cpu_is_protected); /** + * should_export_before_import() - Determine whether an export is needed + * before an import-like operation. + * @uvcb: The Ultravisor control block of the UVC to be performed. + * @mm: The mm of the process. + * + * Returns whether an export is needed before every import-like operation. + * This is needed for shared pages, which don't trigger a secure storage + * exception when accessed from a different guest. + * + * Although considered as one, the Unpin Page UVC is not an actual import, + * so it is not affected. + * + * No export is needed also when there is only one protected VM, because the + * page cannot belong to the wrong VM in that case (there is no "other VM" + * it can belong to). + * + * Return: %true if an export is needed before every import, otherwise %false. + */ +static bool should_export_before_import(struct uv_cb_header *uvcb, struct mm_struct *mm) +{ + /* + * The misc feature indicates, among other things, that importing a + * shared page from a different protected VM will automatically also + * transfer its ownership. + */ + if (uv_has_feature(BIT_UV_FEAT_MISC)) + return false; + if (uvcb->cmd == UVC_CMD_UNPIN_PAGE_SHARED) + return false; + return atomic_read(&mm->context.protected_count) > 1; +} + +struct pv_make_secure { + void *uvcb; + struct folio *folio; + struct kvm *kvm; + int rc; + bool needs_export; +}; + +static int __kvm_s390_pv_make_secure(struct guest_fault *f, struct folio *folio) +{ + struct pv_make_secure *priv = f->priv; + int rc; + + if (priv->needs_export) + uv_convert_from_secure(folio_to_phys(folio)); + + if (folio_test_hugetlb(folio)) + return -EFAULT; + if (folio_test_large(folio)) + return -E2BIG; + + if (!f->page) + folio_get(folio); + rc = __make_folio_secure(folio, priv->uvcb); + if (!f->page) + folio_put(folio); + + return rc; +} + +static void _kvm_s390_pv_make_secure(struct guest_fault *f) +{ + struct pv_make_secure *priv = f->priv; + struct folio *folio; + spinlock_t *ptl; /* pte lock from try_get_locked_pte() */ + pte_t *ptep; + + folio = pfn_folio(f->pfn); + priv->rc = -EAGAIN; + + if (!mmap_read_trylock(priv->kvm->mm)) + return; + + ptep = try_get_locked_pte(priv->kvm->mm, gfn_to_hva(priv->kvm, f->gfn), &ptl); + if (IS_ERR_VALUE(ptep)) { + priv->rc = PTR_ERR(ptep); + goto out; + } + + if (folio_trylock(folio)) { + priv->rc = __kvm_s390_pv_make_secure(f, folio); + if (priv->rc == -E2BIG || priv->rc == -EBUSY) { + priv->folio = folio; + folio_get(folio); + } + folio_unlock(folio); + } + + if (ptep) + pte_unmap_unlock(ptep, ptl); +out: + mmap_read_unlock(priv->kvm->mm); +} + +/** * kvm_s390_pv_make_secure() - make one guest page secure * @kvm: the guest * @gaddr: the guest address that needs to be made secure @@ -45,14 +146,34 @@ EXPORT_SYMBOL_GPL(kvm_s390_pv_cpu_is_protected); */ int kvm_s390_pv_make_secure(struct kvm *kvm, unsigned long gaddr, void *uvcb) { - unsigned long vmaddr; + struct pv_make_secure priv = { .uvcb = uvcb, .kvm = kvm, }; + struct guest_fault f = { + .write_attempt = true, + .gfn = gpa_to_gfn(gaddr), + .callback = _kvm_s390_pv_make_secure, + .priv = &priv, + }; + int rc; lockdep_assert_held(&kvm->srcu); - vmaddr = gfn_to_hva(kvm, gpa_to_gfn(gaddr)); - if (kvm_is_error_hva(vmaddr)) - return -EFAULT; - return make_hva_secure(kvm->mm, vmaddr, uvcb); + priv.needs_export = should_export_before_import(uvcb, kvm->mm); + + scoped_guard(mutex, &kvm->arch.pv.import_lock) { + rc = kvm_s390_faultin_gfn(NULL, kvm, &f); + + if (!rc) { + rc = priv.rc; + if (priv.folio) { + rc = s390_wiggle_split_folio(kvm->mm, priv.folio); + if (!rc) + rc = -EAGAIN; + } + } + } + if (priv.folio) + folio_put(priv.folio); + return rc; } int kvm_s390_pv_convert_to_secure(struct kvm *kvm, unsigned long gaddr) @@ -121,6 +242,28 @@ static void kvm_s390_clear_pv_state(struct kvm *kvm) kvm->arch.pv.guest_len = 0; kvm->arch.pv.stor_base = 0; kvm->arch.pv.stor_var = NULL; + if (kvm->arch.pv.dumping) { + kvm_s390_vcpu_unblock_all(kvm); + kvm->arch.pv.dumping = false; + } +} + +static void kvm_s390_pv_dispose_cpu(struct kvm_vcpu *vcpu, bool free_stor_base) +{ + if (free_stor_base) + free_pages(vcpu->arch.pv.stor_base, get_order(uv_info.guest_cpu_stor_len)); + free_page((unsigned long)sida_addr(vcpu->arch.sie_block)); + vcpu->arch.sie_block->pv_handle_cpu = 0; + vcpu->arch.sie_block->pv_handle_config = 0; + memset(&vcpu->arch.pv, 0, sizeof(vcpu->arch.pv)); + vcpu->arch.sie_block->sdf = 0; + /* + * The sidad field (for sdf == 2) is now the gbea field (for sdf == 0). + * Use the reset value of gbea to avoid leaking the kernel pointer of + * the just freed sida. + */ + vcpu->arch.sie_block->gbea = 1; + kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); } int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) @@ -137,24 +280,9 @@ int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) WARN_ONCE(cc, "protvirt destroy cpu failed rc %x rrc %x", *rc, *rrc); /* Intended memory leak for something that should never happen. */ - if (!cc) - free_pages(vcpu->arch.pv.stor_base, - get_order(uv_info.guest_cpu_stor_len)); - - free_page((unsigned long)sida_addr(vcpu->arch.sie_block)); - vcpu->arch.sie_block->pv_handle_cpu = 0; - vcpu->arch.sie_block->pv_handle_config = 0; - memset(&vcpu->arch.pv, 0, sizeof(vcpu->arch.pv)); - vcpu->arch.sie_block->sdf = 0; - /* - * The sidad field (for sdf == 2) is now the gbea field (for sdf == 0). - * Use the reset value of gbea to avoid leaking the kernel pointer of - * the just freed sida. - */ - vcpu->arch.sie_block->gbea = 1; - kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); + kvm_s390_pv_dispose_cpu(vcpu, !cc); - return cc ? EIO : 0; + return cc ? -EIO : 0; } int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) @@ -198,9 +326,7 @@ int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) uvcb.header.rrc); if (cc) { - u16 dummy; - - kvm_s390_pv_destroy_cpu(vcpu, &dummy, &dummy); + kvm_s390_pv_dispose_cpu(vcpu, true); return -EIO; } @@ -216,7 +342,7 @@ int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) /* only free resources when the destroy was successful */ static void kvm_s390_pv_dealloc_vm(struct kvm *kvm) { - vfree(kvm->arch.pv.stor_var); + uv_free_stor_var(kvm->arch.pv.stor_var); free_pages(kvm->arch.pv.stor_base, get_order(uv_info.guest_base_stor_len)); kvm_s390_clear_pv_state(kvm); @@ -248,7 +374,7 @@ static int kvm_s390_pv_alloc_vm(struct kvm *kvm) /* Allocate variable storage */ vlen = ALIGN(virt * ((npages * PAGE_SIZE) / HPAGE_SIZE), PAGE_SIZE); vlen += uv_info.guest_virt_base_stor_len; - kvm->arch.pv.stor_var = vzalloc(vlen); + kvm->arch.pv.stor_var = uv_alloc_stor_var(vlen); if (!kvm->arch.pv.stor_var) goto out_err; return 0; @@ -293,41 +419,12 @@ static int kvm_s390_pv_dispose_one_leftover(struct kvm *kvm, */ free_pages(leftover->stor_base, get_order(uv_info.guest_base_stor_len)); free_pages(leftover->old_gmap_table, CRST_ALLOC_ORDER); - vfree(leftover->stor_var); + uv_free_stor_var(leftover->stor_var); done_fast: atomic_dec(&kvm->mm->context.protected_count); return 0; } -/** - * kvm_s390_destroy_lower_2g - Destroy the first 2GB of protected guest memory. - * @kvm: the VM whose memory is to be cleared. - * - * Destroy the first 2GB of guest memory, to avoid prefix issues after reboot. - * The CPUs of the protected VM need to be destroyed beforehand. - */ -static void kvm_s390_destroy_lower_2g(struct kvm *kvm) -{ - const unsigned long pages_2g = SZ_2G / PAGE_SIZE; - struct kvm_memory_slot *slot; - unsigned long len; - int srcu_idx; - - srcu_idx = srcu_read_lock(&kvm->srcu); - - /* Take the memslot containing guest absolute address 0 */ - slot = gfn_to_memslot(kvm, 0); - /* Clear all slots or parts thereof that are below 2GB */ - while (slot && slot->base_gfn < pages_2g) { - len = min_t(u64, slot->npages, pages_2g - slot->base_gfn) * PAGE_SIZE; - s390_uv_destroy_range(kvm->mm, slot->userspace_addr, slot->userspace_addr + len); - /* Take the next memslot */ - slot = gfn_to_memslot(kvm, slot->base_gfn + slot->npages); - } - - srcu_read_unlock(&kvm->srcu, srcu_idx); -} - static int kvm_s390_pv_deinit_vm_fast(struct kvm *kvm, u16 *rc, u16 *rrc) { struct uv_cb_destroy_fast uvcb = { @@ -337,12 +434,11 @@ static int kvm_s390_pv_deinit_vm_fast(struct kvm *kvm, u16 *rc, u16 *rrc) }; int cc; - cc = uv_call_sched(0, (u64)&uvcb); + cc = uv_call(0, (u64)&uvcb); if (rc) *rc = uvcb.header.rc; if (rrc) *rrc = uvcb.header.rrc; - WRITE_ONCE(kvm->arch.gmap->guest_handle, 0); KVM_UV_EVENT(kvm, 3, "PROTVIRT DESTROY VM FAST: rc %x rrc %x", uvcb.header.rc, uvcb.header.rrc); WARN_ONCE(cc && uvcb.header.rc != 0x104, @@ -391,10 +487,10 @@ int kvm_s390_pv_set_aside(struct kvm *kvm, u16 *rc, u16 *rrc) return -EINVAL; /* Guest with segment type ASCE, refuse to destroy asynchronously */ - if ((kvm->arch.gmap->asce & _ASCE_TYPE_MASK) == _ASCE_TYPE_SEGMENT) + if (kvm->arch.gmap->asce.dt == TABLE_TYPE_SEGMENT) return -EINVAL; - priv = kzalloc(sizeof(*priv), GFP_KERNEL); + priv = kzalloc_obj(*priv); if (!priv) return -ENOMEM; @@ -404,8 +500,7 @@ int kvm_s390_pv_set_aside(struct kvm *kvm, u16 *rc, u16 *rrc) priv->stor_var = kvm->arch.pv.stor_var; priv->stor_base = kvm->arch.pv.stor_base; priv->handle = kvm_s390_pv_get_handle(kvm); - priv->old_gmap_table = (unsigned long)kvm->arch.gmap->table; - WRITE_ONCE(kvm->arch.gmap->guest_handle, 0); + priv->old_gmap_table = (unsigned long)dereference_asce(kvm->arch.gmap->asce); if (s390_replace_asce(kvm->arch.gmap)) res = -ENOMEM; } @@ -415,7 +510,7 @@ int kvm_s390_pv_set_aside(struct kvm *kvm, u16 *rc, u16 *rrc) return res; } - kvm_s390_destroy_lower_2g(kvm); + gmap_pv_destroy_range(kvm->arch.gmap, 0, gpa_to_gfn(SZ_2G), false); kvm_s390_clear_pv_state(kvm); kvm->arch.pv.set_aside = priv; @@ -449,7 +544,6 @@ int kvm_s390_pv_deinit_vm(struct kvm *kvm, u16 *rc, u16 *rrc) cc = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm), UVC_CMD_DESTROY_SEC_CONF, rc, rrc); - WRITE_ONCE(kvm->arch.gmap->guest_handle, 0); if (!cc) { atomic_dec(&kvm->mm->context.protected_count); kvm_s390_pv_dealloc_vm(kvm); @@ -532,7 +626,7 @@ int kvm_s390_pv_deinit_cleanup_all(struct kvm *kvm, u16 *rc, u16 *rrc) * cleanup has been performed. */ if (need_zap && mmget_not_zero(kvm->mm)) { - s390_uv_destroy_range(kvm->mm, 0, TASK_SIZE); + gmap_pv_destroy_range(kvm->arch.gmap, 0, asce_end(kvm->arch.gmap->asce), false); mmput(kvm->mm); } @@ -570,7 +664,7 @@ int kvm_s390_pv_deinit_aside_vm(struct kvm *kvm, u16 *rc, u16 *rrc) return -EINVAL; /* When a fatal signal is received, stop immediately */ - if (s390_uv_destroy_range_interruptible(kvm->mm, 0, TASK_SIZE_MAX)) + if (gmap_pv_destroy_range(kvm->arch.gmap, 0, asce_end(kvm->arch.gmap->asce), true)) goto done; if (kvm_s390_pv_dispose_one_leftover(kvm, p, rc, rrc)) ret = -EIO; @@ -609,6 +703,7 @@ static void kvm_s390_pv_mmu_notifier_release(struct mmu_notifier *subscription, r = kvm_s390_cpus_from_pv(kvm, &dummy, &dummy); if (!r && is_destroy_fast_available() && kvm_s390_pv_get_handle(kvm)) kvm_s390_pv_deinit_vm_fast(kvm, &dummy, &dummy); + set_bit(GMAP_FLAG_EXPORT_ON_UNMAP, &kvm->arch.gmap->flags); } static const struct mmu_notifier_ops kvm_s390_pv_mmu_notifier_ops = { @@ -642,7 +737,7 @@ int kvm_s390_pv_init_vm(struct kvm *kvm, u16 *rc, u16 *rrc) /* Inputs */ uvcb.guest_stor_origin = 0; /* MSO is 0 for KVM */ uvcb.guest_stor_len = kvm->arch.pv.guest_len; - uvcb.guest_asce = kvm->arch.gmap->asce; + uvcb.guest_asce = kvm->arch.gmap->asce.val; uvcb.guest_sca = virt_to_phys(kvm->arch.sca); uvcb.conf_base_stor_origin = virt_to_phys((void *)kvm->arch.pv.stor_base); @@ -650,7 +745,13 @@ int kvm_s390_pv_init_vm(struct kvm *kvm, u16 *rc, u16 *rrc) uvcb.flags.ap_allow_instr = kvm->arch.model.uv_feat_guest.ap; uvcb.flags.ap_instr_intr = kvm->arch.model.uv_feat_guest.ap_intr; - cc = uv_call_sched(0, (u64)&uvcb); + scoped_guard(write_lock, &kvm->mmu_lock) { + clear_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &kvm->arch.gmap->flags); + clear_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &kvm->arch.gmap->flags); + } + gmap_split_huge_pages(kvm->arch.gmap); + + cc = uv_call(0, (u64)&uvcb); *rc = uvcb.header.rc; *rrc = uvcb.header.rrc; KVM_UV_EVENT(kvm, 3, "PROTVIRT CREATE VM: handle %llx len %llx rc %x rrc %x flags %04x", @@ -669,7 +770,6 @@ int kvm_s390_pv_init_vm(struct kvm *kvm, u16 *rc, u16 *rrc) } return -EIO; } - kvm->arch.gmap->guest_handle = uvcb.guest_handle; return 0; } @@ -704,29 +804,17 @@ static int unpack_one(struct kvm *kvm, unsigned long addr, u64 tweak, .tweak[1] = offset, }; int ret = kvm_s390_pv_make_secure(kvm, addr, &uvcb); - unsigned long vmaddr; - bool unlocked; *rc = uvcb.header.rc; *rrc = uvcb.header.rrc; if (ret == -ENXIO) { - mmap_read_lock(kvm->mm); - vmaddr = gfn_to_hva(kvm, gpa_to_gfn(addr)); - if (kvm_is_error_hva(vmaddr)) { - ret = -EFAULT; - } else { - ret = fixup_user_fault(kvm->mm, vmaddr, FAULT_FLAG_WRITE, &unlocked); - if (!ret) - ret = __gmap_link(kvm->arch.gmap, addr, vmaddr); - } - mmap_read_unlock(kvm->mm); + ret = kvm_s390_faultin_gfn_simple(NULL, kvm, gpa_to_gfn(addr), true); if (!ret) return -EAGAIN; - return ret; } - if (ret && ret != -EAGAIN) + if (ret && ret != -EAGAIN && ret != -EINTR) KVM_UV_EVENT(kvm, 3, "PROTVIRT VM UNPACK: failed addr %llx with rc %x rrc %x", uvcb.gaddr, *rc, *rrc); return ret; @@ -749,7 +837,6 @@ int kvm_s390_pv_unpack(struct kvm *kvm, unsigned long addr, unsigned long size, while (offset < size) { ret = unpack_one(kvm, addr, tweak, offset, rc, rrc); if (ret == -EAGAIN) { - cond_resched(); if (fatal_signal_pending(current)) break; continue; @@ -792,7 +879,7 @@ int kvm_s390_pv_dump_cpu(struct kvm_vcpu *vcpu, void *buff, u16 *rc, u16 *rrc) }; int cc; - cc = uv_call_sched(0, (u64)&uvcb); + cc = uv_call(0, (u64)&uvcb); *rc = uvcb.header.rc; *rrc = uvcb.header.rrc; return cc; @@ -876,7 +963,7 @@ int kvm_s390_pv_dump_stor_state(struct kvm *kvm, void __user *buff_user, /* We will loop until the user buffer is filled or an error occurs */ do { /* Get 1MB worth of guest storage state data */ - cc = uv_call_sched(0, (u64)&uvcb); + cc = uv_call(0, (u64)&uvcb); /* All or nothing */ if (cc) { @@ -954,7 +1041,7 @@ int kvm_s390_pv_dump_complete(struct kvm *kvm, void __user *buff_user, return -ENOMEM; complete.dump_area_origin = (u64)compl_data; - ret = uv_call_sched(0, (u64)&complete); + ret = uv_call(0, (u64)&complete); *rc = complete.header.rc; *rrc = complete.header.rrc; KVM_UV_EVENT(kvm, 3, "PROTVIRT DUMP COMPLETE: rc %x rrc %x", diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/s390/s390.c index 56a50524b3ee..efd77b9cd36c 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/s390/s390.c @@ -40,7 +40,6 @@ #include <asm/lowcore.h> #include <asm/machine.h> #include <asm/stp.h> -#include <asm/gmap.h> #include <asm/gmap_helpers.h> #include <asm/nmi.h> #include <asm/isc.h> @@ -51,9 +50,12 @@ #include <asm/fpu.h> #include <asm/ap.h> #include <asm/uv.h> -#include "kvm-s390.h" +#include "s390.h" #include "gaccess.h" +#include "gmap.h" +#include "faultin.h" #include "pci.h" +#include "kvm_mmu.h" #define CREATE_TRACE_POINTS #include "trace.h" @@ -64,9 +66,15 @@ #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \ (KVM_MAX_VCPUS + LOCAL_IRQS)) -const struct _kvm_stats_desc kvm_vm_stats_desc[] = { +const struct kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS(), STATS_DESC_COUNTER(VM, inject_io), + STATS_DESC_COUNTER(VM, io_390_adapter_map), + STATS_DESC_COUNTER(VM, io_390_adapter_unmap), + STATS_DESC_COUNTER(VM, io_390_inatomic), + STATS_DESC_COUNTER(VM, io_flic_inject_airq), + STATS_DESC_COUNTER(VM, io_set_adapter_int), + STATS_DESC_COUNTER(VM, io_390_inatomic_no_inject), STATS_DESC_COUNTER(VM, inject_float_mchk), STATS_DESC_COUNTER(VM, inject_pfault_done), STATS_DESC_COUNTER(VM, inject_service_signal), @@ -90,7 +98,7 @@ const struct kvm_stats_header kvm_vm_stats_header = { sizeof(kvm_vm_stats_desc), }; -const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), STATS_DESC_COUNTER(VCPU, exit_userspace), STATS_DESC_COUNTER(VCPU, exit_null), @@ -203,11 +211,16 @@ static int nested; module_param(nested, int, S_IRUGO); MODULE_PARM_DESC(nested, "Nested virtualization support"); -/* allow 1m huge page guest backing, if !nested */ +/* allow 1m huge page guest backing */ static int hpage; module_param(hpage, int, 0444); MODULE_PARM_DESC(hpage, "1m huge page backing support"); +/* allow 2g huge page guest backing */ +static int hpage_2g; +module_param(hpage_2g, int, 0444); +MODULE_PARM_DESC(hpage_2g, "2g huge page backing support"); + /* maximum percentage of steal time for polling. >100 is treated like 100 */ static u8 halt_poll_max_steal = 10; module_param(halt_poll_max_steal, byte, 0644); @@ -231,49 +244,36 @@ static int async_destroy = 1; module_param(async_destroy, int, 0444); MODULE_PARM_DESC(async_destroy, "Asynchronous destroy for protected guests"); -/* - * For now we handle at most 16 double words as this is what the s390 base - * kernel handles and stores in the prefix page. If we ever need to go beyond - * this, this requires changes to code, but the external uapi can stay. - */ -#define SIZE_INTERNAL 16 - +#define HMFAI_DWORDS 16 /* * Base feature mask that defines default mask for facilities. Consists of the * defines in FACILITIES_KVM and the non-hypervisor managed bits. */ -static unsigned long kvm_s390_fac_base[SIZE_INTERNAL] = { FACILITIES_KVM }; +static unsigned long kvm_s390_fac_base[HMFAI_DWORDS] = { FACILITIES_KVM }; +static_assert(ARRAY_SIZE(((long[]){ FACILITIES_KVM })) <= HMFAI_DWORDS); +static_assert(ARRAY_SIZE(kvm_s390_fac_base) <= S390_ARCH_FAC_MASK_SIZE_U64); +static_assert(ARRAY_SIZE(kvm_s390_fac_base) <= S390_ARCH_FAC_LIST_SIZE_U64); +static_assert(ARRAY_SIZE(kvm_s390_fac_base) <= ARRAY_SIZE(stfle_fac_list)); + /* * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL * and defines the facilities that can be enabled via a cpu model. */ -static unsigned long kvm_s390_fac_ext[SIZE_INTERNAL] = { FACILITIES_KVM_CPUMODEL }; - -static unsigned long kvm_s390_fac_size(void) -{ - BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_MASK_SIZE_U64); - BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_LIST_SIZE_U64); - BUILD_BUG_ON(SIZE_INTERNAL * sizeof(unsigned long) > - sizeof(stfle_fac_list)); - - return SIZE_INTERNAL; -} +static const unsigned long kvm_s390_fac_ext[] = { FACILITIES_KVM_CPUMODEL }; +static_assert(ARRAY_SIZE(kvm_s390_fac_ext) <= S390_ARCH_FAC_MASK_SIZE_U64); +static_assert(ARRAY_SIZE(kvm_s390_fac_ext) <= S390_ARCH_FAC_LIST_SIZE_U64); +static_assert(ARRAY_SIZE(kvm_s390_fac_ext) <= ARRAY_SIZE(stfle_fac_list)); /* available cpu features supported by kvm */ static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS); /* available subfunctions indicated via query / "test bit" */ static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc; -static struct gmap_notifier gmap_notifier; -static struct gmap_notifier vsie_gmap_notifier; debug_info_t *kvm_s390_dbf; debug_info_t *kvm_s390_dbf_uv; /* Section: not file related */ /* forward declarations */ -static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start, - unsigned long end); - static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta) { u8 delta_idx = 0; @@ -456,8 +456,7 @@ static void __init kvm_s390_cpu_feat_init(void) !test_facility(3) || !nested) return; allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2); - if (sclp.has_64bscao) - allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO); + allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO); if (sclp.has_siif) allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF); if (sclp.has_gpere) @@ -472,6 +471,8 @@ static void __init kvm_s390_cpu_feat_init(void) allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS); if (sclp.has_kss) allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS); + if (sclp.has_astfleie2) + allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ASTFLEIE2); /* * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make * all skey handling functions read/set the skey from the PGSTE @@ -529,10 +530,6 @@ static int __init __kvm_s390_init(void) if (rc) goto err_gib; - gmap_notifier.notifier_call = kvm_gmap_notifier; - gmap_register_pte_notifier(&gmap_notifier); - vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier; - gmap_register_pte_notifier(&vsie_gmap_notifier); atomic_notifier_chain_register(&s390_epoch_delta_notifier, &kvm_clock_notifier); @@ -552,8 +549,6 @@ err_kvm_uv: static void __kvm_s390_exit(void) { - gmap_unregister_pte_notifier(&gmap_notifier); - gmap_unregister_pte_notifier(&vsie_gmap_notifier); atomic_notifier_chain_unregister(&s390_epoch_delta_notifier, &kvm_clock_notifier); @@ -564,12 +559,44 @@ static void __kvm_s390_exit(void) debug_unregister(kvm_s390_dbf_uv); } +static int kvm_s390_keyop(struct kvm_s390_mmu_cache *mc, struct kvm *kvm, int op, + unsigned long addr, union skey skey) +{ + gfn_t gfn = gpa_to_gfn(addr); + union asce asce; + int r; + + guard(read_lock)(&kvm->mmu_lock); + asce = kvm->arch.gmap->asce; + + switch (op) { + case KVM_S390_KEYOP_SSKE: + r = dat_cond_set_storage_key(mc, asce, gfn, skey, &skey, 0, 0, 0); + if (r == 0 || r == 1) + return skey.skey; + break; + case KVM_S390_KEYOP_ISKE: + r = dat_get_storage_key(asce, gfn, &skey); + if (!r) + return skey.skey; + break; + case KVM_S390_KEYOP_RRBE: + r = dat_reset_reference_bit(asce, gfn, &skey); + if (!r) + return skey.skey; + break; + default: + return -EINVAL; + } + return r > 0 ? -EFAULT : r; +} + /* Section: device related */ long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { if (ioctl == KVM_S390_ENABLE_SIE) - return s390_enable_sie(); + return 0; return -EINVAL; } @@ -580,7 +607,6 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) switch (ext) { case KVM_CAP_S390_PSW: case KVM_CAP_S390_GMAP: - case KVM_CAP_SYNC_MMU: #ifdef CONFIG_KVM_S390_UCONTROL case KVM_CAP_S390_UCONTROL: #endif @@ -608,6 +634,9 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_S390_DIAG318: case KVM_CAP_IRQFD_RESAMPLE: case KVM_CAP_S390_USER_OPEREXEC: + case KVM_CAP_S390_KEYOP: + case KVM_CAP_S390_VSIE_ESAMODE: + case KVM_CAP_PRE_FAULT_MEMORY: r = 1; break; case KVM_CAP_SET_GUEST_DEBUG2: @@ -618,6 +647,11 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) if (hpage && !(kvm && kvm_is_ucontrol(kvm))) r = 1; break; + case KVM_CAP_S390_HPAGE_2G: + r = 0; + if (hpage_2g && !(kvm && kvm_is_ucontrol(kvm))) + r = 1; + break; case KVM_CAP_S390_MEM_OP: r = MEM_OP_MAX_SIZE; break; @@ -698,32 +732,10 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) { - int i; - gfn_t cur_gfn, last_gfn; - unsigned long gaddr, vmaddr; - struct gmap *gmap = kvm->arch.gmap; - DECLARE_BITMAP(bitmap, _PAGE_ENTRIES); - - /* Loop over all guest segments */ - cur_gfn = memslot->base_gfn; - last_gfn = memslot->base_gfn + memslot->npages; - for (; cur_gfn <= last_gfn; cur_gfn += _PAGE_ENTRIES) { - gaddr = gfn_to_gpa(cur_gfn); - vmaddr = gfn_to_hva_memslot(memslot, cur_gfn); - if (kvm_is_error_hva(vmaddr)) - continue; - - bitmap_zero(bitmap, _PAGE_ENTRIES); - gmap_sync_dirty_log_pmd(gmap, bitmap, gaddr, vmaddr); - for (i = 0; i < _PAGE_ENTRIES; i++) { - if (test_bit(i, bitmap)) - mark_page_dirty(kvm, cur_gfn + i); - } + gfn_t last_gfn = memslot->base_gfn + memslot->npages; - if (fatal_signal_pending(current)) - return; - cond_resched(); - } + scoped_guard(read_lock, &kvm->mmu_lock) + gmap_sync_dirty_log(kvm->arch.gmap, memslot->base_gfn, last_gfn); } /* Section: vm related */ @@ -735,33 +747,7 @@ static void sca_del_vcpu(struct kvm_vcpu *vcpu); int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log) { - int r; - unsigned long n; - struct kvm_memory_slot *memslot; - int is_dirty; - - if (kvm_is_ucontrol(kvm)) - return -EINVAL; - - mutex_lock(&kvm->slots_lock); - - r = -EINVAL; - if (log->slot >= KVM_USER_MEM_SLOTS) - goto out; - - r = kvm_get_dirty_log(kvm, log, &is_dirty, &memslot); - if (r) - goto out; - - /* Clear the dirty log */ - if (is_dirty) { - n = kvm_dirty_bitmap_bytes(memslot); - memset(memslot->dirty_bitmap, 0, n); - } - r = 0; -out: - mutex_unlock(&kvm->slots_lock); - return r; + return s390_kvm_mmu_get_dirty_log(kvm, log); } static void icpt_operexc_on_all_vcpus(struct kvm *kvm) @@ -883,9 +869,7 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) r = -EINVAL; else { r = 0; - mmap_write_lock(kvm->mm); - kvm->mm->context.allow_gmap_hpage_1m = 1; - mmap_write_unlock(kvm->mm); + set_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &kvm->arch.gmap->flags); /* * We might have to create fake 4k page * tables. To avoid that the hardware works on @@ -898,6 +882,27 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s", r ? "(not available)" : "(success)"); break; + case KVM_CAP_S390_HPAGE_2G: + mutex_lock(&kvm->lock); + if (kvm->created_vcpus) { + r = -EBUSY; + } else if (!hpage_2g || kvm->arch.use_cmma || kvm_is_ucontrol(kvm)) { + r = -EINVAL; + } else { + r = 0; + set_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &kvm->arch.gmap->flags); + /* + * We might have to create fake 4k page + * tables. To avoid that the hardware works on + * stale PGSTEs, we emulate these instructions. + */ + kvm->arch.use_skf = 0; + kvm->arch.use_pfmfi = 0; + } + mutex_unlock(&kvm->lock); + VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE_2G %s", + r ? "(not available)" : "(success)"); + break; case KVM_CAP_S390_USER_STSI: VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI"); kvm->arch.user_stsi = 1; @@ -929,6 +934,11 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) icpt_operexc_on_all_vcpus(kvm); r = 0; break; + case KVM_CAP_S390_VSIE_ESAMODE: + VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_VSIE_ESAMODE"); + kvm->arch.allow_vsie_esamode = 1; + r = 0; + break; default: r = -EINVAL; break; @@ -958,7 +968,7 @@ static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *att static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr) { int ret; - unsigned int idx; + switch (attr->attr) { case KVM_S390_VM_MEM_ENABLE_CMMA: ret = -ENXIO; @@ -969,8 +979,6 @@ static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *att mutex_lock(&kvm->lock); if (kvm->created_vcpus) ret = -EBUSY; - else if (kvm->mm->context.allow_gmap_hpage_1m) - ret = -EINVAL; else { kvm->arch.use_cmma = 1; /* Not compatible with cmma. */ @@ -979,7 +987,9 @@ static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *att } mutex_unlock(&kvm->lock); break; - case KVM_S390_VM_MEM_CLR_CMMA: + case KVM_S390_VM_MEM_CLR_CMMA: { + gfn_t start_gfn = 0; + ret = -ENXIO; if (!sclp.has_cmma) break; @@ -987,16 +997,20 @@ static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *att if (!kvm->arch.use_cmma) break; + guard(mutex)(&kvm->lock); VM_EVENT(kvm, 3, "%s", "RESET: CMMA states"); - mutex_lock(&kvm->lock); - idx = srcu_read_lock(&kvm->srcu); - s390_reset_cmma(kvm->arch.gmap->mm); - srcu_read_unlock(&kvm->srcu, idx); - mutex_unlock(&kvm->lock); + do { + scoped_guard(read_lock, &kvm->mmu_lock) + start_gfn = dat_reset_cmma(kvm->arch.gmap->asce, start_gfn); + } while (start_gfn); ret = 0; break; + } case KVM_S390_VM_MEM_LIMIT_SIZE: { + struct kvm_memslots *slots; + struct kvm_memory_slot *ms; unsigned long new_limit; + int bkt; if (kvm_is_ucontrol(kvm)) return -EINVAL; @@ -1004,6 +1018,9 @@ static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *att if (get_user(new_limit, (u64 __user *)attr->addr)) return -EFAULT; + guard(mutex)(&kvm->lock); + + new_limit = ALIGN(new_limit, HPAGE_SIZE); if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT && new_limit > kvm->arch.mem_limit) return -E2BIG; @@ -1011,29 +1028,27 @@ static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *att if (!new_limit) return -EINVAL; - /* gmap_create takes last usable address */ - if (new_limit != KVM_S390_NO_MEM_LIMIT) - new_limit -= 1; + if (kvm->created_vcpus) + return -EBUSY; - ret = -EBUSY; - mutex_lock(&kvm->lock); - if (!kvm->created_vcpus) { - /* gmap_create will round the limit up */ - struct gmap *new = gmap_create(current->mm, new_limit); - - if (!new) { - ret = -ENOMEM; - } else { - gmap_remove(kvm->arch.gmap); - new->private = kvm; - kvm->arch.gmap = new; - ret = 0; + ret = 0; + scoped_guard(mutex, &kvm->slots_lock) { + slots = kvm_memslots(kvm); + if (slots && !kvm_memslots_empty(slots)) { + kvm_for_each_memslot(ms, bkt, slots) { + if (gpa_to_gfn(new_limit) < ms->base_gfn + ms->npages) { + ret = -EBUSY; + break; + } + } } + if (!ret) + ret = gmap_set_limit(kvm->arch.gmap, gpa_to_gfn(new_limit)); } - mutex_unlock(&kvm->lock); + if (ret) + break; VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit); - VM_EVENT(kvm, 3, "New guest asce: 0x%p", - (void *) kvm->arch.gmap->asce); + VM_EVENT(kvm, 3, "New guest asce: 0x%p", (void *)kvm->arch.gmap->asce.val); break; } default: @@ -1178,13 +1193,13 @@ static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req) /* * Must be called with kvm->srcu held to avoid races on memslots, and with - * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration. + * kvm->slots_arch_lock to avoid races with ourselves, + * kvm_s390_vm_stop_migration(), and kvm_s390_get_cmma_bits(). */ static int kvm_s390_vm_start_migration(struct kvm *kvm) { struct kvm_memory_slot *ms; struct kvm_memslots *slots; - unsigned long ram_pages = 0; int bkt; /* migration mode already enabled */ @@ -1198,37 +1213,59 @@ static int kvm_s390_vm_start_migration(struct kvm *kvm) kvm->arch.migration_mode = 1; return 0; } - /* mark all the pages in active slots as dirty */ kvm_for_each_memslot(ms, bkt, slots) { if (!ms->dirty_bitmap) return -EINVAL; - /* - * The second half of the bitmap is only used on x86, - * and would be wasted otherwise, so we put it to good - * use here to keep track of the state of the storage - * attributes. - */ - memset(kvm_second_dirty_bitmap(ms), 0xff, kvm_dirty_bitmap_bytes(ms)); - ram_pages += ms->npages; } - atomic64_set(&kvm->arch.cmma_dirty_pages, ram_pages); - kvm->arch.migration_mode = 1; + /* + * Set the flag and let KVM handle ESSA manually, potentially setting + * the cmma_d bit in some PGSTEs and increasing cmma_dirty_pages. + * At this point cmma_dirty_pages is still 0, and all existing PGSTEs + * have their cmma_d bit set to 0. + * Any newly allocated page table has its entries marked as cmma-clean, + * which is fine because the CMMA values are not dirty. + */ + WRITE_ONCE(kvm->arch.migration_mode, 1); kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION); + /* + * Mark all PGSTEs as cmma-dirty, increasing cmma_dirty_pages as needed, + * but without double-counting pages that have become dirty on their own + * in the meantime. + * At this point some pages might have become dirty on their own already + * and cmma_dirty_pages might therefore be non-zero. + */ + gmap_set_cmma_all_dirty(kvm->arch.gmap); return 0; } /* - * Must be called with kvm->slots_lock to avoid races with ourselves and - * kvm_s390_vm_start_migration. + * Must be called with kvm->slots_arch_lock to avoid races with ourselves, + * kvm_s390_vm_start_migration() and kvm_s390_get_cmma_bits(). */ -static int kvm_s390_vm_stop_migration(struct kvm *kvm) +int kvm_s390_vm_stop_migration(struct kvm *kvm) { /* migration mode already disabled */ if (!kvm->arch.migration_mode) return 0; - kvm->arch.migration_mode = 0; - if (kvm->arch.use_cmma) - kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION); + /* + * Unset the flag and propagate to all vCPUs. From now on the cmma_d + * bit will not be touched on any PGSTE. + * At this point cmma_dirty_pages is possibly non-zero, and thus some + * PGSTEs might have cmma_d set. + */ + WRITE_ONCE(kvm->arch.migration_mode, 0); + if (!kvm->arch.use_cmma) + return 0; + + kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION); + /* Clear cmma_d on all existing PGSTEs and set cmma_dirty_pages to 0. */ + gmap_set_cmma_all_clean(kvm->arch.gmap); + atomic64_set(&kvm->arch.cmma_dirty_pages, 0); + /* + * At this point the system has the expected state: migration_mode is 0, + * cmma_dirty_pages is 0, and all existing PGSTEs have their cmma_d bit + * set to 0. + */ return 0; } @@ -1237,7 +1274,9 @@ static int kvm_s390_vm_set_migration(struct kvm *kvm, { int res = -ENXIO; - mutex_lock(&kvm->slots_lock); + guard(srcu)(&kvm->srcu); + guard(mutex)(&kvm->slots_arch_lock); + switch (attr->attr) { case KVM_S390_VM_MIGRATION_START: res = kvm_s390_vm_start_migration(kvm); @@ -1248,7 +1287,6 @@ static int kvm_s390_vm_set_migration(struct kvm *kvm, default: break; } - mutex_unlock(&kvm->slots_lock); return res; } @@ -1443,7 +1481,7 @@ static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr) ret = -EBUSY; goto out; } - proc = kzalloc(sizeof(*proc), GFP_KERNEL_ACCOUNT); + proc = kzalloc_obj(*proc, GFP_KERNEL_ACCOUNT); if (!proc) { ret = -ENOMEM; goto out; @@ -1643,7 +1681,7 @@ static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr) struct kvm_s390_vm_cpu_processor *proc; int ret = 0; - proc = kzalloc(sizeof(*proc), GFP_KERNEL_ACCOUNT); + proc = kzalloc_obj(*proc, GFP_KERNEL_ACCOUNT); if (!proc) { ret = -ENOMEM; goto out; @@ -1671,7 +1709,7 @@ static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr) struct kvm_s390_vm_cpu_machine *mach; int ret = 0; - mach = kzalloc(sizeof(*mach), GFP_KERNEL_ACCOUNT); + mach = kzalloc_obj(*mach, GFP_KERNEL_ACCOUNT); if (!mach) { ret = -ENOMEM; goto out; @@ -2116,40 +2154,32 @@ static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr) static int kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) { - uint8_t *keys; - uint64_t hva; - int srcu_idx, i, r = 0; + union skey *keys; + int i, r = 0; if (args->flags != 0) return -EINVAL; /* Is this guest using storage keys? */ - if (!mm_uses_skeys(current->mm)) + if (!uses_skeys(kvm->arch.gmap)) return KVM_S390_GET_SKEYS_NONE; /* Enforce sane limit on memory allocation */ if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX) return -EINVAL; - keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL_ACCOUNT); + keys = kvmalloc_objs(*keys, args->count, GFP_KERNEL_ACCOUNT); if (!keys) return -ENOMEM; - mmap_read_lock(current->mm); - srcu_idx = srcu_read_lock(&kvm->srcu); - for (i = 0; i < args->count; i++) { - hva = gfn_to_hva(kvm, args->start_gfn + i); - if (kvm_is_error_hva(hva)) { - r = -EFAULT; - break; + scoped_guard(read_lock, &kvm->mmu_lock) { + for (i = 0; i < args->count; i++) { + r = dat_get_storage_key(kvm->arch.gmap->asce, + args->start_gfn + i, keys + i); + if (r) + break; } - - r = get_guest_storage_key(current->mm, hva, &keys[i]); - if (r) - break; } - srcu_read_unlock(&kvm->srcu, srcu_idx); - mmap_read_unlock(current->mm); if (!r) { r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys, @@ -2159,15 +2189,14 @@ static int kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) } kvfree(keys); - return r; + return r <= 0 ? r : -EFAULT; } static int kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) { - uint8_t *keys; - uint64_t hva; - int srcu_idx, i, r = 0; - bool unlocked; + struct kvm_s390_mmu_cache *mc; + union skey *keys; + int i, r = 0; if (args->flags != 0) return -EINVAL; @@ -2176,7 +2205,7 @@ static int kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX) return -EINVAL; - keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL_ACCOUNT); + keys = kvmalloc_objs(*keys, args->count, GFP_KERNEL_ACCOUNT); if (!keys) return -ENOMEM; @@ -2188,159 +2217,41 @@ static int kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) } /* Enable storage key handling for the guest */ - r = s390_enable_skey(); + r = gmap_enable_skeys(kvm->arch.gmap); if (r) goto out; - i = 0; - mmap_read_lock(current->mm); - srcu_idx = srcu_read_lock(&kvm->srcu); - while (i < args->count) { - unlocked = false; - hva = gfn_to_hva(kvm, args->start_gfn + i); - if (kvm_is_error_hva(hva)) { - r = -EFAULT; - break; - } - + r = -EINVAL; + for (i = 0; i < args->count; i++) { /* Lowest order bit is reserved */ - if (keys[i] & 0x01) { - r = -EINVAL; - break; - } - - r = set_guest_storage_key(current->mm, hva, keys[i], 0); - if (r) { - r = fixup_user_fault(current->mm, hva, - FAULT_FLAG_WRITE, &unlocked); - if (r) - break; - } - if (!r) - i++; - } - srcu_read_unlock(&kvm->srcu, srcu_idx); - mmap_read_unlock(current->mm); -out: - kvfree(keys); - return r; -} - -/* - * Base address and length must be sent at the start of each block, therefore - * it's cheaper to send some clean data, as long as it's less than the size of - * two longs. - */ -#define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *)) -/* for consistency */ -#define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX) - -static int kvm_s390_peek_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args, - u8 *res, unsigned long bufsize) -{ - unsigned long pgstev, hva, cur_gfn = args->start_gfn; - - args->count = 0; - while (args->count < bufsize) { - hva = gfn_to_hva(kvm, cur_gfn); - /* - * We return an error if the first value was invalid, but we - * return successfully if at least one value was copied. - */ - if (kvm_is_error_hva(hva)) - return args->count ? 0 : -EFAULT; - if (get_pgste(kvm->mm, hva, &pgstev) < 0) - pgstev = 0; - res[args->count++] = (pgstev >> 24) & 0x43; - cur_gfn++; - } - - return 0; -} - -static struct kvm_memory_slot *gfn_to_memslot_approx(struct kvm_memslots *slots, - gfn_t gfn) -{ - return ____gfn_to_memslot(slots, gfn, true); -} - -static unsigned long kvm_s390_next_dirty_cmma(struct kvm_memslots *slots, - unsigned long cur_gfn) -{ - struct kvm_memory_slot *ms = gfn_to_memslot_approx(slots, cur_gfn); - unsigned long ofs = cur_gfn - ms->base_gfn; - struct rb_node *mnode = &ms->gfn_node[slots->node_idx]; - - if (ms->base_gfn + ms->npages <= cur_gfn) { - mnode = rb_next(mnode); - /* If we are above the highest slot, wrap around */ - if (!mnode) - mnode = rb_first(&slots->gfn_tree); - - ms = container_of(mnode, struct kvm_memory_slot, gfn_node[slots->node_idx]); - ofs = 0; + if (keys[i].zero) + goto out; } - if (cur_gfn < ms->base_gfn) - ofs = 0; - - ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, ofs); - while (ofs >= ms->npages && (mnode = rb_next(mnode))) { - ms = container_of(mnode, struct kvm_memory_slot, gfn_node[slots->node_idx]); - ofs = find_first_bit(kvm_second_dirty_bitmap(ms), ms->npages); + mc = kvm_s390_new_mmu_cache(); + if (!mc) { + r = -ENOMEM; + goto out; } - return ms->base_gfn + ofs; -} - -static int kvm_s390_get_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args, - u8 *res, unsigned long bufsize) -{ - unsigned long mem_end, cur_gfn, next_gfn, hva, pgstev; - struct kvm_memslots *slots = kvm_memslots(kvm); - struct kvm_memory_slot *ms; - if (unlikely(kvm_memslots_empty(slots))) - return 0; - - cur_gfn = kvm_s390_next_dirty_cmma(slots, args->start_gfn); - ms = gfn_to_memslot(kvm, cur_gfn); - args->count = 0; - args->start_gfn = cur_gfn; - if (!ms) - return 0; - next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1); - mem_end = kvm_s390_get_gfn_end(slots); - - while (args->count < bufsize) { - hva = gfn_to_hva(kvm, cur_gfn); - if (kvm_is_error_hva(hva)) - return 0; - /* Decrement only if we actually flipped the bit to 0 */ - if (test_and_clear_bit(cur_gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms))) - atomic64_dec(&kvm->arch.cmma_dirty_pages); - if (get_pgste(kvm->mm, hva, &pgstev) < 0) - pgstev = 0; - /* Save the value */ - res[args->count++] = (pgstev >> 24) & 0x43; - /* If the next bit is too far away, stop. */ - if (next_gfn > cur_gfn + KVM_S390_MAX_BIT_DISTANCE) - return 0; - /* If we reached the previous "next", find the next one */ - if (cur_gfn == next_gfn) - next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1); - /* Reached the end of memory or of the buffer, stop */ - if ((next_gfn >= mem_end) || - (next_gfn - args->start_gfn >= bufsize)) - return 0; - cur_gfn++; - /* Reached the end of the current memslot, take the next one. */ - if (cur_gfn - ms->base_gfn >= ms->npages) { - ms = gfn_to_memslot(kvm, cur_gfn); - if (!ms) - return 0; + r = 0; + do { + r = kvm_s390_mmu_cache_topup(mc); + if (r == -ENOMEM) + break; + scoped_guard(read_lock, &kvm->mmu_lock) { + for (i = 0 ; i < args->count; i++) { + r = dat_set_storage_key(mc, kvm->arch.gmap->asce, + args->start_gfn + i, keys[i], 0); + if (r) + break; + } } - } - return 0; + } while (r == -ENOMEM); + kvm_s390_free_mmu_cache(mc); +out: + kvfree(keys); + return r <= 0 ? r : -EFAULT; } /* @@ -2354,8 +2265,7 @@ static int kvm_s390_get_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args, static int kvm_s390_get_cmma_bits(struct kvm *kvm, struct kvm_s390_cmma_log *args) { - unsigned long bufsize; - int srcu_idx, peek, ret; + int peek, ret; u8 *values; if (!kvm->arch.use_cmma) @@ -2368,8 +2278,8 @@ static int kvm_s390_get_cmma_bits(struct kvm *kvm, if (!peek && !kvm->arch.migration_mode) return -EINVAL; /* CMMA is disabled or was not used, or the buffer has length zero */ - bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX); - if (!bufsize || !kvm->mm->context.uses_cmm) { + args->count = min(args->count, KVM_S390_CMMA_SIZE_MAX); + if (!args->count || !uses_cmm(kvm->arch.gmap)) { memset(args, 0, sizeof(*args)); return 0; } @@ -2379,18 +2289,18 @@ static int kvm_s390_get_cmma_bits(struct kvm *kvm, return 0; } - values = vmalloc(bufsize); + values = vzalloc(args->count); if (!values) return -ENOMEM; - mmap_read_lock(kvm->mm); - srcu_idx = srcu_read_lock(&kvm->srcu); - if (peek) - ret = kvm_s390_peek_cmma(kvm, args, values, bufsize); - else - ret = kvm_s390_get_cmma(kvm, args, values, bufsize); - srcu_read_unlock(&kvm->srcu, srcu_idx); - mmap_read_unlock(kvm->mm); + scoped_guard(read_lock, &kvm->mmu_lock) { + if (peek) + ret = dat_peek_cmma(args->start_gfn, kvm->arch.gmap->asce, &args->count, + values); + else + ret = dat_get_cmma(kvm->arch.gmap->asce, &args->start_gfn, &args->count, + values, &kvm->arch.cmma_dirty_pages); + } if (kvm->arch.migration_mode) args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages); @@ -2412,11 +2322,9 @@ static int kvm_s390_get_cmma_bits(struct kvm *kvm, static int kvm_s390_set_cmma_bits(struct kvm *kvm, const struct kvm_s390_cmma_log *args) { - unsigned long hva, mask, pgstev, i; - uint8_t *bits; - int srcu_idx, r = 0; - - mask = args->mask; + struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL; + u8 *bits __free(kvfree) = NULL; + int r = 0; if (!kvm->arch.use_cmma) return -ENXIO; @@ -2430,41 +2338,30 @@ static int kvm_s390_set_cmma_bits(struct kvm *kvm, if (args->count == 0) return 0; + mc = kvm_s390_new_mmu_cache(); + if (!mc) + return -ENOMEM; bits = vmalloc(array_size(sizeof(*bits), args->count)); if (!bits) return -ENOMEM; r = copy_from_user(bits, (void __user *)args->values, args->count); - if (r) { - r = -EFAULT; - goto out; - } + if (r) + return -EFAULT; - mmap_read_lock(kvm->mm); - srcu_idx = srcu_read_lock(&kvm->srcu); - for (i = 0; i < args->count; i++) { - hva = gfn_to_hva(kvm, args->start_gfn + i); - if (kvm_is_error_hva(hva)) { - r = -EFAULT; - break; + do { + r = kvm_s390_mmu_cache_topup(mc); + if (r) + return r; + scoped_guard(read_lock, &kvm->mmu_lock) { + r = dat_set_cmma_bits(mc, kvm->arch.gmap->asce, args->start_gfn, + args->count, args->mask, bits); } + } while (r == -ENOMEM); - pgstev = bits[i]; - pgstev = pgstev << 24; - mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT; - set_pgste_bits(kvm->mm, hva, mask, pgstev); - } - srcu_read_unlock(&kvm->srcu, srcu_idx); - mmap_read_unlock(kvm->mm); + set_bit(GMAP_FLAG_USES_CMM, &kvm->arch.gmap->flags); - if (!kvm->mm->context.uses_cmm) { - mmap_write_lock(kvm->mm); - kvm->mm->context.uses_cmm = 1; - mmap_write_unlock(kvm->mm); - } -out: - vfree(bits); - return r; + return r <= 0 ? r : -EFAULT; } /** @@ -2670,7 +2567,15 @@ static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd) if (kvm_s390_pv_is_protected(kvm)) break; + kvm_s390_unmap_all_adapters(kvm); mmap_write_lock(kvm->mm); + /* + * Disable creation of new THPs. Existing THPs can stay, they + * will be split when any part of them gets imported. + */ + mm_flags_clear(MMF_DISABLE_THP_EXCEPT_ADVISED, kvm->mm); + mm_flags_set(MMF_DISABLE_THP_COMPLETELY, kvm->mm); + set_bit(GMAP_FLAG_EXPORT_ON_UNMAP, &kvm->arch.gmap->flags); r = gmap_helper_disable_cow_sharing(); mmap_write_unlock(kvm->mm); if (r) @@ -2744,9 +2649,9 @@ static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd) if (copy_from_user(&parms, argp, sizeof(parms))) break; - /* Currently restricted to 8KB */ + /* Currently restricted to 1MiB */ r = -EINVAL; - if (parms.length > PAGE_SIZE * 2) + if (parms.length > SZ_1M) break; r = -ENOMEM; @@ -2900,9 +2805,9 @@ static int mem_op_validate_common(struct kvm_s390_mem_op *mop, u64 supported_fla static int kvm_s390_vm_mem_op_abs(struct kvm *kvm, struct kvm_s390_mem_op *mop) { void __user *uaddr = (void __user *)mop->buf; + void *tmpbuf __free(kvfree) = NULL; enum gacc_mode acc_mode; - void *tmpbuf = NULL; - int r, srcu_idx; + int r; r = mem_op_validate_common(mop, KVM_S390_MEMOP_F_SKEY_PROTECTION | KVM_S390_MEMOP_F_CHECK_ONLY); @@ -2915,52 +2820,32 @@ static int kvm_s390_vm_mem_op_abs(struct kvm *kvm, struct kvm_s390_mem_op *mop) return -ENOMEM; } - srcu_idx = srcu_read_lock(&kvm->srcu); + acc_mode = mop->op == KVM_S390_MEMOP_ABSOLUTE_READ ? GACC_FETCH : GACC_STORE; - if (!kvm_is_gpa_in_memslot(kvm, mop->gaddr)) { - r = PGM_ADDRESSING; - goto out_unlock; - } + scoped_guard(srcu, &kvm->srcu) { + if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) + return check_gpa_range(kvm, mop->gaddr, mop->size, acc_mode, mop->key); - acc_mode = mop->op == KVM_S390_MEMOP_ABSOLUTE_READ ? GACC_FETCH : GACC_STORE; - if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) { - r = check_gpa_range(kvm, mop->gaddr, mop->size, acc_mode, mop->key); - goto out_unlock; - } - if (acc_mode == GACC_FETCH) { + if (acc_mode == GACC_STORE && copy_from_user(tmpbuf, uaddr, mop->size)) + return -EFAULT; r = access_guest_abs_with_key(kvm, mop->gaddr, tmpbuf, - mop->size, GACC_FETCH, mop->key); + mop->size, acc_mode, mop->key); if (r) - goto out_unlock; - if (copy_to_user(uaddr, tmpbuf, mop->size)) - r = -EFAULT; - } else { - if (copy_from_user(tmpbuf, uaddr, mop->size)) { - r = -EFAULT; - goto out_unlock; - } - r = access_guest_abs_with_key(kvm, mop->gaddr, tmpbuf, - mop->size, GACC_STORE, mop->key); + return r; + if (acc_mode != GACC_STORE && copy_to_user(uaddr, tmpbuf, mop->size)) + return -EFAULT; } - -out_unlock: - srcu_read_unlock(&kvm->srcu, srcu_idx); - - vfree(tmpbuf); - return r; + return 0; } static int kvm_s390_vm_mem_op_cmpxchg(struct kvm *kvm, struct kvm_s390_mem_op *mop) { void __user *uaddr = (void __user *)mop->buf; void __user *old_addr = (void __user *)mop->old_addr; - union { - __uint128_t quad; - char raw[sizeof(__uint128_t)]; - } old = { .quad = 0}, new = { .quad = 0 }; - unsigned int off_in_quad = sizeof(new) - mop->size; - int r, srcu_idx; - bool success; + union kvm_s390_quad old = { .sixteen = 0 }; + union kvm_s390_quad new = { .sixteen = 0 }; + bool success = false; + int r; r = mem_op_validate_common(mop, KVM_S390_MEMOP_F_SKEY_PROTECTION); if (r) @@ -2972,25 +2857,18 @@ static int kvm_s390_vm_mem_op_cmpxchg(struct kvm *kvm, struct kvm_s390_mem_op *m */ if (mop->size > sizeof(new)) return -EINVAL; - if (copy_from_user(&new.raw[off_in_quad], uaddr, mop->size)) + if (copy_from_user(&new, uaddr, mop->size)) return -EFAULT; - if (copy_from_user(&old.raw[off_in_quad], old_addr, mop->size)) + if (copy_from_user(&old, old_addr, mop->size)) return -EFAULT; - srcu_idx = srcu_read_lock(&kvm->srcu); + scoped_guard(srcu, &kvm->srcu) { + r = cmpxchg_guest_abs_with_key(kvm, mop->gaddr, mop->size, &old, new, + mop->key, &success); - if (!kvm_is_gpa_in_memslot(kvm, mop->gaddr)) { - r = PGM_ADDRESSING; - goto out_unlock; + if (!success && copy_to_user(old_addr, &old, mop->size)) + return -EFAULT; } - - r = cmpxchg_guest_abs_with_key(kvm, mop->gaddr, mop->size, &old.quad, - new.quad, mop->key, &success); - if (!success && copy_to_user(old_addr, &old.raw[off_in_quad], mop->size)) - r = -EFAULT; - -out_unlock: - srcu_read_unlock(&kvm->srcu, srcu_idx); return r; } @@ -3025,15 +2903,24 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) void __user *argp = (void __user *)arg; struct kvm_device_attr attr; int r; + struct kvm_s390_interrupt_info *inti; switch (ioctl) { case KVM_S390_INTERRUPT: { struct kvm_s390_interrupt s390int; + r = -EINVAL; + if (kvm_is_ucontrol(kvm)) + break; r = -EFAULT; if (copy_from_user(&s390int, argp, sizeof(s390int))) break; - r = kvm_s390_inject_vm(kvm, &s390int); + inti = kzalloc_obj(*inti, GFP_KERNEL_ACCOUNT); + if (!inti) + return -ENOMEM; + r = kvm_s390_inject_vm(kvm, &s390int, inti); + if (r) + kfree(inti); break; } case KVM_CREATE_IRQCHIP: { @@ -3089,9 +2976,8 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) r = -EFAULT; if (copy_from_user(&args, argp, sizeof(args))) break; - mutex_lock(&kvm->slots_lock); - r = kvm_s390_get_cmma_bits(kvm, &args); - mutex_unlock(&kvm->slots_lock); + scoped_guard(mutex, &kvm->slots_arch_lock) + r = kvm_s390_get_cmma_bits(kvm, &args); if (!r) { r = copy_to_user(argp, &args, sizeof(args)); if (r) @@ -3105,9 +2991,9 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) r = -EFAULT; if (copy_from_user(&args, argp, sizeof(args))) break; - mutex_lock(&kvm->slots_lock); + mutex_lock(&kvm->slots_arch_lock); r = kvm_s390_set_cmma_bits(kvm, &args); - mutex_unlock(&kvm->slots_lock); + mutex_unlock(&kvm->slots_arch_lock); break; } case KVM_S390_PV_COMMAND: { @@ -3145,6 +3031,32 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) r = -EFAULT; break; } + case KVM_S390_KEYOP: { + struct kvm_s390_mmu_cache *mc; + struct kvm_s390_keyop kop; + union skey skey; + + if (copy_from_user(&kop, argp, sizeof(kop))) { + r = -EFAULT; + break; + } + skey.skey = kop.key; + + mc = kvm_s390_new_mmu_cache(); + if (!mc) + return -ENOMEM; + + r = kvm_s390_keyop(mc, kvm, kop.operation, kop.guest_addr, skey); + kvm_s390_free_mmu_cache(mc); + if (r < 0) + break; + + kop.key = r; + r = 0; + if (copy_to_user(argp, &kop, sizeof(kop))) + r = -EFAULT; + break; + } case KVM_S390_ZPCI_OP: { struct kvm_s390_zpci_op args; @@ -3312,7 +3224,8 @@ static void kvm_s390_crypto_init(struct kvm *kvm) static void sca_dispose(struct kvm *kvm) { - free_pages_exact(kvm->arch.sca, sizeof(*kvm->arch.sca)); + if (kvm->arch.sca) + free_pages_exact(kvm->arch.sca, sizeof(*kvm->arch.sca)); kvm->arch.sca = NULL; } @@ -3330,6 +3243,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) char debug_name[16]; int i, rc; + mutex_init(&kvm->arch.pv.import_lock); + rc = -EINVAL; #ifdef CONFIG_KVM_S390_UCONTROL if (type & ~KVM_VM_S390_UCONTROL) @@ -3340,11 +3255,6 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) if (type) goto out_err; #endif - - rc = s390_enable_sie(); - if (rc) - goto out_err; - rc = -ENOMEM; if (!sclp.has_64bscao) @@ -3371,13 +3281,16 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) kvm->arch.sie_page2->kvm = kvm; kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list; - for (i = 0; i < kvm_s390_fac_size(); i++) { + for (i = 0; i < ARRAY_SIZE(kvm_s390_fac_base); i++) { kvm->arch.model.fac_mask[i] = stfle_fac_list[i] & - (kvm_s390_fac_base[i] | - kvm_s390_fac_ext[i]); + kvm_s390_fac_base[i]; kvm->arch.model.fac_list[i] = stfle_fac_list[i] & kvm_s390_fac_base[i]; } + for (i = 0; i < ARRAY_SIZE(kvm_s390_fac_ext); i++) { + kvm->arch.model.fac_mask[i] |= stfle_fac_list[i] & + kvm_s390_fac_ext[i]; + } kvm->arch.model.subfuncs = kvm_s390_available_subfunc; /* we are always in czam mode - even on pre z14 machines */ @@ -3408,7 +3321,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) mutex_unlock(&kvm->lock); } - mutex_init(&kvm->arch.float_int.ais_lock); + spin_lock_init(&kvm->arch.float_int.ais_lock); spin_lock_init(&kvm->arch.float_int.lock); for (i = 0; i < FIRQ_LIST_COUNT; i++) INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]); @@ -3418,6 +3331,12 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) debug_register_view(kvm->arch.dbf, &debug_sprintf_view); VM_EVENT(kvm, 3, "vm created with type %lu", type); + kvm->arch.mem_limit = type & KVM_VM_S390_UCONTROL ? KVM_S390_NO_MEM_LIMIT : sclp.hamax + 1; + kvm->arch.gmap = gmap_new(kvm, gpa_to_gfn(kvm->arch.mem_limit)); + if (!kvm->arch.gmap) + goto out_err; + clear_bit(GMAP_FLAG_PFAULT_ENABLED, &kvm->arch.gmap->flags); + if (type & KVM_VM_S390_UCONTROL) { struct kvm_userspace_memory_region2 fake_memslot = { .slot = KVM_S390_UCONTROL_MEMSLOT, @@ -3427,23 +3346,15 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) .flags = 0, }; - kvm->arch.gmap = NULL; - kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT; /* one flat fake memslot covering the whole address-space */ mutex_lock(&kvm->slots_lock); KVM_BUG_ON(kvm_set_internal_memslot(kvm, &fake_memslot), kvm); mutex_unlock(&kvm->slots_lock); + set_bit(GMAP_FLAG_IS_UCONTROL, &kvm->arch.gmap->flags); } else { - if (sclp.hamax == U64_MAX) - kvm->arch.mem_limit = TASK_SIZE_MAX; - else - kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX, - sclp.hamax + 1); - kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1); - if (!kvm->arch.gmap) - goto out_err; - kvm->arch.gmap->private = kvm; - kvm->arch.gmap->pfault_enabled = 0; + struct crst_table *table = dereference_asce(kvm->arch.gmap->asce); + + crst_table_init((void *)table, _CRSTE_HOLE(table->crstes[0].h.tt).val); } kvm->arch.use_pfmfi = sclp.has_pfmfi; @@ -3473,12 +3384,16 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id); kvm_s390_clear_local_irqs(vcpu); kvm_clear_async_pf_completion_queue(vcpu); + kvm_s390_clear_bp_data(vcpu); if (!kvm_is_ucontrol(vcpu->kvm)) sca_del_vcpu(vcpu); kvm_s390_update_topology_change_report(vcpu->kvm, 1); - if (kvm_is_ucontrol(vcpu->kvm)) - gmap_remove(vcpu->arch.gmap); + if (kvm_is_ucontrol(vcpu->kvm)) { + scoped_guard(spinlock, &vcpu->kvm->arch.gmap->children_lock) + gmap_remove_child(vcpu->arch.gmap); + vcpu->arch.gmap = gmap_put(vcpu->arch.gmap); + } if (vcpu->kvm->arch.use_cmma) kvm_s390_vcpu_unsetup_cmma(vcpu); @@ -3486,6 +3401,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) if (kvm_s390_pv_cpu_get_handle(vcpu)) kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc); free_page((unsigned long)(vcpu->arch.sie_block)); + kvm_s390_free_mmu_cache(vcpu->arch.mc); } void kvm_arch_destroy_vm(struct kvm *kvm) @@ -3512,30 +3428,19 @@ void kvm_arch_destroy_vm(struct kvm *kvm) debug_unregister(kvm->arch.dbf); free_page((unsigned long)kvm->arch.sie_page2); - if (!kvm_is_ucontrol(kvm)) - gmap_remove(kvm->arch.gmap); kvm_s390_destroy_adapters(kvm); kvm_s390_clear_float_irqs(kvm); kvm_s390_vsie_destroy(kvm); + kvm->arch.gmap = gmap_put(kvm->arch.gmap); KVM_EVENT(3, "vm 0x%p destroyed", kvm); } /* Section: vcpu related */ -static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu) -{ - vcpu->arch.gmap = gmap_create(current->mm, -1UL); - if (!vcpu->arch.gmap) - return -ENOMEM; - vcpu->arch.gmap->private = vcpu->kvm; - - return 0; -} - static void sca_del_vcpu(struct kvm_vcpu *vcpu) { struct esca_block *sca = vcpu->kvm->arch.sca; - if (!kvm_s390_use_sca_entries()) + if (!kvm_s390_use_sca_entries() || !vcpu->arch.initialized) return; clear_bit_inv(vcpu->vcpu_id, (unsigned long *)sca->mcn); @@ -3555,8 +3460,8 @@ static void sca_add_vcpu(struct kvm_vcpu *vcpu) if (!kvm_s390_use_sca_entries()) return; + WRITE_ONCE(sca->cpu[vcpu->vcpu_id].sda, virt_to_phys(vcpu->arch.sie_block)); set_bit_inv(vcpu->vcpu_id, (unsigned long *)sca->mcn); - sca->cpu[vcpu->vcpu_id].sda = virt_to_phys(vcpu->arch.sie_block); } static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id) @@ -3684,8 +3589,12 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) vcpu->arch.gmap = vcpu->kvm->arch.gmap; sca_add_vcpu(vcpu); } - if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0) + if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0 || + vcpu->kvm->arch.user_operexec) vcpu->arch.sie_block->ictl |= ICTL_OPEREXC; + + /* Pairs with smp_load_acquire() in kvm_arch_vcpu_ioctl_run() and kvm_arch_vcpu_ioctl() */ + smp_store_release(&vcpu->arch.initialized, true); } static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr) @@ -3747,7 +3656,8 @@ static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu) void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu) { - free_page((unsigned long)phys_to_virt(vcpu->arch.sie_block->cbrlo)); + if (vcpu->arch.sie_block->cbrlo) + free_page((unsigned long)phys_to_virt(vcpu->arch.sie_block->cbrlo)); vcpu->arch.sie_block->cbrlo = 0; } @@ -3865,12 +3775,18 @@ int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) return 0; } +DEFINE_FREE(sie_page, struct sie_page *, if (_T) free_page((unsigned long)(_T))) + int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) { - struct sie_page *sie_page; + struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL; + struct sie_page *sie_page __free(sie_page) = NULL; int rc; BUILD_BUG_ON(sizeof(struct sie_page) != 4096); + mc = kvm_s390_new_mmu_cache(); + if (!mc) + return -ENOMEM; sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL_ACCOUNT); if (!sie_page) return -ENOMEM; @@ -3915,9 +3831,9 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS; if (kvm_is_ucontrol(vcpu->kvm)) { - rc = __kvm_ucontrol_vcpu_init(vcpu); - if (rc) - goto out_free_sie_block; + vcpu->arch.gmap = gmap_new_child(vcpu->kvm->arch.gmap, -1UL); + if (!vcpu->arch.gmap) + return -ENOMEM; } VM_EVENT(vcpu->kvm, 3, "create cpu %d at 0x%p, sie block at 0x%p", @@ -3925,18 +3841,19 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) trace_kvm_s390_create_vcpu(vcpu->vcpu_id, vcpu, vcpu->arch.sie_block); rc = kvm_s390_vcpu_setup(vcpu); - if (rc) - goto out_ucontrol_uninit; + if (rc) { + if (kvm_is_ucontrol(vcpu->kvm)) { + scoped_guard(spinlock, &vcpu->kvm->arch.gmap->children_lock) + gmap_remove_child(vcpu->arch.gmap); + vcpu->arch.gmap = gmap_put(vcpu->arch.gmap); + } + return rc; + } + vcpu->arch.mc = no_free_ptr(mc); + sie_page = NULL; kvm_s390_update_topology_change_report(vcpu->kvm, 1); return 0; - -out_ucontrol_uninit: - if (kvm_is_ucontrol(vcpu->kvm)) - gmap_remove(vcpu->arch.gmap); -out_free_sie_block: - free_page((unsigned long)(vcpu->arch.sie_block)); - return rc; } int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) @@ -3997,32 +3914,6 @@ void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu) kvm_s390_vcpu_request(vcpu); } -static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start, - unsigned long end) -{ - struct kvm *kvm = gmap->private; - struct kvm_vcpu *vcpu; - unsigned long prefix; - unsigned long i; - - trace_kvm_s390_gmap_notifier(start, end, gmap_is_shadow(gmap)); - - if (gmap_is_shadow(gmap)) - return; - if (start >= 1UL << 31) - /* We are only interested in prefix pages */ - return; - kvm_for_each_vcpu(i, vcpu, kvm) { - /* match against both prefix pages */ - prefix = kvm_s390_get_prefix(vcpu); - if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) { - VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx", - start, end); - kvm_s390_sync_request(KVM_REQ_REFRESH_GUEST_PREFIX, vcpu); - } - } -} - bool kvm_arch_no_poll(struct kvm_vcpu *vcpu) { /* do not poll with more than halt_poll_max_steal percent of steal time */ @@ -4332,8 +4223,10 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, /* enforce guest PER */ kvm_s390_set_cpuflags(vcpu, CPUSTAT_P); - if (dbg->control & KVM_GUESTDBG_USE_HW_BP) - rc = kvm_s390_import_bp_data(vcpu, dbg); + if (dbg->control & KVM_GUESTDBG_USE_HW_BP) { + scoped_guard(srcu, &vcpu->kvm->srcu) + rc = kvm_s390_import_bp_data(vcpu, dbg); + } } else { kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P); vcpu->arch.guestdbg.last_bp = 0; @@ -4404,72 +4297,41 @@ static bool ibs_enabled(struct kvm_vcpu *vcpu) return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS); } -static int __kvm_s390_fixup_fault_sync(struct gmap *gmap, gpa_t gaddr, unsigned int flags) -{ - struct kvm *kvm = gmap->private; - gfn_t gfn = gpa_to_gfn(gaddr); - bool unlocked; - hva_t vmaddr; - gpa_t tmp; - int rc; - - if (kvm_is_ucontrol(kvm)) { - tmp = __gmap_translate(gmap, gaddr); - gfn = gpa_to_gfn(tmp); - } - - vmaddr = gfn_to_hva(kvm, gfn); - rc = fixup_user_fault(gmap->mm, vmaddr, FAULT_FLAG_WRITE, &unlocked); - if (!rc) - rc = __gmap_link(gmap, gaddr, vmaddr); - return rc; -} - -/** - * __kvm_s390_mprotect_many() - Apply specified protection to guest pages - * @gmap: the gmap of the guest - * @gpa: the starting guest address - * @npages: how many pages to protect - * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE - * @bits: pgste notification bits to set - * - * Returns: 0 in case of success, < 0 in case of error - see gmap_protect_one() - * - * Context: kvm->srcu and gmap->mm need to be held in read mode - */ -int __kvm_s390_mprotect_many(struct gmap *gmap, gpa_t gpa, u8 npages, unsigned int prot, - unsigned long bits) +static int vcpu_ucontrol_translate(struct kvm_vcpu *vcpu, gpa_t *gaddr) { - unsigned int fault_flag = (prot & PROT_WRITE) ? FAULT_FLAG_WRITE : 0; - gpa_t end = gpa + npages * PAGE_SIZE; int rc; - for (; gpa < end; gpa = ALIGN(gpa + 1, rc)) { - rc = gmap_protect_one(gmap, gpa, prot, bits); - if (rc == -EAGAIN) { - __kvm_s390_fixup_fault_sync(gmap, gpa, fault_flag); - rc = gmap_protect_one(gmap, gpa, prot, bits); + if (kvm_is_ucontrol(vcpu->kvm)) { + rc = gmap_ucas_translate(vcpu->arch.mc, vcpu->arch.gmap, gaddr); + if (rc == -EREMOTE) { + vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL; + vcpu->run->s390_ucontrol.trans_exc_code = *gaddr; + vcpu->run->s390_ucontrol.pgm_code = PGM_SEGMENT_TRANSLATION; } - if (rc < 0) - return rc; + return rc; } - return 0; } -static int kvm_s390_mprotect_notify_prefix(struct kvm_vcpu *vcpu) +static int kvm_s390_fixup_prefix(struct kvm_vcpu *vcpu) { gpa_t gaddr = kvm_s390_get_prefix(vcpu); - int idx, rc; - - idx = srcu_read_lock(&vcpu->kvm->srcu); - mmap_read_lock(vcpu->arch.gmap->mm); + gfn_t gfn; + int rc; - rc = __kvm_s390_mprotect_many(vcpu->arch.gmap, gaddr, 2, PROT_WRITE, GMAP_NOTIFY_MPROT); + if (vcpu_ucontrol_translate(vcpu, &gaddr)) + return -EREMOTE; + gfn = gpa_to_gfn(gaddr); - mmap_read_unlock(vcpu->arch.gmap->mm); - srcu_read_unlock(&vcpu->kvm->srcu, idx); + rc = kvm_s390_faultin_gfn_simple(vcpu, NULL, gfn, true); + if (rc) + return rc; + rc = kvm_s390_faultin_gfn_simple(vcpu, NULL, gfn + 1, true); + if (rc) + return rc; + scoped_guard(write_lock, &vcpu->kvm->mmu_lock) + rc = dat_set_prefix_notif_bit(vcpu->kvm->arch.gmap->asce, gfn); return rc; } @@ -4489,7 +4351,7 @@ retry: if (kvm_check_request(KVM_REQ_REFRESH_GUEST_PREFIX, vcpu)) { int rc; - rc = kvm_s390_mprotect_notify_prefix(vcpu); + rc = kvm_s390_fixup_prefix(vcpu); if (rc) { kvm_make_request(KVM_REQ_REFRESH_GUEST_PREFIX, vcpu); return rc; @@ -4538,8 +4400,7 @@ retry: * Re-enable CMM virtualization if CMMA is available and * CMM has been used. */ - if ((vcpu->kvm->arch.use_cmma) && - (vcpu->kvm->mm->context.uses_cmm)) + if (vcpu->kvm->arch.use_cmma && uses_cmm(vcpu->arch.gmap)) vcpu->arch.sie_block->ecb2 |= ECB2_CMMA; goto retry; } @@ -4588,19 +4449,28 @@ int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clo } static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token, - unsigned long token) + unsigned long token) { - struct kvm_s390_interrupt inti; - struct kvm_s390_irq irq; + struct kvm_s390_interrupt inti = {}; + struct kvm_s390_irq irq = {}; + struct kvm_s390_interrupt_info *inti_mem = NULL; + int ret = 0; if (start_token) { irq.u.ext.ext_params2 = token; irq.type = KVM_S390_INT_PFAULT_INIT; WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq)); } else { + inti_mem = kzalloc_obj(*inti_mem, GFP_KERNEL_ACCOUNT); + if (WARN_ON_ONCE(!inti_mem)) + return; + inti.type = KVM_S390_INT_PFAULT_DONE; inti.parm64 = token; - WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti)); + ret = kvm_s390_inject_vm(vcpu->kvm, &inti, inti_mem); + if (ret) + kfree(inti_mem); + WARN_ON_ONCE(ret); } } @@ -4635,7 +4505,7 @@ bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu) return true; } -static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu) +bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu) { hva_t hva; struct kvm_arch_async_pf arch; @@ -4651,7 +4521,7 @@ static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu) return false; if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK)) return false; - if (!vcpu->arch.gmap->pfault_enabled) + if (!pfault_enabled(vcpu->arch.gmap)) return false; hva = gfn_to_hva(vcpu->kvm, current->thread.gmap_teid.addr); @@ -4744,98 +4614,25 @@ static void kvm_s390_assert_primary_as(struct kvm_vcpu *vcpu) current->thread.gmap_int_code, current->thread.gmap_teid.val); } -/* - * __kvm_s390_handle_dat_fault() - handle a dat fault for the gmap of a vcpu - * @vcpu: the vCPU whose gmap is to be fixed up - * @gfn: the guest frame number used for memslots (including fake memslots) - * @gaddr: the gmap address, does not have to match @gfn for ucontrol gmaps - * @foll: FOLL_* flags - * - * Return: 0 on success, < 0 in case of error. - * Context: The mm lock must not be held before calling. May sleep. - */ -int __kvm_s390_handle_dat_fault(struct kvm_vcpu *vcpu, gfn_t gfn, gpa_t gaddr, unsigned int foll) -{ - struct kvm_memory_slot *slot; - unsigned int fault_flags; - bool writable, unlocked; - unsigned long vmaddr; - struct page *page; - kvm_pfn_t pfn; +static int vcpu_dat_fault_handler(struct kvm_vcpu *vcpu, gpa_t gaddr, bool wr) +{ + struct guest_fault f = { + .write_attempt = wr, + .attempt_pfault = pfault_enabled(vcpu->arch.gmap), + }; int rc; - slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); - if (!slot || slot->flags & KVM_MEMSLOT_INVALID) - return vcpu_post_run_addressing_exception(vcpu); - - fault_flags = foll & FOLL_WRITE ? FAULT_FLAG_WRITE : 0; - if (vcpu->arch.gmap->pfault_enabled) - foll |= FOLL_NOWAIT; - vmaddr = __gfn_to_hva_memslot(slot, gfn); - -try_again: - pfn = __kvm_faultin_pfn(slot, gfn, foll, &writable, &page); + if (vcpu_ucontrol_translate(vcpu, &gaddr)) + return -EREMOTE; + f.gfn = gpa_to_gfn(gaddr); - /* Access outside memory, inject addressing exception */ - if (is_noslot_pfn(pfn)) + rc = kvm_s390_faultin_gfn(vcpu, NULL, &f); + if (rc <= 0) + return rc; + if (rc == PGM_ADDRESSING) return vcpu_post_run_addressing_exception(vcpu); - /* Signal pending: try again */ - if (pfn == KVM_PFN_ERR_SIGPENDING) - return -EAGAIN; - - /* Needs I/O, try to setup async pfault (only possible with FOLL_NOWAIT) */ - if (pfn == KVM_PFN_ERR_NEEDS_IO) { - trace_kvm_s390_major_guest_pfault(vcpu); - if (kvm_arch_setup_async_pf(vcpu)) - return 0; - vcpu->stat.pfault_sync++; - /* Could not setup async pfault, try again synchronously */ - foll &= ~FOLL_NOWAIT; - goto try_again; - } - /* Any other error */ - if (is_error_pfn(pfn)) - return -EFAULT; - - /* Success */ - mmap_read_lock(vcpu->arch.gmap->mm); - /* Mark the userspace PTEs as young and/or dirty, to avoid page fault loops */ - rc = fixup_user_fault(vcpu->arch.gmap->mm, vmaddr, fault_flags, &unlocked); - if (!rc) - rc = __gmap_link(vcpu->arch.gmap, gaddr, vmaddr); - scoped_guard(spinlock, &vcpu->kvm->mmu_lock) { - kvm_release_faultin_page(vcpu->kvm, page, false, writable); - } - mmap_read_unlock(vcpu->arch.gmap->mm); - return rc; -} - -static int vcpu_dat_fault_handler(struct kvm_vcpu *vcpu, unsigned long gaddr, unsigned int foll) -{ - unsigned long gaddr_tmp; - gfn_t gfn; - - gfn = gpa_to_gfn(gaddr); - if (kvm_is_ucontrol(vcpu->kvm)) { - /* - * This translates the per-vCPU guest address into a - * fake guest address, which can then be used with the - * fake memslots that are identity mapping userspace. - * This allows ucontrol VMs to use the normal fault - * resolution path, like normal VMs. - */ - mmap_read_lock(vcpu->arch.gmap->mm); - gaddr_tmp = __gmap_translate(vcpu->arch.gmap, gaddr); - mmap_read_unlock(vcpu->arch.gmap->mm); - if (gaddr_tmp == -EFAULT) { - vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL; - vcpu->run->s390_ucontrol.trans_exc_code = gaddr; - vcpu->run->s390_ucontrol.pgm_code = PGM_SEGMENT_TRANSLATION; - return -EREMOTE; - } - gfn = gpa_to_gfn(gaddr_tmp); - } - return __kvm_s390_handle_dat_fault(vcpu, gfn, gaddr, foll); + KVM_BUG_ON(rc, vcpu->kvm); + return -EINVAL; } static int vcpu_post_run_handle_fault(struct kvm_vcpu *vcpu) @@ -4908,7 +4705,7 @@ static int vcpu_post_run_handle_fault(struct kvm_vcpu *vcpu) return 0; } -static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason) +static int vcpu_post_run(struct kvm_vcpu *vcpu, int sie_return) { struct mcck_volatile_info *mcck_info; struct sie_page *sie_page; @@ -4924,14 +4721,14 @@ static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason) vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14; vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15; - if (exit_reason == -EINTR) { - VCPU_EVENT(vcpu, 3, "%s", "machine check"); + if (sie_return == SIE64_RETURN_MCCK) { sie_page = container_of(vcpu->arch.sie_block, struct sie_page, sie_block); mcck_info = &sie_page->mcck_info; kvm_s390_reinject_machine_check(vcpu, mcck_info); return 0; } + WARN_ON_ONCE(sie_return != SIE64_RETURN_NORMAL); if (vcpu->arch.sie_block->icptcode > 0) { rc = kvm_handle_sie_intercept(vcpu); @@ -4970,7 +4767,7 @@ int noinstr kvm_s390_enter_exit_sie(struct kvm_s390_sie_block *scb, #define PSW_INT_MASK (PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_MCHECK) static int __vcpu_run(struct kvm_vcpu *vcpu) { - int rc, exit_reason; + int rc, sie_return; struct sie_page *sie_page = (struct sie_page *)vcpu->arch.sie_block; /* @@ -5010,9 +4807,9 @@ xfer_to_guest_mode_check: guest_timing_enter_irqoff(); __disable_cpu_timer_accounting(vcpu); - exit_reason = kvm_s390_enter_exit_sie(vcpu->arch.sie_block, - vcpu->run->s.regs.gprs, - vcpu->arch.gmap->asce); + sie_return = kvm_s390_enter_exit_sie(vcpu->arch.sie_block, + vcpu->run->s.regs.gprs, + vcpu->arch.gmap->asce.val); __enable_cpu_timer_accounting(vcpu); guest_timing_exit_irqoff(); @@ -5035,7 +4832,7 @@ xfer_to_guest_mode_check: } kvm_vcpu_srcu_read_lock(vcpu); - rc = vcpu_post_run(vcpu, exit_reason); + rc = vcpu_post_run(vcpu, sie_return); if (rc || guestdbg_exit_pending(vcpu)) { kvm_vcpu_srcu_read_unlock(vcpu); break; @@ -5225,6 +5022,10 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS) return -EINVAL; + /* Pairs with smp_store_release() in kvm_arch_vcpu_postcreate() */ + if (!smp_load_acquire(&vcpu->arch.initialized)) + return -EINVAL; + vcpu_load(vcpu); if (guestdbg_exit_pending(vcpu)) { @@ -5245,7 +5046,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) pr_err_ratelimited("can't run stopped vcpu %d\n", vcpu->vcpu_id); rc = -EINVAL; - goto out; + goto out_sigset; } kernel_fpu_begin(&fpu, KERNEL_FPC | KERNEL_VXR); @@ -5275,9 +5076,11 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) store_regs(vcpu); kernel_fpu_end(&fpu, KERNEL_FPC | KERNEL_VXR); + vcpu->stat.exit_userspace++; + +out_sigset: kvm_sigset_deactivate(vcpu); - vcpu->stat.exit_userspace++; out: vcpu_put(vcpu); return rc; @@ -5547,8 +5350,8 @@ static long kvm_s390_vcpu_mem_op(struct kvm_vcpu *vcpu, struct kvm_s390_mem_op *mop) { void __user *uaddr = (void __user *)mop->buf; + void *tmpbuf __free(kvfree) = NULL; enum gacc_mode acc_mode; - void *tmpbuf = NULL; int r; r = mem_op_validate_common(mop, KVM_S390_MEMOP_F_INJECT_EXCEPTION | @@ -5570,32 +5373,21 @@ static long kvm_s390_vcpu_mem_op(struct kvm_vcpu *vcpu, if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) { r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, acc_mode, mop->key); - goto out_inject; - } - if (acc_mode == GACC_FETCH) { + } else if (acc_mode == GACC_FETCH) { r = read_guest_with_key(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size, mop->key); - if (r) - goto out_inject; - if (copy_to_user(uaddr, tmpbuf, mop->size)) { - r = -EFAULT; - goto out_free; - } + if (!r && copy_to_user(uaddr, tmpbuf, mop->size)) + return -EFAULT; } else { - if (copy_from_user(tmpbuf, uaddr, mop->size)) { - r = -EFAULT; - goto out_free; - } + if (copy_from_user(tmpbuf, uaddr, mop->size)) + return -EFAULT; r = write_guest_with_key(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size, mop->key); } -out_inject: if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0) kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm); -out_free: - vfree(tmpbuf); return r; } @@ -5637,18 +5429,22 @@ long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl, if (copy_from_user(&s390irq, argp, sizeof(s390irq))) return -EFAULT; - rc = kvm_s390_inject_vcpu(vcpu, &s390irq); + scoped_guard(srcu, &vcpu->kvm->srcu) + rc = kvm_s390_inject_vcpu(vcpu, &s390irq); break; } case KVM_S390_INTERRUPT: { struct kvm_s390_interrupt s390int; struct kvm_s390_irq s390irq = {}; + if (kvm_is_ucontrol(vcpu->kvm)) + return -EINVAL; if (copy_from_user(&s390int, argp, sizeof(s390int))) return -EFAULT; if (s390int_to_s390irq(&s390int, &s390irq)) return -EINVAL; - rc = kvm_s390_inject_vcpu(vcpu, &s390irq); + scoped_guard(srcu, &vcpu->kvm->srcu) + rc = kvm_s390_inject_vcpu(vcpu, &s390irq); break; } default: @@ -5720,6 +5516,10 @@ long kvm_arch_vcpu_ioctl(struct file *filp, long r; u16 rc, rrc; + /* Pairs with smp_store_release() in kvm_arch_vcpu_postcreate() */ + if (!smp_load_acquire(&vcpu->arch.initialized)) + return -EINVAL; + vcpu_load(vcpu); switch (ioctl) { @@ -5785,44 +5585,58 @@ long kvm_arch_vcpu_ioctl(struct file *filp, } #ifdef CONFIG_KVM_S390_UCONTROL case KVM_S390_UCAS_MAP: { - struct kvm_s390_ucas_mapping ucasmap; + struct kvm_s390_ucas_mapping ucas; - if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) { - r = -EFAULT; + r = -EFAULT; + if (copy_from_user(&ucas, argp, sizeof(ucas))) break; - } - if (!kvm_is_ucontrol(vcpu->kvm)) { - r = -EINVAL; + r = -EINVAL; + if (!kvm_is_ucontrol(vcpu->kvm)) + break; + if (!IS_ALIGNED(ucas.user_addr | ucas.vcpu_addr | ucas.length, _SEGMENT_SIZE)) break; - } - r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr, - ucasmap.vcpu_addr, ucasmap.length); + r = gmap_ucas_map(vcpu->arch.gmap, gpa_to_gfn(ucas.user_addr), + gpa_to_gfn(ucas.vcpu_addr), + ucas.length >> _SEGMENT_SHIFT); break; } case KVM_S390_UCAS_UNMAP: { - struct kvm_s390_ucas_mapping ucasmap; + struct kvm_s390_ucas_mapping ucas; - if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) { - r = -EFAULT; + r = -EFAULT; + if (copy_from_user(&ucas, argp, sizeof(ucas))) break; - } - if (!kvm_is_ucontrol(vcpu->kvm)) { - r = -EINVAL; + r = -EINVAL; + if (!kvm_is_ucontrol(vcpu->kvm)) + break; + if (!IS_ALIGNED(ucas.vcpu_addr | ucas.length, _SEGMENT_SIZE)) break; - } - r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr, - ucasmap.length); + gmap_ucas_unmap(vcpu->arch.gmap, gpa_to_gfn(ucas.vcpu_addr), + ucas.length >> _SEGMENT_SHIFT); + r = 0; break; } #endif case KVM_S390_VCPU_FAULT: { - idx = srcu_read_lock(&vcpu->kvm->srcu); - r = vcpu_dat_fault_handler(vcpu, arg, 0); - srcu_read_unlock(&vcpu->kvm->srcu, idx); + gpa_t gaddr = arg; + + scoped_guard(srcu, &vcpu->kvm->srcu) { + r = vcpu_ucontrol_translate(vcpu, &gaddr); + if (r) + break; + + r = kvm_s390_faultin_gfn_simple(vcpu, NULL, gpa_to_gfn(gaddr), false); + if (r == PGM_ADDRESSING) + r = -EFAULT; + if (r <= 0) + break; + r = -EIO; + KVM_BUG_ON(r, vcpu->kvm); + } break; } case KVM_ENABLE_CAP: @@ -5899,7 +5713,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, r = kvm_s390_handle_pv_vcpu_dump(vcpu, &cmd); /* Always copy over UV rc / rrc data */ - if (copy_to_user((__u8 __user *)argp, &cmd.rc, + if (copy_to_user(argp + offsetof(struct kvm_pv_cmd, rc), &cmd.rc, sizeof(cmd.rc) + sizeof(cmd.rrc))) r = -EFAULT; break; @@ -5936,87 +5750,126 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *new, enum kvm_mr_change change) { - gpa_t size; + return s390_kvm_mmu_prepare_memory_region(kvm, old, new, change); +} - if (kvm_is_ucontrol(kvm) && new->id < KVM_USER_MEM_SLOTS) - return -EINVAL; +static long cmma_d_count_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) +{ + union pgste pgste; - /* When we are protected, we should not change the memory slots */ - if (kvm_s390_pv_get_handle(kvm)) - return -EINVAL; + pgste = pgste_get_lock(ptep); + if (pgste.cmma_d) { + pgste.cmma_d = 0; + atomic64_dec(walk->priv); + } + pgste_set_unlock(ptep, pgste); + return 0; +} - if (change != KVM_MR_DELETE && change != KVM_MR_FLAGS_ONLY) { - /* - * A few sanity checks. We can have memory slots which have to be - * located/ended at a segment boundary (1MB). The memory in userland is - * ok to be fragmented into various different vmas. It is okay to mmap() - * and munmap() stuff in this slot after doing this call at any time - */ +void kvm_s390_update_cmma_dirty(struct kvm *kvm, const struct kvm_memory_slot *old) +{ + const struct dat_walk_ops ops = { .pte_entry = cmma_d_count_pte, }; - if (new->userspace_addr & 0xffffful) - return -EINVAL; + if (kvm->arch.migration_mode && kvm->arch.use_cmma && old) { + _dat_walk_gfn_range(old->base_gfn, old->base_gfn + old->npages, + kvm->arch.gmap->asce, &ops, DAT_WALK_IGN_HOLES, + &kvm->arch.cmma_dirty_pages); + } +} - size = new->npages * PAGE_SIZE; - if (size & 0xffffful) - return -EINVAL; +void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ +} - if ((new->base_gfn * PAGE_SIZE) + size > kvm->arch.mem_limit) - return -EINVAL; - } +/** + * kvm_arch_vcpu_pre_fault_memory() -- pre-fault and link gmap dat tables + * @vcpu: the vcpu that shall appear to have generated the fault-in. + * @range: the range that needs to be faulted in. + * + * The first page of the given range is faulted in and the corresponding gmap + * page tables are created, as if the given vCPU had performed a read + * operation. + * If the range starts outside any memslots, an error is returned. An error is + * also returned for UCONTROL VMs, which should instead use the + * KVM_S390_VCPU_FAULT ioctl. + * + * Return: + * * %-ENOENT if the range lies outside of a memslot. + * * %-EINVAL in case of invalid state (for example if the VM is UCONTROL). + * * %-EIO if errors happen while faulting-in the page (will trigger a warning + * in the caller). + * * other error codes < 0 in case of other errors. + * * otherwise a number > 0 of bytes that have been faulted in successfully. + */ +long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range) +{ + struct guest_fault f = { .gfn = gpa_to_gfn(range->gpa), }; + gpa_t end; + int rc; - if (!kvm->arch.migration_mode) - return 0; + if (kvm_is_ucontrol(vcpu->kvm)) + return -EINVAL; - /* - * Turn off migration mode when: - * - userspace creates a new memslot with dirty logging off, - * - userspace modifies an existing memslot (MOVE or FLAGS_ONLY) and - * dirty logging is turned off. - * Migration mode expects dirty page logging being enabled to store - * its dirty bitmap. - */ - if (change != KVM_MR_DELETE && - !(new->flags & KVM_MEM_LOG_DIRTY_PAGES)) - WARN(kvm_s390_vm_stop_migration(kvm), - "Failed to stop migration mode"); + rc = kvm_s390_faultin_gfn(vcpu, NULL, &f); + if (rc == PGM_ADDRESSING) + return -ENOENT; + if (rc > 0) + return -EIO; + if (rc < 0) + return rc; - return 0; + if (f.ptep) + return PAGE_SIZE; + + end = ALIGN(range->gpa + PAGE_SIZE, f.crste_region3 ? _REGION3_SIZE : HPAGE_SIZE); + return min(range->size, end - range->gpa); } -void kvm_arch_commit_memory_region(struct kvm *kvm, - struct kvm_memory_slot *old, - const struct kvm_memory_slot *new, - enum kvm_mr_change change) +/** + * kvm_test_age_gfn() - test young + * @kvm: the kvm instance + * @range: the range of guest addresses whose young status needs to be cleared + * + * Context: called by KVM common code without holding the kvm mmu lock + * Return: true if any page in the given range is young, otherwise 0. + */ +bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) { - int rc = 0; + scoped_guard(read_lock, &kvm->mmu_lock) + return dat_test_age_gfn(kvm->arch.gmap->asce, range->start, range->end); +} - if (kvm_is_ucontrol(kvm)) - return; +/** + * kvm_age_gfn() - clear young + * @kvm: the kvm instance + * @range: the range of guest addresses whose young status needs to be cleared + * + * Context: called by KVM common code without holding the kvm mmu lock + * Return: true if any page in the given range was young, otherwise 0. + */ +bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + scoped_guard(read_lock, &kvm->mmu_lock) + return gmap_age_gfn(kvm->arch.gmap, range->start, range->end); +} - switch (change) { - case KVM_MR_DELETE: - rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE, - old->npages * PAGE_SIZE); - break; - case KVM_MR_MOVE: - rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE, - old->npages * PAGE_SIZE); - if (rc) - break; - fallthrough; - case KVM_MR_CREATE: - rc = gmap_map_segment(kvm->arch.gmap, new->userspace_addr, - new->base_gfn * PAGE_SIZE, - new->npages * PAGE_SIZE); - break; - case KVM_MR_FLAGS_ONLY: - break; - default: - WARN(1, "Unknown KVM MR CHANGE: %d\n", change); - } - if (rc) - pr_warn("failed to commit memory region\n"); - return; +/** + * kvm_unmap_gfn_range() - Unmap a range of guest addresses + * @kvm: the kvm instance + * @range: the range of guest page frames to invalidate + * + * This function always returns false because every DAT table modification + * has to use the appropriate DAT table manipulation instructions, which will + * keep the TLB coherent, hence no additional TLB flush is ever required. + * + * Context: called by KVM common code with the kvm mmu write lock held + * Return: false + */ +bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) +{ + return gmap_unmap_gfn_range(kvm->arch.gmap, range->slot, range->start, range->end); } static inline unsigned long nonhyp_mask(int i) @@ -6035,14 +5888,13 @@ static int __init kvm_s390_init(void) return -ENODEV; } - if (nested && hpage) { - pr_info("A KVM host that supports nesting cannot back its KVM guests with huge pages\n"); - return -EINVAL; + if (hpage_2g && !hpage) { + hpage_2g = 0; + pr_info("Disabling 2G hugepage support, since 1M hugepage support is not enabled.\n"); } - for (i = 0; i < 16; i++) - kvm_s390_fac_base[i] |= - stfle_fac_list[i] & nonhyp_mask(i); + for (i = 0; i < HMFAI_DWORDS; i++) + kvm_s390_fac_base[i] |= nonhyp_mask(i); r = __kvm_s390_init(); if (r) diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/s390/s390.h index 65c950760993..aa0d1d062f8d 100644 --- a/arch/s390/kvm/kvm-s390.h +++ b/arch/s390/kvm/s390/s390.h @@ -19,9 +19,19 @@ #include <asm/facility.h> #include <asm/processor.h> #include <asm/sclp.h> +#include "dat.h" +#include "gmap.h" #define KVM_S390_UCONTROL_MEMSLOT (KVM_USER_MEM_SLOTS + 0) +union kvm_s390_quad { + __uint128_t sixteen; + unsigned long eight; + unsigned int four; + unsigned short two; + unsigned char one; +}; + static inline void kvm_s390_fpu_store(struct kvm_run *run) { fpu_stfpc(&run->s.regs.fpc); @@ -106,15 +116,15 @@ static inline int is_vcpu_idle(struct kvm_vcpu *vcpu) static inline int kvm_is_ucontrol(struct kvm *kvm) { #ifdef CONFIG_KVM_S390_UCONTROL - if (kvm->arch.gmap) - return 0; - return 1; + return test_bit(GMAP_FLAG_IS_UCONTROL, &kvm->arch.gmap->flags); #else return 0; #endif } -#define GUEST_PREFIX_SHIFT 13 +#define GUEST_PREFIX_SHIFT 12 +#define GUEST_PREFIX_MASK_ZARCH 0x7fffe +#define GUEST_PREFIX_MASK_ESA 0x7ffff static inline u32 kvm_s390_get_prefix(struct kvm_vcpu *vcpu) { return vcpu->arch.sie_block->prefix << GUEST_PREFIX_SHIFT; @@ -125,6 +135,7 @@ static inline void kvm_s390_set_prefix(struct kvm_vcpu *vcpu, u32 prefix) VCPU_EVENT(vcpu, 3, "set prefix of cpu %03u to 0x%x", vcpu->vcpu_id, prefix); vcpu->arch.sie_block->prefix = prefix >> GUEST_PREFIX_SHIFT; + vcpu->arch.sie_block->prefix &= GUEST_PREFIX_MASK_ZARCH; kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); kvm_make_request(KVM_REQ_REFRESH_GUEST_PREFIX, vcpu); } @@ -365,7 +376,8 @@ int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu); void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu); void kvm_s390_clear_float_irqs(struct kvm *kvm); int __must_check kvm_s390_inject_vm(struct kvm *kvm, - struct kvm_s390_interrupt *s390int); + struct kvm_s390_interrupt *s390int, + struct kvm_s390_interrupt_info *inti); int __must_check kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq); static inline int kvm_s390_inject_prog_irq(struct kvm_vcpu *vcpu, @@ -432,20 +444,15 @@ int kvm_s390_skey_check_enable(struct kvm_vcpu *vcpu); /* implemented in vsie.c */ int kvm_s390_handle_vsie(struct kvm_vcpu *vcpu); void kvm_s390_vsie_kick(struct kvm_vcpu *vcpu); -void kvm_s390_vsie_gmap_notifier(struct gmap *gmap, unsigned long start, - unsigned long end); +void kvm_s390_vsie_gmap_notifier(struct gmap *gmap, gpa_t start, gpa_t end); void kvm_s390_vsie_init(struct kvm *kvm); void kvm_s390_vsie_destroy(struct kvm *kvm); -int gmap_shadow_valid(struct gmap *sg, unsigned long asce, int edat_level); - -/* implemented in gmap-vsie.c */ -struct gmap *gmap_shadow(struct gmap *parent, unsigned long asce, int edat_level); /* implemented in sigp.c */ int kvm_s390_handle_sigp(struct kvm_vcpu *vcpu); int kvm_s390_handle_sigp_pei(struct kvm_vcpu *vcpu); -/* implemented in kvm-s390.c */ +/* implemented in s390.c */ int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod); int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long addr); int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr); @@ -461,14 +468,13 @@ void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu); void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm); __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu); int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc); -int __kvm_s390_handle_dat_fault(struct kvm_vcpu *vcpu, gfn_t gfn, gpa_t gaddr, unsigned int flags); int __kvm_s390_mprotect_many(struct gmap *gmap, gpa_t gpa, u8 npages, unsigned int prot, unsigned long bits); -static inline int kvm_s390_handle_dat_fault(struct kvm_vcpu *vcpu, gpa_t gaddr, unsigned int flags) -{ - return __kvm_s390_handle_dat_fault(vcpu, gpa_to_gfn(gaddr), gaddr, flags); -} +bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu); +void kvm_s390_update_cmma_dirty(struct kvm *kvm, const struct kvm_memory_slot *old); +int kvm_s390_vm_stop_migration(struct kvm *kvm); + /* implemented in diag.c */ int kvm_s390_handle_diag(struct kvm_vcpu *vcpu); @@ -558,6 +564,8 @@ void kvm_s390_gisa_disable(struct kvm *kvm); void kvm_s390_gisa_enable(struct kvm *kvm); int __init kvm_s390_gib_init(u8 nisc); void kvm_s390_gib_destroy(void); +void kvm_s390_unmap_all_adapters(struct kvm *kvm); + /* implemented in guestdbg.c */ void kvm_s390_backup_guest_per_regs(struct kvm_vcpu *vcpu); @@ -589,6 +597,11 @@ static inline bool kvm_s390_cur_gmap_fault_is_write(void) return test_facility(75) && (current->thread.gmap_teid.fsi == TEID_FSI_STORE); } +static __always_inline int kvm_s390_is_migration_mode(struct kvm *kvm) +{ + return kvm->arch.migration_mode; +} + /** * kvm_s390_vcpu_crypto_reset_all * diff --git a/arch/s390/kvm/sigp.c b/arch/s390/kvm/s390/sigp.c index 55c34cb35428..131b3371ef4f 100644 --- a/arch/s390/kvm/sigp.c +++ b/arch/s390/kvm/s390/sigp.c @@ -14,7 +14,7 @@ #include <linux/slab.h> #include <asm/sigp.h> #include "gaccess.h" -#include "kvm-s390.h" +#include "s390.h" #include "trace.h" static int __sigp_sense(struct kvm_vcpu *vcpu, struct kvm_vcpu *dst_vcpu, diff --git a/arch/s390/kvm/trace-s390.h b/arch/s390/kvm/s390/trace-s390.h index 9e28f165c114..9e28f165c114 100644 --- a/arch/s390/kvm/trace-s390.h +++ b/arch/s390/kvm/s390/trace-s390.h diff --git a/arch/s390/kvm/trace.h b/arch/s390/kvm/s390/trace.h index aa419eb6a0c8..3d2fffb1555a 100644 --- a/arch/s390/kvm/trace.h +++ b/arch/s390/kvm/s390/trace.h @@ -45,20 +45,6 @@ TRACE_EVENT(kvm_s390_skey_related_inst, VCPU_TP_PRINTK("%s", "storage key related instruction") ); -TRACE_EVENT(kvm_s390_major_guest_pfault, - TP_PROTO(VCPU_PROTO_COMMON), - TP_ARGS(VCPU_ARGS_COMMON), - - TP_STRUCT__entry( - VCPU_FIELD_COMMON - ), - - TP_fast_assign( - VCPU_ASSIGN_COMMON - ), - VCPU_TP_PRINTK("%s", "major fault, maybe applicable for pfault") - ); - TRACE_EVENT(kvm_s390_pfault_init, TP_PROTO(VCPU_PROTO_COMMON, long pfault_token), TP_ARGS(VCPU_ARGS_COMMON, pfault_token), @@ -283,6 +269,32 @@ TRACE_EVENT(kvm_s390_handle_diag, __print_symbolic(__entry->code, diagnose_codes)) ); +TRACE_EVENT(kvm_s390_diag_9c, + TP_PROTO(VCPU_PROTO_COMMON, int target_vcpu, int target_cpu, + const char *result), + TP_ARGS(VCPU_ARGS_COMMON, target_vcpu, target_cpu, result), + + TP_STRUCT__entry( + VCPU_FIELD_COMMON + __field(int, target_vcpu) + __field(int, target_cpu) + __string(result, result) + ), + + TP_fast_assign( + VCPU_ASSIGN_COMMON + __entry->target_vcpu = target_vcpu; + __entry->target_cpu = target_cpu; + __assign_str(result); + ), + + VCPU_TP_PRINTK( + "diag=9c target_vcpu=%d target_pcpu=%d result=%s", + __entry->target_vcpu, + __entry->target_cpu, + __get_str(result)) + ); + TRACE_EVENT(kvm_s390_handle_lctl, TP_PROTO(VCPU_PROTO_COMMON, int g, int reg1, int reg3, u64 addr), TP_ARGS(VCPU_ARGS_COMMON, g, reg1, reg3, addr), diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/s390/vsie.c index b526621d2a1b..5926689fa226 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/s390/vsie.c @@ -6,23 +6,26 @@ * * Author(s): David Hildenbrand <dahi@linux.vnet.ibm.com> */ +#include <linux/align.h> #include <linux/vmalloc.h> #include <linux/kvm_host.h> #include <linux/bug.h> +#include <linux/compiler.h> #include <linux/list.h> #include <linux/bitmap.h> #include <linux/sched/signal.h> +#include <linux/stddef.h> #include <linux/io.h> #include <linux/mman.h> -#include <asm/gmap.h> #include <asm/mmu_context.h> #include <asm/sclp.h> #include <asm/nmi.h> #include <asm/dis.h> #include <asm/facility.h> -#include "kvm-s390.h" +#include "s390.h" #include "gaccess.h" +#include "gmap.h" enum vsie_page_flags { VSIE_PAGE_IN_USE = 0, @@ -30,10 +33,7 @@ enum vsie_page_flags { struct vsie_page { struct kvm_s390_sie_block scb_s; /* 0x0000 */ - /* - * the backup info for machine check. ensure it's at - * the same offset as that in struct sie_page! - */ + /* backup info for machine check */ struct mcck_volatile_info mcck_info; /* 0x0200 */ /* * The pinned original scb. Be aware that other VCPUs can modify @@ -41,8 +41,11 @@ struct vsie_page { * are reused conditionally, should be accessed via READ_ONCE. */ struct kvm_s390_sie_block *scb_o; /* 0x0218 */ - /* the shadow gmap in use by the vsie_page */ - struct gmap *gmap; /* 0x0220 */ + /* + * Flags: must be set/cleared atomically after the vsie page can be + * looked up by other CPUs. + */ + unsigned long flags; /* 0x0220 */ /* address of the last reported fault to guest2 */ unsigned long fault_addr; /* 0x0228 */ /* calculated guest addresses of satellite control blocks */ @@ -57,33 +60,16 @@ struct vsie_page { * radix tree. */ gpa_t scb_gpa; /* 0x0258 */ - /* - * Flags: must be set/cleared atomically after the vsie page can be - * looked up by other CPUs. - */ - unsigned long flags; /* 0x0260 */ - __u8 reserved[0x0700 - 0x0268]; /* 0x0268 */ - struct kvm_s390_crypto_cb crycb; /* 0x0700 */ - __u8 fac[S390_ARCH_FAC_LIST_SIZE_BYTE]; /* 0x0800 */ + /* the shadow gmap in use by the vsie_page */ + struct gmap_cache gmap_cache; /* 0x0260 */ + __u8 reserved[0x06f8 - 0x0278]; /* 0x0278 */ + struct kvm_s390_crypto_cb crycb; /* 0x06f8 */ + __u8 fac[8 + S390_ARCH_FAC_LIST_SIZE_BYTE];/* 0x07f8 */ }; -/** - * gmap_shadow_valid() - check if a shadow guest address space matches the - * given properties and is still valid - * @sg: pointer to the shadow guest address space structure - * @asce: ASCE for which the shadow table is requested - * @edat_level: edat level to be used for the shadow translation - * - * Returns 1 if the gmap shadow is still valid and matches the given - * properties, the caller can continue using it. Returns 0 otherwise; the - * caller has to request a new shadow gmap in this case. - */ -int gmap_shadow_valid(struct gmap *sg, unsigned long asce, int edat_level) -{ - if (sg->removed) - return 0; - return sg->orig_asce == asce && sg->edat_level == edat_level; -} +static_assert(sizeof(struct vsie_page) == PAGE_SIZE); +static_assert(offsetof(struct vsie_page, mcck_info) == offsetof(struct sie_page, mcck_info)); +static_assert(IS_ALIGNED(offsetof(struct vsie_page, crycb), 8)); /* trigger a validity icpt for the given scb */ static int set_validity_icpt(struct kvm_s390_sie_block *scb, @@ -141,8 +127,8 @@ static int prepare_cpuflags(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) struct kvm_s390_sie_block *scb_o = vsie_page->scb_o; int newflags, cpuflags = atomic_read(&scb_o->cpuflags); - /* we don't allow ESA/390 guests */ - if (!(cpuflags & CPUSTAT_ZARCH)) + /* we don't allow ESA/390 guests unless explicitly enabled */ + if (!(cpuflags & CPUSTAT_ZARCH) && !vcpu->kvm->arch.allow_vsie_esamode) return set_validity_icpt(scb_s, 0x0001U); if (cpuflags & (CPUSTAT_RRF | CPUSTAT_MCDS)) @@ -151,7 +137,9 @@ static int prepare_cpuflags(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) return set_validity_icpt(scb_s, 0x0007U); /* intervention requests will be set later */ - newflags = CPUSTAT_ZARCH; + newflags = 0; + if (cpuflags & CPUSTAT_ZARCH) + newflags = CPUSTAT_ZARCH; if (cpuflags & CPUSTAT_GED && test_kvm_facility(vcpu->kvm, 8)) newflags |= CPUSTAT_GED; if (cpuflags & CPUSTAT_GED2 && test_kvm_facility(vcpu->kvm, 78)) { @@ -184,6 +172,7 @@ static int setup_apcb10(struct kvm_vcpu *vcpu, struct kvm_s390_apcb1 *apcb_s, sizeof(struct kvm_s390_apcb0))) return -EFAULT; + memset(apcb_s, 0, sizeof(*apcb_s)); apcb_s->apm[0] = apcb_h->apm[0] & tmp.apm[0]; apcb_s->aqm[0] = apcb_h->aqm[0] & tmp.aqm[0] & 0xffff000000000000UL; apcb_s->adm[0] = apcb_h->adm[0] & tmp.adm[0] & 0xffff000000000000UL; @@ -401,6 +390,17 @@ end: return 0; } +static void shadow_esa(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) +{ + struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; + + /* Ensure these bits are indeed turned off */ + scb_s->eca &= ~ECA_VX; + scb_s->ecb &= ~(ECB_GS | ECB_TE); + scb_s->ecb3 &= ~ECB3_RI; + scb_s->ecd &= ~ECD_HOSTREGMGMT; +} + /* shadow (round up/down) the ibc to avoid validity icpt */ static void prepare_ibc(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) { @@ -482,7 +482,7 @@ static int shadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; /* READ_ONCE does not work on bitfields - use a temporary variable */ const uint32_t __new_prefix = scb_o->prefix; - const uint32_t new_prefix = READ_ONCE(__new_prefix); + uint32_t new_prefix = READ_ONCE(__new_prefix); const bool wants_tx = READ_ONCE(scb_o->ecb) & ECB_TE; bool had_tx = scb_s->ecb & ECB_TE; unsigned long new_mso = 0; @@ -530,6 +530,11 @@ static int shadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) scb_s->icpua = scb_o->icpua; + if (!(atomic_read(&scb_s->cpuflags) & CPUSTAT_ZARCH)) + new_prefix &= GUEST_PREFIX_MASK_ESA; + else + new_prefix &= GUEST_PREFIX_MASK_ZARCH; + if (!(atomic_read(&scb_s->cpuflags) & CPUSTAT_SM)) new_mso = READ_ONCE(scb_o->mso) & 0xfffffffffff00000UL; /* if the hva of the prefix changes, we have to remap the prefix */ @@ -604,6 +609,9 @@ static int shadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) scb_s->hpid = HPID_VSIE; scb_s->cpnc = scb_o->cpnc; + if (!(atomic_read(&scb_s->cpuflags) & CPUSTAT_ZARCH)) + shadow_esa(vcpu, vsie_page); + prepare_ibc(vcpu, vsie_page); rc = shadow_crycb(vcpu, vsie_page); out: @@ -612,26 +620,17 @@ out: return rc; } -void kvm_s390_vsie_gmap_notifier(struct gmap *gmap, unsigned long start, - unsigned long end) +void kvm_s390_vsie_gmap_notifier(struct gmap *gmap, gpa_t start, gpa_t end) { - struct kvm *kvm = gmap->private; - struct vsie_page *cur; + struct vsie_page *cur, *next; unsigned long prefix; - int i; - if (!gmap_is_shadow(gmap)) - return; + KVM_BUG_ON(!test_bit(GMAP_FLAG_SHADOW, &gmap->flags), gmap->kvm); /* * Only new shadow blocks are added to the list during runtime, * therefore we can safely reference them all the time. */ - for (i = 0; i < kvm->arch.vsie.page_count; i++) { - cur = READ_ONCE(kvm->arch.vsie.pages[i]); - if (!cur) - continue; - if (READ_ONCE(cur->gmap) != gmap) - continue; + list_for_each_entry_safe(cur, next, &gmap->scb_users, gmap_cache.list) { prefix = cur->scb_s.prefix << GUEST_PREFIX_SHIFT; /* with mso/msl, the prefix lies at an offset */ prefix += cur->scb_s.mso; @@ -652,7 +651,7 @@ void kvm_s390_vsie_gmap_notifier(struct gmap *gmap, unsigned long start, * - -EAGAIN if the caller can retry immediately * - -ENOMEM if out of memory */ -static int map_prefix(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) +static int map_prefix(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct gmap *sg) { struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; u64 prefix = scb_s->prefix << GUEST_PREFIX_SHIFT; @@ -667,10 +666,9 @@ static int map_prefix(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) /* with mso/msl, the prefix lies at offset *mso* */ prefix += scb_s->mso; - rc = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, prefix, NULL); + rc = gaccess_shadow_fault(vcpu, sg, prefix, NULL, true); if (!rc && (scb_s->ecb & ECB_TE)) - rc = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, - prefix + PAGE_SIZE, NULL); + rc = gaccess_shadow_fault(vcpu, sg, prefix + PAGE_SIZE, NULL, true); /* * We don't have to mprotect, we will be called for all unshadows. * SIE will detect if protection applies and trigger a validity. @@ -703,7 +701,7 @@ static int pin_guest_page(struct kvm *kvm, gpa_t gpa, hpa_t *hpa) /* Unpins a page previously pinned via pin_guest_page, marking it as dirty. */ static void unpin_guest_page(struct kvm *kvm, gpa_t gpa, hpa_t hpa) { - kvm_release_page_dirty(pfn_to_page(hpa >> PAGE_SHIFT)); + kvm_release_page_dirty(pfn_to_page(phys_to_pfn(hpa))); /* mark the page always as dirty for migration */ mark_page_dirty(kvm, gpa_to_gfn(gpa)); } @@ -951,8 +949,9 @@ static int inject_fault(struct kvm_vcpu *vcpu, __u16 code, __u64 vaddr, * - > 0 if control has to be given to guest 2 * - < 0 if an error occurred */ -static int handle_fault(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) +static int handle_fault(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct gmap *sg) { + bool wr = kvm_s390_cur_gmap_fault_is_write(); int rc; if ((current->thread.gmap_int_code & PGM_INT_CODE_MASK) == PGM_PROTECTION) @@ -960,12 +959,10 @@ static int handle_fault(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) return inject_fault(vcpu, PGM_PROTECTION, current->thread.gmap_teid.addr * PAGE_SIZE, 1); - rc = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, - current->thread.gmap_teid.addr * PAGE_SIZE, NULL); + rc = gaccess_shadow_fault(vcpu, sg, current->thread.gmap_teid.addr * PAGE_SIZE, NULL, wr); if (rc > 0) { rc = inject_fault(vcpu, rc, - current->thread.gmap_teid.addr * PAGE_SIZE, - kvm_s390_cur_gmap_fault_is_write()); + current->thread.gmap_teid.addr * PAGE_SIZE, wr); if (rc >= 0) vsie_page->fault_addr = current->thread.gmap_teid.addr * PAGE_SIZE; } @@ -978,12 +975,10 @@ static int handle_fault(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) * * Will ignore any errors. The next SIE fault will do proper fault handling. */ -static void handle_last_fault(struct kvm_vcpu *vcpu, - struct vsie_page *vsie_page) +static void handle_last_fault(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct gmap *sg) { if (vsie_page->fault_addr) - kvm_s390_shadow_fault(vcpu, vsie_page->gmap, - vsie_page->fault_addr, NULL); + gaccess_shadow_fault(vcpu, sg, vsie_page->fault_addr, NULL, true); vsie_page->fault_addr = 0; } @@ -1008,6 +1003,45 @@ static void retry_vsie_icpt(struct vsie_page *vsie_page) clear_vsie_icpt(vsie_page); } +static int handle_stfle_0(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, + u32 fac_list_origin) +{ + struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; + + /* + * format-0 -> size of nested guest's facility list == guest's size + * guest's size == host's size, since STFLE is interpretatively executed + * using a format-0 for the guest, too. + */ + if (read_guest_real(vcpu, fac_list_origin, &vsie_page->fac, + stfle_size() * sizeof(u64))) + return set_validity_icpt(scb_s, 0x1090U); + scb_s->fac = (u32)virt_to_phys(&vsie_page->fac); + return 0; +} + +static int handle_stfle_2(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, u32 fac_list_origin) +{ + struct kvm_s390_flcb2 *flcb_s = (struct kvm_s390_flcb2 *)vsie_page->fac; + struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; + u64 len; + + if (read_guest_real(vcpu, fac_list_origin, &len, sizeof(len))) + return set_validity_icpt(scb_s, 0x1090U); + + /* discard reserved bits */ + len = (len & U8_MAX); + flcb_s->header_val = len; + len += 1; + + if (read_guest_real(vcpu, fac_list_origin + offsetof(struct kvm_s390_flcb2, facilities), + &flcb_s->facilities, len * sizeof(u64))) + return set_validity_icpt(scb_s, 0x1090U); + + scb_s->fac = (u32)virt_to_phys(&vsie_page->fac) | S390_ARCH_FAC_FORMAT_2; + return 0; +} + /* * Try to shadow + enable the guest 2 provided facility list. * Retry instruction execution if enabled for and provided by guest 2. @@ -1017,29 +1051,34 @@ static void retry_vsie_icpt(struct vsie_page *vsie_page) */ static int handle_stfle(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) { - struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; - __u32 fac = READ_ONCE(vsie_page->scb_o->fac); + bool has_astfleie2 = test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_ASTFLEIE2); + u32 fac = READ_ONCE(vsie_page->scb_o->fac); + int format_mask, format; + u32 origin; + + /* assert no overflow with maximum len */ + BUILD_BUG_ON(sizeof(vsie_page->fac) < ((S390_ARCH_FAC_LIST_SIZE_U64 + 1) * sizeof(u64))); + BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct vsie_page, fac), 8)); - /* - * Alternate-STFLE-Interpretive-Execution facilities are not supported - * -> format-0 flcb - */ if (fac && test_kvm_facility(vcpu->kvm, 7)) { retry_vsie_icpt(vsie_page); /* * The facility list origin (FLO) is in bits 1 - 28 of the FLD * so we need to mask here before reading. */ - fac = fac & 0x7ffffff8U; - /* - * format-0 -> size of nested guest's facility list == guest's size - * guest's size == host's size, since STFLE is interpretatively executed - * using a format-0 for the guest, too. - */ - if (read_guest_real(vcpu, fac, &vsie_page->fac, - stfle_size() * sizeof(u64))) - return set_validity_icpt(scb_s, 0x1090U); - scb_s->fac = (u32)virt_to_phys(&vsie_page->fac); + origin = fac & 0x7ffffff8U; + format_mask = has_astfleie2 ? 3 : 0; + format = fac & format_mask; + switch (format) { + case 0: + return handle_stfle_0(vcpu, vsie_page, origin); + case 1: + return set_validity_icpt(&vsie_page->scb_s, 0x1330U); + case 2: + return handle_stfle_2(vcpu, vsie_page, origin); + case 3: + return set_validity_icpt(&vsie_page->scb_s, 0x1330U); + } } return 0; } @@ -1065,11 +1104,12 @@ static u64 vsie_get_register(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, } } -static int vsie_handle_mvpg(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) +static int vsie_handle_mvpg(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct gmap *sg) { struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; - unsigned long pei_dest, pei_src, src, dest, mask, prefix; + unsigned long src, dest, mask, prefix; u64 *pei_block = &vsie_page->scb_o->mcic; + union mvpg_pei pei_dest, pei_src; int edat, rc_dest, rc_src; union ctlreg0 cr0; @@ -1083,8 +1123,8 @@ static int vsie_handle_mvpg(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) src = vsie_get_register(vcpu, vsie_page, scb_s->ipb >> 16) & mask; src = _kvm_s390_real_to_abs(prefix, src) + scb_s->mso; - rc_dest = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, dest, &pei_dest); - rc_src = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, src, &pei_src); + rc_dest = gaccess_shadow_fault(vcpu, sg, dest, &pei_dest, true); + rc_src = gaccess_shadow_fault(vcpu, sg, src, &pei_src, false); /* * Either everything went well, or something non-critical went wrong * e.g. because of a race. In either case, simply retry. @@ -1119,8 +1159,8 @@ static int vsie_handle_mvpg(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) rc_src = rc_src != PGM_PAGE_TRANSLATION ? rc_src : 0; } if (!rc_dest && !rc_src) { - pei_block[0] = pei_dest; - pei_block[1] = pei_src; + pei_block[0] = pei_dest.val; + pei_block[1] = pei_src.val; return 1; } @@ -1144,16 +1184,17 @@ static int vsie_handle_mvpg(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) * - > 0 if control has to be given to guest 2 * - < 0 if an error occurred */ -static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) +static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct gmap *sg) __releases(vcpu->kvm->srcu) __acquires(vcpu->kvm->srcu) { struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; struct kvm_s390_sie_block *scb_o = vsie_page->scb_o; + unsigned long sie_return = SIE64_RETURN_NORMAL; int guest_bp_isolation; int rc = 0; - handle_last_fault(vcpu, vsie_page); + handle_last_fault(vcpu, vsie_page, sg); kvm_vcpu_srcu_read_unlock(vcpu); @@ -1191,7 +1232,7 @@ xfer_to_guest_mode_check: goto xfer_to_guest_mode_check; } guest_timing_enter_irqoff(); - rc = kvm_s390_enter_exit_sie(scb_s, vcpu->run->s.regs.gprs, vsie_page->gmap->asce); + sie_return = kvm_s390_enter_exit_sie(scb_s, vcpu->run->s.regs.gprs, sg->asce.val); guest_timing_exit_irqoff(); local_irq_enable(); } @@ -1206,16 +1247,17 @@ skip_sie: kvm_vcpu_srcu_read_lock(vcpu); - if (rc == -EINTR) { - VCPU_EVENT(vcpu, 3, "%s", "machine check"); + if (sie_return == SIE64_RETURN_MCCK) { kvm_s390_reinject_machine_check(vcpu, &vsie_page->mcck_info); return 0; } + WARN_ON_ONCE(sie_return != SIE64_RETURN_NORMAL); + if (rc > 0) rc = 0; /* we could still have an icpt */ else if (current->thread.gmap_int_code) - return handle_fault(vcpu, vsie_page); + return handle_fault(vcpu, vsie_page, sg); switch (scb_s->icptcode) { case ICPT_INST: @@ -1233,7 +1275,7 @@ skip_sie: break; case ICPT_PARTEXEC: if (scb_s->ipa == 0xb254) - rc = vsie_handle_mvpg(vcpu, vsie_page); + rc = vsie_handle_mvpg(vcpu, vsie_page, sg); break; } return rc; @@ -1241,43 +1283,67 @@ skip_sie: static void release_gmap_shadow(struct vsie_page *vsie_page) { - if (vsie_page->gmap) - gmap_put(vsie_page->gmap); - WRITE_ONCE(vsie_page->gmap, NULL); + struct gmap *gmap = vsie_page->gmap_cache.gmap; + + lockdep_assert_held(&gmap->kvm->arch.gmap->children_lock); + + list_del(&vsie_page->gmap_cache.list); + vsie_page->gmap_cache.gmap = NULL; prefix_unmapped(vsie_page); + + if (list_empty(&gmap->scb_users)) { + gmap_remove_child(gmap); + gmap_put(gmap); + } } -static int acquire_gmap_shadow(struct kvm_vcpu *vcpu, - struct vsie_page *vsie_page) +static struct gmap *acquire_gmap_shadow(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) { - unsigned long asce; union ctlreg0 cr0; struct gmap *gmap; + union asce asce; int edat; - asce = vcpu->arch.sie_block->gcr[1]; + asce.val = vcpu->arch.sie_block->gcr[1]; cr0.val = vcpu->arch.sie_block->gcr[0]; edat = cr0.edat && test_kvm_facility(vcpu->kvm, 8); edat += edat && test_kvm_facility(vcpu->kvm, 78); - /* - * ASCE or EDAT could have changed since last icpt, or the gmap - * we're holding has been unshadowed. If the gmap is still valid, - * we can safely reuse it. - */ - if (vsie_page->gmap && gmap_shadow_valid(vsie_page->gmap, asce, edat)) { - vcpu->kvm->stat.gmap_shadow_reuse++; - return 0; + scoped_guard(spinlock, &vcpu->kvm->arch.gmap->children_lock) { + gmap = vsie_page->gmap_cache.gmap; + if (gmap) { + /* + * ASCE or EDAT could have changed since last icpt, or the gmap + * we're holding has been unshadowed. If the gmap is still valid, + * we can safely reuse it. + */ + if (gmap_is_shadow_valid(gmap, asce, edat)) { + vcpu->kvm->stat.gmap_shadow_reuse++; + gmap_get(gmap); + return gmap; + } + /* release the old shadow and mark the prefix as unmapped */ + release_gmap_shadow(vsie_page); + } } - - /* release the old shadow - if any, and mark the prefix as unmapped */ - release_gmap_shadow(vsie_page); - gmap = gmap_shadow(vcpu->arch.gmap, asce, edat); +again: + gmap = gmap_create_shadow(vcpu->arch.mc, vcpu->kvm->arch.gmap, asce, edat); if (IS_ERR(gmap)) - return PTR_ERR(gmap); - vcpu->kvm->stat.gmap_shadow_create++; - WRITE_ONCE(vsie_page->gmap, gmap); - return 0; + return gmap; + scoped_guard(spinlock, &vcpu->kvm->arch.gmap->children_lock) { + /* unlikely race condition, remove the previous shadow */ + if (vsie_page->gmap_cache.gmap) + release_gmap_shadow(vsie_page); + if (!gmap->parent) { + gmap_put(gmap); + goto again; + } + vcpu->kvm->stat.gmap_shadow_create++; + list_add(&vsie_page->gmap_cache.list, &gmap->scb_users); + vsie_page->gmap_cache.gmap = gmap; + prefix_unmapped(vsie_page); + } + return gmap; } /* @@ -1330,15 +1396,20 @@ static void unregister_shadow_scb(struct kvm_vcpu *vcpu) static int vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) { struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; + struct gmap *sg = NULL; int rc = 0; while (1) { - rc = acquire_gmap_shadow(vcpu, vsie_page); + sg = acquire_gmap_shadow(vcpu, vsie_page); + if (IS_ERR(sg)) { + rc = PTR_ERR(sg); + sg = NULL; + } if (!rc) - rc = map_prefix(vcpu, vsie_page); + rc = map_prefix(vcpu, vsie_page, sg); if (!rc) { update_intervention_requests(vsie_page); - rc = do_vsie_run(vcpu, vsie_page); + rc = do_vsie_run(vcpu, vsie_page, sg); } atomic_andnot(PROG_BLOCK_SIE, &scb_s->prog20); @@ -1361,7 +1432,11 @@ static int vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) kvm_s390_rewind_psw(vcpu, 4); break; } + if (sg) + sg = gmap_put(sg); } + if (sg) + sg = gmap_put(sg); if (rc == -EFAULT) { /* @@ -1410,7 +1485,7 @@ static struct vsie_page *get_vsie_page(struct kvm *kvm, unsigned long addr) int nr_vcpus; rcu_read_lock(); - vsie_page = radix_tree_lookup(&kvm->arch.vsie.addr_to_page, addr >> 9); + vsie_page = radix_tree_lookup(&kvm->arch.vsie.addr_to_page, addr >> SCB_ALIGNMENT_SHIFT); rcu_read_unlock(); if (vsie_page) { if (try_get_vsie_page(vsie_page)) { @@ -1451,13 +1526,13 @@ static struct vsie_page *get_vsie_page(struct kvm *kvm, unsigned long addr) } if (vsie_page->scb_gpa != ULONG_MAX) radix_tree_delete(&kvm->arch.vsie.addr_to_page, - vsie_page->scb_gpa >> 9); + vsie_page->scb_gpa >> SCB_ALIGNMENT_SHIFT); } /* Mark it as invalid until it resides in the tree. */ vsie_page->scb_gpa = ULONG_MAX; /* Double use of the same address or allocation failure. */ - if (radix_tree_insert(&kvm->arch.vsie.addr_to_page, addr >> 9, + if (radix_tree_insert(&kvm->arch.vsie.addr_to_page, addr >> SCB_ALIGNMENT_SHIFT, vsie_page)) { put_vsie_page(vsie_page); mutex_unlock(&kvm->arch.vsie.mutex); @@ -1467,7 +1542,12 @@ static struct vsie_page *get_vsie_page(struct kvm *kvm, unsigned long addr) mutex_unlock(&kvm->arch.vsie.mutex); memset(&vsie_page->scb_s, 0, sizeof(struct kvm_s390_sie_block)); - release_gmap_shadow(vsie_page); + if (vsie_page->gmap_cache.gmap) { + scoped_guard(spinlock, &kvm->arch.gmap->children_lock) + if (vsie_page->gmap_cache.gmap) + release_gmap_shadow(vsie_page); + } + prefix_unmapped(vsie_page); vsie_page->fault_addr = 0; vsie_page->scb_s.ihcpu = 0xffffU; return vsie_page; @@ -1485,7 +1565,6 @@ int kvm_s390_handle_vsie(struct kvm_vcpu *vcpu) if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); - BUILD_BUG_ON(sizeof(struct vsie_page) != PAGE_SIZE); scb_addr = kvm_s390_get_base_disp_s(vcpu, NULL); /* 512 byte alignment */ @@ -1498,11 +1577,13 @@ int kvm_s390_handle_vsie(struct kvm_vcpu *vcpu) } vsie_page = get_vsie_page(vcpu->kvm, scb_addr); - if (IS_ERR(vsie_page)) + if (IS_ERR(vsie_page)) { return PTR_ERR(vsie_page); - else if (!vsie_page) + } else if (!vsie_page) { /* double use of sie control block - simply do nothing */ + kvm_s390_rewind_psw(vcpu, 4); return 0; + } rc = pin_scb(vcpu, vsie_page, scb_addr); if (rc) @@ -1543,12 +1624,14 @@ void kvm_s390_vsie_destroy(struct kvm *kvm) mutex_lock(&kvm->arch.vsie.mutex); for (i = 0; i < kvm->arch.vsie.page_count; i++) { vsie_page = kvm->arch.vsie.pages[i]; + scoped_guard(spinlock, &kvm->arch.gmap->children_lock) + if (vsie_page->gmap_cache.gmap) + release_gmap_shadow(vsie_page); kvm->arch.vsie.pages[i] = NULL; - release_gmap_shadow(vsie_page); /* free the radix tree entry */ if (vsie_page->scb_gpa != ULONG_MAX) radix_tree_delete(&kvm->arch.vsie.addr_to_page, - vsie_page->scb_gpa >> 9); + vsie_page->scb_gpa >> SCB_ALIGNMENT_SHIFT); free_page((unsigned long)vsie_page); } kvm->arch.vsie.page_count = 0; diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile index f43f897d3fc0..bb707700d7af 100644 --- a/arch/s390/lib/Makefile +++ b/arch/s390/lib/Makefile @@ -3,9 +3,15 @@ # Makefile for s390-specific library files.. # +CONTEXT_ANALYSIS := y + +# string.o implements standard library functions like memset/memcpy etc. +# Use -ffreestanding to ensure that the compiler does not try to "optimize" +# them into calls to themselves. +CFLAGS_string.o = -ffreestanding + lib-y += delay.o string.o uaccess.o find.o spinlock.o tishift.o lib-y += csum-partial.o -obj-y += mem.o xor.o lib-$(CONFIG_KPROBES) += probes.o lib-$(CONFIG_UPROBES) += probes.o obj-$(CONFIG_S390_KPROBES_SANITY_TEST) += test_kprobes_s390.o diff --git a/arch/s390/lib/csum-partial.c b/arch/s390/lib/csum-partial.c index 458abd9bac70..9d74ceff136c 100644 --- a/arch/s390/lib/csum-partial.c +++ b/arch/s390/lib/csum-partial.c @@ -23,7 +23,7 @@ static __always_inline __wsum csum_copy(void *dst, const void *src, int len, __w if (!cpu_has_vx()) { if (copy) memcpy(dst, src, len); - return cksm(dst, len, sum); + return cksm(src, len, sum); } kernel_fpu_begin(&vxstate, KERNEL_VXR_V16V23); fpu_vlvgf(16, (__force u32)sum, 1); diff --git a/arch/s390/lib/mem.S b/arch/s390/lib/mem.S deleted file mode 100644 index d026debf250c..000000000000 --- a/arch/s390/lib/mem.S +++ /dev/null @@ -1,192 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * String handling functions. - * - * Copyright IBM Corp. 2012 - */ - -#include <linux/export.h> -#include <linux/linkage.h> -#include <asm/nospec-insn.h> - - GEN_BR_THUNK %r14 - -/* - * void *memmove(void *dest, const void *src, size_t n) - */ -SYM_FUNC_START(__memmove) - ltgr %r4,%r4 - lgr %r1,%r2 - jz .Lmemmove_exit - aghi %r4,-1 - clgr %r2,%r3 - jnh .Lmemmove_forward - la %r5,1(%r4,%r3) - clgr %r2,%r5 - jl .Lmemmove_reverse -.Lmemmove_forward: - srlg %r0,%r4,8 - ltgr %r0,%r0 - jz .Lmemmove_forward_remainder -.Lmemmove_forward_loop: - mvc 0(256,%r1),0(%r3) - la %r1,256(%r1) - la %r3,256(%r3) - brctg %r0,.Lmemmove_forward_loop -.Lmemmove_forward_remainder: - exrl %r4,.Lmemmove_mvc -.Lmemmove_exit: - BR_EX %r14 -.Lmemmove_reverse: - ic %r0,0(%r4,%r3) - stc %r0,0(%r4,%r1) - brctg %r4,.Lmemmove_reverse - ic %r0,0(%r4,%r3) - stc %r0,0(%r4,%r1) - BR_EX %r14 -.Lmemmove_mvc: - mvc 0(1,%r1),0(%r3) -SYM_FUNC_END(__memmove) -EXPORT_SYMBOL(__memmove) - -SYM_FUNC_ALIAS(memmove, __memmove) -EXPORT_SYMBOL(memmove) - -/* - * memset implementation - * - * This code corresponds to the C construct below. We do distinguish - * between clearing (c == 0) and setting a memory array (c != 0) simply - * because nearly all memset invocations in the kernel clear memory and - * the xc instruction is preferred in such cases. - * - * void *memset(void *s, int c, size_t n) - * { - * if (likely(c == 0)) - * return __builtin_memset(s, 0, n); - * return __builtin_memset(s, c, n); - * } - */ -SYM_FUNC_START(__memset) - ltgr %r4,%r4 - jz .Lmemset_exit - ltgr %r3,%r3 - jnz .Lmemset_fill - aghi %r4,-1 - srlg %r3,%r4,8 - ltgr %r3,%r3 - lgr %r1,%r2 - jz .Lmemset_clear_remainder -.Lmemset_clear_loop: - xc 0(256,%r1),0(%r1) - la %r1,256(%r1) - brctg %r3,.Lmemset_clear_loop -.Lmemset_clear_remainder: - exrl %r4,.Lmemset_xc -.Lmemset_exit: - BR_EX %r14 -.Lmemset_fill: - cghi %r4,1 - lgr %r1,%r2 - je .Lmemset_fill_exit - aghi %r4,-2 - srlg %r5,%r4,8 - ltgr %r5,%r5 - jz .Lmemset_fill_remainder -.Lmemset_fill_loop: - stc %r3,0(%r1) - mvc 1(255,%r1),0(%r1) - la %r1,256(%r1) - brctg %r5,.Lmemset_fill_loop -.Lmemset_fill_remainder: - stc %r3,0(%r1) - exrl %r4,.Lmemset_mvc - BR_EX %r14 -.Lmemset_fill_exit: - stc %r3,0(%r1) - BR_EX %r14 -.Lmemset_xc: - xc 0(1,%r1),0(%r1) -.Lmemset_mvc: - mvc 1(1,%r1),0(%r1) -SYM_FUNC_END(__memset) -EXPORT_SYMBOL(__memset) - -SYM_FUNC_ALIAS(memset, __memset) -EXPORT_SYMBOL(memset) - -/* - * memcpy implementation - * - * void *memcpy(void *dest, const void *src, size_t n) - */ -SYM_FUNC_START(__memcpy) - ltgr %r4,%r4 - jz .Lmemcpy_exit - aghi %r4,-1 - srlg %r5,%r4,8 - ltgr %r5,%r5 - lgr %r1,%r2 - jnz .Lmemcpy_loop -.Lmemcpy_remainder: - exrl %r4,.Lmemcpy_mvc -.Lmemcpy_exit: - BR_EX %r14 -.Lmemcpy_loop: - mvc 0(256,%r1),0(%r3) - la %r1,256(%r1) - la %r3,256(%r3) - brctg %r5,.Lmemcpy_loop - j .Lmemcpy_remainder -.Lmemcpy_mvc: - mvc 0(1,%r1),0(%r3) -SYM_FUNC_END(__memcpy) -EXPORT_SYMBOL(__memcpy) - -SYM_FUNC_ALIAS(memcpy, __memcpy) -EXPORT_SYMBOL(memcpy) - -/* - * __memset16/32/64 - * - * void *__memset16(uint16_t *s, uint16_t v, size_t count) - * void *__memset32(uint32_t *s, uint32_t v, size_t count) - * void *__memset64(uint64_t *s, uint64_t v, size_t count) - */ -.macro __MEMSET bits,bytes,insn -SYM_FUNC_START(__memset\bits) - ltgr %r4,%r4 - jz .L__memset_exit\bits - cghi %r4,\bytes - je .L__memset_store\bits - aghi %r4,-(\bytes+1) - srlg %r5,%r4,8 - ltgr %r5,%r5 - lgr %r1,%r2 - jz .L__memset_remainder\bits -.L__memset_loop\bits: - \insn %r3,0(%r1) - mvc \bytes(256-\bytes,%r1),0(%r1) - la %r1,256(%r1) - brctg %r5,.L__memset_loop\bits -.L__memset_remainder\bits: - \insn %r3,0(%r1) - exrl %r4,.L__memset_mvc\bits - BR_EX %r14 -.L__memset_store\bits: - \insn %r3,0(%r2) -.L__memset_exit\bits: - BR_EX %r14 -.L__memset_mvc\bits: - mvc \bytes(1,%r1),0(%r1) -SYM_FUNC_END(__memset\bits) -.endm - -__MEMSET 16,2,sth -EXPORT_SYMBOL(__memset16) - -__MEMSET 32,4,st -EXPORT_SYMBOL(__memset32) - -__MEMSET 64,8,stg -EXPORT_SYMBOL(__memset64) diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c index 10db1e56a811..dbabca35c008 100644 --- a/arch/s390/lib/spinlock.c +++ b/arch/s390/lib/spinlock.c @@ -18,6 +18,7 @@ #include <asm/alternative.h> #include <asm/machine.h> #include <asm/asm.h> +#include <trace/events/lock.h> int spin_retry = -1; @@ -281,10 +282,12 @@ static inline void arch_spin_lock_classic(arch_spinlock_t *lp) void arch_spin_lock_wait(arch_spinlock_t *lp) { + trace_contention_begin(lp, LCB_F_SPIN); if (test_cpu_flag(CIF_DEDICATED_CPU)) arch_spin_lock_queued(lp); else arch_spin_lock_classic(lp); + trace_contention_end(lp, 0); } EXPORT_SYMBOL(arch_spin_lock_wait); diff --git a/arch/s390/lib/string.c b/arch/s390/lib/string.c index 757f58960198..32e0e6b1e623 100644 --- a/arch/s390/lib/string.c +++ b/arch/s390/lib/string.c @@ -15,8 +15,218 @@ #include <linux/types.h> #include <linux/string.h> #include <linux/export.h> +#include <asm/facility.h> #include <asm/asm.h> +#define SYMBOL_FUNCTION_ALIAS(alias, name) \ +asm(".globl " __stringify(alias) "\n\t" \ + ".set " __stringify(alias) "," __stringify(name)) + +#ifdef __HAVE_ARCH_MEMMOVE +noinstr void *__memmove(void *dest, const void *src, size_t n) +{ + const char *s = src; + char *d = dest; + + if (!n) + return dest; + if ((d <= s || d >= s + n)) { + /* Forward copy */ + while (n >= 256) { + asm volatile( + " mvc 0(256,%[d]),0(%[s])\n" + : + : [d] "a" (d), [s] "a" (s) + : "memory"); + d += 256; + s += 256; + n -= 256; + } + if (n) { + asm volatile( + " exrl %[n],0f\n" + " j 1f\n" + "0: mvc 0(1,%[d]),0(%[s])\n" + "1:" + : + : [d] "a" (d), [s] "a" (s), [n] "a" (n - 1) + : "memory"); + } + return dest; + } + /* Backward copy */ + if (test_facility(61)) { + /* Use mvcrl instruction if available */ + while (n >= 256) { + asm volatile( + " lghi %%r0,255\n" + " .insn sse,0xe50a00000000,%[d],%[s]\n" + : [d] "=Q" (*(d + n - 256)) + : [s] "Q" (*(s + n - 256)) + : "0", "memory"); + n -= 256; + } + if (n) { + asm volatile( + " lgr %%r0,%[n]\n" + " .insn sse,0xe50a00000000,%[d],%[s]\n" + : [d] "=Q" (*d) + : [s] "Q" (*s), [n] "d" (n - 1) + : "0", "memory"); + } + } else { + while (n--) + d[n] = s[n]; + } + return dest; +} +SYMBOL_FUNCTION_ALIAS(memmove, __memmove); +EXPORT_SYMBOL(__memmove); +EXPORT_SYMBOL(memmove); +#endif + +#ifdef __HAVE_ARCH_MEMSET +noinstr void *__memset(void *s, int c, size_t n) +{ + char *xs = s; + + if (!n) + return s; + if (!c) { + /* Clear memory */ + while (n >= 256) { + asm volatile( + " xc 0(256,%[xs]),0(%[xs])" + : + : [xs] "a" (xs) + : "cc", "memory"); + xs += 256; + n -= 256; + } + if (!n) + return s; + asm volatile( + " exrl %[n],0f\n" + " j 1f\n" + "0: xc 0(1,%[xs]),0(%[xs])\n" + "1:" + : + : [xs] "a" (xs), [n] "a" (n - 1) + : "cc", "memory"); + } else { + /* Fill memory */ + while (n >= 256) { + *xs = c; + asm volatile( + " mvc 1(255,%[xs]),0(%[xs])" + : + : [xs] "a" (xs) + : "memory"); + xs += 256; + n -= 256; + } + if (!n) + return s; + *xs = c; + if (n == 1) + return s; + asm volatile( + " exrl %[n],0f\n" + " j 1f\n" + "0: mvc 1(1,%[xs]),0(%[xs])\n" + "1:" + : + : [xs] "a" (xs), [n] "a" (n - 2) + : "memory"); + } + return s; +} +SYMBOL_FUNCTION_ALIAS(memset, __memset); +EXPORT_SYMBOL(__memset); +EXPORT_SYMBOL(memset); +#endif + +#ifdef __HAVE_ARCH_MEMCPY +noinstr void *__memcpy(void *dest, const void *src, size_t n) +{ + void *d = dest; + + if (!n) + return d; + while (n >= 256) { + asm volatile( + " mvc 0(256,%[dest]),0(%[src])" + : + : [dest] "a" (dest), [src] "a" (src) + : "memory"); + dest += 256; + src += 256; + n -= 256; + } + if (!n) + return d; + asm volatile( + " exrl %[n],1f\n" + " j 2f\n" + "1: mvc 0(1,%[dest]),0(%[src])\n" + "2:" + : + : [dest] "a" (dest), [src] "a" (src), [n] "a" (n - 1) + : "memory"); + return d; +} +SYMBOL_FUNCTION_ALIAS(memcpy, __memcpy); +EXPORT_SYMBOL(__memcpy); +EXPORT_SYMBOL(memcpy); +#endif + +#define DEFINE_MEMSET(_bits, _bytes, _type) \ +void *__memset##_bits(_type *s, _type v, size_t n) \ +{ \ + _type *xs = s; \ + \ + if (!n) \ + return s; \ + while (n >= 256) { \ + *xs = v; \ + asm volatile( \ + " mvc %[_b](256-%[_b],%[xs]),0(%[xs])\n" \ + : \ + : [xs] "a" (xs), [_b] "i" (_bytes) \ + : "memory"); \ + xs = (_type *)((char *)xs + 256); \ + n -= 256; \ + } \ + if (!n) \ + return s; \ + *xs = v; \ + if (n == _bytes) \ + return s; \ + n -= _bytes + 1; \ + asm volatile( \ + " exrl %[n],1f\n" \ + " j 2f\n" \ + "1: mvc %[_b](1,%[xs]),0(%[xs])\n" \ + "2:" \ + : \ + : [n] "a" (n), [xs] "a" (xs), [_b] "i" (_bytes) \ + : "memory"); \ + return s; \ +} \ +EXPORT_SYMBOL(__memset##_bits) + +#ifdef __HAVE_ARCH_MEMSET16 +DEFINE_MEMSET(16, 2, uint16_t); +#endif + +#ifdef __HAVE_ARCH_MEMSET32 +DEFINE_MEMSET(32, 4, uint32_t); +#endif + +#ifdef __HAVE_ARCH_MEMSET64 +DEFINE_MEMSET(64, 8, uint64_t); +#endif + /* * Helper functions to find the end of a string */ @@ -105,32 +315,6 @@ EXPORT_SYMBOL(strcat); #endif /** - * strlcat - Append a length-limited, %NUL-terminated string to another - * @dest: The string to be appended to - * @src: The string to append to it - * @n: The size of the destination buffer. - */ -#ifdef __HAVE_ARCH_STRLCAT -size_t strlcat(char *dest, const char *src, size_t n) -{ - size_t dsize = __strend(dest) - dest; - size_t len = __strend(src) - src; - size_t res = dsize + len; - - if (dsize < n) { - dest += dsize; - n -= dsize; - if (len >= n) - len = n - 1; - dest[len] = '\0'; - memcpy(dest, src, len); - } - return res; -} -EXPORT_SYMBOL(strlcat); -#endif - -/** * strncat - Append a length-limited, %NUL-terminated string to another * @dest: The string to be appended to * @src: The string to append to it diff --git a/arch/s390/lib/tishift.S b/arch/s390/lib/tishift.S deleted file mode 100644 index 96214f51f49b..000000000000 --- a/arch/s390/lib/tishift.S +++ /dev/null @@ -1,63 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -#include <linux/export.h> -#include <linux/linkage.h> -#include <asm/nospec-insn.h> - - .section .noinstr.text, "ax" - - GEN_BR_THUNK %r14 - -SYM_FUNC_START(__ashlti3) - lmg %r0,%r1,0(%r3) - cije %r4,0,1f - lhi %r3,64 - sr %r3,%r4 - jnh 0f - srlg %r3,%r1,0(%r3) - sllg %r0,%r0,0(%r4) - sllg %r1,%r1,0(%r4) - ogr %r0,%r3 - j 1f -0: sllg %r0,%r1,-64(%r4) - lghi %r1,0 -1: stmg %r0,%r1,0(%r2) - BR_EX %r14 -SYM_FUNC_END(__ashlti3) -EXPORT_SYMBOL(__ashlti3) - -SYM_FUNC_START(__ashrti3) - lmg %r0,%r1,0(%r3) - cije %r4,0,1f - lhi %r3,64 - sr %r3,%r4 - jnh 0f - sllg %r3,%r0,0(%r3) - srlg %r1,%r1,0(%r4) - srag %r0,%r0,0(%r4) - ogr %r1,%r3 - j 1f -0: srag %r1,%r0,-64(%r4) - srag %r0,%r0,63 -1: stmg %r0,%r1,0(%r2) - BR_EX %r14 -SYM_FUNC_END(__ashrti3) -EXPORT_SYMBOL(__ashrti3) - -SYM_FUNC_START(__lshrti3) - lmg %r0,%r1,0(%r3) - cije %r4,0,1f - lhi %r3,64 - sr %r3,%r4 - jnh 0f - sllg %r3,%r0,0(%r3) - srlg %r1,%r1,0(%r4) - srlg %r0,%r0,0(%r4) - ogr %r1,%r3 - j 1f -0: srlg %r1,%r0,-64(%r4) - lghi %r0,0 -1: stmg %r0,%r1,0(%r2) - BR_EX %r14 -SYM_FUNC_END(__lshrti3) -EXPORT_SYMBOL(__lshrti3) diff --git a/arch/s390/lib/tishift.c b/arch/s390/lib/tishift.c new file mode 100644 index 000000000000..bb16cf639af3 --- /dev/null +++ b/arch/s390/lib/tishift.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/export.h> +#include <linux/types.h> +#include "tishift.h" + +union ti { + __int128_t val; + struct { + u64 high; + u64 low; + }; +}; + +noinstr __int128_t __ashlti3(__int128_t a, int shift) +{ + union ti ti = { .val = a }; + + if (!shift) + return ti.val; + if (shift < 64) { + ti.high = (ti.high << shift) | (ti.low >> (64 - shift)); + ti.low = ti.low << shift; + } else { + ti.high = ti.low << (shift - 64); + ti.low = 0; + } + return ti.val; +} +EXPORT_SYMBOL(__ashlti3); + +noinstr __int128_t __ashrti3(__int128_t a, int shift) +{ + union ti ti = { .val = a }; + + if (!shift) + return ti.val; + if (shift < 64) { + ti.low = (ti.low >> shift) | (ti.high << (64 - shift)); + ti.high = (int64_t)ti.high >> shift; + } else { + ti.low = (int64_t)ti.high >> (shift - 64); + ti.high = (int64_t)ti.high >> 63; + } + return ti.val; +} +EXPORT_SYMBOL(__ashrti3); + +noinstr __int128_t __lshrti3(__int128_t a, int shift) +{ + union ti ti = { .val = a }; + + if (!shift) + return ti.val; + if (shift < 64) { + ti.low = (ti.low >> shift) | (ti.high << (64 - shift)); + ti.high = ti.high >> shift; + } else { + ti.low = ti.high >> (shift - 64); + ti.high = 0; + } + return ti.val; +} +EXPORT_SYMBOL(__lshrti3); diff --git a/arch/s390/lib/tishift.h b/arch/s390/lib/tishift.h new file mode 100644 index 000000000000..43a9b8c8e545 --- /dev/null +++ b/arch/s390/lib/tishift.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _S390_LIB_TISHIFT_H +#define _S390_LIB_TISHIFT_H + +__int128_t __ashlti3(__int128_t a, int b); +__int128_t __ashrti3(__int128_t a, int b); +__int128_t __lshrti3(__int128_t a, int b); + +#endif /* _S390_LIB_TISHIFT_H */ diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index 1a6ba105e071..0ac2f3998b14 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -34,136 +34,19 @@ void debug_user_asce(int exit) } #endif /*CONFIG_DEBUG_ENTRY */ -union oac { - unsigned int val; - struct { - struct { - unsigned short key : 4; - unsigned short : 4; - unsigned short as : 2; - unsigned short : 4; - unsigned short k : 1; - unsigned short a : 1; - } oac1; - struct { - unsigned short key : 4; - unsigned short : 4; - unsigned short as : 2; - unsigned short : 4; - unsigned short k : 1; - unsigned short a : 1; - } oac2; - }; -}; - -static uaccess_kmsan_or_inline __must_check unsigned long -raw_copy_from_user_key(void *to, const void __user *from, unsigned long size, unsigned long key) -{ - unsigned long osize; - union oac spec = { - .oac2.key = key, - .oac2.as = PSW_BITS_AS_SECONDARY, - .oac2.k = 1, - .oac2.a = 1, - }; - int cc; - - while (1) { - osize = size; - asm_inline volatile( - " lr %%r0,%[spec]\n" - "0: mvcos %[to],%[from],%[size]\n" - "1: nopr %%r7\n" - CC_IPM(cc) - EX_TABLE_UA_MVCOS_FROM(0b, 0b) - EX_TABLE_UA_MVCOS_FROM(1b, 0b) - : CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char *)to) - : [spec] "d" (spec.val), [from] "Q" (*(const char __user *)from) - : CC_CLOBBER_LIST("memory", "0")); - if (CC_TRANSFORM(cc) == 0) - return osize - size; - size -= 4096; - to += 4096; - from += 4096; - } -} - -unsigned long _copy_from_user_key(void *to, const void __user *from, - unsigned long n, unsigned long key) -{ - unsigned long res = n; - - might_fault(); - if (!should_fail_usercopy()) { - instrument_copy_from_user_before(to, from, n); - res = raw_copy_from_user_key(to, from, n, key); - instrument_copy_from_user_after(to, from, n, res); - } - if (unlikely(res)) - memset(to + (n - res), 0, res); - return res; -} -EXPORT_SYMBOL(_copy_from_user_key); - -static uaccess_kmsan_or_inline __must_check unsigned long -raw_copy_to_user_key(void __user *to, const void *from, unsigned long size, unsigned long key) -{ - unsigned long osize; - union oac spec = { - .oac1.key = key, - .oac1.as = PSW_BITS_AS_SECONDARY, - .oac1.k = 1, - .oac1.a = 1, - }; - int cc; - - while (1) { - osize = size; - asm_inline volatile( - " lr %%r0,%[spec]\n" - "0: mvcos %[to],%[from],%[size]\n" - "1: nopr %%r7\n" - CC_IPM(cc) - EX_TABLE_UA_MVCOS_TO(0b, 0b) - EX_TABLE_UA_MVCOS_TO(1b, 0b) - : CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char __user *)to) - : [spec] "d" (spec.val), [from] "Q" (*(const char *)from) - : CC_CLOBBER_LIST("memory", "0")); - if (CC_TRANSFORM(cc) == 0) - return osize - size; - size -= 4096; - to += 4096; - from += 4096; - } -} - -unsigned long _copy_to_user_key(void __user *to, const void *from, - unsigned long n, unsigned long key) -{ - might_fault(); - if (should_fail_usercopy()) - return n; - instrument_copy_to_user(to, from, n); - return raw_copy_to_user_key(to, from, n, key); -} -EXPORT_SYMBOL(_copy_to_user_key); - #define CMPXCHG_USER_KEY_MAX_LOOPS 128 -static nokprobe_inline int __cmpxchg_user_key_small(unsigned long address, unsigned int *uval, - unsigned int old, unsigned int new, - unsigned int mask, unsigned long key) +static nokprobe_inline int __cmpxchg_key_small(void *address, unsigned int *uval, + unsigned int old, unsigned int new, + unsigned int mask, unsigned long key) { unsigned long count; unsigned int prev; - bool sacf_flag; int rc = 0; skey_regions_initialize(); - sacf_flag = enable_sacf_uaccess(); asm_inline volatile( "20: spka 0(%[key])\n" - " sacf 256\n" " llill %[count],%[max_loops]\n" "0: l %[prev],%[address]\n" "1: nr %[prev],%[mask]\n" @@ -178,8 +61,7 @@ static nokprobe_inline int __cmpxchg_user_key_small(unsigned long address, unsig " nr %[tmp],%[mask]\n" " jnz 5f\n" " brct %[count],2b\n" - "5: sacf 768\n" - " spka %[default_key]\n" + "5: spka %[default_key]\n" "21:\n" EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev]) EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev]) @@ -197,16 +79,16 @@ static nokprobe_inline int __cmpxchg_user_key_small(unsigned long address, unsig [default_key] "J" (PAGE_DEFAULT_KEY), [max_loops] "J" (CMPXCHG_USER_KEY_MAX_LOOPS) : "memory", "cc"); - disable_sacf_uaccess(sacf_flag); *uval = prev; if (!count) rc = -EAGAIN; return rc; } -int __kprobes __cmpxchg_user_key1(unsigned long address, unsigned char *uval, - unsigned char old, unsigned char new, unsigned long key) +int __kprobes __cmpxchg_key1(void *addr, unsigned char *uval, unsigned char old, + unsigned char new, unsigned long key) { + unsigned long address = (unsigned long)addr; unsigned int prev, shift, mask, _old, _new; int rc; @@ -215,15 +97,16 @@ int __kprobes __cmpxchg_user_key1(unsigned long address, unsigned char *uval, _old = (unsigned int)old << shift; _new = (unsigned int)new << shift; mask = ~(0xff << shift); - rc = __cmpxchg_user_key_small(address, &prev, _old, _new, mask, key); + rc = __cmpxchg_key_small((void *)address, &prev, _old, _new, mask, key); *uval = prev >> shift; return rc; } -EXPORT_SYMBOL(__cmpxchg_user_key1); +EXPORT_SYMBOL(__cmpxchg_key1); -int __kprobes __cmpxchg_user_key2(unsigned long address, unsigned short *uval, - unsigned short old, unsigned short new, unsigned long key) +int __kprobes __cmpxchg_key2(void *addr, unsigned short *uval, unsigned short old, + unsigned short new, unsigned long key) { + unsigned long address = (unsigned long)addr; unsigned int prev, shift, mask, _old, _new; int rc; @@ -232,27 +115,23 @@ int __kprobes __cmpxchg_user_key2(unsigned long address, unsigned short *uval, _old = (unsigned int)old << shift; _new = (unsigned int)new << shift; mask = ~(0xffff << shift); - rc = __cmpxchg_user_key_small(address, &prev, _old, _new, mask, key); + rc = __cmpxchg_key_small((void *)address, &prev, _old, _new, mask, key); *uval = prev >> shift; return rc; } -EXPORT_SYMBOL(__cmpxchg_user_key2); +EXPORT_SYMBOL(__cmpxchg_key2); -int __kprobes __cmpxchg_user_key4(unsigned long address, unsigned int *uval, - unsigned int old, unsigned int new, unsigned long key) +int __kprobes __cmpxchg_key4(void *address, unsigned int *uval, unsigned int old, + unsigned int new, unsigned long key) { unsigned int prev = old; - bool sacf_flag; int rc = 0; skey_regions_initialize(); - sacf_flag = enable_sacf_uaccess(); asm_inline volatile( "20: spka 0(%[key])\n" - " sacf 256\n" "0: cs %[prev],%[new],%[address]\n" - "1: sacf 768\n" - " spka %[default_key]\n" + "1: spka %[default_key]\n" "21:\n" EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev]) @@ -264,27 +143,22 @@ int __kprobes __cmpxchg_user_key4(unsigned long address, unsigned int *uval, [key] "a" (key << 4), [default_key] "J" (PAGE_DEFAULT_KEY) : "memory", "cc"); - disable_sacf_uaccess(sacf_flag); *uval = prev; return rc; } -EXPORT_SYMBOL(__cmpxchg_user_key4); +EXPORT_SYMBOL(__cmpxchg_key4); -int __kprobes __cmpxchg_user_key8(unsigned long address, unsigned long *uval, - unsigned long old, unsigned long new, unsigned long key) +int __kprobes __cmpxchg_key8(void *address, unsigned long *uval, unsigned long old, + unsigned long new, unsigned long key) { unsigned long prev = old; - bool sacf_flag; int rc = 0; skey_regions_initialize(); - sacf_flag = enable_sacf_uaccess(); asm_inline volatile( "20: spka 0(%[key])\n" - " sacf 256\n" "0: csg %[prev],%[new],%[address]\n" - "1: sacf 768\n" - " spka %[default_key]\n" + "1: spka %[default_key]\n" "21:\n" EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev]) @@ -296,27 +170,22 @@ int __kprobes __cmpxchg_user_key8(unsigned long address, unsigned long *uval, [key] "a" (key << 4), [default_key] "J" (PAGE_DEFAULT_KEY) : "memory", "cc"); - disable_sacf_uaccess(sacf_flag); *uval = prev; return rc; } -EXPORT_SYMBOL(__cmpxchg_user_key8); +EXPORT_SYMBOL(__cmpxchg_key8); -int __kprobes __cmpxchg_user_key16(unsigned long address, __uint128_t *uval, - __uint128_t old, __uint128_t new, unsigned long key) +int __kprobes __cmpxchg_key16(void *address, __uint128_t *uval, __uint128_t old, + __uint128_t new, unsigned long key) { __uint128_t prev = old; - bool sacf_flag; int rc = 0; skey_regions_initialize(); - sacf_flag = enable_sacf_uaccess(); asm_inline volatile( "20: spka 0(%[key])\n" - " sacf 256\n" "0: cdsg %[prev],%[new],%[address]\n" - "1: sacf 768\n" - " spka %[default_key]\n" + "1: spka %[default_key]\n" "21:\n" EX_TABLE_UA_LOAD_REGPAIR(0b, 1b, %[rc], %[prev]) EX_TABLE_UA_LOAD_REGPAIR(1b, 1b, %[rc], %[prev]) @@ -328,8 +197,7 @@ int __kprobes __cmpxchg_user_key16(unsigned long address, __uint128_t *uval, [key] "a" (key << 4), [default_key] "J" (PAGE_DEFAULT_KEY) : "memory", "cc"); - disable_sacf_uaccess(sacf_flag); *uval = prev; return rc; } -EXPORT_SYMBOL(__cmpxchg_user_key16); +EXPORT_SYMBOL(__cmpxchg_key16); diff --git a/arch/s390/lib/xor.c b/arch/s390/lib/xor.c deleted file mode 100644 index 1721b73b7803..000000000000 --- a/arch/s390/lib/xor.c +++ /dev/null @@ -1,137 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Optimized xor_block operation for RAID4/5 - * - * Copyright IBM Corp. 2016 - * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> - */ - -#include <linux/types.h> -#include <linux/export.h> -#include <linux/raid/xor.h> -#include <asm/xor.h> - -static void xor_xc_2(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2) -{ - asm volatile( - " aghi %0,-1\n" - " jm 3f\n" - " srlg 0,%0,8\n" - " ltgr 0,0\n" - " jz 1f\n" - "0: xc 0(256,%1),0(%2)\n" - " la %1,256(%1)\n" - " la %2,256(%2)\n" - " brctg 0,0b\n" - "1: exrl %0,2f\n" - " j 3f\n" - "2: xc 0(1,%1),0(%2)\n" - "3:" - : : "d" (bytes), "a" (p1), "a" (p2) - : "0", "cc", "memory"); -} - -static void xor_xc_3(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2, - const unsigned long * __restrict p3) -{ - asm volatile( - " aghi %0,-1\n" - " jm 4f\n" - " srlg 0,%0,8\n" - " ltgr 0,0\n" - " jz 1f\n" - "0: xc 0(256,%1),0(%2)\n" - " xc 0(256,%1),0(%3)\n" - " la %1,256(%1)\n" - " la %2,256(%2)\n" - " la %3,256(%3)\n" - " brctg 0,0b\n" - "1: exrl %0,2f\n" - " exrl %0,3f\n" - " j 4f\n" - "2: xc 0(1,%1),0(%2)\n" - "3: xc 0(1,%1),0(%3)\n" - "4:" - : "+d" (bytes), "+a" (p1), "+a" (p2), "+a" (p3) - : : "0", "cc", "memory"); -} - -static void xor_xc_4(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2, - const unsigned long * __restrict p3, - const unsigned long * __restrict p4) -{ - asm volatile( - " aghi %0,-1\n" - " jm 5f\n" - " srlg 0,%0,8\n" - " ltgr 0,0\n" - " jz 1f\n" - "0: xc 0(256,%1),0(%2)\n" - " xc 0(256,%1),0(%3)\n" - " xc 0(256,%1),0(%4)\n" - " la %1,256(%1)\n" - " la %2,256(%2)\n" - " la %3,256(%3)\n" - " la %4,256(%4)\n" - " brctg 0,0b\n" - "1: exrl %0,2f\n" - " exrl %0,3f\n" - " exrl %0,4f\n" - " j 5f\n" - "2: xc 0(1,%1),0(%2)\n" - "3: xc 0(1,%1),0(%3)\n" - "4: xc 0(1,%1),0(%4)\n" - "5:" - : "+d" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4) - : : "0", "cc", "memory"); -} - -static void xor_xc_5(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2, - const unsigned long * __restrict p3, - const unsigned long * __restrict p4, - const unsigned long * __restrict p5) -{ - asm volatile( - " larl 1,2f\n" - " aghi %0,-1\n" - " jm 6f\n" - " srlg 0,%0,8\n" - " ltgr 0,0\n" - " jz 1f\n" - "0: xc 0(256,%1),0(%2)\n" - " xc 0(256,%1),0(%3)\n" - " xc 0(256,%1),0(%4)\n" - " xc 0(256,%1),0(%5)\n" - " la %1,256(%1)\n" - " la %2,256(%2)\n" - " la %3,256(%3)\n" - " la %4,256(%4)\n" - " la %5,256(%5)\n" - " brctg 0,0b\n" - "1: exrl %0,2f\n" - " exrl %0,3f\n" - " exrl %0,4f\n" - " exrl %0,5f\n" - " j 6f\n" - "2: xc 0(1,%1),0(%2)\n" - "3: xc 0(1,%1),0(%3)\n" - "4: xc 0(1,%1),0(%4)\n" - "5: xc 0(1,%1),0(%5)\n" - "6:" - : "+d" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4), - "+a" (p5) - : : "0", "cc", "memory"); -} - -struct xor_block_template xor_block_xc = { - .name = "xc", - .do_2 = xor_xc_2, - .do_3 = xor_xc_3, - .do_4 = xor_xc_4, - .do_5 = xor_xc_5, -}; -EXPORT_SYMBOL(xor_block_xc); diff --git a/arch/s390/mm/Makefile b/arch/s390/mm/Makefile index bd0401cc7ca5..7dea37a5ad3b 100644 --- a/arch/s390/mm/Makefile +++ b/arch/s390/mm/Makefile @@ -3,6 +3,8 @@ # Makefile for the linux s390-specific parts of the memory manager. # +CONTEXT_ANALYSIS := y + obj-y := init.o fault.o extmem.o mmap.o vmem.o maccess.o obj-y += page-states.o pageattr.o pgtable.o pgalloc.o extable.o @@ -10,7 +12,6 @@ obj-$(CONFIG_CMM) += cmm.o obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o obj-$(CONFIG_PTDUMP) += dump_pagetables.o -obj-$(CONFIG_PGSTE) += gmap.o obj-$(CONFIG_PFAULT) += pfault.o obj-$(subst m,y,$(CONFIG_KVM)) += gmap_helpers.o diff --git a/arch/s390/mm/cmm.c b/arch/s390/mm/cmm.c index eb7ef63fab1e..d342b68ee9be 100644 --- a/arch/s390/mm/cmm.c +++ b/arch/s390/mm/cmm.c @@ -95,7 +95,6 @@ static long cmm_alloc_pages(long nr, long *counter, (*counter)++; spin_unlock(&cmm_lock); nr--; - cond_resched(); } return nr; } @@ -134,7 +133,6 @@ static long cmm_free_pages(long nr, long *counter, struct cmm_page_array **list) inc = __cmm_free_pages(inc, counter, list); if (inc) break; - cond_resched(); } return nr + inc; } diff --git a/arch/s390/mm/extmem.c b/arch/s390/mm/extmem.c index 6cc33c705de2..1c3a1eed4381 100644 --- a/arch/s390/mm/extmem.c +++ b/arch/s390/mm/extmem.c @@ -171,8 +171,8 @@ query_segment_type (struct dcss_segment *seg) struct qout64 *qout; struct qin64 *qin; - qin = kmalloc(sizeof(*qin), GFP_KERNEL | GFP_DMA); - qout = kmalloc(sizeof(*qout), GFP_KERNEL | GFP_DMA); + qin = kmalloc_obj(*qin, GFP_KERNEL | GFP_DMA); + qout = kmalloc_obj(*qout, GFP_KERNEL | GFP_DMA); if ((qin == NULL) || (qout == NULL)) { rc = -ENOMEM; goto out_free; @@ -302,7 +302,7 @@ __segment_load (char *name, int do_nonshared, unsigned long *addr, unsigned long start_addr = end_addr = 0; segtype = -1; - seg = kmalloc(sizeof(*seg), GFP_KERNEL | GFP_DMA); + seg = kmalloc_obj(*seg, GFP_KERNEL | GFP_DMA); if (seg == NULL) { rc = -ENOMEM; goto out; @@ -317,7 +317,7 @@ __segment_load (char *name, int do_nonshared, unsigned long *addr, unsigned long goto out_free; } - seg->res = kzalloc(sizeof(struct resource), GFP_KERNEL); + seg->res = kzalloc_obj(struct resource); if (seg->res == NULL) { rc = -ENOMEM; goto out_free; diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index e2e13778c36a..46d828926009 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -403,15 +403,16 @@ void do_dat_exception(struct pt_regs *regs) } NOKPROBE_SYMBOL(do_dat_exception); -#if IS_ENABLED(CONFIG_PGSTE) +#if IS_ENABLED(CONFIG_KVM) void do_secure_storage_access(struct pt_regs *regs) +__context_unsafe(/* folio_walk_end() not instrumented */) { union teid teid = { .val = regs->int_parm_long }; unsigned long addr = get_fault_address(regs); + struct mm_struct *mm = current->mm; struct vm_area_struct *vma; struct folio_walk fw; - struct mm_struct *mm; struct folio *folio; int rc; @@ -427,10 +428,8 @@ void do_secure_storage_access(struct pt_regs *regs) * was not supposed to do, e.g. branching into secure * memory. Trigger a segmentation fault. */ - if (user_mode(regs)) { - send_sig(SIGSEGV, current, 0); - return; - } + if (user_mode(regs)) + return handle_fault_error_nolock(regs, SEGV_ACCERR); /* * The kernel should never run into this case and * there is no way out of this situation. @@ -438,36 +437,36 @@ void do_secure_storage_access(struct pt_regs *regs) panic("Unexpected PGM 0x3d with TEID bit 61=0"); } if (is_kernel_fault(regs)) { - folio = phys_to_folio(addr); - if (unlikely(!folio_try_get(folio))) - return; - rc = arch_make_folio_accessible(folio); - folio_put(folio); + if (is_vmalloc_addr((void *)addr)) + return handle_fault_error_nolock(regs, 0); + rc = uv_convert_from_secure(__pa(addr)); + /* + * There are some valid fixup types for kernel + * accesses to donated secure memory. zeropad is one + * of them. + */ if (rc) - BUG(); + return handle_fault_error_nolock(regs, 0); } else { - if (faulthandler_disabled()) + if (faulthandler_disabled() || !mm) return handle_fault_error_nolock(regs, 0); - mm = current->mm; - mmap_read_lock(mm); - vma = find_vma(mm, addr); + vma = lock_mm_and_find_vma(mm, addr, regs); if (!vma) - return handle_fault_error(regs, SEGV_MAPERR); + return handle_fault_error_nolock(regs, SEGV_MAPERR); folio = folio_walk_start(&fw, vma, addr, 0); - if (!folio) { - mmap_read_unlock(mm); - return; - } + if (!folio) + goto out; /* arch_make_folio_accessible() needs a raised refcount. */ folio_get(folio); rc = arch_make_folio_accessible(folio); folio_put(folio); folio_walk_end(&fw, vma); if (rc) - send_sig(SIGSEGV, current, 0); + return handle_fault_error(regs, SEGV_ACCERR); +out: mmap_read_unlock(mm); } } NOKPROBE_SYMBOL(do_secure_storage_access); -#endif /* CONFIG_PGSTE */ +#endif /* CONFIG_KVM */ diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c deleted file mode 100644 index dd85bcca817d..000000000000 --- a/arch/s390/mm/gmap.c +++ /dev/null @@ -1,2436 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * KVM guest address space mapping code - * - * Copyright IBM Corp. 2007, 2020 - * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> - * David Hildenbrand <david@redhat.com> - * Janosch Frank <frankja@linux.vnet.ibm.com> - */ - -#include <linux/cpufeature.h> -#include <linux/export.h> -#include <linux/kernel.h> -#include <linux/pagewalk.h> -#include <linux/swap.h> -#include <linux/smp.h> -#include <linux/spinlock.h> -#include <linux/slab.h> -#include <linux/swapops.h> -#include <linux/ksm.h> -#include <linux/mman.h> -#include <linux/pgtable.h> -#include <asm/page-states.h> -#include <asm/pgalloc.h> -#include <asm/machine.h> -#include <asm/gmap_helpers.h> -#include <asm/gmap.h> -#include <asm/page.h> - -/* - * The address is saved in a radix tree directly; NULL would be ambiguous, - * since 0 is a valid address, and NULL is returned when nothing was found. - * The lower bits are ignored by all users of the macro, so it can be used - * to distinguish a valid address 0 from a NULL. - */ -#define VALID_GADDR_FLAG 1 -#define IS_GADDR_VALID(gaddr) ((gaddr) & VALID_GADDR_FLAG) -#define MAKE_VALID_GADDR(gaddr) (((gaddr) & HPAGE_MASK) | VALID_GADDR_FLAG) - -#define GMAP_SHADOW_FAKE_TABLE 1ULL - -static struct page *gmap_alloc_crst(void) -{ - struct page *page; - - page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); - if (!page) - return NULL; - __arch_set_page_dat(page_to_virt(page), 1UL << CRST_ALLOC_ORDER); - return page; -} - -/** - * gmap_alloc - allocate and initialize a guest address space - * @limit: maximum address of the gmap address space - * - * Returns a guest address space structure. - */ -struct gmap *gmap_alloc(unsigned long limit) -{ - struct gmap *gmap; - struct page *page; - unsigned long *table; - unsigned long etype, atype; - - if (limit < _REGION3_SIZE) { - limit = _REGION3_SIZE - 1; - atype = _ASCE_TYPE_SEGMENT; - etype = _SEGMENT_ENTRY_EMPTY; - } else if (limit < _REGION2_SIZE) { - limit = _REGION2_SIZE - 1; - atype = _ASCE_TYPE_REGION3; - etype = _REGION3_ENTRY_EMPTY; - } else if (limit < _REGION1_SIZE) { - limit = _REGION1_SIZE - 1; - atype = _ASCE_TYPE_REGION2; - etype = _REGION2_ENTRY_EMPTY; - } else { - limit = -1UL; - atype = _ASCE_TYPE_REGION1; - etype = _REGION1_ENTRY_EMPTY; - } - gmap = kzalloc(sizeof(struct gmap), GFP_KERNEL_ACCOUNT); - if (!gmap) - goto out; - INIT_LIST_HEAD(&gmap->children); - INIT_RADIX_TREE(&gmap->guest_to_host, GFP_KERNEL_ACCOUNT); - INIT_RADIX_TREE(&gmap->host_to_guest, GFP_ATOMIC | __GFP_ACCOUNT); - INIT_RADIX_TREE(&gmap->host_to_rmap, GFP_ATOMIC | __GFP_ACCOUNT); - spin_lock_init(&gmap->guest_table_lock); - spin_lock_init(&gmap->shadow_lock); - refcount_set(&gmap->ref_count, 1); - page = gmap_alloc_crst(); - if (!page) - goto out_free; - table = page_to_virt(page); - crst_table_init(table, etype); - gmap->table = table; - gmap->asce = atype | _ASCE_TABLE_LENGTH | - _ASCE_USER_BITS | __pa(table); - gmap->asce_end = limit; - return gmap; - -out_free: - kfree(gmap); -out: - return NULL; -} -EXPORT_SYMBOL_GPL(gmap_alloc); - -/** - * gmap_create - create a guest address space - * @mm: pointer to the parent mm_struct - * @limit: maximum size of the gmap address space - * - * Returns a guest address space structure. - */ -struct gmap *gmap_create(struct mm_struct *mm, unsigned long limit) -{ - struct gmap *gmap; - unsigned long gmap_asce; - - gmap = gmap_alloc(limit); - if (!gmap) - return NULL; - gmap->mm = mm; - spin_lock(&mm->context.lock); - list_add_rcu(&gmap->list, &mm->context.gmap_list); - if (list_is_singular(&mm->context.gmap_list)) - gmap_asce = gmap->asce; - else - gmap_asce = -1UL; - WRITE_ONCE(mm->context.gmap_asce, gmap_asce); - spin_unlock(&mm->context.lock); - return gmap; -} -EXPORT_SYMBOL_GPL(gmap_create); - -static void gmap_flush_tlb(struct gmap *gmap) -{ - __tlb_flush_idte(gmap->asce); -} - -static void gmap_radix_tree_free(struct radix_tree_root *root) -{ - struct radix_tree_iter iter; - unsigned long indices[16]; - unsigned long index; - void __rcu **slot; - int i, nr; - - /* A radix tree is freed by deleting all of its entries */ - index = 0; - do { - nr = 0; - radix_tree_for_each_slot(slot, root, &iter, index) { - indices[nr] = iter.index; - if (++nr == 16) - break; - } - for (i = 0; i < nr; i++) { - index = indices[i]; - radix_tree_delete(root, index); - } - } while (nr > 0); -} - -static void gmap_rmap_radix_tree_free(struct radix_tree_root *root) -{ - struct gmap_rmap *rmap, *rnext, *head; - struct radix_tree_iter iter; - unsigned long indices[16]; - unsigned long index; - void __rcu **slot; - int i, nr; - - /* A radix tree is freed by deleting all of its entries */ - index = 0; - do { - nr = 0; - radix_tree_for_each_slot(slot, root, &iter, index) { - indices[nr] = iter.index; - if (++nr == 16) - break; - } - for (i = 0; i < nr; i++) { - index = indices[i]; - head = radix_tree_delete(root, index); - gmap_for_each_rmap_safe(rmap, rnext, head) - kfree(rmap); - } - } while (nr > 0); -} - -static void gmap_free_crst(unsigned long *table, bool free_ptes) -{ - bool is_segment = (table[0] & _SEGMENT_ENTRY_TYPE_MASK) == 0; - int i; - - if (is_segment) { - if (!free_ptes) - goto out; - for (i = 0; i < _CRST_ENTRIES; i++) - if (!(table[i] & _SEGMENT_ENTRY_INVALID)) - page_table_free_pgste(page_ptdesc(phys_to_page(table[i]))); - } else { - for (i = 0; i < _CRST_ENTRIES; i++) - if (!(table[i] & _REGION_ENTRY_INVALID)) - gmap_free_crst(__va(table[i] & PAGE_MASK), free_ptes); - } - -out: - free_pages((unsigned long)table, CRST_ALLOC_ORDER); -} - -/** - * gmap_free - free a guest address space - * @gmap: pointer to the guest address space structure - * - * No locks required. There are no references to this gmap anymore. - */ -void gmap_free(struct gmap *gmap) -{ - /* Flush tlb of all gmaps (if not already done for shadows) */ - if (!(gmap_is_shadow(gmap) && gmap->removed)) - gmap_flush_tlb(gmap); - /* Free all segment & region tables. */ - gmap_free_crst(gmap->table, gmap_is_shadow(gmap)); - - gmap_radix_tree_free(&gmap->guest_to_host); - gmap_radix_tree_free(&gmap->host_to_guest); - - /* Free additional data for a shadow gmap */ - if (gmap_is_shadow(gmap)) { - gmap_rmap_radix_tree_free(&gmap->host_to_rmap); - /* Release reference to the parent */ - gmap_put(gmap->parent); - } - - kfree(gmap); -} -EXPORT_SYMBOL_GPL(gmap_free); - -/** - * gmap_get - increase reference counter for guest address space - * @gmap: pointer to the guest address space structure - * - * Returns the gmap pointer - */ -struct gmap *gmap_get(struct gmap *gmap) -{ - refcount_inc(&gmap->ref_count); - return gmap; -} -EXPORT_SYMBOL_GPL(gmap_get); - -/** - * gmap_put - decrease reference counter for guest address space - * @gmap: pointer to the guest address space structure - * - * If the reference counter reaches zero the guest address space is freed. - */ -void gmap_put(struct gmap *gmap) -{ - if (refcount_dec_and_test(&gmap->ref_count)) - gmap_free(gmap); -} -EXPORT_SYMBOL_GPL(gmap_put); - -/** - * gmap_remove - remove a guest address space but do not free it yet - * @gmap: pointer to the guest address space structure - */ -void gmap_remove(struct gmap *gmap) -{ - struct gmap *sg, *next; - unsigned long gmap_asce; - - /* Remove all shadow gmaps linked to this gmap */ - if (!list_empty(&gmap->children)) { - spin_lock(&gmap->shadow_lock); - list_for_each_entry_safe(sg, next, &gmap->children, list) { - list_del(&sg->list); - gmap_put(sg); - } - spin_unlock(&gmap->shadow_lock); - } - /* Remove gmap from the pre-mm list */ - spin_lock(&gmap->mm->context.lock); - list_del_rcu(&gmap->list); - if (list_empty(&gmap->mm->context.gmap_list)) - gmap_asce = 0; - else if (list_is_singular(&gmap->mm->context.gmap_list)) - gmap_asce = list_first_entry(&gmap->mm->context.gmap_list, - struct gmap, list)->asce; - else - gmap_asce = -1UL; - WRITE_ONCE(gmap->mm->context.gmap_asce, gmap_asce); - spin_unlock(&gmap->mm->context.lock); - synchronize_rcu(); - /* Put reference */ - gmap_put(gmap); -} -EXPORT_SYMBOL_GPL(gmap_remove); - -/* - * gmap_alloc_table is assumed to be called with mmap_lock held - */ -static int gmap_alloc_table(struct gmap *gmap, unsigned long *table, - unsigned long init, unsigned long gaddr) -{ - struct page *page; - unsigned long *new; - - /* since we dont free the gmap table until gmap_free we can unlock */ - page = gmap_alloc_crst(); - if (!page) - return -ENOMEM; - new = page_to_virt(page); - crst_table_init(new, init); - spin_lock(&gmap->guest_table_lock); - if (*table & _REGION_ENTRY_INVALID) { - *table = __pa(new) | _REGION_ENTRY_LENGTH | - (*table & _REGION_ENTRY_TYPE_MASK); - page = NULL; - } - spin_unlock(&gmap->guest_table_lock); - if (page) - __free_pages(page, CRST_ALLOC_ORDER); - return 0; -} - -static unsigned long host_to_guest_lookup(struct gmap *gmap, unsigned long vmaddr) -{ - return (unsigned long)radix_tree_lookup(&gmap->host_to_guest, vmaddr >> PMD_SHIFT); -} - -static unsigned long host_to_guest_delete(struct gmap *gmap, unsigned long vmaddr) -{ - return (unsigned long)radix_tree_delete(&gmap->host_to_guest, vmaddr >> PMD_SHIFT); -} - -static pmd_t *host_to_guest_pmd_delete(struct gmap *gmap, unsigned long vmaddr, - unsigned long *gaddr) -{ - *gaddr = host_to_guest_delete(gmap, vmaddr); - if (IS_GADDR_VALID(*gaddr)) - return (pmd_t *)gmap_table_walk(gmap, *gaddr, 1); - return NULL; -} - -/** - * __gmap_unlink_by_vmaddr - unlink a single segment via a host address - * @gmap: pointer to the guest address space structure - * @vmaddr: address in the host process address space - * - * Returns 1 if a TLB flush is required - */ -static int __gmap_unlink_by_vmaddr(struct gmap *gmap, unsigned long vmaddr) -{ - unsigned long gaddr; - int flush = 0; - pmd_t *pmdp; - - BUG_ON(gmap_is_shadow(gmap)); - spin_lock(&gmap->guest_table_lock); - - pmdp = host_to_guest_pmd_delete(gmap, vmaddr, &gaddr); - if (pmdp) { - flush = (pmd_val(*pmdp) != _SEGMENT_ENTRY_EMPTY); - *pmdp = __pmd(_SEGMENT_ENTRY_EMPTY); - } - - spin_unlock(&gmap->guest_table_lock); - return flush; -} - -/** - * __gmap_unmap_by_gaddr - unmap a single segment via a guest address - * @gmap: pointer to the guest address space structure - * @gaddr: address in the guest address space - * - * Returns 1 if a TLB flush is required - */ -static int __gmap_unmap_by_gaddr(struct gmap *gmap, unsigned long gaddr) -{ - unsigned long vmaddr; - - vmaddr = (unsigned long) radix_tree_delete(&gmap->guest_to_host, - gaddr >> PMD_SHIFT); - return vmaddr ? __gmap_unlink_by_vmaddr(gmap, vmaddr) : 0; -} - -/** - * gmap_unmap_segment - unmap segment from the guest address space - * @gmap: pointer to the guest address space structure - * @to: address in the guest address space - * @len: length of the memory area to unmap - * - * Returns 0 if the unmap succeeded, -EINVAL if not. - */ -int gmap_unmap_segment(struct gmap *gmap, unsigned long to, unsigned long len) -{ - unsigned long off; - int flush; - - BUG_ON(gmap_is_shadow(gmap)); - if ((to | len) & (PMD_SIZE - 1)) - return -EINVAL; - if (len == 0 || to + len < to) - return -EINVAL; - - flush = 0; - mmap_write_lock(gmap->mm); - for (off = 0; off < len; off += PMD_SIZE) - flush |= __gmap_unmap_by_gaddr(gmap, to + off); - mmap_write_unlock(gmap->mm); - if (flush) - gmap_flush_tlb(gmap); - return 0; -} -EXPORT_SYMBOL_GPL(gmap_unmap_segment); - -/** - * gmap_map_segment - map a segment to the guest address space - * @gmap: pointer to the guest address space structure - * @from: source address in the parent address space - * @to: target address in the guest address space - * @len: length of the memory area to map - * - * Returns 0 if the mmap succeeded, -EINVAL or -ENOMEM if not. - */ -int gmap_map_segment(struct gmap *gmap, unsigned long from, - unsigned long to, unsigned long len) -{ - unsigned long off; - int flush; - - BUG_ON(gmap_is_shadow(gmap)); - if ((from | to | len) & (PMD_SIZE - 1)) - return -EINVAL; - if (len == 0 || from + len < from || to + len < to || - from + len - 1 > TASK_SIZE_MAX || to + len - 1 > gmap->asce_end) - return -EINVAL; - - flush = 0; - mmap_write_lock(gmap->mm); - for (off = 0; off < len; off += PMD_SIZE) { - /* Remove old translation */ - flush |= __gmap_unmap_by_gaddr(gmap, to + off); - /* Store new translation */ - if (radix_tree_insert(&gmap->guest_to_host, - (to + off) >> PMD_SHIFT, - (void *) from + off)) - break; - } - mmap_write_unlock(gmap->mm); - if (flush) - gmap_flush_tlb(gmap); - if (off >= len) - return 0; - gmap_unmap_segment(gmap, to, len); - return -ENOMEM; -} -EXPORT_SYMBOL_GPL(gmap_map_segment); - -/** - * __gmap_translate - translate a guest address to a user space address - * @gmap: pointer to guest mapping meta data structure - * @gaddr: guest address - * - * Returns user space address which corresponds to the guest address or - * -EFAULT if no such mapping exists. - * This function does not establish potentially missing page table entries. - * The mmap_lock of the mm that belongs to the address space must be held - * when this function gets called. - * - * Note: Can also be called for shadow gmaps. - */ -unsigned long __gmap_translate(struct gmap *gmap, unsigned long gaddr) -{ - unsigned long vmaddr; - - vmaddr = (unsigned long) - radix_tree_lookup(&gmap->guest_to_host, gaddr >> PMD_SHIFT); - /* Note: guest_to_host is empty for a shadow gmap */ - return vmaddr ? (vmaddr | (gaddr & ~PMD_MASK)) : -EFAULT; -} -EXPORT_SYMBOL_GPL(__gmap_translate); - -/** - * gmap_unlink - disconnect a page table from the gmap shadow tables - * @mm: pointer to the parent mm_struct - * @table: pointer to the host page table - * @vmaddr: vm address associated with the host page table - */ -void gmap_unlink(struct mm_struct *mm, unsigned long *table, - unsigned long vmaddr) -{ - struct gmap *gmap; - int flush; - - rcu_read_lock(); - list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { - flush = __gmap_unlink_by_vmaddr(gmap, vmaddr); - if (flush) - gmap_flush_tlb(gmap); - } - rcu_read_unlock(); -} - -static void gmap_pmdp_xchg(struct gmap *gmap, pmd_t *old, pmd_t new, - unsigned long gaddr); - -/** - * __gmap_link - set up shadow page tables to connect a host to a guest address - * @gmap: pointer to guest mapping meta data structure - * @gaddr: guest address - * @vmaddr: vm address - * - * Returns 0 on success, -ENOMEM for out of memory conditions, and -EFAULT - * if the vm address is already mapped to a different guest segment. - * The mmap_lock of the mm that belongs to the address space must be held - * when this function gets called. - */ -int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr) -{ - struct mm_struct *mm; - unsigned long *table; - spinlock_t *ptl; - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; - u64 unprot; - int rc; - - BUG_ON(gmap_is_shadow(gmap)); - /* Create higher level tables in the gmap page table */ - table = gmap->table; - if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION1) { - table += (gaddr & _REGION1_INDEX) >> _REGION1_SHIFT; - if ((*table & _REGION_ENTRY_INVALID) && - gmap_alloc_table(gmap, table, _REGION2_ENTRY_EMPTY, - gaddr & _REGION1_MASK)) - return -ENOMEM; - table = __va(*table & _REGION_ENTRY_ORIGIN); - } - if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION2) { - table += (gaddr & _REGION2_INDEX) >> _REGION2_SHIFT; - if ((*table & _REGION_ENTRY_INVALID) && - gmap_alloc_table(gmap, table, _REGION3_ENTRY_EMPTY, - gaddr & _REGION2_MASK)) - return -ENOMEM; - table = __va(*table & _REGION_ENTRY_ORIGIN); - } - if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION3) { - table += (gaddr & _REGION3_INDEX) >> _REGION3_SHIFT; - if ((*table & _REGION_ENTRY_INVALID) && - gmap_alloc_table(gmap, table, _SEGMENT_ENTRY_EMPTY, - gaddr & _REGION3_MASK)) - return -ENOMEM; - table = __va(*table & _REGION_ENTRY_ORIGIN); - } - table += (gaddr & _SEGMENT_INDEX) >> _SEGMENT_SHIFT; - /* Walk the parent mm page table */ - mm = gmap->mm; - pgd = pgd_offset(mm, vmaddr); - VM_BUG_ON(pgd_none(*pgd)); - p4d = p4d_offset(pgd, vmaddr); - VM_BUG_ON(p4d_none(*p4d)); - pud = pud_offset(p4d, vmaddr); - VM_BUG_ON(pud_none(*pud)); - /* large puds cannot yet be handled */ - if (pud_leaf(*pud)) - return -EFAULT; - pmd = pmd_offset(pud, vmaddr); - VM_BUG_ON(pmd_none(*pmd)); - /* Are we allowed to use huge pages? */ - if (pmd_leaf(*pmd) && !gmap->mm->context.allow_gmap_hpage_1m) - return -EFAULT; - /* Link gmap segment table entry location to page table. */ - rc = radix_tree_preload(GFP_KERNEL_ACCOUNT); - if (rc) - return rc; - ptl = pmd_lock(mm, pmd); - spin_lock(&gmap->guest_table_lock); - if (*table == _SEGMENT_ENTRY_EMPTY) { - rc = radix_tree_insert(&gmap->host_to_guest, - vmaddr >> PMD_SHIFT, - (void *)MAKE_VALID_GADDR(gaddr)); - if (!rc) { - if (pmd_leaf(*pmd)) { - *table = (pmd_val(*pmd) & - _SEGMENT_ENTRY_HARDWARE_BITS_LARGE) - | _SEGMENT_ENTRY_GMAP_UC - | _SEGMENT_ENTRY; - } else - *table = (pmd_val(*pmd) & - _SEGMENT_ENTRY_HARDWARE_BITS) - | _SEGMENT_ENTRY; - } - } else if (*table & _SEGMENT_ENTRY_PROTECT && - !(pmd_val(*pmd) & _SEGMENT_ENTRY_PROTECT)) { - unprot = (u64)*table; - unprot &= ~_SEGMENT_ENTRY_PROTECT; - unprot |= _SEGMENT_ENTRY_GMAP_UC; - gmap_pmdp_xchg(gmap, (pmd_t *)table, __pmd(unprot), gaddr); - } - spin_unlock(&gmap->guest_table_lock); - spin_unlock(ptl); - radix_tree_preload_end(); - return rc; -} -EXPORT_SYMBOL(__gmap_link); - -/* - * this function is assumed to be called with mmap_lock held - */ -void __gmap_zap(struct gmap *gmap, unsigned long gaddr) -{ - unsigned long vmaddr; - - mmap_assert_locked(gmap->mm); - - /* Find the vm address for the guest address */ - vmaddr = (unsigned long) radix_tree_lookup(&gmap->guest_to_host, - gaddr >> PMD_SHIFT); - if (vmaddr) { - vmaddr |= gaddr & ~PMD_MASK; - gmap_helper_zap_one_page(gmap->mm, vmaddr); - } -} -EXPORT_SYMBOL_GPL(__gmap_zap); - -static LIST_HEAD(gmap_notifier_list); -static DEFINE_SPINLOCK(gmap_notifier_lock); - -/** - * gmap_register_pte_notifier - register a pte invalidation callback - * @nb: pointer to the gmap notifier block - */ -void gmap_register_pte_notifier(struct gmap_notifier *nb) -{ - spin_lock(&gmap_notifier_lock); - list_add_rcu(&nb->list, &gmap_notifier_list); - spin_unlock(&gmap_notifier_lock); -} -EXPORT_SYMBOL_GPL(gmap_register_pte_notifier); - -/** - * gmap_unregister_pte_notifier - remove a pte invalidation callback - * @nb: pointer to the gmap notifier block - */ -void gmap_unregister_pte_notifier(struct gmap_notifier *nb) -{ - spin_lock(&gmap_notifier_lock); - list_del_rcu(&nb->list); - spin_unlock(&gmap_notifier_lock); - synchronize_rcu(); -} -EXPORT_SYMBOL_GPL(gmap_unregister_pte_notifier); - -/** - * gmap_call_notifier - call all registered invalidation callbacks - * @gmap: pointer to guest mapping meta data structure - * @start: start virtual address in the guest address space - * @end: end virtual address in the guest address space - */ -static void gmap_call_notifier(struct gmap *gmap, unsigned long start, - unsigned long end) -{ - struct gmap_notifier *nb; - - list_for_each_entry(nb, &gmap_notifier_list, list) - nb->notifier_call(gmap, start, end); -} - -/** - * gmap_table_walk - walk the gmap page tables - * @gmap: pointer to guest mapping meta data structure - * @gaddr: virtual address in the guest address space - * @level: page table level to stop at - * - * Returns a table entry pointer for the given guest address and @level - * @level=0 : returns a pointer to a page table table entry (or NULL) - * @level=1 : returns a pointer to a segment table entry (or NULL) - * @level=2 : returns a pointer to a region-3 table entry (or NULL) - * @level=3 : returns a pointer to a region-2 table entry (or NULL) - * @level=4 : returns a pointer to a region-1 table entry (or NULL) - * - * Returns NULL if the gmap page tables could not be walked to the - * requested level. - * - * Note: Can also be called for shadow gmaps. - */ -unsigned long *gmap_table_walk(struct gmap *gmap, unsigned long gaddr, int level) -{ - const int asce_type = gmap->asce & _ASCE_TYPE_MASK; - unsigned long *table = gmap->table; - - if (gmap_is_shadow(gmap) && gmap->removed) - return NULL; - - if (WARN_ON_ONCE(level > (asce_type >> 2) + 1)) - return NULL; - - if (asce_type != _ASCE_TYPE_REGION1 && - gaddr & (-1UL << (31 + (asce_type >> 2) * 11))) - return NULL; - - switch (asce_type) { - case _ASCE_TYPE_REGION1: - table += (gaddr & _REGION1_INDEX) >> _REGION1_SHIFT; - if (level == 4) - break; - if (*table & _REGION_ENTRY_INVALID) - return NULL; - table = __va(*table & _REGION_ENTRY_ORIGIN); - fallthrough; - case _ASCE_TYPE_REGION2: - table += (gaddr & _REGION2_INDEX) >> _REGION2_SHIFT; - if (level == 3) - break; - if (*table & _REGION_ENTRY_INVALID) - return NULL; - table = __va(*table & _REGION_ENTRY_ORIGIN); - fallthrough; - case _ASCE_TYPE_REGION3: - table += (gaddr & _REGION3_INDEX) >> _REGION3_SHIFT; - if (level == 2) - break; - if (*table & _REGION_ENTRY_INVALID) - return NULL; - table = __va(*table & _REGION_ENTRY_ORIGIN); - fallthrough; - case _ASCE_TYPE_SEGMENT: - table += (gaddr & _SEGMENT_INDEX) >> _SEGMENT_SHIFT; - if (level == 1) - break; - if (*table & _REGION_ENTRY_INVALID) - return NULL; - table = __va(*table & _SEGMENT_ENTRY_ORIGIN); - table += (gaddr & _PAGE_INDEX) >> PAGE_SHIFT; - } - return table; -} -EXPORT_SYMBOL(gmap_table_walk); - -/** - * gmap_pte_op_walk - walk the gmap page table, get the page table lock - * and return the pte pointer - * @gmap: pointer to guest mapping meta data structure - * @gaddr: virtual address in the guest address space - * @ptl: pointer to the spinlock pointer - * - * Returns a pointer to the locked pte for a guest address, or NULL - */ -static pte_t *gmap_pte_op_walk(struct gmap *gmap, unsigned long gaddr, - spinlock_t **ptl) -{ - unsigned long *table; - - BUG_ON(gmap_is_shadow(gmap)); - /* Walk the gmap page table, lock and get pte pointer */ - table = gmap_table_walk(gmap, gaddr, 1); /* get segment pointer */ - if (!table || *table & _SEGMENT_ENTRY_INVALID) - return NULL; - return pte_alloc_map_lock(gmap->mm, (pmd_t *) table, gaddr, ptl); -} - -/** - * gmap_pte_op_fixup - force a page in and connect the gmap page table - * @gmap: pointer to guest mapping meta data structure - * @gaddr: virtual address in the guest address space - * @vmaddr: address in the host process address space - * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE - * - * Returns 0 if the caller can retry __gmap_translate (might fail again), - * -ENOMEM if out of memory and -EFAULT if anything goes wrong while fixing - * up or connecting the gmap page table. - */ -static int gmap_pte_op_fixup(struct gmap *gmap, unsigned long gaddr, - unsigned long vmaddr, int prot) -{ - struct mm_struct *mm = gmap->mm; - unsigned int fault_flags; - bool unlocked = false; - - BUG_ON(gmap_is_shadow(gmap)); - fault_flags = (prot == PROT_WRITE) ? FAULT_FLAG_WRITE : 0; - if (fixup_user_fault(mm, vmaddr, fault_flags, &unlocked)) - return -EFAULT; - if (unlocked) - /* lost mmap_lock, caller has to retry __gmap_translate */ - return 0; - /* Connect the page tables */ - return __gmap_link(gmap, gaddr, vmaddr); -} - -/** - * gmap_pte_op_end - release the page table lock - * @ptep: pointer to the locked pte - * @ptl: pointer to the page table spinlock - */ -static void gmap_pte_op_end(pte_t *ptep, spinlock_t *ptl) -{ - pte_unmap_unlock(ptep, ptl); -} - -/** - * gmap_pmd_op_walk - walk the gmap tables, get the guest table lock - * and return the pmd pointer - * @gmap: pointer to guest mapping meta data structure - * @gaddr: virtual address in the guest address space - * - * Returns a pointer to the pmd for a guest address, or NULL - */ -static inline pmd_t *gmap_pmd_op_walk(struct gmap *gmap, unsigned long gaddr) -{ - pmd_t *pmdp; - - BUG_ON(gmap_is_shadow(gmap)); - pmdp = (pmd_t *) gmap_table_walk(gmap, gaddr, 1); - if (!pmdp) - return NULL; - - /* without huge pages, there is no need to take the table lock */ - if (!gmap->mm->context.allow_gmap_hpage_1m) - return pmd_none(*pmdp) ? NULL : pmdp; - - spin_lock(&gmap->guest_table_lock); - if (pmd_none(*pmdp)) { - spin_unlock(&gmap->guest_table_lock); - return NULL; - } - - /* 4k page table entries are locked via the pte (pte_alloc_map_lock). */ - if (!pmd_leaf(*pmdp)) - spin_unlock(&gmap->guest_table_lock); - return pmdp; -} - -/** - * gmap_pmd_op_end - release the guest_table_lock if needed - * @gmap: pointer to the guest mapping meta data structure - * @pmdp: pointer to the pmd - */ -static inline void gmap_pmd_op_end(struct gmap *gmap, pmd_t *pmdp) -{ - if (pmd_leaf(*pmdp)) - spin_unlock(&gmap->guest_table_lock); -} - -/* - * gmap_protect_pmd - remove access rights to memory and set pmd notification bits - * @pmdp: pointer to the pmd to be protected - * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE - * @bits: notification bits to set - * - * Returns: - * 0 if successfully protected - * -EAGAIN if a fixup is needed - * -EINVAL if unsupported notifier bits have been specified - * - * Expected to be called with sg->mm->mmap_lock in read and - * guest_table_lock held. - */ -static int gmap_protect_pmd(struct gmap *gmap, unsigned long gaddr, - pmd_t *pmdp, int prot, unsigned long bits) -{ - int pmd_i = pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID; - int pmd_p = pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT; - pmd_t new = *pmdp; - - /* Fixup needed */ - if ((pmd_i && (prot != PROT_NONE)) || (pmd_p && (prot == PROT_WRITE))) - return -EAGAIN; - - if (prot == PROT_NONE && !pmd_i) { - new = set_pmd_bit(new, __pgprot(_SEGMENT_ENTRY_INVALID)); - gmap_pmdp_xchg(gmap, pmdp, new, gaddr); - } - - if (prot == PROT_READ && !pmd_p) { - new = clear_pmd_bit(new, __pgprot(_SEGMENT_ENTRY_INVALID)); - new = set_pmd_bit(new, __pgprot(_SEGMENT_ENTRY_PROTECT)); - gmap_pmdp_xchg(gmap, pmdp, new, gaddr); - } - - if (bits & GMAP_NOTIFY_MPROT) - set_pmd(pmdp, set_pmd_bit(*pmdp, __pgprot(_SEGMENT_ENTRY_GMAP_IN))); - - /* Shadow GMAP protection needs split PMDs */ - if (bits & GMAP_NOTIFY_SHADOW) - return -EINVAL; - - return 0; -} - -/* - * gmap_protect_pte - remove access rights to memory and set pgste bits - * @gmap: pointer to guest mapping meta data structure - * @gaddr: virtual address in the guest address space - * @pmdp: pointer to the pmd associated with the pte - * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE - * @bits: notification bits to set - * - * Returns 0 if successfully protected, -ENOMEM if out of memory and - * -EAGAIN if a fixup is needed. - * - * Expected to be called with sg->mm->mmap_lock in read - */ -static int gmap_protect_pte(struct gmap *gmap, unsigned long gaddr, - pmd_t *pmdp, int prot, unsigned long bits) -{ - int rc; - pte_t *ptep; - spinlock_t *ptl; - unsigned long pbits = 0; - - if (pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID) - return -EAGAIN; - - ptep = pte_alloc_map_lock(gmap->mm, pmdp, gaddr, &ptl); - if (!ptep) - return -ENOMEM; - - pbits |= (bits & GMAP_NOTIFY_MPROT) ? PGSTE_IN_BIT : 0; - pbits |= (bits & GMAP_NOTIFY_SHADOW) ? PGSTE_VSIE_BIT : 0; - /* Protect and unlock. */ - rc = ptep_force_prot(gmap->mm, gaddr, ptep, prot, pbits); - gmap_pte_op_end(ptep, ptl); - return rc; -} - -/* - * gmap_protect_range - remove access rights to memory and set pgste bits - * @gmap: pointer to guest mapping meta data structure - * @gaddr: virtual address in the guest address space - * @len: size of area - * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE - * @bits: pgste notification bits to set - * - * Returns: - * PAGE_SIZE if a small page was successfully protected; - * HPAGE_SIZE if a large page was successfully protected; - * -ENOMEM if out of memory; - * -EFAULT if gaddr is invalid (or mapping for shadows is missing); - * -EAGAIN if the guest mapping is missing and should be fixed by the caller. - * - * Context: Called with sg->mm->mmap_lock in read. - */ -int gmap_protect_one(struct gmap *gmap, unsigned long gaddr, int prot, unsigned long bits) -{ - pmd_t *pmdp; - int rc = 0; - - BUG_ON(gmap_is_shadow(gmap)); - - pmdp = gmap_pmd_op_walk(gmap, gaddr); - if (!pmdp) - return -EAGAIN; - - if (!pmd_leaf(*pmdp)) { - rc = gmap_protect_pte(gmap, gaddr, pmdp, prot, bits); - if (!rc) - rc = PAGE_SIZE; - } else { - rc = gmap_protect_pmd(gmap, gaddr, pmdp, prot, bits); - if (!rc) - rc = HPAGE_SIZE; - } - gmap_pmd_op_end(gmap, pmdp); - - return rc; -} -EXPORT_SYMBOL_GPL(gmap_protect_one); - -/** - * gmap_read_table - get an unsigned long value from a guest page table using - * absolute addressing, without marking the page referenced. - * @gmap: pointer to guest mapping meta data structure - * @gaddr: virtual address in the guest address space - * @val: pointer to the unsigned long value to return - * - * Returns 0 if the value was read, -ENOMEM if out of memory and -EFAULT - * if reading using the virtual address failed. -EINVAL if called on a gmap - * shadow. - * - * Called with gmap->mm->mmap_lock in read. - */ -int gmap_read_table(struct gmap *gmap, unsigned long gaddr, unsigned long *val) -{ - unsigned long address, vmaddr; - spinlock_t *ptl; - pte_t *ptep, pte; - int rc; - - if (gmap_is_shadow(gmap)) - return -EINVAL; - - while (1) { - rc = -EAGAIN; - ptep = gmap_pte_op_walk(gmap, gaddr, &ptl); - if (ptep) { - pte = *ptep; - if (pte_present(pte) && (pte_val(pte) & _PAGE_READ)) { - address = pte_val(pte) & PAGE_MASK; - address += gaddr & ~PAGE_MASK; - *val = *(unsigned long *)__va(address); - set_pte(ptep, set_pte_bit(*ptep, __pgprot(_PAGE_YOUNG))); - /* Do *NOT* clear the _PAGE_INVALID bit! */ - rc = 0; - } - gmap_pte_op_end(ptep, ptl); - } - if (!rc) - break; - vmaddr = __gmap_translate(gmap, gaddr); - if (IS_ERR_VALUE(vmaddr)) { - rc = vmaddr; - break; - } - rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, PROT_READ); - if (rc) - break; - } - return rc; -} -EXPORT_SYMBOL_GPL(gmap_read_table); - -/** - * gmap_insert_rmap - add a rmap to the host_to_rmap radix tree - * @sg: pointer to the shadow guest address space structure - * @vmaddr: vm address associated with the rmap - * @rmap: pointer to the rmap structure - * - * Called with the sg->guest_table_lock - */ -static inline void gmap_insert_rmap(struct gmap *sg, unsigned long vmaddr, - struct gmap_rmap *rmap) -{ - struct gmap_rmap *temp; - void __rcu **slot; - - BUG_ON(!gmap_is_shadow(sg)); - slot = radix_tree_lookup_slot(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT); - if (slot) { - rmap->next = radix_tree_deref_slot_protected(slot, - &sg->guest_table_lock); - for (temp = rmap->next; temp; temp = temp->next) { - if (temp->raddr == rmap->raddr) { - kfree(rmap); - return; - } - } - radix_tree_replace_slot(&sg->host_to_rmap, slot, rmap); - } else { - rmap->next = NULL; - radix_tree_insert(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT, - rmap); - } -} - -/** - * gmap_protect_rmap - restrict access rights to memory (RO) and create an rmap - * @sg: pointer to the shadow guest address space structure - * @raddr: rmap address in the shadow gmap - * @paddr: address in the parent guest address space - * @len: length of the memory area to protect - * - * Returns 0 if successfully protected and the rmap was created, -ENOMEM - * if out of memory and -EFAULT if paddr is invalid. - */ -static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr, - unsigned long paddr, unsigned long len) -{ - struct gmap *parent; - struct gmap_rmap *rmap; - unsigned long vmaddr; - spinlock_t *ptl; - pte_t *ptep; - int rc; - - BUG_ON(!gmap_is_shadow(sg)); - parent = sg->parent; - while (len) { - vmaddr = __gmap_translate(parent, paddr); - if (IS_ERR_VALUE(vmaddr)) - return vmaddr; - rmap = kzalloc(sizeof(*rmap), GFP_KERNEL_ACCOUNT); - if (!rmap) - return -ENOMEM; - rmap->raddr = raddr; - rc = radix_tree_preload(GFP_KERNEL_ACCOUNT); - if (rc) { - kfree(rmap); - return rc; - } - rc = -EAGAIN; - ptep = gmap_pte_op_walk(parent, paddr, &ptl); - if (ptep) { - spin_lock(&sg->guest_table_lock); - rc = ptep_force_prot(parent->mm, paddr, ptep, PROT_READ, - PGSTE_VSIE_BIT); - if (!rc) - gmap_insert_rmap(sg, vmaddr, rmap); - spin_unlock(&sg->guest_table_lock); - gmap_pte_op_end(ptep, ptl); - } - radix_tree_preload_end(); - if (rc) { - kfree(rmap); - rc = gmap_pte_op_fixup(parent, paddr, vmaddr, PROT_READ); - if (rc) - return rc; - continue; - } - paddr += PAGE_SIZE; - len -= PAGE_SIZE; - } - return 0; -} - -#define _SHADOW_RMAP_MASK 0x7 -#define _SHADOW_RMAP_REGION1 0x5 -#define _SHADOW_RMAP_REGION2 0x4 -#define _SHADOW_RMAP_REGION3 0x3 -#define _SHADOW_RMAP_SEGMENT 0x2 -#define _SHADOW_RMAP_PGTABLE 0x1 - -/** - * gmap_idte_one - invalidate a single region or segment table entry - * @asce: region or segment table *origin* + table-type bits - * @vaddr: virtual address to identify the table entry to flush - * - * The invalid bit of a single region or segment table entry is set - * and the associated TLB entries depending on the entry are flushed. - * The table-type of the @asce identifies the portion of the @vaddr - * that is used as the invalidation index. - */ -static inline void gmap_idte_one(unsigned long asce, unsigned long vaddr) -{ - asm volatile( - " idte %0,0,%1" - : : "a" (asce), "a" (vaddr) : "cc", "memory"); -} - -/** - * gmap_unshadow_page - remove a page from a shadow page table - * @sg: pointer to the shadow guest address space structure - * @raddr: rmap address in the shadow guest address space - * - * Called with the sg->guest_table_lock - */ -static void gmap_unshadow_page(struct gmap *sg, unsigned long raddr) -{ - unsigned long *table; - - BUG_ON(!gmap_is_shadow(sg)); - table = gmap_table_walk(sg, raddr, 0); /* get page table pointer */ - if (!table || *table & _PAGE_INVALID) - return; - gmap_call_notifier(sg, raddr, raddr + PAGE_SIZE - 1); - ptep_unshadow_pte(sg->mm, raddr, (pte_t *) table); -} - -/** - * __gmap_unshadow_pgt - remove all entries from a shadow page table - * @sg: pointer to the shadow guest address space structure - * @raddr: rmap address in the shadow guest address space - * @pgt: pointer to the start of a shadow page table - * - * Called with the sg->guest_table_lock - */ -static void __gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr, - unsigned long *pgt) -{ - int i; - - BUG_ON(!gmap_is_shadow(sg)); - for (i = 0; i < _PAGE_ENTRIES; i++, raddr += PAGE_SIZE) - pgt[i] = _PAGE_INVALID; -} - -/** - * gmap_unshadow_pgt - remove a shadow page table from a segment entry - * @sg: pointer to the shadow guest address space structure - * @raddr: address in the shadow guest address space - * - * Called with the sg->guest_table_lock - */ -static void gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr) -{ - unsigned long *ste; - phys_addr_t sto, pgt; - struct ptdesc *ptdesc; - - BUG_ON(!gmap_is_shadow(sg)); - ste = gmap_table_walk(sg, raddr, 1); /* get segment pointer */ - if (!ste || !(*ste & _SEGMENT_ENTRY_ORIGIN)) - return; - gmap_call_notifier(sg, raddr, raddr + _SEGMENT_SIZE - 1); - sto = __pa(ste - ((raddr & _SEGMENT_INDEX) >> _SEGMENT_SHIFT)); - gmap_idte_one(sto | _ASCE_TYPE_SEGMENT, raddr); - pgt = *ste & _SEGMENT_ENTRY_ORIGIN; - *ste = _SEGMENT_ENTRY_EMPTY; - __gmap_unshadow_pgt(sg, raddr, __va(pgt)); - /* Free page table */ - ptdesc = page_ptdesc(phys_to_page(pgt)); - page_table_free_pgste(ptdesc); -} - -/** - * __gmap_unshadow_sgt - remove all entries from a shadow segment table - * @sg: pointer to the shadow guest address space structure - * @raddr: rmap address in the shadow guest address space - * @sgt: pointer to the start of a shadow segment table - * - * Called with the sg->guest_table_lock - */ -static void __gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr, - unsigned long *sgt) -{ - struct ptdesc *ptdesc; - phys_addr_t pgt; - int i; - - BUG_ON(!gmap_is_shadow(sg)); - for (i = 0; i < _CRST_ENTRIES; i++, raddr += _SEGMENT_SIZE) { - if (!(sgt[i] & _SEGMENT_ENTRY_ORIGIN)) - continue; - pgt = sgt[i] & _REGION_ENTRY_ORIGIN; - sgt[i] = _SEGMENT_ENTRY_EMPTY; - __gmap_unshadow_pgt(sg, raddr, __va(pgt)); - /* Free page table */ - ptdesc = page_ptdesc(phys_to_page(pgt)); - page_table_free_pgste(ptdesc); - } -} - -/** - * gmap_unshadow_sgt - remove a shadow segment table from a region-3 entry - * @sg: pointer to the shadow guest address space structure - * @raddr: rmap address in the shadow guest address space - * - * Called with the shadow->guest_table_lock - */ -static void gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr) -{ - unsigned long r3o, *r3e; - phys_addr_t sgt; - struct page *page; - - BUG_ON(!gmap_is_shadow(sg)); - r3e = gmap_table_walk(sg, raddr, 2); /* get region-3 pointer */ - if (!r3e || !(*r3e & _REGION_ENTRY_ORIGIN)) - return; - gmap_call_notifier(sg, raddr, raddr + _REGION3_SIZE - 1); - r3o = (unsigned long) (r3e - ((raddr & _REGION3_INDEX) >> _REGION3_SHIFT)); - gmap_idte_one(__pa(r3o) | _ASCE_TYPE_REGION3, raddr); - sgt = *r3e & _REGION_ENTRY_ORIGIN; - *r3e = _REGION3_ENTRY_EMPTY; - __gmap_unshadow_sgt(sg, raddr, __va(sgt)); - /* Free segment table */ - page = phys_to_page(sgt); - __free_pages(page, CRST_ALLOC_ORDER); -} - -/** - * __gmap_unshadow_r3t - remove all entries from a shadow region-3 table - * @sg: pointer to the shadow guest address space structure - * @raddr: address in the shadow guest address space - * @r3t: pointer to the start of a shadow region-3 table - * - * Called with the sg->guest_table_lock - */ -static void __gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr, - unsigned long *r3t) -{ - struct page *page; - phys_addr_t sgt; - int i; - - BUG_ON(!gmap_is_shadow(sg)); - for (i = 0; i < _CRST_ENTRIES; i++, raddr += _REGION3_SIZE) { - if (!(r3t[i] & _REGION_ENTRY_ORIGIN)) - continue; - sgt = r3t[i] & _REGION_ENTRY_ORIGIN; - r3t[i] = _REGION3_ENTRY_EMPTY; - __gmap_unshadow_sgt(sg, raddr, __va(sgt)); - /* Free segment table */ - page = phys_to_page(sgt); - __free_pages(page, CRST_ALLOC_ORDER); - } -} - -/** - * gmap_unshadow_r3t - remove a shadow region-3 table from a region-2 entry - * @sg: pointer to the shadow guest address space structure - * @raddr: rmap address in the shadow guest address space - * - * Called with the sg->guest_table_lock - */ -static void gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr) -{ - unsigned long r2o, *r2e; - phys_addr_t r3t; - struct page *page; - - BUG_ON(!gmap_is_shadow(sg)); - r2e = gmap_table_walk(sg, raddr, 3); /* get region-2 pointer */ - if (!r2e || !(*r2e & _REGION_ENTRY_ORIGIN)) - return; - gmap_call_notifier(sg, raddr, raddr + _REGION2_SIZE - 1); - r2o = (unsigned long) (r2e - ((raddr & _REGION2_INDEX) >> _REGION2_SHIFT)); - gmap_idte_one(__pa(r2o) | _ASCE_TYPE_REGION2, raddr); - r3t = *r2e & _REGION_ENTRY_ORIGIN; - *r2e = _REGION2_ENTRY_EMPTY; - __gmap_unshadow_r3t(sg, raddr, __va(r3t)); - /* Free region 3 table */ - page = phys_to_page(r3t); - __free_pages(page, CRST_ALLOC_ORDER); -} - -/** - * __gmap_unshadow_r2t - remove all entries from a shadow region-2 table - * @sg: pointer to the shadow guest address space structure - * @raddr: rmap address in the shadow guest address space - * @r2t: pointer to the start of a shadow region-2 table - * - * Called with the sg->guest_table_lock - */ -static void __gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr, - unsigned long *r2t) -{ - phys_addr_t r3t; - struct page *page; - int i; - - BUG_ON(!gmap_is_shadow(sg)); - for (i = 0; i < _CRST_ENTRIES; i++, raddr += _REGION2_SIZE) { - if (!(r2t[i] & _REGION_ENTRY_ORIGIN)) - continue; - r3t = r2t[i] & _REGION_ENTRY_ORIGIN; - r2t[i] = _REGION2_ENTRY_EMPTY; - __gmap_unshadow_r3t(sg, raddr, __va(r3t)); - /* Free region 3 table */ - page = phys_to_page(r3t); - __free_pages(page, CRST_ALLOC_ORDER); - } -} - -/** - * gmap_unshadow_r2t - remove a shadow region-2 table from a region-1 entry - * @sg: pointer to the shadow guest address space structure - * @raddr: rmap address in the shadow guest address space - * - * Called with the sg->guest_table_lock - */ -static void gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr) -{ - unsigned long r1o, *r1e; - struct page *page; - phys_addr_t r2t; - - BUG_ON(!gmap_is_shadow(sg)); - r1e = gmap_table_walk(sg, raddr, 4); /* get region-1 pointer */ - if (!r1e || !(*r1e & _REGION_ENTRY_ORIGIN)) - return; - gmap_call_notifier(sg, raddr, raddr + _REGION1_SIZE - 1); - r1o = (unsigned long) (r1e - ((raddr & _REGION1_INDEX) >> _REGION1_SHIFT)); - gmap_idte_one(__pa(r1o) | _ASCE_TYPE_REGION1, raddr); - r2t = *r1e & _REGION_ENTRY_ORIGIN; - *r1e = _REGION1_ENTRY_EMPTY; - __gmap_unshadow_r2t(sg, raddr, __va(r2t)); - /* Free region 2 table */ - page = phys_to_page(r2t); - __free_pages(page, CRST_ALLOC_ORDER); -} - -/** - * __gmap_unshadow_r1t - remove all entries from a shadow region-1 table - * @sg: pointer to the shadow guest address space structure - * @raddr: rmap address in the shadow guest address space - * @r1t: pointer to the start of a shadow region-1 table - * - * Called with the shadow->guest_table_lock - */ -static void __gmap_unshadow_r1t(struct gmap *sg, unsigned long raddr, - unsigned long *r1t) -{ - unsigned long asce; - struct page *page; - phys_addr_t r2t; - int i; - - BUG_ON(!gmap_is_shadow(sg)); - asce = __pa(r1t) | _ASCE_TYPE_REGION1; - for (i = 0; i < _CRST_ENTRIES; i++, raddr += _REGION1_SIZE) { - if (!(r1t[i] & _REGION_ENTRY_ORIGIN)) - continue; - r2t = r1t[i] & _REGION_ENTRY_ORIGIN; - __gmap_unshadow_r2t(sg, raddr, __va(r2t)); - /* Clear entry and flush translation r1t -> r2t */ - gmap_idte_one(asce, raddr); - r1t[i] = _REGION1_ENTRY_EMPTY; - /* Free region 2 table */ - page = phys_to_page(r2t); - __free_pages(page, CRST_ALLOC_ORDER); - } -} - -/** - * gmap_unshadow - remove a shadow page table completely - * @sg: pointer to the shadow guest address space structure - * - * Called with sg->guest_table_lock - */ -void gmap_unshadow(struct gmap *sg) -{ - unsigned long *table; - - BUG_ON(!gmap_is_shadow(sg)); - if (sg->removed) - return; - sg->removed = 1; - gmap_call_notifier(sg, 0, -1UL); - gmap_flush_tlb(sg); - table = __va(sg->asce & _ASCE_ORIGIN); - switch (sg->asce & _ASCE_TYPE_MASK) { - case _ASCE_TYPE_REGION1: - __gmap_unshadow_r1t(sg, 0, table); - break; - case _ASCE_TYPE_REGION2: - __gmap_unshadow_r2t(sg, 0, table); - break; - case _ASCE_TYPE_REGION3: - __gmap_unshadow_r3t(sg, 0, table); - break; - case _ASCE_TYPE_SEGMENT: - __gmap_unshadow_sgt(sg, 0, table); - break; - } -} -EXPORT_SYMBOL(gmap_unshadow); - -/** - * gmap_shadow_r2t - create an empty shadow region 2 table - * @sg: pointer to the shadow guest address space structure - * @saddr: faulting address in the shadow gmap - * @r2t: parent gmap address of the region 2 table to get shadowed - * @fake: r2t references contiguous guest memory block, not a r2t - * - * The r2t parameter specifies the address of the source table. The - * four pages of the source table are made read-only in the parent gmap - * address space. A write to the source table area @r2t will automatically - * remove the shadow r2 table and all of its descendants. - * - * Returns 0 if successfully shadowed or already shadowed, -EAGAIN if the - * shadow table structure is incomplete, -ENOMEM if out of memory and - * -EFAULT if an address in the parent gmap could not be resolved. - * - * Called with sg->mm->mmap_lock in read. - */ -int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t, - int fake) -{ - unsigned long raddr, origin, offset, len; - unsigned long *table; - phys_addr_t s_r2t; - struct page *page; - int rc; - - BUG_ON(!gmap_is_shadow(sg)); - /* Allocate a shadow region second table */ - page = gmap_alloc_crst(); - if (!page) - return -ENOMEM; - s_r2t = page_to_phys(page); - /* Install shadow region second table */ - spin_lock(&sg->guest_table_lock); - table = gmap_table_walk(sg, saddr, 4); /* get region-1 pointer */ - if (!table) { - rc = -EAGAIN; /* Race with unshadow */ - goto out_free; - } - if (!(*table & _REGION_ENTRY_INVALID)) { - rc = 0; /* Already established */ - goto out_free; - } else if (*table & _REGION_ENTRY_ORIGIN) { - rc = -EAGAIN; /* Race with shadow */ - goto out_free; - } - crst_table_init(__va(s_r2t), _REGION2_ENTRY_EMPTY); - /* mark as invalid as long as the parent table is not protected */ - *table = s_r2t | _REGION_ENTRY_LENGTH | - _REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_INVALID; - if (sg->edat_level >= 1) - *table |= (r2t & _REGION_ENTRY_PROTECT); - if (fake) { - /* nothing to protect for fake tables */ - *table &= ~_REGION_ENTRY_INVALID; - spin_unlock(&sg->guest_table_lock); - return 0; - } - spin_unlock(&sg->guest_table_lock); - /* Make r2t read-only in parent gmap page table */ - raddr = (saddr & _REGION1_MASK) | _SHADOW_RMAP_REGION1; - origin = r2t & _REGION_ENTRY_ORIGIN; - offset = ((r2t & _REGION_ENTRY_OFFSET) >> 6) * PAGE_SIZE; - len = ((r2t & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset; - rc = gmap_protect_rmap(sg, raddr, origin + offset, len); - spin_lock(&sg->guest_table_lock); - if (!rc) { - table = gmap_table_walk(sg, saddr, 4); - if (!table || (*table & _REGION_ENTRY_ORIGIN) != s_r2t) - rc = -EAGAIN; /* Race with unshadow */ - else - *table &= ~_REGION_ENTRY_INVALID; - } else { - gmap_unshadow_r2t(sg, raddr); - } - spin_unlock(&sg->guest_table_lock); - return rc; -out_free: - spin_unlock(&sg->guest_table_lock); - __free_pages(page, CRST_ALLOC_ORDER); - return rc; -} -EXPORT_SYMBOL_GPL(gmap_shadow_r2t); - -/** - * gmap_shadow_r3t - create a shadow region 3 table - * @sg: pointer to the shadow guest address space structure - * @saddr: faulting address in the shadow gmap - * @r3t: parent gmap address of the region 3 table to get shadowed - * @fake: r3t references contiguous guest memory block, not a r3t - * - * Returns 0 if successfully shadowed or already shadowed, -EAGAIN if the - * shadow table structure is incomplete, -ENOMEM if out of memory and - * -EFAULT if an address in the parent gmap could not be resolved. - * - * Called with sg->mm->mmap_lock in read. - */ -int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t, - int fake) -{ - unsigned long raddr, origin, offset, len; - unsigned long *table; - phys_addr_t s_r3t; - struct page *page; - int rc; - - BUG_ON(!gmap_is_shadow(sg)); - /* Allocate a shadow region second table */ - page = gmap_alloc_crst(); - if (!page) - return -ENOMEM; - s_r3t = page_to_phys(page); - /* Install shadow region second table */ - spin_lock(&sg->guest_table_lock); - table = gmap_table_walk(sg, saddr, 3); /* get region-2 pointer */ - if (!table) { - rc = -EAGAIN; /* Race with unshadow */ - goto out_free; - } - if (!(*table & _REGION_ENTRY_INVALID)) { - rc = 0; /* Already established */ - goto out_free; - } else if (*table & _REGION_ENTRY_ORIGIN) { - rc = -EAGAIN; /* Race with shadow */ - goto out_free; - } - crst_table_init(__va(s_r3t), _REGION3_ENTRY_EMPTY); - /* mark as invalid as long as the parent table is not protected */ - *table = s_r3t | _REGION_ENTRY_LENGTH | - _REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_INVALID; - if (sg->edat_level >= 1) - *table |= (r3t & _REGION_ENTRY_PROTECT); - if (fake) { - /* nothing to protect for fake tables */ - *table &= ~_REGION_ENTRY_INVALID; - spin_unlock(&sg->guest_table_lock); - return 0; - } - spin_unlock(&sg->guest_table_lock); - /* Make r3t read-only in parent gmap page table */ - raddr = (saddr & _REGION2_MASK) | _SHADOW_RMAP_REGION2; - origin = r3t & _REGION_ENTRY_ORIGIN; - offset = ((r3t & _REGION_ENTRY_OFFSET) >> 6) * PAGE_SIZE; - len = ((r3t & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset; - rc = gmap_protect_rmap(sg, raddr, origin + offset, len); - spin_lock(&sg->guest_table_lock); - if (!rc) { - table = gmap_table_walk(sg, saddr, 3); - if (!table || (*table & _REGION_ENTRY_ORIGIN) != s_r3t) - rc = -EAGAIN; /* Race with unshadow */ - else - *table &= ~_REGION_ENTRY_INVALID; - } else { - gmap_unshadow_r3t(sg, raddr); - } - spin_unlock(&sg->guest_table_lock); - return rc; -out_free: - spin_unlock(&sg->guest_table_lock); - __free_pages(page, CRST_ALLOC_ORDER); - return rc; -} -EXPORT_SYMBOL_GPL(gmap_shadow_r3t); - -/** - * gmap_shadow_sgt - create a shadow segment table - * @sg: pointer to the shadow guest address space structure - * @saddr: faulting address in the shadow gmap - * @sgt: parent gmap address of the segment table to get shadowed - * @fake: sgt references contiguous guest memory block, not a sgt - * - * Returns: 0 if successfully shadowed or already shadowed, -EAGAIN if the - * shadow table structure is incomplete, -ENOMEM if out of memory and - * -EFAULT if an address in the parent gmap could not be resolved. - * - * Called with sg->mm->mmap_lock in read. - */ -int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt, - int fake) -{ - unsigned long raddr, origin, offset, len; - unsigned long *table; - phys_addr_t s_sgt; - struct page *page; - int rc; - - BUG_ON(!gmap_is_shadow(sg) || (sgt & _REGION3_ENTRY_LARGE)); - /* Allocate a shadow segment table */ - page = gmap_alloc_crst(); - if (!page) - return -ENOMEM; - s_sgt = page_to_phys(page); - /* Install shadow region second table */ - spin_lock(&sg->guest_table_lock); - table = gmap_table_walk(sg, saddr, 2); /* get region-3 pointer */ - if (!table) { - rc = -EAGAIN; /* Race with unshadow */ - goto out_free; - } - if (!(*table & _REGION_ENTRY_INVALID)) { - rc = 0; /* Already established */ - goto out_free; - } else if (*table & _REGION_ENTRY_ORIGIN) { - rc = -EAGAIN; /* Race with shadow */ - goto out_free; - } - crst_table_init(__va(s_sgt), _SEGMENT_ENTRY_EMPTY); - /* mark as invalid as long as the parent table is not protected */ - *table = s_sgt | _REGION_ENTRY_LENGTH | - _REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_INVALID; - if (sg->edat_level >= 1) - *table |= sgt & _REGION_ENTRY_PROTECT; - if (fake) { - /* nothing to protect for fake tables */ - *table &= ~_REGION_ENTRY_INVALID; - spin_unlock(&sg->guest_table_lock); - return 0; - } - spin_unlock(&sg->guest_table_lock); - /* Make sgt read-only in parent gmap page table */ - raddr = (saddr & _REGION3_MASK) | _SHADOW_RMAP_REGION3; - origin = sgt & _REGION_ENTRY_ORIGIN; - offset = ((sgt & _REGION_ENTRY_OFFSET) >> 6) * PAGE_SIZE; - len = ((sgt & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset; - rc = gmap_protect_rmap(sg, raddr, origin + offset, len); - spin_lock(&sg->guest_table_lock); - if (!rc) { - table = gmap_table_walk(sg, saddr, 2); - if (!table || (*table & _REGION_ENTRY_ORIGIN) != s_sgt) - rc = -EAGAIN; /* Race with unshadow */ - else - *table &= ~_REGION_ENTRY_INVALID; - } else { - gmap_unshadow_sgt(sg, raddr); - } - spin_unlock(&sg->guest_table_lock); - return rc; -out_free: - spin_unlock(&sg->guest_table_lock); - __free_pages(page, CRST_ALLOC_ORDER); - return rc; -} -EXPORT_SYMBOL_GPL(gmap_shadow_sgt); - -static void gmap_pgste_set_pgt_addr(struct ptdesc *ptdesc, unsigned long pgt_addr) -{ - unsigned long *pgstes = page_to_virt(ptdesc_page(ptdesc)); - - pgstes += _PAGE_ENTRIES; - - pgstes[0] &= ~PGSTE_ST2_MASK; - pgstes[1] &= ~PGSTE_ST2_MASK; - pgstes[2] &= ~PGSTE_ST2_MASK; - pgstes[3] &= ~PGSTE_ST2_MASK; - - pgstes[0] |= (pgt_addr >> 16) & PGSTE_ST2_MASK; - pgstes[1] |= pgt_addr & PGSTE_ST2_MASK; - pgstes[2] |= (pgt_addr << 16) & PGSTE_ST2_MASK; - pgstes[3] |= (pgt_addr << 32) & PGSTE_ST2_MASK; -} - -/** - * gmap_shadow_pgt - instantiate a shadow page table - * @sg: pointer to the shadow guest address space structure - * @saddr: faulting address in the shadow gmap - * @pgt: parent gmap address of the page table to get shadowed - * @fake: pgt references contiguous guest memory block, not a pgtable - * - * Returns 0 if successfully shadowed or already shadowed, -EAGAIN if the - * shadow table structure is incomplete, -ENOMEM if out of memory, - * -EFAULT if an address in the parent gmap could not be resolved and - * - * Called with gmap->mm->mmap_lock in read - */ -int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt, - int fake) -{ - unsigned long raddr, origin; - unsigned long *table; - struct ptdesc *ptdesc; - phys_addr_t s_pgt; - int rc; - - BUG_ON(!gmap_is_shadow(sg) || (pgt & _SEGMENT_ENTRY_LARGE)); - /* Allocate a shadow page table */ - ptdesc = page_table_alloc_pgste(sg->mm); - if (!ptdesc) - return -ENOMEM; - origin = pgt & _SEGMENT_ENTRY_ORIGIN; - if (fake) - origin |= GMAP_SHADOW_FAKE_TABLE; - gmap_pgste_set_pgt_addr(ptdesc, origin); - s_pgt = page_to_phys(ptdesc_page(ptdesc)); - /* Install shadow page table */ - spin_lock(&sg->guest_table_lock); - table = gmap_table_walk(sg, saddr, 1); /* get segment pointer */ - if (!table) { - rc = -EAGAIN; /* Race with unshadow */ - goto out_free; - } - if (!(*table & _SEGMENT_ENTRY_INVALID)) { - rc = 0; /* Already established */ - goto out_free; - } else if (*table & _SEGMENT_ENTRY_ORIGIN) { - rc = -EAGAIN; /* Race with shadow */ - goto out_free; - } - /* mark as invalid as long as the parent table is not protected */ - *table = (unsigned long) s_pgt | _SEGMENT_ENTRY | - (pgt & _SEGMENT_ENTRY_PROTECT) | _SEGMENT_ENTRY_INVALID; - if (fake) { - /* nothing to protect for fake tables */ - *table &= ~_SEGMENT_ENTRY_INVALID; - spin_unlock(&sg->guest_table_lock); - return 0; - } - spin_unlock(&sg->guest_table_lock); - /* Make pgt read-only in parent gmap page table (not the pgste) */ - raddr = (saddr & _SEGMENT_MASK) | _SHADOW_RMAP_SEGMENT; - origin = pgt & _SEGMENT_ENTRY_ORIGIN & PAGE_MASK; - rc = gmap_protect_rmap(sg, raddr, origin, PAGE_SIZE); - spin_lock(&sg->guest_table_lock); - if (!rc) { - table = gmap_table_walk(sg, saddr, 1); - if (!table || (*table & _SEGMENT_ENTRY_ORIGIN) != s_pgt) - rc = -EAGAIN; /* Race with unshadow */ - else - *table &= ~_SEGMENT_ENTRY_INVALID; - } else { - gmap_unshadow_pgt(sg, raddr); - } - spin_unlock(&sg->guest_table_lock); - return rc; -out_free: - spin_unlock(&sg->guest_table_lock); - page_table_free_pgste(ptdesc); - return rc; - -} -EXPORT_SYMBOL_GPL(gmap_shadow_pgt); - -/** - * gmap_shadow_page - create a shadow page mapping - * @sg: pointer to the shadow guest address space structure - * @saddr: faulting address in the shadow gmap - * @pte: pte in parent gmap address space to get shadowed - * - * Returns 0 if successfully shadowed or already shadowed, -EAGAIN if the - * shadow table structure is incomplete, -ENOMEM if out of memory and - * -EFAULT if an address in the parent gmap could not be resolved. - * - * Called with sg->mm->mmap_lock in read. - */ -int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte) -{ - struct gmap *parent; - struct gmap_rmap *rmap; - unsigned long vmaddr, paddr; - spinlock_t *ptl; - pte_t *sptep, *tptep; - int prot; - int rc; - - BUG_ON(!gmap_is_shadow(sg)); - parent = sg->parent; - prot = (pte_val(pte) & _PAGE_PROTECT) ? PROT_READ : PROT_WRITE; - - rmap = kzalloc(sizeof(*rmap), GFP_KERNEL_ACCOUNT); - if (!rmap) - return -ENOMEM; - rmap->raddr = (saddr & PAGE_MASK) | _SHADOW_RMAP_PGTABLE; - - while (1) { - paddr = pte_val(pte) & PAGE_MASK; - vmaddr = __gmap_translate(parent, paddr); - if (IS_ERR_VALUE(vmaddr)) { - rc = vmaddr; - break; - } - rc = radix_tree_preload(GFP_KERNEL_ACCOUNT); - if (rc) - break; - rc = -EAGAIN; - sptep = gmap_pte_op_walk(parent, paddr, &ptl); - if (sptep) { - spin_lock(&sg->guest_table_lock); - /* Get page table pointer */ - tptep = (pte_t *) gmap_table_walk(sg, saddr, 0); - if (!tptep) { - spin_unlock(&sg->guest_table_lock); - gmap_pte_op_end(sptep, ptl); - radix_tree_preload_end(); - break; - } - rc = ptep_shadow_pte(sg->mm, saddr, sptep, tptep, pte); - if (rc > 0) { - /* Success and a new mapping */ - gmap_insert_rmap(sg, vmaddr, rmap); - rmap = NULL; - rc = 0; - } - gmap_pte_op_end(sptep, ptl); - spin_unlock(&sg->guest_table_lock); - } - radix_tree_preload_end(); - if (!rc) - break; - rc = gmap_pte_op_fixup(parent, paddr, vmaddr, prot); - if (rc) - break; - } - kfree(rmap); - return rc; -} -EXPORT_SYMBOL_GPL(gmap_shadow_page); - -/* - * gmap_shadow_notify - handle notifications for shadow gmap - * - * Called with sg->parent->shadow_lock. - */ -static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr, - unsigned long gaddr) -{ - struct gmap_rmap *rmap, *rnext, *head; - unsigned long start, end, bits, raddr; - - BUG_ON(!gmap_is_shadow(sg)); - - spin_lock(&sg->guest_table_lock); - if (sg->removed) { - spin_unlock(&sg->guest_table_lock); - return; - } - /* Check for top level table */ - start = sg->orig_asce & _ASCE_ORIGIN; - end = start + ((sg->orig_asce & _ASCE_TABLE_LENGTH) + 1) * PAGE_SIZE; - if (!(sg->orig_asce & _ASCE_REAL_SPACE) && gaddr >= start && - gaddr < end) { - /* The complete shadow table has to go */ - gmap_unshadow(sg); - spin_unlock(&sg->guest_table_lock); - list_del(&sg->list); - gmap_put(sg); - return; - } - /* Remove the page table tree from on specific entry */ - head = radix_tree_delete(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT); - gmap_for_each_rmap_safe(rmap, rnext, head) { - bits = rmap->raddr & _SHADOW_RMAP_MASK; - raddr = rmap->raddr ^ bits; - switch (bits) { - case _SHADOW_RMAP_REGION1: - gmap_unshadow_r2t(sg, raddr); - break; - case _SHADOW_RMAP_REGION2: - gmap_unshadow_r3t(sg, raddr); - break; - case _SHADOW_RMAP_REGION3: - gmap_unshadow_sgt(sg, raddr); - break; - case _SHADOW_RMAP_SEGMENT: - gmap_unshadow_pgt(sg, raddr); - break; - case _SHADOW_RMAP_PGTABLE: - gmap_unshadow_page(sg, raddr); - break; - } - kfree(rmap); - } - spin_unlock(&sg->guest_table_lock); -} - -/** - * ptep_notify - call all invalidation callbacks for a specific pte. - * @mm: pointer to the process mm_struct - * @vmaddr: virtual address in the process address space - * @pte: pointer to the page table entry - * @bits: bits from the pgste that caused the notify call - * - * This function is assumed to be called with the page table lock held - * for the pte to notify. - */ -void ptep_notify(struct mm_struct *mm, unsigned long vmaddr, - pte_t *pte, unsigned long bits) -{ - unsigned long offset, gaddr = 0; - struct gmap *gmap, *sg, *next; - - offset = ((unsigned long) pte) & (255 * sizeof(pte_t)); - offset = offset * (PAGE_SIZE / sizeof(pte_t)); - rcu_read_lock(); - list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { - spin_lock(&gmap->guest_table_lock); - gaddr = host_to_guest_lookup(gmap, vmaddr) + offset; - spin_unlock(&gmap->guest_table_lock); - if (!IS_GADDR_VALID(gaddr)) - continue; - - if (!list_empty(&gmap->children) && (bits & PGSTE_VSIE_BIT)) { - spin_lock(&gmap->shadow_lock); - list_for_each_entry_safe(sg, next, - &gmap->children, list) - gmap_shadow_notify(sg, vmaddr, gaddr); - spin_unlock(&gmap->shadow_lock); - } - if (bits & PGSTE_IN_BIT) - gmap_call_notifier(gmap, gaddr, gaddr + PAGE_SIZE - 1); - } - rcu_read_unlock(); -} -EXPORT_SYMBOL_GPL(ptep_notify); - -static void pmdp_notify_gmap(struct gmap *gmap, pmd_t *pmdp, - unsigned long gaddr) -{ - set_pmd(pmdp, clear_pmd_bit(*pmdp, __pgprot(_SEGMENT_ENTRY_GMAP_IN))); - gmap_call_notifier(gmap, gaddr, gaddr + HPAGE_SIZE - 1); -} - -/** - * gmap_pmdp_xchg - exchange a gmap pmd with another - * @gmap: pointer to the guest address space structure - * @pmdp: pointer to the pmd entry - * @new: replacement entry - * @gaddr: the affected guest address - * - * This function is assumed to be called with the guest_table_lock - * held. - */ -static void gmap_pmdp_xchg(struct gmap *gmap, pmd_t *pmdp, pmd_t new, - unsigned long gaddr) -{ - gaddr &= HPAGE_MASK; - pmdp_notify_gmap(gmap, pmdp, gaddr); - new = clear_pmd_bit(new, __pgprot(_SEGMENT_ENTRY_GMAP_IN)); - if (machine_has_tlb_guest()) - __pmdp_idte(gaddr, (pmd_t *)pmdp, IDTE_GUEST_ASCE, gmap->asce, - IDTE_GLOBAL); - else - __pmdp_idte(gaddr, (pmd_t *)pmdp, 0, 0, IDTE_GLOBAL); - set_pmd(pmdp, new); -} - -static void gmap_pmdp_clear(struct mm_struct *mm, unsigned long vmaddr, - int purge) -{ - pmd_t *pmdp; - struct gmap *gmap; - unsigned long gaddr; - - rcu_read_lock(); - list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { - spin_lock(&gmap->guest_table_lock); - pmdp = host_to_guest_pmd_delete(gmap, vmaddr, &gaddr); - if (pmdp) { - pmdp_notify_gmap(gmap, pmdp, gaddr); - WARN_ON(pmd_val(*pmdp) & ~(_SEGMENT_ENTRY_HARDWARE_BITS_LARGE | - _SEGMENT_ENTRY_GMAP_UC | - _SEGMENT_ENTRY)); - if (purge) - __pmdp_cspg(pmdp); - set_pmd(pmdp, __pmd(_SEGMENT_ENTRY_EMPTY)); - } - spin_unlock(&gmap->guest_table_lock); - } - rcu_read_unlock(); -} - -/** - * gmap_pmdp_invalidate - invalidate all affected guest pmd entries without - * flushing - * @mm: pointer to the process mm_struct - * @vmaddr: virtual address in the process address space - */ -void gmap_pmdp_invalidate(struct mm_struct *mm, unsigned long vmaddr) -{ - gmap_pmdp_clear(mm, vmaddr, 0); -} -EXPORT_SYMBOL_GPL(gmap_pmdp_invalidate); - -/** - * gmap_pmdp_idte_local - invalidate and clear a guest pmd entry - * @mm: pointer to the process mm_struct - * @vmaddr: virtual address in the process address space - */ -void gmap_pmdp_idte_local(struct mm_struct *mm, unsigned long vmaddr) -{ - unsigned long gaddr; - struct gmap *gmap; - pmd_t *pmdp; - - rcu_read_lock(); - list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { - spin_lock(&gmap->guest_table_lock); - pmdp = host_to_guest_pmd_delete(gmap, vmaddr, &gaddr); - if (pmdp) { - pmdp_notify_gmap(gmap, pmdp, gaddr); - WARN_ON(pmd_val(*pmdp) & ~(_SEGMENT_ENTRY_HARDWARE_BITS_LARGE | - _SEGMENT_ENTRY_GMAP_UC | - _SEGMENT_ENTRY)); - if (machine_has_tlb_guest()) - __pmdp_idte(gaddr, pmdp, IDTE_GUEST_ASCE, - gmap->asce, IDTE_LOCAL); - else - __pmdp_idte(gaddr, pmdp, 0, 0, IDTE_LOCAL); - *pmdp = __pmd(_SEGMENT_ENTRY_EMPTY); - } - spin_unlock(&gmap->guest_table_lock); - } - rcu_read_unlock(); -} -EXPORT_SYMBOL_GPL(gmap_pmdp_idte_local); - -/** - * gmap_pmdp_idte_global - invalidate and clear a guest pmd entry - * @mm: pointer to the process mm_struct - * @vmaddr: virtual address in the process address space - */ -void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr) -{ - unsigned long gaddr; - struct gmap *gmap; - pmd_t *pmdp; - - rcu_read_lock(); - list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { - spin_lock(&gmap->guest_table_lock); - pmdp = host_to_guest_pmd_delete(gmap, vmaddr, &gaddr); - if (pmdp) { - pmdp_notify_gmap(gmap, pmdp, gaddr); - WARN_ON(pmd_val(*pmdp) & ~(_SEGMENT_ENTRY_HARDWARE_BITS_LARGE | - _SEGMENT_ENTRY_GMAP_UC | - _SEGMENT_ENTRY)); - if (machine_has_tlb_guest()) - __pmdp_idte(gaddr, pmdp, IDTE_GUEST_ASCE, - gmap->asce, IDTE_GLOBAL); - else - __pmdp_idte(gaddr, pmdp, 0, 0, IDTE_GLOBAL); - *pmdp = __pmd(_SEGMENT_ENTRY_EMPTY); - } - spin_unlock(&gmap->guest_table_lock); - } - rcu_read_unlock(); -} -EXPORT_SYMBOL_GPL(gmap_pmdp_idte_global); - -/** - * gmap_test_and_clear_dirty_pmd - test and reset segment dirty status - * @gmap: pointer to guest address space - * @pmdp: pointer to the pmd to be tested - * @gaddr: virtual address in the guest address space - * - * This function is assumed to be called with the guest_table_lock - * held. - */ -static bool gmap_test_and_clear_dirty_pmd(struct gmap *gmap, pmd_t *pmdp, - unsigned long gaddr) -{ - if (pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID) - return false; - - /* Already protected memory, which did not change is clean */ - if (pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT && - !(pmd_val(*pmdp) & _SEGMENT_ENTRY_GMAP_UC)) - return false; - - /* Clear UC indication and reset protection */ - set_pmd(pmdp, clear_pmd_bit(*pmdp, __pgprot(_SEGMENT_ENTRY_GMAP_UC))); - gmap_protect_pmd(gmap, gaddr, pmdp, PROT_READ, 0); - return true; -} - -/** - * gmap_sync_dirty_log_pmd - set bitmap based on dirty status of segment - * @gmap: pointer to guest address space - * @bitmap: dirty bitmap for this pmd - * @gaddr: virtual address in the guest address space - * @vmaddr: virtual address in the host address space - * - * This function is assumed to be called with the guest_table_lock - * held. - */ -void gmap_sync_dirty_log_pmd(struct gmap *gmap, unsigned long bitmap[4], - unsigned long gaddr, unsigned long vmaddr) -{ - int i; - pmd_t *pmdp; - pte_t *ptep; - spinlock_t *ptl; - - pmdp = gmap_pmd_op_walk(gmap, gaddr); - if (!pmdp) - return; - - if (pmd_leaf(*pmdp)) { - if (gmap_test_and_clear_dirty_pmd(gmap, pmdp, gaddr)) - bitmap_fill(bitmap, _PAGE_ENTRIES); - } else { - for (i = 0; i < _PAGE_ENTRIES; i++, vmaddr += PAGE_SIZE) { - ptep = pte_alloc_map_lock(gmap->mm, pmdp, vmaddr, &ptl); - if (!ptep) - continue; - if (ptep_test_and_clear_uc(gmap->mm, vmaddr, ptep)) - set_bit(i, bitmap); - pte_unmap_unlock(ptep, ptl); - } - } - gmap_pmd_op_end(gmap, pmdp); -} -EXPORT_SYMBOL_GPL(gmap_sync_dirty_log_pmd); - -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -static int thp_split_walk_pmd_entry(pmd_t *pmd, unsigned long addr, - unsigned long end, struct mm_walk *walk) -{ - struct vm_area_struct *vma = walk->vma; - - split_huge_pmd(vma, pmd, addr); - return 0; -} - -static const struct mm_walk_ops thp_split_walk_ops = { - .pmd_entry = thp_split_walk_pmd_entry, - .walk_lock = PGWALK_WRLOCK_VERIFY, -}; - -static inline void thp_split_mm(struct mm_struct *mm) -{ - struct vm_area_struct *vma; - VMA_ITERATOR(vmi, mm, 0); - - for_each_vma(vmi, vma) { - vm_flags_mod(vma, VM_NOHUGEPAGE, VM_HUGEPAGE); - walk_page_vma(vma, &thp_split_walk_ops, NULL); - } - mm->def_flags |= VM_NOHUGEPAGE; -} -#else -static inline void thp_split_mm(struct mm_struct *mm) -{ -} -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ - -/* - * switch on pgstes for its userspace process (for kvm) - */ -int s390_enable_sie(void) -{ - struct mm_struct *mm = current->mm; - - /* Do we have pgstes? if yes, we are done */ - if (mm_has_pgste(mm)) - return 0; - mmap_write_lock(mm); - mm->context.has_pgste = 1; - /* split thp mappings and disable thp for future mappings */ - thp_split_mm(mm); - mmap_write_unlock(mm); - return 0; -} -EXPORT_SYMBOL_GPL(s390_enable_sie); - -/* - * Enable storage key handling from now on and initialize the storage - * keys with the default key. - */ -static int __s390_enable_skey_pte(pte_t *pte, unsigned long addr, - unsigned long next, struct mm_walk *walk) -{ - /* Clear storage key */ - ptep_zap_key(walk->mm, addr, pte); - return 0; -} - -/* - * Give a chance to schedule after setting a key to 256 pages. - * We only hold the mm lock, which is a rwsem and the kvm srcu. - * Both can sleep. - */ -static int __s390_enable_skey_pmd(pmd_t *pmd, unsigned long addr, - unsigned long next, struct mm_walk *walk) -{ - cond_resched(); - return 0; -} - -static int __s390_enable_skey_hugetlb(pte_t *pte, unsigned long addr, - unsigned long hmask, unsigned long next, - struct mm_walk *walk) -{ - pmd_t *pmd = (pmd_t *)pte; - unsigned long start, end; - struct folio *folio = page_folio(pmd_page(*pmd)); - - /* - * The write check makes sure we do not set a key on shared - * memory. This is needed as the walker does not differentiate - * between actual guest memory and the process executable or - * shared libraries. - */ - if (pmd_val(*pmd) & _SEGMENT_ENTRY_INVALID || - !(pmd_val(*pmd) & _SEGMENT_ENTRY_WRITE)) - return 0; - - start = pmd_val(*pmd) & HPAGE_MASK; - end = start + HPAGE_SIZE; - __storage_key_init_range(start, end); - set_bit(PG_arch_1, &folio->flags.f); - cond_resched(); - return 0; -} - -static const struct mm_walk_ops enable_skey_walk_ops = { - .hugetlb_entry = __s390_enable_skey_hugetlb, - .pte_entry = __s390_enable_skey_pte, - .pmd_entry = __s390_enable_skey_pmd, - .walk_lock = PGWALK_WRLOCK, -}; - -int s390_enable_skey(void) -{ - struct mm_struct *mm = current->mm; - int rc = 0; - - mmap_write_lock(mm); - if (mm_uses_skeys(mm)) - goto out_up; - - mm->context.uses_skeys = 1; - rc = gmap_helper_disable_cow_sharing(); - if (rc) { - mm->context.uses_skeys = 0; - goto out_up; - } - walk_page_range(mm, 0, TASK_SIZE, &enable_skey_walk_ops, NULL); - -out_up: - mmap_write_unlock(mm); - return rc; -} -EXPORT_SYMBOL_GPL(s390_enable_skey); - -/* - * Reset CMMA state, make all pages stable again. - */ -static int __s390_reset_cmma(pte_t *pte, unsigned long addr, - unsigned long next, struct mm_walk *walk) -{ - ptep_zap_unused(walk->mm, addr, pte, 1); - return 0; -} - -static const struct mm_walk_ops reset_cmma_walk_ops = { - .pte_entry = __s390_reset_cmma, - .walk_lock = PGWALK_WRLOCK, -}; - -void s390_reset_cmma(struct mm_struct *mm) -{ - mmap_write_lock(mm); - walk_page_range(mm, 0, TASK_SIZE, &reset_cmma_walk_ops, NULL); - mmap_write_unlock(mm); -} -EXPORT_SYMBOL_GPL(s390_reset_cmma); - -#define GATHER_GET_PAGES 32 - -struct reset_walk_state { - unsigned long next; - unsigned long count; - unsigned long pfns[GATHER_GET_PAGES]; -}; - -static int s390_gather_pages(pte_t *ptep, unsigned long addr, - unsigned long next, struct mm_walk *walk) -{ - struct reset_walk_state *p = walk->private; - pte_t pte = READ_ONCE(*ptep); - - if (pte_present(pte)) { - /* we have a reference from the mapping, take an extra one */ - get_page(phys_to_page(pte_val(pte))); - p->pfns[p->count] = phys_to_pfn(pte_val(pte)); - p->next = next; - p->count++; - } - return p->count >= GATHER_GET_PAGES; -} - -static const struct mm_walk_ops gather_pages_ops = { - .pte_entry = s390_gather_pages, - .walk_lock = PGWALK_RDLOCK, -}; - -/* - * Call the Destroy secure page UVC on each page in the given array of PFNs. - * Each page needs to have an extra reference, which will be released here. - */ -void s390_uv_destroy_pfns(unsigned long count, unsigned long *pfns) -{ - struct folio *folio; - unsigned long i; - - for (i = 0; i < count; i++) { - folio = pfn_folio(pfns[i]); - /* we always have an extra reference */ - uv_destroy_folio(folio); - /* get rid of the extra reference */ - folio_put(folio); - cond_resched(); - } -} -EXPORT_SYMBOL_GPL(s390_uv_destroy_pfns); - -/** - * __s390_uv_destroy_range - Call the destroy secure page UVC on each page - * in the given range of the given address space. - * @mm: the mm to operate on - * @start: the start of the range - * @end: the end of the range - * @interruptible: if not 0, stop when a fatal signal is received - * - * Walk the given range of the given address space and call the destroy - * secure page UVC on each page. Optionally exit early if a fatal signal is - * pending. - * - * Return: 0 on success, -EINTR if the function stopped before completing - */ -int __s390_uv_destroy_range(struct mm_struct *mm, unsigned long start, - unsigned long end, bool interruptible) -{ - struct reset_walk_state state = { .next = start }; - int r = 1; - - while (r > 0) { - state.count = 0; - mmap_read_lock(mm); - r = walk_page_range(mm, state.next, end, &gather_pages_ops, &state); - mmap_read_unlock(mm); - cond_resched(); - s390_uv_destroy_pfns(state.count, state.pfns); - if (interruptible && fatal_signal_pending(current)) - return -EINTR; - } - return 0; -} -EXPORT_SYMBOL_GPL(__s390_uv_destroy_range); - -/** - * s390_replace_asce - Try to replace the current ASCE of a gmap with a copy - * @gmap: the gmap whose ASCE needs to be replaced - * - * If the ASCE is a SEGMENT type then this function will return -EINVAL, - * otherwise the pointers in the host_to_guest radix tree will keep pointing - * to the wrong pages, causing use-after-free and memory corruption. - * If the allocation of the new top level page table fails, the ASCE is not - * replaced. - * In any case, the old ASCE is always removed from the gmap CRST list. - * Therefore the caller has to make sure to save a pointer to it - * beforehand, unless a leak is actually intended. - */ -int s390_replace_asce(struct gmap *gmap) -{ - unsigned long asce; - struct page *page; - void *table; - - /* Replacing segment type ASCEs would cause serious issues */ - if ((gmap->asce & _ASCE_TYPE_MASK) == _ASCE_TYPE_SEGMENT) - return -EINVAL; - - page = gmap_alloc_crst(); - if (!page) - return -ENOMEM; - table = page_to_virt(page); - memcpy(table, gmap->table, 1UL << (CRST_ALLOC_ORDER + PAGE_SHIFT)); - - /* Set new table origin while preserving existing ASCE control bits */ - asce = (gmap->asce & ~_ASCE_ORIGIN) | __pa(table); - WRITE_ONCE(gmap->asce, asce); - WRITE_ONCE(gmap->mm->context.gmap_asce, asce); - WRITE_ONCE(gmap->table, table); - - return 0; -} -EXPORT_SYMBOL_GPL(s390_replace_asce); diff --git a/arch/s390/mm/gmap_helpers.c b/arch/s390/mm/gmap_helpers.c index d41b19925a5a..ff63ffb1dbd2 100644 --- a/arch/s390/mm/gmap_helpers.c +++ b/arch/s390/mm/gmap_helpers.c @@ -15,25 +15,71 @@ #include <linux/pagewalk.h> #include <linux/ksm.h> #include <asm/gmap_helpers.h> -#include <asm/pgtable.h> /** - * ptep_zap_softleaf_entry() - discard a software leaf entry. + * try_get_locked_pte() - like get_locked_pte(), but atomic and with trylock * @mm: the mm - * @entry: the software leaf entry that needs to be zapped + * @vmaddr: the userspace virtual address whose pte is to be found + * @ptl: will be set to the pointer to the lock used to lock the pte in case + * of success. * - * Discards the given software leaf entry. If the leaf entry was an actual - * swap entry (and not a migration entry, for example), the actual swapped - * page is also discarded from swap. + * This function returns the pointer to the pte corresponding to @addr in @mm, + * similarly to get_locked_pte(). Unlike get_locked_pte(), no attempt is made + * to allocate missing page tables. If a missing or large entry is found, the + * function will return NULL. If the ptl lock is contended, %-EAGAIN is + * returned. + * + * In case of success, *@ptl will point to the locked pte lock for the returned + * pte, like get_locked_pte() does. + * + * Context: mmap_lock or vma lock for read or for write needs to be held. + * Return: + * * %NULL if the pte cannot be reached. + * * %-EAGAIN if the pte can be reached, but cannot be locked. + * * the pointer to the pte corresponding to @addr in @mm, if it can be reached + * and locked. */ -static void ptep_zap_softleaf_entry(struct mm_struct *mm, softleaf_t entry) +pte_t *try_get_locked_pte(struct mm_struct *mm, unsigned long vmaddr, spinlock_t **ptl) +__context_unsafe(/* Returns nonnull if lock taken or not taken */) { - if (softleaf_is_swap(entry)) - dec_mm_counter(mm, MM_SWAPENTS); - else if (softleaf_is_migration(entry)) - dec_mm_counter(mm, mm_counter(softleaf_to_folio(entry))); - free_swap_and_cache(entry); + pmd_t *pmdp, pmd, pmdval; + pud_t *pudp, pud; + p4d_t *p4dp, p4d; + pgd_t *pgdp, pgd; + pte_t *ptep; + + pgdp = pgd_offset(mm, vmaddr); + pgd = pgdp_get(pgdp); + if (pgd_none(pgd) || !pgd_present(pgd)) + return NULL; + p4dp = p4d_offset_lockless(pgdp, pgd, vmaddr); + p4d = p4dp_get(p4dp); + if (p4d_none(p4d) || !p4d_present(p4d)) + return NULL; + pudp = pud_offset_lockless(p4dp, p4d, vmaddr); + pud = pudp_get(pudp); + if (pud_none(pud) || pud_leaf(pud) || !pud_present(pud)) + return NULL; + pmdp = pmd_offset_lockless(pudp, pud, vmaddr); + pmd = pmdp_get_lockless(pmdp); + if (pmd_none(pmd) || pmd_leaf(pmd) || !pmd_present(pmd)) + return NULL; + ptep = pte_offset_map_rw_nolock(mm, pmdp, vmaddr, &pmdval, ptl); + if (!ptep) + return NULL; + + if (spin_trylock(*ptl)) { + if (unlikely(!pmd_same(pmdval, pmdp_get_lockless(pmdp)))) { + pte_unmap_unlock(ptep, *ptl); + return ERR_PTR(-EAGAIN); + } + return ptep; + } + + pte_unmap(ptep); + return ERR_PTR(-EAGAIN); } +EXPORT_SYMBOL_GPL(try_get_locked_pte); /** * gmap_helper_zap_one_page() - discard a page if it was swapped. @@ -45,11 +91,11 @@ static void ptep_zap_softleaf_entry(struct mm_struct *mm, softleaf_t entry) * Context: needs to be called while holding the mmap lock. */ void gmap_helper_zap_one_page(struct mm_struct *mm, unsigned long vmaddr) +__context_unsafe(/* pte_unmap_unlock() not instrumented */) { struct vm_area_struct *vma; - unsigned long pgstev; - spinlock_t *ptl; - pgste_t pgste; + spinlock_t *ptl; /* Lock for the host (userspace) page table */ + softleaf_t sl; pte_t *ptep; mmap_assert_locked(mm); @@ -60,22 +106,14 @@ void gmap_helper_zap_one_page(struct mm_struct *mm, unsigned long vmaddr) return; /* Get pointer to the page table entry */ - ptep = get_locked_pte(mm, vmaddr, &ptl); - if (unlikely(!ptep)) + ptep = try_get_locked_pte(mm, vmaddr, &ptl); + if (IS_ERR_OR_NULL(ptep)) return; - if (pte_swap(*ptep)) { - preempt_disable(); - pgste = pgste_get_lock(ptep); - pgstev = pgste_val(pgste); - - if ((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED || - (pgstev & _PGSTE_GPS_ZERO)) { - ptep_zap_softleaf_entry(mm, softleaf_from_pte(*ptep)); - pte_clear(mm, vmaddr, ptep); - } - - pgste_set_unlock(ptep, pgste); - preempt_enable(); + sl = softleaf_from_pte(*ptep); + if (pte_swap(*ptep) && softleaf_is_swap(sl)) { + dec_mm_counter(mm, MM_SWAPENTS); + swap_put_entries_direct(sl, 1); + pte_clear(mm, vmaddr, ptep); } pte_unmap_unlock(ptep, ptl); } @@ -102,12 +140,56 @@ void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned lo if (!vma) return; if (!is_vm_hugetlb_page(vma)) - zap_page_range_single(vma, vmaddr, min(end, vma->vm_end) - vmaddr, NULL); + zap_vma_range(vma, vmaddr, min(end, vma->vm_end) - vmaddr); vmaddr = vma->vm_end; } } EXPORT_SYMBOL_GPL(gmap_helper_discard); +/** + * gmap_helper_try_set_pte_unused() - mark a pte entry as unused + * @mm: the mm + * @vmaddr: the userspace address whose pte is to be marked + * + * Mark the pte corresponding the given address as unused. This will cause + * core mm code to just drop this page instead of swapping it. + * + * This function needs to be called with interrupts disabled (for example + * while holding a spinlock), or while holding the mmap lock. Normally this + * function is called as a result of an unmap operation, and thus KVM common + * code will already hold kvm->mmu_lock in write mode. + * + * Context: Needs to be called while holding the mmap lock or with interrupts + * disabled. + */ +void gmap_helper_try_set_pte_unused(struct mm_struct *mm, unsigned long vmaddr) +__context_unsafe(/* pte_unmap_unlock() not instrumented */) +{ + spinlock_t *ptl; /* Lock for the host (userspace) page table */ + pte_t *ptep; + + /* + * Several paths exists that takes the ptl lock and then call the + * mmu_notifier, which takes the mmu_lock. The unmap path, instead, + * takes the mmu_lock in write mode first, and then potentially + * calls this function, which takes the ptl lock. This can lead to a + * deadlock. + * The unused page mechanism is only an optimization, if the + * _PAGE_UNUSED bit is not set, the unused page is swapped as normal + * instead of being discarded. + * If the lock is contended the bit is not set and the deadlock is + * avoided. + */ + ptep = try_get_locked_pte(mm, vmaddr, &ptl); + if (IS_ERR_OR_NULL(ptep)) + return; + + if (pte_present(*ptep)) + __atomic64_or(_PAGE_UNUSED, (long *)ptep); + pte_unmap_unlock(ptep, ptl); +} +EXPORT_SYMBOL_GPL(gmap_helper_try_set_pte_unused); + static int find_zeropage_pte_entry(pte_t *pte, unsigned long addr, unsigned long end, struct mm_walk *walk) { @@ -121,7 +203,7 @@ static int find_zeropage_pte_entry(pte_t *pte, unsigned long addr, * currently only works in COW mappings, which is also where * mm_forbids_zeropage() is checked. */ - if (!is_cow_mapping(walk->vma->vm_flags)) + if (!vma_is_cow_mapping(walk->vma)) return -EFAULT; *found_addr = addr; @@ -193,7 +275,6 @@ retry: * truncation. In that case, the shared zeropage would be gone * and we can simply retry and make progress. */ - cond_resched(); goto retry; } diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c index d42e61c7594e..f84aa9265430 100644 --- a/arch/s390/mm/hugetlbpage.c +++ b/arch/s390/mm/hugetlbpage.c @@ -135,29 +135,6 @@ static inline pte_t __rste_to_pte(unsigned long rste) return __pte(pteval); } -static void clear_huge_pte_skeys(struct mm_struct *mm, unsigned long rste) -{ - struct folio *folio; - unsigned long size, paddr; - - if (!mm_uses_skeys(mm) || - rste & _SEGMENT_ENTRY_INVALID) - return; - - if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) { - folio = page_folio(pud_page(__pud(rste))); - size = PUD_SIZE; - paddr = rste & PUD_MASK; - } else { - folio = page_folio(pmd_page(__pmd(rste))); - size = PMD_SIZE; - paddr = rste & PMD_MASK; - } - - if (!test_and_set_bit(PG_arch_1, &folio->flags.f)) - __storage_key_init_range(paddr, paddr + size); -} - void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) { @@ -166,15 +143,16 @@ void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr, rste = __pte_to_rste(pte); /* Set correct table type for 2G hugepages */ - if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) { + if ((pte_val(ptep_get(ptep)) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) { if (likely(pte_present(pte))) rste |= _REGION3_ENTRY_LARGE; rste |= _REGION_ENTRY_TYPE_R3; - } else if (likely(pte_present(pte))) - rste |= _SEGMENT_ENTRY_LARGE; - - clear_huge_pte_skeys(mm, rste); - set_pte(ptep, __pte(rste)); + set_pud((pud_t *)ptep, __pud(rste)); + } else { + if (likely(pte_present(pte))) + rste |= _SEGMENT_ENTRY_LARGE; + set_pmd((pmd_t *)ptep, __pmd(rste)); + } } void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, @@ -185,7 +163,7 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - return __rste_to_pte(pte_val(*ptep)); + return __rste_to_pte(pte_val(ptep_get(ptep))); } pte_t __huge_ptep_get_and_clear(struct mm_struct *mm, @@ -195,7 +173,7 @@ pte_t __huge_ptep_get_and_clear(struct mm_struct *mm, pmd_t *pmdp = (pmd_t *) ptep; pud_t *pudp = (pud_t *) ptep; - if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) + if ((pte_val(ptep_get(ptep)) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) pudp_xchg_direct(mm, addr, pudp, __pud(_REGION3_ENTRY_EMPTY)); else pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY)); @@ -233,13 +211,13 @@ pte_t *huge_pte_offset(struct mm_struct *mm, pmd_t *pmdp = NULL; pgdp = pgd_offset(mm, addr); - if (pgd_present(*pgdp)) { + if (pgd_present(pgdp_get(pgdp))) { p4dp = p4d_offset(pgdp, addr); - if (p4d_present(*p4dp)) { + if (p4d_present(p4dp_get(p4dp))) { pudp = pud_offset(p4dp, addr); if (sz == PUD_SIZE) return (pte_t *)pudp; - if (pud_present(*pudp)) + if (pud_present(pudp_get(pudp))) pmdp = pmd_offset(pudp, addr); } } @@ -255,3 +233,11 @@ bool __init arch_hugetlb_valid_size(unsigned long size) else return false; } + +unsigned int __init arch_hugetlb_cma_order(void) +{ + if (cpu_has_edat2()) + return PUD_SHIFT - PAGE_SHIFT; + + return 0; +} diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index e4953453d254..be7e009e7b59 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -50,6 +50,7 @@ #include <linux/virtio_anchor.h> #include <linux/virtio_config.h> #include <linux/execmem.h> +#include <linux/cc_platform.h> pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(".bss..swapper_pg_dir"); pgd_t invalid_pg_dir[PTRS_PER_PGD] __section(".bss..invalid_pg_dir"); @@ -69,7 +70,7 @@ unsigned long empty_zero_page, zero_page_mask; EXPORT_SYMBOL(empty_zero_page); EXPORT_SYMBOL(zero_page_mask); -static void __init setup_zero_pages(void) +void __init arch_setup_zero_pages(void) { unsigned long total_pages = memblock_estimated_nr_free_pages(); unsigned int order; @@ -84,6 +85,14 @@ static void __init setup_zero_pages(void) empty_zero_page = (unsigned long)memblock_alloc_or_panic(PAGE_SIZE << order, PAGE_SIZE); zero_page_mask = ((PAGE_SIZE << order) - 1) & PAGE_MASK; + + set_memory_ro(empty_zero_page, 1UL << order); +} + +void __init arch_zone_limits_init(unsigned long *max_zone_pfns) +{ + max_zone_pfns[ZONE_DMA] = virt_to_pfn(MAX_DMA_ADDRESS); + max_zone_pfns[ZONE_NORMAL] = max_low_pfn; } /* @@ -91,15 +100,8 @@ static void __init setup_zero_pages(void) */ void __init paging_init(void) { - unsigned long max_zone_pfns[MAX_NR_ZONES]; - vmem_map_init(); - sparse_init(); zone_dma_limit = DMA_BIT_MASK(31); - memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); - max_zone_pfns[ZONE_DMA] = virt_to_pfn(MAX_DMA_ADDRESS); - max_zone_pfns[ZONE_NORMAL] = max_low_pfn; - free_area_init(max_zone_pfns); } void mark_rodata_ro(void) @@ -141,6 +143,20 @@ bool force_dma_unencrypted(struct device *dev) return is_prot_virt_guest(); } + +bool cc_platform_has(enum cc_attr attr) +{ + switch (attr) { + case CC_ATTR_MEM_ENCRYPT: + case CC_ATTR_GUEST_MEM_ENCRYPT: + return is_prot_virt_guest(); + + default: + return false; + } +} +EXPORT_SYMBOL_GPL(cc_platform_has); + /* protected virtualization */ static void __init pv_init(void) { @@ -150,7 +166,7 @@ static void __init pv_init(void) virtio_set_mem_acc_cb(virtio_require_restricted_mem_acc); /* make sure bounce buffers are shared */ - swiotlb_init(true, SWIOTLB_FORCE | SWIOTLB_VERBOSE); + swiotlb_init(true, SWIOTLB_VERBOSE | SWIOTLB_ANY); swiotlb_update_mem_attributes(); } @@ -160,8 +176,6 @@ void __init arch_mm_preinit(void) cpumask_set_cpu(0, mm_cpumask(&init_mm)); pv_init(); - - setup_zero_pages(); /* Setup zeroed pages. */ } unsigned long memory_block_size_bytes(void) @@ -279,12 +293,13 @@ int arch_add_memory(int nid, u64 start, u64 size, return rc; } -void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap) +void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap, + struct dev_pagemap *pgmap) { unsigned long start_pfn = start >> PAGE_SHIFT; unsigned long nr_pages = size >> PAGE_SHIFT; - __remove_pages(start_pfn, nr_pages, altmap); + __remove_pages(start_pfn, nr_pages, altmap, pgmap); vmem_remove_mapping(start, size); } #endif /* CONFIG_MEMORY_HOTPLUG */ diff --git a/arch/s390/mm/maccess.c b/arch/s390/mm/maccess.c index cfd219fe495c..f39968dd8063 100644 --- a/arch/s390/mm/maccess.c +++ b/arch/s390/mm/maccess.c @@ -96,7 +96,7 @@ size_t memcpy_real_iter(struct iov_iter *iter, unsigned long src, size_t count) pte = mk_pte_phys(phys, PAGE_KERNEL_RO); mutex_lock(&memcpy_real_mutex); - if (pte_val(pte) != pte_val(*memcpy_real_ptep)) { + if (pte_val(pte) != pte_val(ptep_get(memcpy_real_ptep))) { __ptep_ipte(__memcpy_real_area, memcpy_real_ptep, 0, 0, IPTE_GLOBAL); set_pte(memcpy_real_ptep, pte); } diff --git a/arch/s390/mm/mmap.c b/arch/s390/mm/mmap.c index 2a222a7e14f4..ef7bfc87758c 100644 --- a/arch/s390/mm/mmap.c +++ b/arch/s390/mm/mmap.c @@ -64,7 +64,7 @@ static inline unsigned long mmap_base(unsigned long rnd, return PAGE_ALIGN(STACK_TOP - gap - rnd); } -static int get_align_mask(struct file *filp, unsigned long flags) +static unsigned long get_align_mask(struct file *filp, unsigned long flags) { if (filp && is_file_hugepages(filp)) return huge_page_mask_align(filp); diff --git a/arch/s390/mm/page-states.c b/arch/s390/mm/page-states.c index 01f9b39e65f5..5bee173db72e 100644 --- a/arch/s390/mm/page-states.c +++ b/arch/s390/mm/page-states.c @@ -13,6 +13,7 @@ #include <asm/page.h> int __bootdata_preserved(cmma_flag); +EXPORT_SYMBOL(cmma_flag); void arch_free_page(struct page *page, int order) { diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c index d3ce04a4b248..1e202e3d08e7 100644 --- a/arch/s390/mm/pageattr.c +++ b/arch/s390/mm/pageattr.c @@ -16,13 +16,6 @@ #include <asm/asm.h> #include <asm/set_memory.h> -static inline unsigned long sske_frame(unsigned long addr, unsigned char skey) -{ - asm volatile(".insn rrf,0xb22b0000,%[skey],%[addr],1,0" - : [addr] "+a" (addr) : [skey] "d" (skey)); - return addr; -} - void __storage_key_init_range(unsigned long start, unsigned long end) { unsigned long boundary, size; @@ -92,7 +85,7 @@ static int walk_pte_level(pmd_t *pmdp, unsigned long addr, unsigned long end, return 0; ptep = pte_offset_kernel(pmdp, addr); do { - new = *ptep; + new = ptep_get(ptep); if (pte_none(new)) return -EINVAL; if (flags & SET_MEMORY_RO) @@ -112,7 +105,6 @@ static int walk_pte_level(pmd_t *pmdp, unsigned long addr, unsigned long end, pgt_set((unsigned long *)ptep, pte_val(new), addr, CRDTE_DTT_PAGE); ptep++; addr += PAGE_SIZE; - cond_resched(); } while (addr < end); return 0; } @@ -121,15 +113,16 @@ static int split_pmd_page(pmd_t *pmdp, unsigned long addr) { unsigned long pte_addr, prot; pte_t *pt_dir, *ptep; - pmd_t new; + pmd_t new, pmd; int i, ro, nx; pt_dir = vmem_pte_alloc(); if (!pt_dir) return -ENOMEM; - pte_addr = pmd_pfn(*pmdp) << PAGE_SHIFT; - ro = !!(pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT); - nx = !!(pmd_val(*pmdp) & _SEGMENT_ENTRY_NOEXEC); + pmd = pmdp_get(pmdp); + pte_addr = pmd_pfn(pmd) << PAGE_SHIFT; + ro = !!(pmd_val(pmd) & _SEGMENT_ENTRY_PROTECT); + nx = !!(pmd_val(pmd) & _SEGMENT_ENTRY_NOEXEC); prot = pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL); if (!nx) prot &= ~_PAGE_NOEXEC; @@ -149,7 +142,7 @@ static int split_pmd_page(pmd_t *pmdp, unsigned long addr) static void modify_pmd_page(pmd_t *pmdp, unsigned long addr, unsigned long flags) { - pmd_t new = *pmdp; + pmd_t new = pmdp_get(pmdp); if (flags & SET_MEMORY_RO) new = pmd_wrprotect(new); @@ -172,16 +165,17 @@ static int walk_pmd_level(pud_t *pudp, unsigned long addr, unsigned long end, unsigned long flags) { unsigned long next; + pmd_t *pmdp, pmd; int need_split; - pmd_t *pmdp; int rc = 0; pmdp = pmd_offset(pudp, addr); do { - if (pmd_none(*pmdp)) + pmd = pmdp_get(pmdp); + if (pmd_none(pmd)) return -EINVAL; next = pmd_addr_end(addr, end); - if (pmd_leaf(*pmdp)) { + if (pmd_leaf(pmd)) { need_split = !!(flags & SET_MEMORY_4K); need_split |= !!(addr & ~PMD_MASK); need_split |= !!(addr + PMD_SIZE > next); @@ -199,7 +193,6 @@ static int walk_pmd_level(pud_t *pudp, unsigned long addr, unsigned long end, } pmdp++; addr = next; - cond_resched(); } while (addr < end); return rc; } @@ -208,15 +201,16 @@ int split_pud_page(pud_t *pudp, unsigned long addr) { unsigned long pmd_addr, prot; pmd_t *pm_dir, *pmdp; - pud_t new; + pud_t new, pud; int i, ro, nx; pm_dir = vmem_crst_alloc(_SEGMENT_ENTRY_EMPTY); if (!pm_dir) return -ENOMEM; - pmd_addr = pud_pfn(*pudp) << PAGE_SHIFT; - ro = !!(pud_val(*pudp) & _REGION_ENTRY_PROTECT); - nx = !!(pud_val(*pudp) & _REGION_ENTRY_NOEXEC); + pud = pudp_get(pudp); + pmd_addr = pud_pfn(pud) << PAGE_SHIFT; + ro = !!(pud_val(pud) & _REGION_ENTRY_PROTECT); + nx = !!(pud_val(pud) & _REGION_ENTRY_NOEXEC); prot = pgprot_val(ro ? SEGMENT_KERNEL_RO : SEGMENT_KERNEL); if (!nx) prot &= ~_SEGMENT_ENTRY_NOEXEC; @@ -236,7 +230,7 @@ int split_pud_page(pud_t *pudp, unsigned long addr) static void modify_pud_page(pud_t *pudp, unsigned long addr, unsigned long flags) { - pud_t new = *pudp; + pud_t new = pudp_get(pudp); if (flags & SET_MEMORY_RO) new = pud_wrprotect(new); @@ -259,16 +253,17 @@ static int walk_pud_level(p4d_t *p4d, unsigned long addr, unsigned long end, unsigned long flags) { unsigned long next; + pud_t *pudp, pud; int need_split; - pud_t *pudp; int rc = 0; pudp = pud_offset(p4d, addr); do { - if (pud_none(*pudp)) + pud = pudp_get(pudp); + if (pud_none(pud)) return -EINVAL; next = pud_addr_end(addr, end); - if (pud_leaf(*pudp)) { + if (pud_leaf(pud)) { need_split = !!(flags & SET_MEMORY_4K); need_split |= !!(addr & ~PUD_MASK); need_split |= !!(addr + PUD_SIZE > next); @@ -284,7 +279,6 @@ static int walk_pud_level(p4d_t *p4d, unsigned long addr, unsigned long end, } pudp++; addr = next; - cond_resched(); } while (addr < end && !rc); return rc; } @@ -298,13 +292,12 @@ static int walk_p4d_level(pgd_t *pgd, unsigned long addr, unsigned long end, p4dp = p4d_offset(pgd, addr); do { - if (p4d_none(*p4dp)) + if (p4d_none(p4dp_get(p4dp))) return -EINVAL; next = p4d_addr_end(addr, end); rc = walk_pud_level(p4dp, addr, next, flags); p4dp++; addr = next; - cond_resched(); } while (addr < end && !rc); return rc; } @@ -320,13 +313,12 @@ static int change_page_attr(unsigned long addr, unsigned long end, pgdp = pgd_offset_k(addr); do { - if (pgd_none(*pgdp)) + if (pgd_none(pgdp_get(pgdp))) break; next = pgd_addr_end(addr, end); rc = walk_p4d_level(pgdp, addr, next, flags); if (rc) break; - cond_resched(); } while (pgdp++, addr = next, addr < end && !rc); return rc; } @@ -458,7 +450,8 @@ void __kernel_map_pages(struct page *page, int numpages, int enable) nr = min(numpages - i, nr); if (enable) { for (j = 0; j < nr; j++) { - pte = clear_pte_bit(*ptep, __pgprot(_PAGE_INVALID)); + pte = ptep_get(ptep); + pte = clear_pte_bit(pte, __pgprot(_PAGE_INVALID)); set_pte(ptep, pte); address += PAGE_SIZE; ptep++; diff --git a/arch/s390/mm/pfault.c b/arch/s390/mm/pfault.c index 2f829448c719..6ecd6b0a22a8 100644 --- a/arch/s390/mm/pfault.c +++ b/arch/s390/mm/pfault.c @@ -62,7 +62,7 @@ int __pfault_init(void) "0: nopr %%r7\n" EX_TABLE(0b, 0b) : [rc] "+d" (rc) - : [refbk] "a" (&pfault_init_refbk), "m" (pfault_init_refbk) + : [refbk] "a" (virt_to_phys(&pfault_init_refbk)), "m" (pfault_init_refbk) : "cc"); return rc; } @@ -84,7 +84,7 @@ void __pfault_fini(void) "0: nopr %%r7\n" EX_TABLE(0b, 0b) : - : [refbk] "a" (&pfault_fini_refbk), "m" (pfault_fini_refbk) + : [refbk] "a" (virt_to_phys(&pfault_fini_refbk)), "m" (pfault_fini_refbk) : "cc"); } diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index 7df23528c01b..4b160eedc5a0 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -55,89 +55,48 @@ static void __crst_table_upgrade(void *arg) int crst_table_upgrade(struct mm_struct *mm, unsigned long end) { - unsigned long *pgd = NULL, *p4d = NULL, *__pgd; - unsigned long asce_limit = mm->context.asce_limit; + unsigned long *table, *pgd; + int rc, notify; mmap_assert_write_locked(mm); - /* upgrade should only happen from 3 to 4, 3 to 5, or 4 to 5 levels */ - VM_BUG_ON(asce_limit < _REGION2_SIZE); - - if (end <= asce_limit) - return 0; - - if (asce_limit == _REGION2_SIZE) { - p4d = crst_table_alloc(mm); - if (unlikely(!p4d)) - goto err_p4d; - crst_table_init(p4d, _REGION2_ENTRY_EMPTY); - pagetable_p4d_ctor(virt_to_ptdesc(p4d)); - } - if (end > _REGION1_SIZE) { - pgd = crst_table_alloc(mm); - if (unlikely(!pgd)) - goto err_pgd; - crst_table_init(pgd, _REGION1_ENTRY_EMPTY); - pagetable_pgd_ctor(virt_to_ptdesc(pgd)); - } - - spin_lock_bh(&mm->page_table_lock); - - if (p4d) { - __pgd = (unsigned long *) mm->pgd; - p4d_populate(mm, (p4d_t *) p4d, (pud_t *) __pgd); - mm->pgd = (pgd_t *) p4d; - mm->context.asce_limit = _REGION1_SIZE; - mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | - _ASCE_USER_BITS | _ASCE_TYPE_REGION2; - mm_inc_nr_puds(mm); - } - if (pgd) { - __pgd = (unsigned long *) mm->pgd; - pgd_populate(mm, (pgd_t *) pgd, (p4d_t *) __pgd); - mm->pgd = (pgd_t *) pgd; - mm->context.asce_limit = TASK_SIZE_MAX; - mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | - _ASCE_USER_BITS | _ASCE_TYPE_REGION1; - } - - spin_unlock_bh(&mm->page_table_lock); - - on_each_cpu(__crst_table_upgrade, mm, 0); - - return 0; - -err_pgd: - pagetable_dtor(virt_to_ptdesc(p4d)); - crst_table_free(mm, p4d); -err_p4d: - return -ENOMEM; -} - -#ifdef CONFIG_PGSTE - -struct ptdesc *page_table_alloc_pgste_noprof(struct mm_struct *mm) -{ - struct ptdesc *ptdesc; - u64 *table; - - ptdesc = pagetable_alloc_noprof(GFP_KERNEL_ACCOUNT, 0); - if (ptdesc) { - table = (u64 *)ptdesc_address(ptdesc); - __arch_set_page_dat(table, 1); - memset64(table, _PAGE_INVALID, PTRS_PER_PTE); - memset64(table + PTRS_PER_PTE, 0, PTRS_PER_PTE); + VM_BUG_ON(mm->context.asce_limit < _REGION2_SIZE); + rc = 0; + notify = 0; + while (mm->context.asce_limit < end) { + table = crst_table_alloc(mm); + if (!table) { + rc = -ENOMEM; + break; + } + spin_lock_bh(&mm->page_table_lock); + pgd = (unsigned long *)mm->pgd; + if (mm->context.asce_limit == _REGION2_SIZE) { + crst_table_init(table, _REGION2_ENTRY_EMPTY); + p4d_populate(mm, (p4d_t *)table, (pud_t *)pgd); + pagetable_p4d_ctor(virt_to_ptdesc(table)); + mm->pgd = (pgd_t *)table; + mm->context.asce_limit = _REGION1_SIZE; + mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | + _ASCE_USER_BITS | _ASCE_TYPE_REGION2; + mm_inc_nr_puds(mm); + } else { + crst_table_init(table, _REGION1_ENTRY_EMPTY); + pgd_populate(mm, (pgd_t *)table, (p4d_t *)pgd); + pagetable_pgd_ctor(virt_to_ptdesc(table)); + mm->pgd = (pgd_t *)table; + mm->context.asce_limit = TASK_SIZE_MAX; + mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | + _ASCE_USER_BITS | _ASCE_TYPE_REGION1; + } + notify = 1; + spin_unlock_bh(&mm->page_table_lock); } - return ptdesc; -} - -void page_table_free_pgste(struct ptdesc *ptdesc) -{ - pagetable_free(ptdesc); + if (notify) + on_each_cpu(__crst_table_upgrade, mm, 0); + return rc; } -#endif /* CONFIG_PGSTE */ - unsigned long *page_table_alloc_noprof(struct mm_struct *mm) { gfp_t gfp = GFP_KERNEL_ACCOUNT; @@ -292,7 +251,6 @@ static int base_segment_walk(unsigned long *origin, unsigned long addr, return rc; if (!alloc) base_pgt_free(table); - cond_resched(); } while (ste++, addr = next, addr < end); return 0; } diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c index 666adcd681ab..4acd8b140c4b 100644 --- a/arch/s390/mm/pgtable.c +++ b/arch/s390/mm/pgtable.c @@ -24,7 +24,6 @@ #include <asm/tlbflush.h> #include <asm/mmu_context.h> #include <asm/page-states.h> -#include <asm/pgtable.h> #include <asm/machine.h> pgprot_t pgprot_writecombine(pgprot_t prot) @@ -116,149 +115,14 @@ static inline pte_t ptep_flush_lazy(struct mm_struct *mm, return old; } -static inline pgste_t pgste_get(pte_t *ptep) -{ - unsigned long pgste = 0; -#ifdef CONFIG_PGSTE - pgste = *(unsigned long *)(ptep + PTRS_PER_PTE); -#endif - return __pgste(pgste); -} - -static inline void pgste_set(pte_t *ptep, pgste_t pgste) -{ -#ifdef CONFIG_PGSTE - *(pgste_t *)(ptep + PTRS_PER_PTE) = pgste; -#endif -} - -static inline pgste_t pgste_update_all(pte_t pte, pgste_t pgste, - struct mm_struct *mm) -{ -#ifdef CONFIG_PGSTE - unsigned long address, bits, skey; - - if (!mm_uses_skeys(mm) || pte_val(pte) & _PAGE_INVALID) - return pgste; - address = pte_val(pte) & PAGE_MASK; - skey = (unsigned long) page_get_storage_key(address); - bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); - /* Transfer page changed & referenced bit to guest bits in pgste */ - pgste = set_pgste_bit(pgste, bits << 48); /* GR bit & GC bit */ - /* Copy page access key and fetch protection bit to pgste */ - pgste = clear_pgste_bit(pgste, PGSTE_ACC_BITS | PGSTE_FP_BIT); - pgste = set_pgste_bit(pgste, (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56); -#endif - return pgste; - -} - -static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry, - struct mm_struct *mm) -{ -#ifdef CONFIG_PGSTE - unsigned long address; - unsigned long nkey; - - if (!mm_uses_skeys(mm) || pte_val(entry) & _PAGE_INVALID) - return; - VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID)); - address = pte_val(entry) & PAGE_MASK; - /* - * Set page access key and fetch protection bit from pgste. - * The guest C/R information is still in the PGSTE, set real - * key C/R to 0. - */ - nkey = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56; - nkey |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48; - page_set_storage_key(address, nkey, 0); -#endif -} - -static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry) -{ -#ifdef CONFIG_PGSTE - if ((pte_val(entry) & _PAGE_PRESENT) && - (pte_val(entry) & _PAGE_WRITE) && - !(pte_val(entry) & _PAGE_INVALID)) { - if (!machine_has_esop()) { - /* - * Without enhanced suppression-on-protection force - * the dirty bit on for all writable ptes. - */ - entry = set_pte_bit(entry, __pgprot(_PAGE_DIRTY)); - entry = clear_pte_bit(entry, __pgprot(_PAGE_PROTECT)); - } - if (!(pte_val(entry) & _PAGE_PROTECT)) - /* This pte allows write access, set user-dirty */ - pgste = set_pgste_bit(pgste, PGSTE_UC_BIT); - } -#endif - set_pte(ptep, entry); - return pgste; -} - -static inline pgste_t pgste_pte_notify(struct mm_struct *mm, - unsigned long addr, - pte_t *ptep, pgste_t pgste) -{ -#ifdef CONFIG_PGSTE - unsigned long bits; - - bits = pgste_val(pgste) & (PGSTE_IN_BIT | PGSTE_VSIE_BIT); - if (bits) { - pgste = __pgste(pgste_val(pgste) ^ bits); - ptep_notify(mm, addr, ptep, bits); - } -#endif - return pgste; -} - -static inline pgste_t ptep_xchg_start(struct mm_struct *mm, - unsigned long addr, pte_t *ptep) -{ - pgste_t pgste = __pgste(0); - - if (mm_has_pgste(mm)) { - pgste = pgste_get_lock(ptep); - pgste = pgste_pte_notify(mm, addr, ptep, pgste); - } - return pgste; -} - -static inline pte_t ptep_xchg_commit(struct mm_struct *mm, - unsigned long addr, pte_t *ptep, - pgste_t pgste, pte_t old, pte_t new) -{ - if (mm_has_pgste(mm)) { - if (pte_val(old) & _PAGE_INVALID) - pgste_set_key(ptep, pgste, new, mm); - if (pte_val(new) & _PAGE_INVALID) { - pgste = pgste_update_all(old, pgste, mm); - if ((pgste_val(pgste) & _PGSTE_GPS_USAGE_MASK) == - _PGSTE_GPS_USAGE_UNUSED) - old = set_pte_bit(old, __pgprot(_PAGE_UNUSED)); - } - pgste = pgste_set_pte(ptep, pgste, new); - pgste_set_unlock(ptep, pgste); - } else { - set_pte(ptep, new); - } - return old; -} - pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t new) { - pgste_t pgste; pte_t old; - int nodat; preempt_disable(); - pgste = ptep_xchg_start(mm, addr, ptep); - nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); - old = ptep_flush_direct(mm, addr, ptep, nodat); - old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); + old = ptep_flush_direct(mm, addr, ptep, 1); + set_pte(ptep, new); preempt_enable(); return old; } @@ -292,15 +156,11 @@ EXPORT_SYMBOL(ptep_reset_dat_prot); pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t new) { - pgste_t pgste; pte_t old; - int nodat; preempt_disable(); - pgste = ptep_xchg_start(mm, addr, ptep); - nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); - old = ptep_flush_lazy(mm, addr, ptep, nodat); - old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); + old = ptep_flush_lazy(mm, addr, ptep, 1); + set_pte(ptep, new); preempt_enable(); return old; } @@ -309,47 +169,22 @@ EXPORT_SYMBOL(ptep_xchg_lazy); pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { - pgste_t pgste; - pte_t old; - int nodat; - struct mm_struct *mm = vma->vm_mm; - - pgste = ptep_xchg_start(mm, addr, ptep); - nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); - old = ptep_flush_lazy(mm, addr, ptep, nodat); - if (mm_has_pgste(mm)) { - pgste = pgste_update_all(old, pgste, mm); - pgste_set(ptep, pgste); - } - return old; + return ptep_flush_lazy(vma->vm_mm, addr, ptep, 1); } void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t old_pte, pte_t pte) { - pgste_t pgste; - struct mm_struct *mm = vma->vm_mm; - - if (mm_has_pgste(mm)) { - pgste = pgste_get(ptep); - pgste_set_key(ptep, pgste, pte, mm); - pgste = pgste_set_pte(ptep, pgste, pte); - pgste_set_unlock(ptep, pgste); - } else { - set_pte(ptep, pte); - } + set_pte(ptep, pte); } static inline void pmdp_idte_local(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) { if (machine_has_tlb_guest()) - __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE, - mm->context.asce, IDTE_LOCAL); + __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE, mm->context.asce, IDTE_LOCAL); else __pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL); - if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) - gmap_pmdp_idte_local(mm, addr); } static inline void pmdp_idte_global(struct mm_struct *mm, @@ -358,12 +193,8 @@ static inline void pmdp_idte_global(struct mm_struct *mm, if (machine_has_tlb_guest()) { __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE, mm->context.asce, IDTE_GLOBAL); - if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) - gmap_pmdp_idte_global(mm, addr); } else { __pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL); - if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) - gmap_pmdp_idte_global(mm, addr); } } @@ -398,8 +229,6 @@ static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, cpumask_of(smp_processor_id()))) { set_pmd(pmdp, set_pmd_bit(*pmdp, __pgprot(_SEGMENT_ENTRY_INVALID))); mm->context.flush_mm = 1; - if (mm_has_pgste(mm)) - gmap_pmdp_invalidate(mm, addr); } else { pmdp_idte_global(mm, addr, pmdp); } @@ -407,40 +236,6 @@ static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, return old; } -#ifdef CONFIG_PGSTE -static int pmd_lookup(struct mm_struct *mm, unsigned long addr, pmd_t **pmdp) -{ - struct vm_area_struct *vma; - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - - /* We need a valid VMA, otherwise this is clearly a fault. */ - vma = vma_lookup(mm, addr); - if (!vma) - return -EFAULT; - - pgd = pgd_offset(mm, addr); - if (!pgd_present(*pgd)) - return -ENOENT; - - p4d = p4d_offset(pgd, addr); - if (!p4d_present(*p4d)) - return -ENOENT; - - pud = pud_offset(p4d, addr); - if (!pud_present(*pud)) - return -ENOENT; - - /* Large PUDs are not supported yet. */ - if (pud_leaf(*pud)) - return -EFAULT; - - *pmdp = pmd_offset(pud, addr); - return 0; -} -#endif - pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t new) { @@ -558,598 +353,3 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) return pgtable; } #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ - -#ifdef CONFIG_PGSTE -void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t entry) -{ - pgste_t pgste; - - /* the mm_has_pgste() check is done in set_pte_at() */ - preempt_disable(); - pgste = pgste_get_lock(ptep); - pgste = clear_pgste_bit(pgste, _PGSTE_GPS_ZERO); - pgste_set_key(ptep, pgste, entry, mm); - pgste = pgste_set_pte(ptep, pgste, entry); - pgste_set_unlock(ptep, pgste); - preempt_enable(); -} - -void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep) -{ - pgste_t pgste; - - preempt_disable(); - pgste = pgste_get_lock(ptep); - pgste = set_pgste_bit(pgste, PGSTE_IN_BIT); - pgste_set_unlock(ptep, pgste); - preempt_enable(); -} - -/** - * ptep_force_prot - change access rights of a locked pte - * @mm: pointer to the process mm_struct - * @addr: virtual address in the guest address space - * @ptep: pointer to the page table entry - * @prot: indicates guest access rights: PROT_NONE, PROT_READ or PROT_WRITE - * @bit: pgste bit to set (e.g. for notification) - * - * Returns 0 if the access rights were changed and -EAGAIN if the current - * and requested access rights are incompatible. - */ -int ptep_force_prot(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, int prot, unsigned long bit) -{ - pte_t entry; - pgste_t pgste; - int pte_i, pte_p, nodat; - - pgste = pgste_get_lock(ptep); - entry = *ptep; - /* Check pte entry after all locks have been acquired */ - pte_i = pte_val(entry) & _PAGE_INVALID; - pte_p = pte_val(entry) & _PAGE_PROTECT; - if ((pte_i && (prot != PROT_NONE)) || - (pte_p && (prot & PROT_WRITE))) { - pgste_set_unlock(ptep, pgste); - return -EAGAIN; - } - /* Change access rights and set pgste bit */ - nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); - if (prot == PROT_NONE && !pte_i) { - ptep_flush_direct(mm, addr, ptep, nodat); - pgste = pgste_update_all(entry, pgste, mm); - entry = set_pte_bit(entry, __pgprot(_PAGE_INVALID)); - } - if (prot == PROT_READ && !pte_p) { - ptep_flush_direct(mm, addr, ptep, nodat); - entry = clear_pte_bit(entry, __pgprot(_PAGE_INVALID)); - entry = set_pte_bit(entry, __pgprot(_PAGE_PROTECT)); - } - pgste = set_pgste_bit(pgste, bit); - pgste = pgste_set_pte(ptep, pgste, entry); - pgste_set_unlock(ptep, pgste); - return 0; -} - -int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr, - pte_t *sptep, pte_t *tptep, pte_t pte) -{ - pgste_t spgste, tpgste; - pte_t spte, tpte; - int rc = -EAGAIN; - - if (!(pte_val(*tptep) & _PAGE_INVALID)) - return 0; /* already shadowed */ - spgste = pgste_get_lock(sptep); - spte = *sptep; - if (!(pte_val(spte) & _PAGE_INVALID) && - !((pte_val(spte) & _PAGE_PROTECT) && - !(pte_val(pte) & _PAGE_PROTECT))) { - spgste = set_pgste_bit(spgste, PGSTE_VSIE_BIT); - tpgste = pgste_get_lock(tptep); - tpte = __pte((pte_val(spte) & PAGE_MASK) | - (pte_val(pte) & _PAGE_PROTECT)); - /* don't touch the storage key - it belongs to parent pgste */ - tpgste = pgste_set_pte(tptep, tpgste, tpte); - pgste_set_unlock(tptep, tpgste); - rc = 1; - } - pgste_set_unlock(sptep, spgste); - return rc; -} - -void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep) -{ - pgste_t pgste; - int nodat; - - pgste = pgste_get_lock(ptep); - /* notifier is called by the caller */ - nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); - ptep_flush_direct(mm, saddr, ptep, nodat); - /* don't touch the storage key - it belongs to parent pgste */ - pgste = pgste_set_pte(ptep, pgste, __pte(_PAGE_INVALID)); - pgste_set_unlock(ptep, pgste); -} - -static void ptep_zap_softleaf_entry(struct mm_struct *mm, softleaf_t entry) -{ - if (softleaf_is_swap(entry)) - dec_mm_counter(mm, MM_SWAPENTS); - else if (softleaf_is_migration(entry)) { - struct folio *folio = softleaf_to_folio(entry); - - dec_mm_counter(mm, mm_counter(folio)); - } - free_swap_and_cache(entry); -} - -void ptep_zap_unused(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, int reset) -{ - unsigned long pgstev; - pgste_t pgste; - pte_t pte; - - /* Zap unused and logically-zero pages */ - preempt_disable(); - pgste = pgste_get_lock(ptep); - pgstev = pgste_val(pgste); - pte = *ptep; - if (!reset && pte_swap(pte) && - ((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED || - (pgstev & _PGSTE_GPS_ZERO))) { - ptep_zap_softleaf_entry(mm, softleaf_from_pte(pte)); - pte_clear(mm, addr, ptep); - } - if (reset) - pgste = clear_pgste_bit(pgste, _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT); - pgste_set_unlock(ptep, pgste); - preempt_enable(); -} - -void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep) -{ - unsigned long ptev; - pgste_t pgste; - - /* Clear storage key ACC and F, but set R/C */ - preempt_disable(); - pgste = pgste_get_lock(ptep); - pgste = clear_pgste_bit(pgste, PGSTE_ACC_BITS | PGSTE_FP_BIT); - pgste = set_pgste_bit(pgste, PGSTE_GR_BIT | PGSTE_GC_BIT); - ptev = pte_val(*ptep); - if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE)) - page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 0); - pgste_set_unlock(ptep, pgste); - preempt_enable(); -} - -/* - * Test and reset if a guest page is dirty - */ -bool ptep_test_and_clear_uc(struct mm_struct *mm, unsigned long addr, - pte_t *ptep) -{ - pgste_t pgste; - pte_t pte; - bool dirty; - int nodat; - - pgste = pgste_get_lock(ptep); - dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT); - pgste = clear_pgste_bit(pgste, PGSTE_UC_BIT); - pte = *ptep; - if (dirty && (pte_val(pte) & _PAGE_PRESENT)) { - pgste = pgste_pte_notify(mm, addr, ptep, pgste); - nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); - ptep_ipte_global(mm, addr, ptep, nodat); - if (machine_has_esop() || !(pte_val(pte) & _PAGE_WRITE)) - pte = set_pte_bit(pte, __pgprot(_PAGE_PROTECT)); - else - pte = set_pte_bit(pte, __pgprot(_PAGE_INVALID)); - set_pte(ptep, pte); - } - pgste_set_unlock(ptep, pgste); - return dirty; -} -EXPORT_SYMBOL_GPL(ptep_test_and_clear_uc); - -int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, - unsigned char key, bool nq) -{ - unsigned long keyul, paddr; - spinlock_t *ptl; - pgste_t old, new; - pmd_t *pmdp; - pte_t *ptep; - - /* - * If we don't have a PTE table and if there is no huge page mapped, - * we can ignore attempts to set the key to 0, because it already is 0. - */ - switch (pmd_lookup(mm, addr, &pmdp)) { - case -ENOENT: - return key ? -EFAULT : 0; - case 0: - break; - default: - return -EFAULT; - } -again: - ptl = pmd_lock(mm, pmdp); - if (!pmd_present(*pmdp)) { - spin_unlock(ptl); - return key ? -EFAULT : 0; - } - - if (pmd_leaf(*pmdp)) { - paddr = pmd_val(*pmdp) & HPAGE_MASK; - paddr |= addr & ~HPAGE_MASK; - /* - * Huge pmds need quiescing operations, they are - * always mapped. - */ - page_set_storage_key(paddr, key, 1); - spin_unlock(ptl); - return 0; - } - spin_unlock(ptl); - - ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl); - if (!ptep) - goto again; - new = old = pgste_get_lock(ptep); - new = clear_pgste_bit(new, PGSTE_GR_BIT | PGSTE_GC_BIT | - PGSTE_ACC_BITS | PGSTE_FP_BIT); - keyul = (unsigned long) key; - new = set_pgste_bit(new, (keyul & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48); - new = set_pgste_bit(new, (keyul & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56); - if (!(pte_val(*ptep) & _PAGE_INVALID)) { - unsigned long bits, skey; - - paddr = pte_val(*ptep) & PAGE_MASK; - skey = (unsigned long) page_get_storage_key(paddr); - bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); - skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT); - /* Set storage key ACC and FP */ - page_set_storage_key(paddr, skey, !nq); - /* Merge host changed & referenced into pgste */ - new = set_pgste_bit(new, bits << 52); - } - /* changing the guest storage key is considered a change of the page */ - if ((pgste_val(new) ^ pgste_val(old)) & - (PGSTE_ACC_BITS | PGSTE_FP_BIT | PGSTE_GR_BIT | PGSTE_GC_BIT)) - new = set_pgste_bit(new, PGSTE_UC_BIT); - - pgste_set_unlock(ptep, new); - pte_unmap_unlock(ptep, ptl); - return 0; -} -EXPORT_SYMBOL(set_guest_storage_key); - -/* - * Conditionally set a guest storage key (handling csske). - * oldkey will be updated when either mr or mc is set and a pointer is given. - * - * Returns 0 if a guests storage key update wasn't necessary, 1 if the guest - * storage key was updated and -EFAULT on access errors. - */ -int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr, - unsigned char key, unsigned char *oldkey, - bool nq, bool mr, bool mc) -{ - unsigned char tmp, mask = _PAGE_ACC_BITS | _PAGE_FP_BIT; - int rc; - - /* we can drop the pgste lock between getting and setting the key */ - if (mr | mc) { - rc = get_guest_storage_key(current->mm, addr, &tmp); - if (rc) - return rc; - if (oldkey) - *oldkey = tmp; - if (!mr) - mask |= _PAGE_REFERENCED; - if (!mc) - mask |= _PAGE_CHANGED; - if (!((tmp ^ key) & mask)) - return 0; - } - rc = set_guest_storage_key(current->mm, addr, key, nq); - return rc < 0 ? rc : 1; -} -EXPORT_SYMBOL(cond_set_guest_storage_key); - -/* - * Reset a guest reference bit (rrbe), returning the reference and changed bit. - * - * Returns < 0 in case of error, otherwise the cc to be reported to the guest. - */ -int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr) -{ - spinlock_t *ptl; - unsigned long paddr; - pgste_t old, new; - pmd_t *pmdp; - pte_t *ptep; - int cc = 0; - - /* - * If we don't have a PTE table and if there is no huge page mapped, - * the storage key is 0 and there is nothing for us to do. - */ - switch (pmd_lookup(mm, addr, &pmdp)) { - case -ENOENT: - return 0; - case 0: - break; - default: - return -EFAULT; - } -again: - ptl = pmd_lock(mm, pmdp); - if (!pmd_present(*pmdp)) { - spin_unlock(ptl); - return 0; - } - - if (pmd_leaf(*pmdp)) { - paddr = pmd_val(*pmdp) & HPAGE_MASK; - paddr |= addr & ~HPAGE_MASK; - cc = page_reset_referenced(paddr); - spin_unlock(ptl); - return cc; - } - spin_unlock(ptl); - - ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl); - if (!ptep) - goto again; - new = old = pgste_get_lock(ptep); - /* Reset guest reference bit only */ - new = clear_pgste_bit(new, PGSTE_GR_BIT); - - if (!(pte_val(*ptep) & _PAGE_INVALID)) { - paddr = pte_val(*ptep) & PAGE_MASK; - cc = page_reset_referenced(paddr); - /* Merge real referenced bit into host-set */ - new = set_pgste_bit(new, ((unsigned long)cc << 53) & PGSTE_HR_BIT); - } - /* Reflect guest's logical view, not physical */ - cc |= (pgste_val(old) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 49; - /* Changing the guest storage key is considered a change of the page */ - if ((pgste_val(new) ^ pgste_val(old)) & PGSTE_GR_BIT) - new = set_pgste_bit(new, PGSTE_UC_BIT); - - pgste_set_unlock(ptep, new); - pte_unmap_unlock(ptep, ptl); - return cc; -} -EXPORT_SYMBOL(reset_guest_reference_bit); - -int get_guest_storage_key(struct mm_struct *mm, unsigned long addr, - unsigned char *key) -{ - unsigned long paddr; - spinlock_t *ptl; - pgste_t pgste; - pmd_t *pmdp; - pte_t *ptep; - - /* - * If we don't have a PTE table and if there is no huge page mapped, - * the storage key is 0. - */ - *key = 0; - - switch (pmd_lookup(mm, addr, &pmdp)) { - case -ENOENT: - return 0; - case 0: - break; - default: - return -EFAULT; - } -again: - ptl = pmd_lock(mm, pmdp); - if (!pmd_present(*pmdp)) { - spin_unlock(ptl); - return 0; - } - - if (pmd_leaf(*pmdp)) { - paddr = pmd_val(*pmdp) & HPAGE_MASK; - paddr |= addr & ~HPAGE_MASK; - *key = page_get_storage_key(paddr); - spin_unlock(ptl); - return 0; - } - spin_unlock(ptl); - - ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl); - if (!ptep) - goto again; - pgste = pgste_get_lock(ptep); - *key = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56; - paddr = pte_val(*ptep) & PAGE_MASK; - if (!(pte_val(*ptep) & _PAGE_INVALID)) - *key = page_get_storage_key(paddr); - /* Reflect guest's logical view, not physical */ - *key |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48; - pgste_set_unlock(ptep, pgste); - pte_unmap_unlock(ptep, ptl); - return 0; -} -EXPORT_SYMBOL(get_guest_storage_key); - -/** - * pgste_perform_essa - perform ESSA actions on the PGSTE. - * @mm: the memory context. It must have PGSTEs, no check is performed here! - * @hva: the host virtual address of the page whose PGSTE is to be processed - * @orc: the specific action to perform, see the ESSA_SET_* macros. - * @oldpte: the PTE will be saved there if the pointer is not NULL. - * @oldpgste: the old PGSTE will be saved there if the pointer is not NULL. - * - * Return: 1 if the page is to be added to the CBRL, otherwise 0, - * or < 0 in case of error. -EINVAL is returned for invalid values - * of orc, -EFAULT for invalid addresses. - */ -int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc, - unsigned long *oldpte, unsigned long *oldpgste) -{ - struct vm_area_struct *vma; - unsigned long pgstev; - spinlock_t *ptl; - pgste_t pgste; - pte_t *ptep; - int res = 0; - - WARN_ON_ONCE(orc > ESSA_MAX); - if (unlikely(orc > ESSA_MAX)) - return -EINVAL; - - vma = vma_lookup(mm, hva); - if (!vma || is_vm_hugetlb_page(vma)) - return -EFAULT; - ptep = get_locked_pte(mm, hva, &ptl); - if (unlikely(!ptep)) - return -EFAULT; - pgste = pgste_get_lock(ptep); - pgstev = pgste_val(pgste); - if (oldpte) - *oldpte = pte_val(*ptep); - if (oldpgste) - *oldpgste = pgstev; - - switch (orc) { - case ESSA_GET_STATE: - break; - case ESSA_SET_STABLE: - pgstev &= ~(_PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT); - pgstev |= _PGSTE_GPS_USAGE_STABLE; - break; - case ESSA_SET_UNUSED: - pgstev &= ~_PGSTE_GPS_USAGE_MASK; - pgstev |= _PGSTE_GPS_USAGE_UNUSED; - if (pte_val(*ptep) & _PAGE_INVALID) - res = 1; - break; - case ESSA_SET_VOLATILE: - pgstev &= ~_PGSTE_GPS_USAGE_MASK; - pgstev |= _PGSTE_GPS_USAGE_VOLATILE; - if (pte_val(*ptep) & _PAGE_INVALID) - res = 1; - break; - case ESSA_SET_POT_VOLATILE: - pgstev &= ~_PGSTE_GPS_USAGE_MASK; - if (!(pte_val(*ptep) & _PAGE_INVALID)) { - pgstev |= _PGSTE_GPS_USAGE_POT_VOLATILE; - break; - } - if (pgstev & _PGSTE_GPS_ZERO) { - pgstev |= _PGSTE_GPS_USAGE_VOLATILE; - break; - } - if (!(pgstev & PGSTE_GC_BIT)) { - pgstev |= _PGSTE_GPS_USAGE_VOLATILE; - res = 1; - break; - } - break; - case ESSA_SET_STABLE_RESIDENT: - pgstev &= ~_PGSTE_GPS_USAGE_MASK; - pgstev |= _PGSTE_GPS_USAGE_STABLE; - /* - * Since the resident state can go away any time after this - * call, we will not make this page resident. We can revisit - * this decision if a guest will ever start using this. - */ - break; - case ESSA_SET_STABLE_IF_RESIDENT: - if (!(pte_val(*ptep) & _PAGE_INVALID)) { - pgstev &= ~_PGSTE_GPS_USAGE_MASK; - pgstev |= _PGSTE_GPS_USAGE_STABLE; - } - break; - case ESSA_SET_STABLE_NODAT: - pgstev &= ~_PGSTE_GPS_USAGE_MASK; - pgstev |= _PGSTE_GPS_USAGE_STABLE | _PGSTE_GPS_NODAT; - break; - default: - /* we should never get here! */ - break; - } - /* If we are discarding a page, set it to logical zero */ - if (res) - pgstev |= _PGSTE_GPS_ZERO; - - pgste = __pgste(pgstev); - pgste_set_unlock(ptep, pgste); - pte_unmap_unlock(ptep, ptl); - return res; -} -EXPORT_SYMBOL(pgste_perform_essa); - -/** - * set_pgste_bits - set specific PGSTE bits. - * @mm: the memory context. It must have PGSTEs, no check is performed here! - * @hva: the host virtual address of the page whose PGSTE is to be processed - * @bits: a bitmask representing the bits that will be touched - * @value: the values of the bits to be written. Only the bits in the mask - * will be written. - * - * Return: 0 on success, < 0 in case of error. - */ -int set_pgste_bits(struct mm_struct *mm, unsigned long hva, - unsigned long bits, unsigned long value) -{ - struct vm_area_struct *vma; - spinlock_t *ptl; - pgste_t new; - pte_t *ptep; - - vma = vma_lookup(mm, hva); - if (!vma || is_vm_hugetlb_page(vma)) - return -EFAULT; - ptep = get_locked_pte(mm, hva, &ptl); - if (unlikely(!ptep)) - return -EFAULT; - new = pgste_get_lock(ptep); - - new = clear_pgste_bit(new, bits); - new = set_pgste_bit(new, value & bits); - - pgste_set_unlock(ptep, new); - pte_unmap_unlock(ptep, ptl); - return 0; -} -EXPORT_SYMBOL(set_pgste_bits); - -/** - * get_pgste - get the current PGSTE for the given address. - * @mm: the memory context. It must have PGSTEs, no check is performed here! - * @hva: the host virtual address of the page whose PGSTE is to be processed - * @pgstep: will be written with the current PGSTE for the given address. - * - * Return: 0 on success, < 0 in case of error. - */ -int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep) -{ - struct vm_area_struct *vma; - spinlock_t *ptl; - pte_t *ptep; - - vma = vma_lookup(mm, hva); - if (!vma || is_vm_hugetlb_page(vma)) - return -EFAULT; - ptep = get_locked_pte(mm, hva, &ptl); - if (unlikely(!ptep)) - return -EFAULT; - *pgstep = pgste_val(pgste_get(ptep)); - pte_unmap_unlock(ptep, ptl); - return 0; -} -EXPORT_SYMBOL(get_pgste); -#endif diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index eeadff45e0e1..d2879ce860a1 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -4,7 +4,6 @@ */ #include <linux/memory_hotplug.h> -#include <linux/bootmem_info.h> #include <linux/cpufeature.h> #include <linux/memblock.h> #include <linux/pfn.h> @@ -40,7 +39,6 @@ static void __ref *vmem_alloc_pages(unsigned int order) static void vmem_free_pages(unsigned long addr, int order, struct vmem_altmap *altmap) { - unsigned int nr_pages = 1 << order; struct page *page; if (altmap) { @@ -50,8 +48,7 @@ static void vmem_free_pages(unsigned long addr, int order, struct vmem_altmap *a page = virt_to_page((void *)addr); if (PageReserved(page)) { /* allocated from memblock */ - while (nr_pages--) - free_bootmem_page(page++); + free_reserved_pages(page, order); } else { free_pages(addr, order); } @@ -171,18 +168,19 @@ static int __ref modify_pte_table(pmd_t *pmd, unsigned long addr, { unsigned long prot, pages = 0; int ret = -ENOMEM; - pte_t *pte; + pte_t *pte, entry; prot = pgprot_val(PAGE_KERNEL); pte = pte_offset_kernel(pmd, addr); for (; addr < end; addr += PAGE_SIZE, pte++) { + entry = ptep_get(pte); if (!add) { - if (pte_none(*pte)) + if (pte_none(entry)) continue; if (!direct) - vmem_free_pages((unsigned long)pfn_to_virt(pte_pfn(*pte)), get_order(PAGE_SIZE), altmap); + vmem_free_pages((unsigned long)pfn_to_virt(pte_pfn(entry)), get_order(PAGE_SIZE), altmap); pte_clear(&init_mm, addr, pte); - } else if (pte_none(*pte)) { + } else if (pte_none(entry)) { if (!direct) { void *new_page = vmemmap_alloc_block_buf(PAGE_SIZE, NUMA_NO_NODE, altmap); @@ -212,10 +210,10 @@ static void try_free_pte_table(pmd_t *pmd, unsigned long start) /* We can safely assume this is fully in 1:1 mapping & vmemmap area */ pte = pte_offset_kernel(pmd, start); for (i = 0; i < PTRS_PER_PTE; i++, pte++) { - if (!pte_none(*pte)) + if (!pte_none(ptep_get(pte))) return; } - vmem_pte_free((unsigned long *) pmd_deref(*pmd)); + vmem_pte_free((unsigned long *)pmd_deref(pmdp_get(pmd))); pmd_clear(pmd); } @@ -226,6 +224,7 @@ static int __ref modify_pmd_table(pud_t *pud, unsigned long addr, { unsigned long next, prot, pages = 0; int ret = -ENOMEM; + pmd_t entry; pmd_t *pmd; pte_t *pte; @@ -233,23 +232,24 @@ static int __ref modify_pmd_table(pud_t *pud, unsigned long addr, pmd = pmd_offset(pud, addr); for (; addr < end; addr = next, pmd++) { next = pmd_addr_end(addr, end); + entry = pmdp_get(pmd); if (!add) { - if (pmd_none(*pmd)) + if (pmd_none(entry)) continue; - if (pmd_leaf(*pmd)) { + if (pmd_leaf(entry)) { if (IS_ALIGNED(addr, PMD_SIZE) && IS_ALIGNED(next, PMD_SIZE)) { if (!direct) - vmem_free_pages(pmd_deref(*pmd), get_order(PMD_SIZE), altmap); + vmem_free_pages(pmd_deref(entry), get_order(PMD_SIZE), altmap); pmd_clear(pmd); pages++; } else if (!direct && vmemmap_unuse_sub_pmd(addr, next)) { - vmem_free_pages(pmd_deref(*pmd), get_order(PMD_SIZE), altmap); + vmem_free_pages(pmd_deref(entry), get_order(PMD_SIZE), altmap); pmd_clear(pmd); } continue; } - } else if (pmd_none(*pmd)) { + } else if (pmd_none(entry)) { if (IS_ALIGNED(addr, PMD_SIZE) && IS_ALIGNED(next, PMD_SIZE) && cpu_has_edat1() && direct && @@ -281,7 +281,7 @@ static int __ref modify_pmd_table(pud_t *pud, unsigned long addr, if (!pte) goto out; pmd_populate(&init_mm, pmd, pte); - } else if (pmd_leaf(*pmd)) { + } else if (pmd_leaf(entry)) { if (!direct) vmemmap_use_sub_pmd(addr, next); continue; @@ -306,9 +306,9 @@ static void try_free_pmd_table(pud_t *pud, unsigned long start) pmd = pmd_offset(pud, start); for (i = 0; i < PTRS_PER_PMD; i++, pmd++) - if (!pmd_none(*pmd)) + if (!pmd_none(pmdp_get(pmd))) return; - vmem_free_pages(pud_deref(*pud), CRST_ALLOC_ORDER, NULL); + vmem_free_pages(pud_deref(pudp_get(pud)), CRST_ALLOC_ORDER, NULL); pud_clear(pud); } @@ -317,21 +317,22 @@ static int modify_pud_table(p4d_t *p4d, unsigned long addr, unsigned long end, { unsigned long next, prot, pages = 0; int ret = -ENOMEM; - pud_t *pud; + pud_t *pud, entry; pmd_t *pmd; prot = pgprot_val(REGION3_KERNEL); pud = pud_offset(p4d, addr); for (; addr < end; addr = next, pud++) { next = pud_addr_end(addr, end); + entry = pudp_get(pud); if (!add) { - if (pud_none(*pud)) + if (pud_none(entry)) continue; - if (pud_leaf(*pud)) { + if (pud_leaf(entry)) { if (IS_ALIGNED(addr, PUD_SIZE) && IS_ALIGNED(next, PUD_SIZE)) { if (!direct) - vmem_free_pages(pud_deref(*pud), get_order(PUD_SIZE), altmap); + vmem_free_pages(pud_deref(entry), get_order(PUD_SIZE), altmap); pud_clear(pud); pages++; continue; @@ -339,7 +340,7 @@ static int modify_pud_table(p4d_t *p4d, unsigned long addr, unsigned long end, split_pud_page(pud, addr & PUD_MASK); } } - } else if (pud_none(*pud)) { + } else if (pud_none(entry)) { if (IS_ALIGNED(addr, PUD_SIZE) && IS_ALIGNED(next, PUD_SIZE) && cpu_has_edat2() && direct && @@ -352,7 +353,7 @@ static int modify_pud_table(p4d_t *p4d, unsigned long addr, unsigned long end, if (!pmd) goto out; pud_populate(&init_mm, pud, pmd); - } else if (pud_leaf(*pud)) { + } else if (pud_leaf(entry)) { continue; } ret = modify_pmd_table(pud, addr, next, add, direct, altmap); @@ -375,10 +376,10 @@ static void try_free_pud_table(p4d_t *p4d, unsigned long start) pud = pud_offset(p4d, start); for (i = 0; i < PTRS_PER_PUD; i++, pud++) { - if (!pud_none(*pud)) + if (!pud_none(pudp_get(pud))) return; } - vmem_free_pages(p4d_deref(*p4d), CRST_ALLOC_ORDER, NULL); + vmem_free_pages(p4d_deref(p4dp_get(p4d)), CRST_ALLOC_ORDER, NULL); p4d_clear(p4d); } @@ -387,16 +388,17 @@ static int modify_p4d_table(pgd_t *pgd, unsigned long addr, unsigned long end, { unsigned long next; int ret = -ENOMEM; - p4d_t *p4d; + p4d_t *p4d, entry; pud_t *pud; p4d = p4d_offset(pgd, addr); for (; addr < end; addr = next, p4d++) { next = p4d_addr_end(addr, end); + entry = p4dp_get(p4d); if (!add) { - if (p4d_none(*p4d)) + if (p4d_none(entry)) continue; - } else if (p4d_none(*p4d)) { + } else if (p4d_none(entry)) { pud = vmem_crst_alloc(_REGION3_ENTRY_EMPTY); if (!pud) goto out; @@ -420,10 +422,10 @@ static void try_free_p4d_table(pgd_t *pgd, unsigned long start) p4d = p4d_offset(pgd, start); for (i = 0; i < PTRS_PER_P4D; i++, p4d++) { - if (!p4d_none(*p4d)) + if (!p4d_none(p4dp_get(p4d))) return; } - vmem_free_pages(pgd_deref(*pgd), CRST_ALLOC_ORDER, NULL); + vmem_free_pages(pgd_deref(pgdp_get(pgd)), CRST_ALLOC_ORDER, NULL); pgd_clear(pgd); } @@ -432,7 +434,7 @@ static int modify_pagetable(unsigned long start, unsigned long end, bool add, { unsigned long addr, next; int ret = -ENOMEM; - pgd_t *pgd; + pgd_t *pgd, entry; p4d_t *p4d; if (WARN_ON_ONCE(!PAGE_ALIGNED(start | end))) @@ -449,11 +451,12 @@ static int modify_pagetable(unsigned long start, unsigned long end, bool add, for (addr = start; addr < end; addr = next) { next = pgd_addr_end(addr, end); pgd = pgd_offset_k(addr); + entry = pgdp_get(pgd); if (!add) { - if (pgd_none(*pgd)) + if (pgd_none(entry)) continue; - } else if (pgd_none(*pgd)) { + } else if (pgd_none(entry)) { p4d = vmem_crst_alloc(_REGION2_ENTRY_EMPTY); if (!p4d) goto out; @@ -575,6 +578,8 @@ int vmem_add_mapping(unsigned long start, unsigned long size) pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc) { pte_t *ptep = NULL; + pud_t pud_entry; + pmd_t pmd_entry; pgd_t *pgd; p4d_t *p4d; pud_t *pud; @@ -582,7 +587,7 @@ pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc) pte_t *pte; pgd = pgd_offset_k(addr); - if (pgd_none(*pgd)) { + if (pgd_none(pgdp_get(pgd))) { if (!alloc) goto out; p4d = vmem_crst_alloc(_REGION2_ENTRY_EMPTY); @@ -591,7 +596,7 @@ pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc) pgd_populate(&init_mm, pgd, p4d); } p4d = p4d_offset(pgd, addr); - if (p4d_none(*p4d)) { + if (p4d_none(p4dp_get(p4d))) { if (!alloc) goto out; pud = vmem_crst_alloc(_REGION3_ENTRY_EMPTY); @@ -600,25 +605,27 @@ pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc) p4d_populate(&init_mm, p4d, pud); } pud = pud_offset(p4d, addr); - if (pud_none(*pud)) { + pud_entry = pudp_get(pud); + if (pud_none(pud_entry)) { if (!alloc) goto out; pmd = vmem_crst_alloc(_SEGMENT_ENTRY_EMPTY); if (!pmd) goto out; pud_populate(&init_mm, pud, pmd); - } else if (WARN_ON_ONCE(pud_leaf(*pud))) { + } else if (WARN_ON_ONCE(pud_leaf(pud_entry))) { goto out; } pmd = pmd_offset(pud, addr); - if (pmd_none(*pmd)) { + pmd_entry = pmdp_get(pmd); + if (pmd_none(pmd_entry)) { if (!alloc) goto out; pte = vmem_pte_alloc(); if (!pte) goto out; pmd_populate(&init_mm, pmd, pte); - } else if (WARN_ON_ONCE(pmd_leaf(*pmd))) { + } else if (WARN_ON_ONCE(pmd_leaf(pmd_entry))) { goto out; } ptep = pte_offset_kernel(pmd, addr); diff --git a/arch/s390/net/Makefile b/arch/s390/net/Makefile index 9275cf63192a..0a7323c9b372 100644 --- a/arch/s390/net/Makefile +++ b/arch/s390/net/Makefile @@ -2,5 +2,8 @@ # # Arch-specific network modules # + +CONTEXT_ANALYSIS := y + obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o bpf_timed_may_goto.o obj-$(CONFIG_HAVE_PNETID) += pnet.o diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 579461d471bb..c4b47070bb59 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -21,12 +21,14 @@ #include <linux/filter.h> #include <linux/init.h> #include <linux/bpf.h> +#include <linux/cfi.h> #include <linux/mm.h> #include <linux/kernel.h> #include <asm/cacheflush.h> #include <asm/extable.h> #include <asm/dis.h> #include <asm/facility.h> +#include <asm/lowcore.h> #include <asm/nospec-branch.h> #include <asm/set_memory.h> #include <asm/text-patching.h> @@ -355,6 +357,19 @@ static void emit6_pcrel_rilc(struct bpf_jit *jit, u32 op, u8 mask, s64 pcrel) } \ }) +static inline void emit_u32_data(const u32 data, struct bpf_jit *jit) +{ + if (jit->prg_buf) + *(u32 *)(jit->prg_buf + jit->prg) = data; + jit->prg += 4; +} + +static inline void emit_kcfi(u32 hash, struct bpf_jit *jit) +{ + if (IS_ENABLED(CONFIG_CFI)) + emit_u32_data(hash, jit); +} + /* * Return whether this is the first pass. The first pass is special, since we * don't know any sizes yet, and thus must be conservative. @@ -596,6 +611,8 @@ static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp) { BUILD_BUG_ON(sizeof(struct prog_frame) != STACK_FRAME_OVERHEAD); + emit_kcfi(bpf_is_subprog(fp) ? cfi_bpf_subprog_hash : cfi_bpf_hash, jit); + /* No-op for hotpatching */ /* brcl 0,prologue_plt */ EMIT6_PCREL_RILC(0xc0040000, 0, jit->prologue_plt); @@ -615,7 +632,7 @@ static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp) bpf_skip(jit, 6); } /* Tail calls have to skip above initialization */ - jit->tail_call_start = jit->prg; + jit->tail_call_start = jit->prg - cfi_get_offset(); if (fp->aux->exception_cb) { /* * Switch stack, the new address is in the 2nd parameter. @@ -742,10 +759,12 @@ static void bpf_jit_probe_load_pre(struct bpf_jit *jit, struct bpf_insn *insn, { if (BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX && - BPF_MODE(insn->code) != BPF_PROBE_MEM32) + BPF_MODE(insn->code) != BPF_PROBE_MEM32 && + BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) return; - if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) { + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) { /* lgrl %r1,kern_arena */ EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); probe->arena_reg = REG_W1; @@ -757,7 +776,8 @@ static void bpf_jit_probe_load_pre(struct bpf_jit *jit, struct bpf_insn *insn, static void bpf_jit_probe_store_pre(struct bpf_jit *jit, struct bpf_insn *insn, struct bpf_jit_probe *probe) { - if (BPF_MODE(insn->code) != BPF_PROBE_MEM32) + if (BPF_MODE(insn->code) != BPF_PROBE_MEM32 && + BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) return; /* lgrl %r1,kern_arena */ @@ -770,6 +790,8 @@ static void bpf_jit_probe_atomic_pre(struct bpf_jit *jit, struct bpf_insn *insn, struct bpf_jit_probe *probe) { + int load_reg; + if (BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) return; @@ -779,6 +801,14 @@ static void bpf_jit_probe_atomic_pre(struct bpf_jit *jit, EMIT4(0xb9080000, REG_W1, insn->dst_reg); probe->arena_reg = REG_W1; probe->prg = jit->prg; + /* + * A read-modify-write carrying BPF_FETCH reads the old value into + * src_reg, or into r0 for a BPF_CMPXCHG. Clear that register on + * fault, the remaining atomics only write memory. + */ + load_reg = bpf_atomic_load_reg(insn); + if (load_reg >= 0) + probe->reg = reg2hex[load_reg]; } static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp, @@ -829,26 +859,101 @@ static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp, return 0; } +static int emit_ldx(struct bpf_jit *jit, struct bpf_prog *fp, struct bpf_insn *insn) +{ + struct bpf_jit_probe probe; + + bpf_jit_probe_init(&probe); + bpf_jit_probe_load_pre(jit, insn, &probe); + + switch (BPF_SIZE(insn->code)) { + case BPF_B: /* dst = *(u8 *)(ul) (src + off) */ + /* llgc %dst,off(%src,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0090, insn->dst_reg, insn->src_reg, + probe.arena_reg, insn->off); + break; + case BPF_H: /* dst = *(u16 *)(ul) (src + off) */ + /* llgh %dst,off(%src,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0091, insn->dst_reg, insn->src_reg, + probe.arena_reg, insn->off); + break; + case BPF_W: /* dst = *(u32 *)(ul) (src + off) */ + /* llgf %dst,off(%src,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0016, insn->dst_reg, insn->src_reg, + probe.arena_reg, insn->off); + break; + case BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ + /* lg %dst,off(%src,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0004, insn->dst_reg, insn->src_reg, + probe.arena_reg, insn->off); + break; + } + + return bpf_jit_probe_post(jit, fp, &probe); +} + +static int emit_stx(struct bpf_jit *jit, struct bpf_prog *fp, struct bpf_insn *insn) +{ + struct bpf_jit_probe probe; + + bpf_jit_probe_init(&probe); + bpf_jit_probe_store_pre(jit, insn, &probe); + + switch (BPF_SIZE(insn->code)) { + case BPF_B: /* *(u8 *)(dst + off) = src_reg */ + /* stcy %src,off(%dst,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0072, insn->src_reg, insn->dst_reg, + probe.arena_reg, insn->off); + break; + case BPF_H: /* (u16 *)(dst + off) = src */ + /* sthy %src,off(%dst,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0070, insn->src_reg, insn->dst_reg, + probe.arena_reg, insn->off); + break; + case BPF_W: /* *(u32 *)(dst + off) = src */ + /* sty %src,off(%dst,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0050, insn->src_reg, insn->dst_reg, + probe.arena_reg, insn->off); + break; + case BPF_DW: /* (u64 *)(dst + off) = src */ + /* stg %src,off(%dst,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0024, insn->src_reg, insn->dst_reg, + probe.arena_reg, insn->off); + break; + } + + return bpf_jit_probe_post(jit, fp, &probe); +} + /* - * Sign-extend the register if necessary + * Sign- or zero-extend the register if necessary */ -static int sign_extend(struct bpf_jit *jit, int r, u8 size, u8 flags) +static int sign_zero_extend(struct bpf_jit *jit, int r, u8 size, u8 flags) { - if (!(flags & BTF_FMODEL_SIGNED_ARG)) - return 0; - switch (size) { case 1: - /* lgbr %r,%r */ - EMIT4(0xb9060000, r, r); + if (flags & BTF_FMODEL_SIGNED_ARG) + /* lgbr %r,%r */ + EMIT4(0xb9060000, r, r); + else + /* llgcr %r,%r */ + EMIT4(0xb9840000, r, r); return 0; case 2: - /* lghr %r,%r */ - EMIT4(0xb9070000, r, r); + if (flags & BTF_FMODEL_SIGNED_ARG) + /* lghr %r,%r */ + EMIT4(0xb9070000, r, r); + else + /* llghr %r,%r */ + EMIT4(0xb9850000, r, r); return 0; case 4: - /* lgfr %r,%r */ - EMIT4(0xb9140000, r, r); + if (flags & BTF_FMODEL_SIGNED_ARG) + /* lgfr %r,%r */ + EMIT4(0xb9140000, r, r); + else + /* llgfr %r,%r */ + EMIT4(0xb9160000, r, r); return 0; case 8: return 0; @@ -1467,44 +1572,13 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, */ case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src_reg */ case BPF_STX | BPF_PROBE_MEM32 | BPF_B: - bpf_jit_probe_store_pre(jit, insn, &probe); - /* stcy %src,off(%dst,%arena) */ - EMIT6_DISP_LH(0xe3000000, 0x0072, src_reg, dst_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); - if (err < 0) - return err; - jit->seen |= SEEN_MEM; - break; case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */ case BPF_STX | BPF_PROBE_MEM32 | BPF_H: - bpf_jit_probe_store_pre(jit, insn, &probe); - /* sthy %src,off(%dst,%arena) */ - EMIT6_DISP_LH(0xe3000000, 0x0070, src_reg, dst_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); - if (err < 0) - return err; - jit->seen |= SEEN_MEM; - break; case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */ case BPF_STX | BPF_PROBE_MEM32 | BPF_W: - bpf_jit_probe_store_pre(jit, insn, &probe); - /* sty %src,off(%dst,%arena) */ - EMIT6_DISP_LH(0xe3000000, 0x0050, src_reg, dst_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); - if (err < 0) - return err; - jit->seen |= SEEN_MEM; - break; case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */ case BPF_STX | BPF_PROBE_MEM32 | BPF_DW: - bpf_jit_probe_store_pre(jit, insn, &probe); - /* stg %src,off(%dst,%arena) */ - EMIT6_DISP_LH(0xe3000000, 0x0024, src_reg, dst_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); + err = emit_stx(jit, fp, insn); if (err < 0) return err; jit->seen |= SEEN_MEM; @@ -1564,19 +1638,23 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, /* * BPF_ATOMIC */ + case BPF_STX | BPF_ATOMIC | BPF_B: + case BPF_STX | BPF_ATOMIC | BPF_H: case BPF_STX | BPF_ATOMIC | BPF_DW: case BPF_STX | BPF_ATOMIC | BPF_W: + case BPF_STX | BPF_PROBE_ATOMIC | BPF_B: + case BPF_STX | BPF_PROBE_ATOMIC | BPF_H: case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW: case BPF_STX | BPF_PROBE_ATOMIC | BPF_W: { bool is32 = BPF_SIZE(insn->code) == BPF_W; /* - * Unlike loads and stores, atomics have only a base register, - * but no index register. For the non-arena case, simply use - * %dst as a base. For the arena case, use the work register - * %r1: first, load the arena base into it, and then add %dst - * to it. + * Unlike loads and stores, s390 atomics have only a base + * register, but no index register. For the non-arena case, + * simply use %dst as a base. For the arena case, use the + * work register %r1: first, load the arena base into it, + * and then add %dst to it. */ probe.arena_reg = dst_reg; @@ -1632,6 +1710,7 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, if (load_probe.prg != -1) { probe.prg = jit->prg; probe.arena_reg = load_probe.arena_reg; + probe.reg = load_probe.reg; } loop_start = jit->prg; /* 0: {csy|csg} %w0,%src,off(%arena) */ @@ -1663,6 +1742,18 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, if (err < 0) return err; break; + case BPF_LOAD_ACQ: + /* s390 has strong ordering, just use load */ + err = emit_ldx(jit, fp, insn); + if (err < 0) + return err; + break; + case BPF_STORE_REL: + /* s390 has strong ordering, just use store */ + err = emit_stx(jit, fp, insn); + if (err < 0) + return err; + break; default: pr_err("Unknown atomic operation %02x\n", insn->imm); return -1; @@ -1677,15 +1768,20 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */ case BPF_LDX | BPF_PROBE_MEM | BPF_B: case BPF_LDX | BPF_PROBE_MEM32 | BPF_B: - bpf_jit_probe_load_pre(jit, insn, &probe); - /* llgc %dst,off(%src,%arena) */ - EMIT6_DISP_LH(0xe3000000, 0x0090, dst_reg, src_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); + case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */ + case BPF_LDX | BPF_PROBE_MEM | BPF_H: + case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: + case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */ + case BPF_LDX | BPF_PROBE_MEM | BPF_W: + case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: + case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ + case BPF_LDX | BPF_PROBE_MEM | BPF_DW: + case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: + err = emit_ldx(jit, fp, insn); if (err < 0) return err; jit->seen |= SEEN_MEM; - if (insn_is_zext(&insn[1])) + if (BPF_SIZE(insn->code) != BPF_DW && insn_is_zext(&insn[1])) insn_count = 2; break; case BPF_LDX | BPF_MEMSX | BPF_B: /* dst = *(s8 *)(ul) (src + off) */ @@ -1698,20 +1794,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, return err; jit->seen |= SEEN_MEM; break; - case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */ - case BPF_LDX | BPF_PROBE_MEM | BPF_H: - case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: - bpf_jit_probe_load_pre(jit, insn, &probe); - /* llgh %dst,off(%src,%arena) */ - EMIT6_DISP_LH(0xe3000000, 0x0091, dst_reg, src_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); - if (err < 0) - return err; - jit->seen |= SEEN_MEM; - if (insn_is_zext(&insn[1])) - insn_count = 2; - break; case BPF_LDX | BPF_MEMSX | BPF_H: /* dst = *(s16 *)(ul) (src + off) */ case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: bpf_jit_probe_load_pre(jit, insn, &probe); @@ -1722,20 +1804,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, return err; jit->seen |= SEEN_MEM; break; - case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */ - case BPF_LDX | BPF_PROBE_MEM | BPF_W: - case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: - bpf_jit_probe_load_pre(jit, insn, &probe); - /* llgf %dst,off(%src) */ - jit->seen |= SEEN_MEM; - EMIT6_DISP_LH(0xe3000000, 0x0016, dst_reg, src_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); - if (err < 0) - return err; - if (insn_is_zext(&insn[1])) - insn_count = 2; - break; case BPF_LDX | BPF_MEMSX | BPF_W: /* dst = *(s32 *)(ul) (src + off) */ case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: bpf_jit_probe_load_pre(jit, insn, &probe); @@ -1746,18 +1814,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, if (err < 0) return err; break; - case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ - case BPF_LDX | BPF_PROBE_MEM | BPF_DW: - case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: - bpf_jit_probe_load_pre(jit, insn, &probe); - /* lg %dst,off(%src,%arena) */ - jit->seen |= SEEN_MEM; - EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, src_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); - if (err < 0) - return err; - break; /* * BPF_JMP / CALL */ @@ -1768,6 +1824,30 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int j, ret; u64 func; + /* Implement helper call to bpf_get_smp_processor_id() inline */ + if (insn->src_reg == 0 && + insn->imm == BPF_FUNC_get_smp_processor_id) { + const u32 *cpu_nr = &get_lowcore()->cpu_nr; + + /* llgf %b0, cpu_nr */ + EMIT6_DISP_LH(0xe3000000, 0x0016, BPF_REG_0, REG_0, REG_0, + (unsigned long)cpu_nr); + break; + } + + /* Implement helper call to bpf_get_current_task/_btf() inline */ + if (insn->src_reg == 0 && + (insn->imm == BPF_FUNC_get_current_task || + insn->imm == BPF_FUNC_get_current_task_btf)) { + const u64 *current_task = + &get_lowcore()->current_task; + + /* lg %b0, current_task */ + EMIT6_DISP_LH(0xe3000000, 0x0004, BPF_REG_0, REG_0, REG_0, + (unsigned long)current_task); + break; + } + ret = bpf_jit_get_func_addr(fp, insn, extra_pass, &func, &func_addr_fixed); if (ret < 0) @@ -1798,9 +1878,9 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, return -1; for (j = 0; j < m->nr_args; j++) { - if (sign_extend(jit, BPF_REG_1 + j, - m->arg_size[j], - m->arg_flags[j])) + if (sign_zero_extend(jit, BPF_REG_1 + j, + m->arg_size[j], + m->arg_flags[j])) return -1; } } @@ -1862,20 +1942,21 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, jit->prg); /* - * if (tail_call_cnt++ >= MAX_TAIL_CALL_CNT) + * if (tail_call_cnt >= MAX_TAIL_CALL_CNT) * goto out; + * + * tail_call_cnt is read into %w0, which needs to be preserved + * until it's incremented and flushed. */ off = jit->frame_off + offsetof(struct prog_frame, tail_call_cnt); - /* lhi %w0,1 */ - EMIT4_IMM(0xa7080000, REG_W0, 1); - /* laal %w1,%w0,off(%r15) */ - EMIT6_DISP_LH(0xeb000000, 0x00fa, REG_W1, REG_W0, REG_15, off); - /* clij %w1,MAX_TAIL_CALL_CNT-1,0x2,out */ + /* ly %w0,off(%r15) */ + EMIT6_DISP_LH(0xe3000000, 0x0058, REG_W0, REG_0, REG_15, off); + /* clij %w0,MAX_TAIL_CALL_CNT,0xa,out */ patch_2_clij = jit->prg; - EMIT6_PCREL_RIEC(0xec000000, 0x007f, REG_W1, MAX_TAIL_CALL_CNT - 1, - 2, jit->prg); + EMIT6_PCREL_RIEC(0xec000000, 0x007f, REG_W0, MAX_TAIL_CALL_CNT, + 0xa, jit->prg); /* * prog = array->ptrs[index]; @@ -1894,6 +1975,12 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, patch_3_brc = jit->prg; EMIT4_PCREL_RIC(0xa7040000, 8, jit->prg); + /* tail_call_cnt++; */ + /* ahi %w0,1 */ + EMIT4_IMM(0xa70a0000, REG_W0, 1); + /* sty %w0,off(%r15) */ + EMIT6_DISP_LH(0xe3000000, 0x0050, REG_W0, REG_0, REG_15, off); + /* * Restore registers before calling function */ @@ -2296,38 +2383,22 @@ static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit, /* * Compile eBPF program "fp" */ -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp) { - struct bpf_prog *tmp, *orig_fp = fp; struct bpf_binary_header *header; struct s390_jit_data *jit_data; - bool tmp_blinded = false; bool extra_pass = false; struct bpf_jit jit; int pass; if (!fp->jit_requested) - return orig_fp; - - tmp = bpf_jit_blind_constants(fp); - /* - * If blinding was requested and we failed during blinding, - * we must fall back to the interpreter. - */ - if (IS_ERR(tmp)) - return orig_fp; - if (tmp != fp) { - tmp_blinded = true; - fp = tmp; - } + return fp; jit_data = fp->aux->jit_data; if (!jit_data) { - jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); - if (!jit_data) { - fp = orig_fp; - goto out; - } + jit_data = kzalloc_obj(*jit_data); + if (!jit_data) + return fp; fp->aux->jit_data = jit_data; } if (jit_data->ctx.addrs) { @@ -2340,34 +2411,27 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) memset(&jit, 0, sizeof(jit)); jit.addrs = kvcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL); - if (jit.addrs == NULL) { - fp = orig_fp; - goto free_addrs; - } + if (jit.addrs == NULL) + goto out_err; /* * Three initial passes: * - 1/2: Determine clobbered registers * - 3: Calculate program size and addrs array */ for (pass = 1; pass <= 3; pass++) { - if (bpf_jit_prog(&jit, fp, extra_pass)) { - fp = orig_fp; - goto free_addrs; - } + if (bpf_jit_prog(&jit, fp, extra_pass)) + goto out_err; } /* * Final pass: Allocate and generate program */ header = bpf_jit_alloc(&jit, fp); - if (!header) { - fp = orig_fp; - goto free_addrs; - } + if (!header) + goto out_err; skip_init_ctx: if (bpf_jit_prog(&jit, fp, extra_pass)) { bpf_jit_binary_free(header); - fp = orig_fp; - goto free_addrs; + goto out_err; } if (bpf_jit_enable > 1) { bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf); @@ -2376,30 +2440,36 @@ skip_init_ctx: if (!fp->is_func || extra_pass) { if (bpf_jit_binary_lock_ro(header)) { bpf_jit_binary_free(header); - fp = orig_fp; - goto free_addrs; + goto out_err; } } else { jit_data->header = header; jit_data->ctx = jit; jit_data->pass = pass; } - fp->bpf_func = (void *) jit.prg_buf; + fp->bpf_func = (void *)jit.prg_buf + cfi_get_offset(); fp->jited = 1; - fp->jited_len = jit.size; + fp->jited_len = jit.size - cfi_get_offset(); if (!fp->is_func || extra_pass) { + for (int i = 0; i < fp->len; i++) + jit.addrs[i] -= cfi_get_offset(); bpf_prog_fill_jited_linfo(fp, jit.addrs + 1); free_addrs: kvfree(jit.addrs); kfree(jit_data); fp->aux->jit_data = NULL; } -out: - if (tmp_blinded) - bpf_jit_prog_release_other(fp, fp == orig_fp ? - tmp : orig_fp); + return fp; + +out_err: + if (extra_pass) { + fp->bpf_func = NULL; + fp->jited = 0; + fp->jited_len = 0; + } + goto free_addrs; } bool bpf_jit_supports_kfunc_call(void) @@ -2480,8 +2550,8 @@ struct bpf_tramp_jit { int ip_off; /* For bpf_get_func_ip(), has to be at * (ctx - 16) */ - int arg_cnt_off; /* For bpf_get_func_arg_cnt(), has to be at - * (ctx - 8) + int func_meta_off; /* For bpf_get_func_arg_cnt()/fsession, has + * to be at (ctx - 8) */ int bpf_args_off; /* Offset of BPF_PROG context, which consists * of BPF arguments followed by return value @@ -2506,24 +2576,28 @@ static void load_imm64(struct bpf_jit *jit, int dst_reg, u64 val) EMIT6_IMM(0xc00d0000, dst_reg, val); } +static void emit_store_stack_imm64(struct bpf_jit *jit, int tmp_reg, int stack_off, u64 imm) +{ + load_imm64(jit, tmp_reg, imm); + /* stg %tmp_reg,stack_off(%r15) */ + EMIT6_DISP_LH(0xe3000000, 0x0024, tmp_reg, REG_0, REG_15, stack_off); +} + static int invoke_bpf_prog(struct bpf_tramp_jit *tjit, const struct btf_func_model *m, - struct bpf_tramp_link *tlink, bool save_ret) + struct bpf_tramp_node *node, bool save_ret) { struct bpf_jit *jit = &tjit->common; int cookie_off = tjit->run_ctx_off + offsetof(struct bpf_tramp_run_ctx, bpf_cookie); - struct bpf_prog *p = tlink->link.prog; + struct bpf_prog *p = node->link->prog; int patch; /* - * run_ctx.cookie = tlink->cookie; + * run_ctx.cookie = node->cookie; */ - /* %r0 = tlink->cookie */ - load_imm64(jit, REG_W0, tlink->cookie); - /* stg %r0,cookie_off(%r15) */ - EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, REG_0, REG_15, cookie_off); + emit_store_stack_imm64(jit, REG_W0, cookie_off, node->cookie); /* * if ((start = __bpf_prog_enter(p, &run_ctx)) == 0) @@ -2555,7 +2629,7 @@ static int invoke_bpf_prog(struct bpf_tramp_jit *tjit, EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, p->bpf_func); /* stg %r2,retval_off(%r15) */ if (save_ret) { - if (sign_extend(jit, REG_2, m->ret_size, m->ret_flags)) + if (sign_zero_extend(jit, REG_2, m->ret_size, m->ret_flags)) return -1; EMIT6_DISP_LH(0xe3000000, 0x0024, REG_2, REG_0, REG_15, tjit->retval_off); @@ -2581,6 +2655,28 @@ static int invoke_bpf_prog(struct bpf_tramp_jit *tjit, return 0; } +static int invoke_bpf(struct bpf_tramp_jit *tjit, + const struct btf_func_model *m, + struct bpf_tramp_nodes *tn, bool save_ret, + u64 func_meta, int cookie_off) +{ + int i, cur_cookie = (tjit->bpf_args_off - cookie_off) / sizeof(u64); + struct bpf_jit *jit = &tjit->common; + + for (i = 0; i < tn->nr_nodes; i++) { + if (bpf_prog_calls_session_cookie(tn->nodes[i])) { + u64 meta = func_meta | ((u64)cur_cookie << BPF_TRAMP_COOKIE_INDEX_SHIFT); + + emit_store_stack_imm64(jit, REG_0, tjit->func_meta_off, meta); + cur_cookie--; + } + if (invoke_bpf_prog(tjit, m, tn->nodes[i], save_ret)) + return -EINVAL; + } + + return 0; +} + static int alloc_stack(struct bpf_tramp_jit *tjit, size_t size) { int stack_offset = tjit->stack_size; @@ -2603,15 +2699,17 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, struct bpf_tramp_jit *tjit, const struct btf_func_model *m, u32 flags, - struct bpf_tramp_links *tlinks, + struct bpf_tramp_nodes *tnodes, void *func_addr) { - struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; - struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; - struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; + struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN]; + struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY]; + struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT]; int nr_bpf_args, nr_reg_args, nr_stack_args; + int cookie_cnt, cookie_off, fsession_cnt; struct bpf_jit *jit = &tjit->common; int arg, bpf_arg_off; + u64 func_meta; int i, j; /* Support as many stack arguments as "mvc" instruction can handle. */ @@ -2621,8 +2719,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, return -ENOTSUPP; /* Return to %r14 in the struct_ops case. */ - if (flags & BPF_TRAMP_F_INDIRECT) + if (flags & BPF_TRAMP_F_INDIRECT) { flags |= BPF_TRAMP_F_SKIP_FRAME; + emit_kcfi(cfi_get_func_hash(func_addr), jit); + } /* * Compute how many arguments we need to pass to BPF programs. @@ -2643,6 +2743,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, return -ENOTSUPP; } + cookie_cnt = bpf_fsession_cookie_cnt(tnodes); + fsession_cnt = bpf_fsession_cnt(tnodes); + /* * Calculate the stack layout. */ @@ -2655,8 +2758,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, tjit->backchain_off = tjit->stack_size - sizeof(u64); tjit->stack_args_off = alloc_stack(tjit, nr_stack_args * sizeof(u64)); tjit->reg_args_off = alloc_stack(tjit, nr_reg_args * sizeof(u64)); + cookie_off = alloc_stack(tjit, cookie_cnt * sizeof(u64)); tjit->ip_off = alloc_stack(tjit, sizeof(u64)); - tjit->arg_cnt_off = alloc_stack(tjit, sizeof(u64)); + tjit->func_meta_off = alloc_stack(tjit, sizeof(u64)); tjit->bpf_args_off = alloc_stack(tjit, nr_bpf_args * sizeof(u64)); tjit->retval_off = alloc_stack(tjit, sizeof(u64)); tjit->r7_r8_off = alloc_stack(tjit, 2 * sizeof(u64)); @@ -2743,18 +2847,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, * arg_cnt = m->nr_args; */ - if (flags & BPF_TRAMP_F_IP_ARG) { - /* %r0 = func_addr */ - load_imm64(jit, REG_0, (u64)func_addr); - /* stg %r0,ip_off(%r15) */ - EMIT6_DISP_LH(0xe3000000, 0x0024, REG_0, REG_0, REG_15, - tjit->ip_off); - } - /* lghi %r0,nr_bpf_args */ - EMIT4_IMM(0xa7090000, REG_0, nr_bpf_args); - /* stg %r0,arg_cnt_off(%r15) */ + if (flags & BPF_TRAMP_F_IP_ARG) + emit_store_stack_imm64(jit, REG_0, tjit->ip_off, (u64)func_addr); + func_meta = nr_bpf_args; + /* lghi %r0,func_meta */ + EMIT4_IMM(0xa7090000, REG_0, func_meta); + /* stg %r0,func_meta_off(%r15) */ EMIT6_DISP_LH(0xe3000000, 0x0024, REG_0, REG_0, REG_15, - tjit->arg_cnt_off); + tjit->func_meta_off); if (flags & BPF_TRAMP_F_CALL_ORIG) { /* @@ -2767,12 +2867,19 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, __bpf_tramp_enter); } - for (i = 0; i < fentry->nr_links; i++) - if (invoke_bpf_prog(tjit, m, fentry->links[i], - flags & BPF_TRAMP_F_RET_FENTRY_RET)) - return -EINVAL; + if (fsession_cnt) { + /* Clear all the session cookies' value. */ + for (i = 0; i < cookie_cnt; i++) + emit_store_stack_imm64(jit, REG_0, cookie_off + 8 * i, 0); + /* Clear the return value to make sure fentry always gets 0. */ + emit_store_stack_imm64(jit, REG_0, tjit->retval_off, 0); + } - if (fmod_ret->nr_links) { + if (invoke_bpf(tjit, m, fentry, flags & BPF_TRAMP_F_RET_FENTRY_RET, + func_meta, cookie_off)) + return -EINVAL; + + if (fmod_ret->nr_nodes) { /* * retval = 0; */ @@ -2781,8 +2888,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, _EMIT6(0xd707f000 | tjit->retval_off, 0xf000 | tjit->retval_off); - for (i = 0; i < fmod_ret->nr_links; i++) { - if (invoke_bpf_prog(tjit, m, fmod_ret->links[i], true)) + for (i = 0; i < fmod_ret->nr_nodes; i++) { + if (invoke_bpf_prog(tjit, m, fmod_ret->nodes[i], true)) return -EINVAL; /* @@ -2847,11 +2954,16 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, EMIT6_PCREL_RILC(0xc0040000, 0, (u64)im->ip_epilogue); } + /* Set the "is_return" flag for fsession. */ + func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT); + if (fsession_cnt) + emit_store_stack_imm64(jit, REG_W0, tjit->func_meta_off, + func_meta); + /* do_fexit: */ tjit->do_fexit = jit->prg; - for (i = 0; i < fexit->nr_links; i++) - if (invoke_bpf_prog(tjit, m, fexit->links[i], false)) - return -EINVAL; + if (invoke_bpf(tjit, m, fexit, false, func_meta, cookie_off)) + return -EINVAL; if (flags & BPF_TRAMP_F_CALL_ORIG) { im->ip_epilogue = jit->prg_buf + jit->prg; @@ -2902,7 +3014,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, } int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, - struct bpf_tramp_links *tlinks, void *orig_call) + struct bpf_tramp_nodes *tnodes, void *orig_call) { struct bpf_tramp_image im; struct bpf_tramp_jit tjit; @@ -2911,14 +3023,14 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, memset(&tjit, 0, sizeof(tjit)); ret = __arch_prepare_bpf_trampoline(&im, &tjit, m, flags, - tlinks, orig_call); + tnodes, orig_call); return ret < 0 ? ret : tjit.common.prg; } int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, const struct btf_func_model *m, - u32 flags, struct bpf_tramp_links *tlinks, + u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr) { struct bpf_tramp_jit tjit; @@ -2927,7 +3039,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, /* Compute offsets, check whether the code fits. */ memset(&tjit, 0, sizeof(tjit)); ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, - tlinks, func_addr); + tnodes, func_addr); if (ret < 0) return ret; @@ -2941,7 +3053,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, tjit.common.prg = 0; tjit.common.prg_buf = image; ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, - tlinks, func_addr); + tnodes, func_addr); return ret < 0 ? ret : tjit.common.prg; } @@ -2956,18 +3068,16 @@ bool bpf_jit_supports_arena(void) return true; } +bool bpf_jit_supports_fsession(void) +{ + return true; +} + bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { if (!in_arena) return true; switch (insn->code) { - case BPF_STX | BPF_ATOMIC | BPF_B: - case BPF_STX | BPF_ATOMIC | BPF_H: - case BPF_STX | BPF_ATOMIC | BPF_W: - case BPF_STX | BPF_ATOMIC | BPF_DW: - if (bpf_atomic_is_load_store(insn)) - return false; - break; case BPF_LDX | BPF_MEMSX | BPF_B: case BPF_LDX | BPF_MEMSX | BPF_H: case BPF_LDX | BPF_MEMSX | BPF_W: @@ -3015,3 +3125,15 @@ bool bpf_jit_supports_timed_may_goto(void) { return true; } + +bool bpf_jit_inlines_helper_call(s32 imm) +{ + switch (imm) { + case BPF_FUNC_get_smp_processor_id: + case BPF_FUNC_get_current_task: + case BPF_FUNC_get_current_task_btf: + return true; + default: + return false; + } +} diff --git a/arch/s390/pci/Makefile b/arch/s390/pci/Makefile index 1810e0944a4e..b8be1316491b 100644 --- a/arch/s390/pci/Makefile +++ b/arch/s390/pci/Makefile @@ -3,6 +3,8 @@ # Makefile for the s390 PCI subsystem. # +CONTEXT_ANALYSIS := y + obj-$(CONFIG_PCI) += pci.o pci_irq.o pci_clp.o \ pci_event.o pci_debug.o pci_insn.o pci_mmio.o \ pci_bus.o pci_kvm_hook.o pci_report.o pci_fixup.o diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index 57f3980b98a9..d64c544b32b0 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -231,24 +231,33 @@ int zpci_fmb_disable_device(struct zpci_dev *zdev) static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len) { u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); + int rc = -ENODEV; u64 data; - int rc; + + if (!zdev_enabled(zdev)) + goto out_err; rc = __zpci_load(&data, req, offset); - if (!rc) { - data = le64_to_cpu((__force __le64) data); - data >>= (8 - len) * 8; - *val = (u32) data; - } else - *val = 0xffffffff; + if (rc) + goto out_err; + data = le64_to_cpu((__force __le64)data); + data >>= (8 - len) * 8; + *val = (u32)data; + return 0; + +out_err: + PCI_SET_ERROR_RESPONSE(val); return rc; } static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len) { u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); + int rc = -ENODEV; u64 data = val; - int rc; + + if (!zdev_enabled(zdev)) + return rc; data <<= (8 - len) * 8; data = (__force u64) cpu_to_le64(data); @@ -257,6 +266,7 @@ static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len) } resource_size_t pcibios_align_resource(void *data, const struct resource *res, + const struct resource *empty_res, resource_size_t size, resource_size_t align) { @@ -397,7 +407,9 @@ static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, { struct zpci_dev *zdev = zdev_from_bus(bus, devfn); - return (zdev) ? zpci_cfg_load(zdev, where, val, size) : -ENODEV; + if (!zdev || zpci_cfg_load(zdev, where, val, size)) + return PCIBIOS_DEVICE_NOT_FOUND; + return PCIBIOS_SUCCESSFUL; } static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, @@ -405,7 +417,9 @@ static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, { struct zpci_dev *zdev = zdev_from_bus(bus, devfn); - return (zdev) ? zpci_cfg_store(zdev, where, val, size) : -ENODEV; + if (!zdev || zpci_cfg_store(zdev, where, val, size)) + return PCIBIOS_DEVICE_NOT_FOUND; + return PCIBIOS_SUCCESSFUL; } static struct pci_ops pci_root_ops = { @@ -510,7 +524,7 @@ static struct resource *__alloc_res(struct zpci_dev *zdev, unsigned long start, { struct resource *r; - r = kzalloc(sizeof(*r), GFP_KERNEL); + r = kzalloc_obj(*r); if (!r) return NULL; @@ -811,7 +825,7 @@ struct zpci_dev *zpci_create_device(u32 fid, u32 fh, enum zpci_state state) struct zpci_dev *zdev; int rc; - zdev = kzalloc(sizeof(*zdev), GFP_KERNEL); + zdev = kzalloc_obj(*zdev); if (!zdev) return ERR_PTR(-ENOMEM); @@ -828,6 +842,7 @@ struct zpci_dev *zpci_create_device(u32 fid, u32 fh, enum zpci_state state) mutex_init(&zdev->state_lock); mutex_init(&zdev->fmb_lock); mutex_init(&zdev->kzdev_lock); + mutex_init(&zdev->pending_errs_lock); return zdev; @@ -1052,14 +1067,15 @@ static int zpci_mem_init(void) { BUILD_BUG_ON(!is_power_of_2(__alignof__(struct zpci_fmb)) || __alignof__(struct zpci_fmb) < sizeof(struct zpci_fmb)); + BUILD_BUG_ON((CONFIG_ILLEGAL_POINTER_VALUE + 0x10000 > ZPCI_IOMAP_ADDR_BASE) && + (CONFIG_ILLEGAL_POINTER_VALUE <= ZPCI_IOMAP_ADDR_MAX)); zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb), __alignof__(struct zpci_fmb), 0, NULL); if (!zdev_fmb_cache) goto error_fmb; - zpci_iomap_start = kcalloc(ZPCI_IOMAP_ENTRIES, - sizeof(*zpci_iomap_start), GFP_KERNEL); + zpci_iomap_start = kzalloc_objs(*zpci_iomap_start, ZPCI_IOMAP_ENTRIES); if (!zpci_iomap_start) goto error_iomap; @@ -1158,10 +1174,8 @@ int zpci_scan_devices(void) return rc; zpci_add_devices(&scan_list); - zpci_bus_for_each(zbus) { + zpci_bus_for_each(zbus) zpci_bus_scan_bus(zbus); - cond_resched(); - } return 0; } diff --git a/arch/s390/pci/pci_bus.c b/arch/s390/pci/pci_bus.c index 42a13e451f64..36a4807285fa 100644 --- a/arch/s390/pci/pci_bus.c +++ b/arch/s390/pci/pci_bus.c @@ -315,7 +315,7 @@ static struct zpci_bus *zpci_bus_alloc(int topo, bool topo_is_tid) { struct zpci_bus *zbus; - zbus = kzalloc(sizeof(*zbus), GFP_KERNEL); + zbus = kzalloc_obj(*zbus); if (!zbus) return NULL; diff --git a/arch/s390/pci/pci_event.c b/arch/s390/pci/pci_event.c index 839bd91c056e..d6af4015223e 100644 --- a/arch/s390/pci/pci_event.c +++ b/arch/s390/pci/pci_event.c @@ -17,23 +17,6 @@ #include "pci_bus.h" #include "pci_report.h" -/* Content Code Description for PCI Function Error */ -struct zpci_ccdf_err { - u32 reserved1; - u32 fh; /* function handle */ - u32 fid; /* function id */ - u32 ett : 4; /* expected table type */ - u32 mvn : 12; /* MSI vector number */ - u32 dmaas : 8; /* DMA address space */ - u32 : 6; - u32 q : 1; /* event qualifier */ - u32 rw : 1; /* read/write */ - u64 faddr; /* failing address */ - u32 reserved3; - u16 reserved4; - u16 pec; /* PCI event code */ -} __packed; - /* Content Code Description for PCI Function Availability */ struct zpci_ccdf_avail { u32 reserved1; @@ -60,18 +43,6 @@ static inline bool ers_result_indicates_abort(pci_ers_result_t ers_res) } } -static bool is_passed_through(struct pci_dev *pdev) -{ - struct zpci_dev *zdev = to_zpci(pdev); - bool ret; - - mutex_lock(&zdev->kzdev_lock); - ret = !!zdev->kzdev; - mutex_unlock(&zdev->kzdev_lock); - - return ret; -} - static bool is_driver_supported(struct pci_driver *driver) { if (!driver || !driver->err_handler) @@ -81,6 +52,68 @@ static bool is_driver_supported(struct pci_driver *driver) return true; } +static int zpci_store_pci_error(struct pci_dev *pdev, + struct zpci_ccdf_err *ccdf) +{ + struct zpci_dev *zdev = to_zpci(pdev); + int i; + + guard(mutex)(&zdev->pending_errs_lock); + if (!zdev->pending_errs.mediated_recovery) + return -EINVAL; + + if (zdev->pending_errs.count >= ZPCI_ERR_PENDING_MAX) { + dev_warn_ratelimited(&pdev->dev, + "%s: Maximum number (%d) of pending error events queued\n", + pci_name(pdev), + ZPCI_ERR_PENDING_MAX); + return -ENOMEM; + } + + i = zdev->pending_errs.tail % ZPCI_ERR_PENDING_MAX; + memcpy(&zdev->pending_errs.err[i], ccdf, sizeof(struct zpci_ccdf_err)); + zdev->pending_errs.tail++; + zdev->pending_errs.count++; + return 0; +} + +int zpci_get_pending_error(struct zpci_dev *zdev, + struct zpci_ccdf_err *ccdf) +{ + int head; + + guard(mutex)(&zdev->pending_errs_lock); + + if (!zdev->pending_errs.count) + return -ENOMSG; + + head = zdev->pending_errs.head % ZPCI_ERR_PENDING_MAX; + memcpy(ccdf, &zdev->pending_errs.err[head], + sizeof(struct zpci_ccdf_err)); + zdev->pending_errs.head++; + zdev->pending_errs.count--; + return 0; +} +EXPORT_SYMBOL_GPL(zpci_get_pending_error); + +void zpci_start_mediated_recovery(struct zpci_dev *zdev) +{ + guard(mutex)(&zdev->pending_errs_lock); + zdev->pending_errs.mediated_recovery = true; +} +EXPORT_SYMBOL_GPL(zpci_start_mediated_recovery); + +void zpci_stop_mediated_recovery(struct zpci_dev *zdev) +{ + guard(mutex)(&zdev->pending_errs_lock); + zdev->pending_errs.mediated_recovery = false; + if (zdev->pending_errs.count) + pr_info("Unhandled PCI error events count=%d for PCI function 0x%x\n", + zdev->pending_errs.count, zdev->fid); + memset(&zdev->pending_errs, 0, sizeof(struct zpci_ccdf_pending)); +} +EXPORT_SYMBOL_GPL(zpci_stop_mediated_recovery); + static pci_ers_result_t zpci_event_notify_error_detected(struct pci_dev *pdev, struct pci_driver *driver) { @@ -175,12 +208,15 @@ static pci_ers_result_t zpci_event_do_reset(struct pci_dev *pdev, * and the platform determines which functions are affected for * multi-function devices. */ -static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev) +static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev, + struct zpci_ccdf_err *ccdf) { pci_ers_result_t ers_res = PCI_ERS_RESULT_DISCONNECT; struct zpci_dev *zdev = to_zpci(pdev); + bool mediated_recovery = false; char *status_str = "success"; struct pci_driver *driver; + int rc; /* * Ensure that the PCI function is not removed concurrently, no driver @@ -190,17 +226,11 @@ static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev) device_lock(&pdev->dev); if (pdev->error_state == pci_channel_io_perm_failure) { ers_res = PCI_ERS_RESULT_DISCONNECT; + status_str = "skipped (permanent failure)"; goto out_unlock; } pdev->error_state = pci_channel_io_frozen; - if (is_passed_through(pdev)) { - pr_info("%s: Cannot be recovered in the host because it is a pass-through device\n", - pci_name(pdev)); - status_str = "failed (pass-through)"; - goto out_unlock; - } - driver = to_pci_driver(pdev->dev.driver); if (!is_driver_supported(driver)) { if (!driver) { @@ -216,12 +246,24 @@ static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev) goto out_unlock; } + rc = zpci_store_pci_error(pdev, ccdf); + if (!rc || rc == -ENOMEM) + mediated_recovery = true; + ers_res = zpci_event_notify_error_detected(pdev, driver); if (ers_result_indicates_abort(ers_res)) { status_str = "failed (abort on detection)"; goto out_unlock; } + if (mediated_recovery) { + pr_info("%s: Leaving recovery of pass-through device to user-space\n", + pci_name(pdev)); + ers_res = PCI_ERS_RESULT_RECOVERED; + status_str = "in progress"; + goto out_unlock; + } + if (ers_res != PCI_ERS_RESULT_NEED_RESET) { ers_res = zpci_event_do_error_state_clear(pdev, driver); if (ers_result_indicates_abort(ers_res)) { @@ -256,8 +298,8 @@ static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev) driver->err_handler->resume(pdev); pci_uevent_ers(pdev, PCI_ERS_RESULT_RECOVERED); out_unlock: + zpci_report_status(zdev, pdev, "recovery", status_str); device_unlock(&pdev->dev); - zpci_report_status(zdev, "recovery", status_str); return ers_res; } @@ -266,28 +308,28 @@ out_unlock: * @pdev: PCI function for which to report * @es: PCI channel failure state to report */ -static void zpci_event_io_failure(struct pci_dev *pdev, pci_channel_state_t es) +static void zpci_event_io_failure(struct pci_dev *pdev, pci_channel_state_t es, + struct zpci_ccdf_err *ccdf) { struct pci_driver *driver; pci_dev_lock(pdev); pdev->error_state = es; - /** - * While vfio-pci's error_detected callback notifies user-space QEMU - * reacts to this by freezing the guest. In an s390 environment PCI - * errors are rarely fatal so this is overkill. Instead in the future - * we will inject the error event and let the guest recover the device - * itself. - */ - if (is_passed_through(pdev)) - goto out; + + zpci_store_pci_error(pdev, ccdf); driver = to_pci_driver(pdev->dev.driver); if (driver && driver->err_handler && driver->err_handler->error_detected) driver->err_handler->error_detected(pdev, pdev->error_state); -out: + pci_dev_unlock(pdev); } +static void __zpci_event_print_error(struct pci_dev *pdev, struct zpci_ccdf_err *ccdf) +{ + pr_err("%s: Event 0x%x reports an error for PCI function 0x%x\n", + pdev ? pci_name(pdev) : "n/a", ccdf->pec, ccdf->fid); +} + static void __zpci_event_error(struct zpci_ccdf_err *ccdf) { struct zpci_dev *zdev = get_zdev_by_fid(ccdf->fid); @@ -301,27 +343,29 @@ static void __zpci_event_error(struct zpci_ccdf_err *ccdf) zpci_err("error CCDF:\n"); zpci_err_hex(ccdf, sizeof(*ccdf)); - if (zdev) { - mutex_lock(&zdev->state_lock); - rc = clp_refresh_fh(zdev->fid, &fh); - if (rc) - goto no_pdev; - if (!fh || ccdf->fh != fh) { - /* Ignore events with stale handles */ - zpci_dbg(3, "err fid:%x, fh:%x (stale %x)\n", - ccdf->fid, fh, ccdf->fh); - goto no_pdev; - } - zpci_update_fh(zdev, ccdf->fh); - if (zdev->zbus->bus) - pdev = pci_get_slot(zdev->zbus->bus, zdev->devfn); + if (!zdev) + return __zpci_event_print_error(NULL, ccdf); + + mutex_lock(&zdev->state_lock); + rc = clp_refresh_fh(zdev->fid, &fh); + if (rc) + goto no_pdev; + if (!fh || ccdf->fh != fh) { + /* Ignore events with stale handles */ + zpci_dbg(3, "err fid:%x, fh:%x (stale %x)\n", + ccdf->fid, fh, ccdf->fh); + goto no_pdev; } + zpci_update_fh(zdev, ccdf->fh); + if (zdev->zbus->bus) + pdev = pci_get_slot(zdev->zbus->bus, zdev->devfn); - pr_err("%s: Event 0x%x reports an error for PCI function 0x%x\n", - pdev ? pci_name(pdev) : "n/a", ccdf->pec, ccdf->fid); + __zpci_event_print_error(pdev, ccdf); - if (!pdev) + if (!pdev) { + zpci_report_status(zdev, NULL, "error event", "no pdev bound"); goto no_pdev; + } switch (ccdf->pec) { case 0x002a: /* Error event concerns FMB */ @@ -330,18 +374,17 @@ static void __zpci_event_error(struct zpci_ccdf_err *ccdf) break; case 0x0040: /* Service Action or Error Recovery Failed */ case 0x003b: - zpci_event_io_failure(pdev, pci_channel_io_perm_failure); + zpci_event_io_failure(pdev, pci_channel_io_perm_failure, ccdf); break; default: /* PCI function left in the error state attempt to recover */ - ers_res = zpci_event_attempt_error_recovery(pdev); + ers_res = zpci_event_attempt_error_recovery(pdev, ccdf); if (ers_res != PCI_ERS_RESULT_RECOVERED) - zpci_event_io_failure(pdev, pci_channel_io_perm_failure); + zpci_event_io_failure(pdev, pci_channel_io_perm_failure, ccdf); break; } pci_dev_put(pdev); no_pdev: - if (zdev) - mutex_unlock(&zdev->state_lock); + mutex_unlock(&zdev->state_lock); zpci_zdev_put(zdev); } @@ -382,98 +425,106 @@ static void zpci_event_reappear(struct zpci_dev *zdev) zpci_dbg(1, "rea fid:%x, fh:%x\n", zdev->fid, zdev->fh); } -static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf) +static bool zpci_event_avail_any_device(struct zpci_ccdf_avail *ccdf) { - struct zpci_dev *zdev = get_zdev_by_fid(ccdf->fid); - bool existing_zdev = !!zdev; - enum zpci_state state; - - zpci_dbg(3, "avl fid:%x, fh:%x, pec:%x\n", - ccdf->fid, ccdf->fh, ccdf->pec); + /* 0x0306 - No handle or fid stored */ + if (ccdf->pec != 0x0306) + return false; + /* 0x308 or 0x302 for multiple devices */ + zpci_remove_reserved_devices(); + zpci_scan_devices(); + return true; +} - if (existing_zdev) - mutex_lock(&zdev->state_lock); +static void zpci_event_avail_new_device(struct zpci_ccdf_avail *ccdf) +{ + struct zpci_dev *zdev; switch (ccdf->pec) { case 0x0301: /* Reserved|Standby -> Configured */ - if (!zdev) { - zdev = zpci_create_device(ccdf->fid, ccdf->fh, ZPCI_FN_STATE_CONFIGURED); - if (IS_ERR(zdev)) - break; - if (zpci_add_device(zdev)) { - kfree(zdev); - break; - } - } else { - if (zdev->state == ZPCI_FN_STATE_RESERVED) - zpci_event_reappear(zdev); - /* the configuration request may be stale */ - else if (zdev->state != ZPCI_FN_STATE_STANDBY) - break; - zdev->state = ZPCI_FN_STATE_CONFIGURED; + zdev = zpci_create_device(ccdf->fid, ccdf->fh, ZPCI_FN_STATE_CONFIGURED); + if (IS_ERR(zdev)) + break; + if (zpci_add_device(zdev)) { + kfree(zdev); + break; } zpci_scan_configured_device(zdev, ccdf->fh); break; case 0x0302: /* Reserved -> Standby */ - if (!zdev) { - zdev = zpci_create_device(ccdf->fid, ccdf->fh, ZPCI_FN_STATE_STANDBY); - if (IS_ERR(zdev)) - break; - if (zpci_add_device(zdev)) { - kfree(zdev); - break; - } - } else { - if (zdev->state == ZPCI_FN_STATE_RESERVED) - zpci_event_reappear(zdev); - zpci_update_fh(zdev, ccdf->fh); + zdev = zpci_create_device(ccdf->fid, ccdf->fh, ZPCI_FN_STATE_STANDBY); + if (IS_ERR(zdev)) + break; + if (zpci_add_device(zdev)) { + kfree(zdev); + break; } break; + } +} + +static void zpci_event_avail_existing_device(struct zpci_dev *zdev, struct zpci_ccdf_avail *ccdf) +{ + enum zpci_state state; + + switch (ccdf->pec) { + case 0x0301: /* Reserved|Standby -> Configured */ + if (zdev->state == ZPCI_FN_STATE_RESERVED) + zpci_event_reappear(zdev); + /* the configuration request may be stale */ + else if (zdev->state != ZPCI_FN_STATE_STANDBY) + break; + zdev->state = ZPCI_FN_STATE_CONFIGURED; + zpci_scan_configured_device(zdev, ccdf->fh); + break; + case 0x0302: /* Reserved -> Standby */ + if (zdev->state == ZPCI_FN_STATE_RESERVED) + zpci_event_reappear(zdev); + zpci_update_fh(zdev, ccdf->fh); + break; case 0x0303: /* Deconfiguration requested */ - if (zdev) { - /* The event may have been queued before we configured - * the device. - */ - if (zdev->state != ZPCI_FN_STATE_CONFIGURED) - break; - zpci_update_fh(zdev, ccdf->fh); - zpci_deconfigure_device(zdev); - } + /* The event may have been queued before we configured + * the device. + */ + if (zdev->state != ZPCI_FN_STATE_CONFIGURED) + break; + zpci_update_fh(zdev, ccdf->fh); + zpci_deconfigure_device(zdev); break; case 0x0304: /* Configured -> Standby|Reserved */ - if (zdev) { - /* The event may have been queued before we configured - * the device.: - */ - if (zdev->state == ZPCI_FN_STATE_CONFIGURED) - zpci_event_hard_deconfigured(zdev, ccdf->fh); - /* The 0x0304 event may immediately reserve the device */ - if (!clp_get_state(zdev->fid, &state) && - state == ZPCI_FN_STATE_RESERVED) { - zpci_device_reserved(zdev); - } + /* The event may have been queued before we configured + * the device.: + */ + if (zdev->state == ZPCI_FN_STATE_CONFIGURED) + zpci_event_hard_deconfigured(zdev, ccdf->fh); + /* The 0x0304 event may immediately reserve the device */ + if (!clp_get_state(zdev->fid, &state) && + state == ZPCI_FN_STATE_RESERVED) { + zpci_device_reserved(zdev); } break; - case 0x0306: /* 0x308 or 0x302 for multiple devices */ - zpci_remove_reserved_devices(); - zpci_scan_devices(); - break; case 0x0308: /* Standby -> Reserved */ - if (!zdev) - break; zpci_device_reserved(zdev); break; - default: - break; - } - if (existing_zdev) { - mutex_unlock(&zdev->state_lock); - zpci_zdev_put(zdev); } } void zpci_event_availability(void *data) { - if (zpci_is_enabled()) - __zpci_event_availability(data); + struct zpci_ccdf_avail *ccdf = data; + struct zpci_dev *zdev; + + if (!zpci_is_enabled()) + return; + zpci_dbg(3, "avl fid:%x, fh:%x, pec:%x\n", + ccdf->fid, ccdf->fh, ccdf->pec); + if (zpci_event_avail_any_device(ccdf)) + return; + zdev = get_zdev_by_fid(ccdf->fid); + if (!zdev) + return zpci_event_avail_new_device(ccdf); + mutex_lock(&zdev->state_lock); + zpci_event_avail_existing_device(zdev, ccdf); + mutex_unlock(&zdev->state_lock); + zpci_zdev_put(zdev); } diff --git a/arch/s390/pci/pci_irq.c b/arch/s390/pci/pci_irq.c index e9dd45f3c09d..9c9ed3d8d959 100644 --- a/arch/s390/pci/pci_irq.c +++ b/arch/s390/pci/pci_irq.c @@ -563,8 +563,7 @@ static int __init zpci_directed_irq_init(void) iib.diib.disb_addr = virt_to_phys(zpci_sbv->vector); zpci_set_irq_ctrl(SIC_IRQ_MODE_DIRECT, 0, &iib); - zpci_ibv = kcalloc(num_possible_cpus(), sizeof(*zpci_ibv), - GFP_KERNEL); + zpci_ibv = kzalloc_objs(*zpci_ibv, num_possible_cpus()); if (!zpci_ibv) return -ENOMEM; @@ -590,7 +589,7 @@ static int __init zpci_directed_irq_init(void) static int __init zpci_floating_irq_init(void) { - zpci_ibv = kcalloc(ZPCI_NR_DEVICES, sizeof(*zpci_ibv), GFP_KERNEL); + zpci_ibv = kzalloc_objs(*zpci_ibv, ZPCI_NR_DEVICES); if (!zpci_ibv) return -ENOMEM; diff --git a/arch/s390/pci/pci_mmio.c b/arch/s390/pci/pci_mmio.c index 51e7a28af899..f3f79ba78410 100644 --- a/arch/s390/pci/pci_mmio.c +++ b/arch/s390/pci/pci_mmio.c @@ -188,6 +188,7 @@ SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, goto out_unlock_mmap; } + ret = -EFAULT; io_addr = (void __iomem *)((args.pfn << PAGE_SHIFT) | (mmio_addr & ~PAGE_MASK)); diff --git a/arch/s390/pci/pci_report.c b/arch/s390/pci/pci_report.c index 7030f7052926..72ecabf7003c 100644 --- a/arch/s390/pci/pci_report.c +++ b/arch/s390/pci/pci_report.c @@ -87,9 +87,23 @@ static struct debug_view debug_log_view = { NULL }; +static ssize_t zpci_report_pdev(struct pci_dev *pdev, char *buf, size_t size) +{ + struct pci_driver *driver; + const char *start = buf; + char *end = buf + size; + + device_lock_assert(&pdev->dev); + buf += scnprintf(buf, end - buf, "state: %s\n", zpci_state_str(pdev->error_state)); + driver = to_pci_driver(pdev->dev.driver); + buf += scnprintf(buf, end - buf, "driver: %s\n", (driver) ? driver->name : "n/a"); + return buf - start; +} + /** * zpci_report_status - Report the status of operations on a PCI device - * @zdev: The PCI device for which to report status + * @zdev: The zPCI device for which to report status + * @pdev: The PCI device associated with the zdev if any, NULL otherwise * @operation: A string representing the operation reported * @status: A string representing the status of the operation * @@ -103,15 +117,14 @@ static struct debug_view debug_log_view = { * * Return: 0 on success an error code < 0 otherwise. */ -int zpci_report_status(struct zpci_dev *zdev, const char *operation, const char *status) +int zpci_report_status(struct zpci_dev *zdev, struct pci_dev *pdev, + const char *operation, const char *status) { struct zpci_report_error *report; - struct pci_driver *driver = NULL; - struct pci_dev *pdev = NULL; char *buf, *end; int ret; - if (!zdev || !zdev->zbus) + if (!zdev) return -ENODEV; /* Protected virtualization hosts get nothing from us */ @@ -121,18 +134,13 @@ int zpci_report_status(struct zpci_dev *zdev, const char *operation, const char report = (void *)get_zeroed_page(GFP_KERNEL); if (!report) return -ENOMEM; - if (zdev->zbus->bus) - pdev = pci_get_slot(zdev->zbus->bus, zdev->devfn); - if (pdev) - driver = to_pci_driver(pdev->dev.driver); buf = report->data.log_data; end = report->data.log_data + ZPCI_REPORT_DATA_SIZE; buf += scnprintf(buf, end - buf, "report: %s\n", operation); buf += scnprintf(buf, end - buf, "status: %s\n", status); - buf += scnprintf(buf, end - buf, "state: %s\n", - (pdev) ? zpci_state_str(pdev->error_state) : "n/a"); - buf += scnprintf(buf, end - buf, "driver: %s\n", (driver) ? driver->name : "n/a"); + if (pdev) + buf += zpci_report_pdev(pdev, buf, end - buf); ret = debug_dump(pci_debug_msg_id, &debug_log_view, buf, end - buf, true); if (ret < 0) pr_err("Reading PCI debug messages failed with code %d\n", ret); diff --git a/arch/s390/pci/pci_report.h b/arch/s390/pci/pci_report.h index e08003d51a97..dd7b0b05001c 100644 --- a/arch/s390/pci/pci_report.h +++ b/arch/s390/pci/pci_report.h @@ -8,9 +8,11 @@ */ #ifndef __S390_PCI_REPORT_H #define __S390_PCI_REPORT_H +#include <linux/pci.h> struct zpci_dev; -int zpci_report_status(struct zpci_dev *zdev, const char *operation, const char *status); +int zpci_report_status(struct zpci_dev *zdev, struct pci_dev *pdev, + const char *operation, const char *status); #endif /* __S390_PCI_REPORT_H */ diff --git a/arch/s390/pci/pci_sysfs.c b/arch/s390/pci/pci_sysfs.c index c2444a23e26c..bbb76113a4d0 100644 --- a/arch/s390/pci/pci_sysfs.c +++ b/arch/s390/pci/pci_sysfs.c @@ -153,6 +153,9 @@ static ssize_t report_error_write(struct file *filp, struct kobject *kobj, if (off || (count < sizeof(*report))) return -EINVAL; + if (count < (report->length + sizeof(*report))) + return -EINVAL; + ret = sclp_pci_report(report, zdev->fh, zdev->fid); return ret ? ret : count; @@ -187,6 +190,17 @@ static ssize_t index_show(struct device *dev, } static DEVICE_ATTR_RO(index); +static ssize_t zpci_uid_slot_show(struct pci_slot *slot, char *buf) +{ + struct zpci_dev *zdev = container_of(slot->hotplug, struct zpci_dev, + hotplug_slot); + + return sysfs_emit(buf, "0x%x\n", zdev->uid); +} + +static struct pci_slot_attribute zpci_slot_attr_uid = + __ATTR(uid, 0444, zpci_uid_slot_show, NULL); + static umode_t zpci_index_is_visible(struct kobject *kobj, struct attribute *attr, int n) { @@ -243,6 +257,15 @@ const struct attribute_group pfip_attr_group = { .attrs = pfip_attrs, }; +static struct attribute *zpci_slot_attrs[] = { + &zpci_slot_attr_uid.attr, + NULL, +}; + +const struct attribute_group zpci_slot_attr_group = { + .attrs = zpci_slot_attrs, +}; + static struct attribute *clp_fw_attrs[] = { &uid_checking_attr.attr, NULL, diff --git a/arch/s390/purgatory/Makefile b/arch/s390/purgatory/Makefile index 0c196a5b194a..b2f17e0d7fd6 100644 --- a/arch/s390/purgatory/Makefile +++ b/arch/s390/purgatory/Makefile @@ -1,6 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 -purgatory-y := head.o purgatory.o string.o sha256.o mem.o +CONTEXT_ANALYSIS := y + +purgatory-y := head.o purgatory.o string.o sha256.o targets += $(purgatory-y) purgatory.lds purgatory purgatory.chk purgatory.ro PURGATORY_OBJS = $(addprefix $(obj)/,$(purgatory-y)) @@ -10,21 +12,20 @@ $(obj)/sha256.o: $(srctree)/lib/crypto/sha256.c FORCE CFLAGS_sha256.o := -D__NO_FORTIFY -$(obj)/mem.o: $(srctree)/arch/s390/lib/mem.S FORCE - $(call if_changed_rule,as_o_S) +CC_FLAGS_MARCH_MINIMUM := -march=z10 -KBUILD_CFLAGS := -std=gnu11 -fms-extensions -fno-strict-aliasing -Wall -Wstrict-prototypes +KBUILD_CFLAGS := $(CC_FLAGS_DIALECT) -fno-strict-aliasing -Wall -Wstrict-prototypes KBUILD_CFLAGS += -Wno-pointer-sign -Wno-sign-compare KBUILD_CFLAGS += -fno-zero-initialized-in-bss -fno-builtin -ffreestanding KBUILD_CFLAGS += -Os -m64 -msoft-float -fno-common KBUILD_CFLAGS += -fno-stack-protector KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING -KBUILD_CFLAGS += -D__DISABLE_EXPORTS +KBUILD_CFLAGS += $(CC_FLAGS_MARCH_MINIMUM) -D__DISABLE_EXPORTS KBUILD_CFLAGS += $(CLANG_FLAGS) -KBUILD_CFLAGS += $(if $(CONFIG_CC_IS_CLANG),-Wno-microsoft-anon-tag) KBUILD_CFLAGS += $(call cc-option,-fno-PIE) +KBUILD_CFLAGS += $(call cc-option, -Wno-default-const-init-unsafe) KBUILD_AFLAGS := $(filter-out -DCC_USING_EXPOLINE,$(KBUILD_AFLAGS)) -KBUILD_AFLAGS += -D__DISABLE_EXPORTS +KBUILD_AFLAGS += $(CC_FLAGS_MARCH_MINIMUM) -D__DISABLE_EXPORTS # Since we link purgatory with -r unresolved symbols are not checked, so we # also link a purgatory.chk binary without -r to check for unresolved symbols. diff --git a/arch/s390/purgatory/purgatory.lds.S b/arch/s390/purgatory/purgatory.lds.S index 482eb4fbcef1..387d0db4085f 100644 --- a/arch/s390/purgatory/purgatory.lds.S +++ b/arch/s390/purgatory/purgatory.lds.S @@ -19,6 +19,7 @@ SECTIONS _text = .; /* Text */ *(.text) *(.text.*) + *(.noinstr.text) _etext = . ; } .rodata : { diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c index 2d28a569f793..32dd5a57240d 100644 --- a/arch/s390/tools/gen_facilities.c +++ b/arch/s390/tools/gen_facilities.c @@ -96,6 +96,13 @@ static struct facility_def facility_defs[] = { 150, /* enhanced sort */ 151, /* deflate conversion */ 155, /* msa extension 9 */ + 165, /* nnpa facility */ + 170, /* ineffective-nonconstrained-transaction facility */ + 193, /* bear enhancement facility */ + 194, /* rdp enhancement facility */ + 196, /* processor activity instrumentation facility */ + 197, /* processor activity instrumentation extension 1 */ + 201, /* concurrent-functions facility */ -1 /* END */ } }, @@ -112,13 +119,6 @@ static struct facility_def facility_defs[] = { 12, /* AP Query Configuration Information */ 15, /* AP Facilities Test */ 156, /* etoken facility */ - 165, /* nnpa facility */ - 170, /* ineffective-nonconstrained-transaction facility */ - 193, /* bear enhancement facility */ - 194, /* rdp enhancement facility */ - 196, /* processor activity instrumentation facility */ - 197, /* processor activity instrumentation extension 1 */ - 201, /* concurrent-functions facility */ -1 /* END */ } }, diff --git a/arch/s390/tools/relocs.c b/arch/s390/tools/relocs.c index 30a732c808f3..72178dc90a4c 100644 --- a/arch/s390/tools/relocs.c +++ b/arch/s390/tools/relocs.c @@ -41,6 +41,10 @@ static Elf_Ehdr ehdr; static unsigned long shnum; static unsigned int shstrndx; +static unsigned int shsymtabndx; +static unsigned int shxsymtabndx; + +static int sym_index(Elf_Sym *sym); struct relocs { uint32_t *offset; @@ -54,11 +58,40 @@ static struct relocs relocs64; struct section { Elf_Shdr shdr; struct section *link; + Elf_Sym *symtab; + Elf32_Word *xsymtab; Elf_Rel *reltab; + char *strtab; }; static struct section *secs; +static const char *sec_name(unsigned shndx) +{ + const char *sec_strtab; + const char *name = "<noname>"; + sec_strtab = secs[shstrndx].strtab; + + if (shndx < shnum) + name = sec_strtab + secs[shndx].shdr.sh_name; + else if (shndx == SHN_ABS) + name = "ABSOLUTE"; + else if (shndx == SHN_COMMON) + name = "COMMON"; + return name; +} + +static const char *sym_name(const char *sym_strtab, Elf_Sym *sym) +{ + const char *name; + + if (sym->st_name) + name = sym_strtab + sym->st_name; + else + name = sec_name(sym_index(sym)); + return name; +} + #if BYTE_ORDER == LITTLE_ENDIAN #define le16_to_cpu(val) (val) #define le32_to_cpu(val) (val) @@ -105,6 +138,23 @@ static uint64_t elf64_to_cpu(uint64_t val) #define elf_off_to_cpu(x) elf64_to_cpu(x) #define elf_xword_to_cpu(x) elf64_to_cpu(x) +static int sym_index(Elf_Sym *sym) +{ + Elf_Sym *symtab = secs[shsymtabndx].symtab; + Elf32_Word *xsymtab = secs[shxsymtabndx].xsymtab; + unsigned long offset; + int index; + + if (sym->st_shndx != SHN_XINDEX) + return sym->st_shndx; + + /* calculate offset of sym from head of table. */ + offset = (unsigned long)sym - (unsigned long)symtab; + index = offset / sizeof(*sym); + + return elf32_to_cpu(xsymtab[index]); +} + static void die(char *fmt, ...) { va_list ap; @@ -216,6 +266,81 @@ static void read_shdrs(FILE *fp) } +static void read_strtabs(FILE *fp) +{ + int i; + + for (i = 0; i < shnum; i++) { + struct section *sec = &secs[i]; + + if (sec->shdr.sh_type != SHT_STRTAB) + continue; + + sec->strtab = malloc(sec->shdr.sh_size); + if (!sec->strtab) + die("malloc of %" FMT " bytes for strtab failed\n", sec->shdr.sh_size); + + if (fseek(fp, sec->shdr.sh_offset, SEEK_SET) < 0) + die("Seek to %" FMT " failed: %s\n", sec->shdr.sh_offset, strerror(errno)); + + if (fread(sec->strtab, 1, sec->shdr.sh_size, fp) != sec->shdr.sh_size) + die("Cannot read symbol table: %s\n", strerror(errno)); + } +} + +static void read_symtabs(FILE *fp) +{ + int i, j; + + for (i = 0; i < shnum; i++) { + struct section *sec = &secs[i]; + int num_syms; + + switch (sec->shdr.sh_type) { + case SHT_SYMTAB_SHNDX: + sec->xsymtab = malloc(sec->shdr.sh_size); + if (!sec->xsymtab) + die("malloc of %" FMT " bytes for xsymtab failed\n", sec->shdr.sh_size); + + if (fseek(fp, sec->shdr.sh_offset, SEEK_SET) < 0) + die("Seek to %" FMT " failed: %s\n", sec->shdr.sh_offset, strerror(errno)); + + if (fread(sec->xsymtab, 1, sec->shdr.sh_size, fp) != sec->shdr.sh_size) + die("Cannot read extended symbol table: %s\n", strerror(errno)); + + shxsymtabndx = i; + continue; + + case SHT_SYMTAB: + num_syms = sec->shdr.sh_size / sizeof(Elf_Sym); + + sec->symtab = malloc(sec->shdr.sh_size); + if (!sec->symtab) + die("malloc of %" FMT " bytes for symtab failed\n", sec->shdr.sh_size); + + if (fseek(fp, sec->shdr.sh_offset, SEEK_SET) < 0) + die("Seek to %" FMT " failed: %s\n", sec->shdr.sh_offset, strerror(errno)); + + if (fread(sec->symtab, 1, sec->shdr.sh_size, fp) != sec->shdr.sh_size) + die("Cannot read symbol table: %s\n", strerror(errno)); + + for (j = 0; j < num_syms; j++) { + Elf_Sym *sym = &sec->symtab[j]; + + sym->st_name = elf_word_to_cpu(sym->st_name); + sym->st_value = elf_addr_to_cpu(sym->st_value); + sym->st_size = elf_xword_to_cpu(sym->st_size); + sym->st_shndx = elf_half_to_cpu(sym->st_shndx); + } + shsymtabndx = i; + continue; + + default: + continue; + } + } +} + static void read_relocs(FILE *fp) { int i, j; @@ -263,7 +388,8 @@ static void add_reloc(struct relocs *r, uint32_t offset) r->offset[r->count++] = offset; } -static int do_reloc(struct section *sec, Elf_Rel *rel) +static int do_reloc(struct section *sec, Elf_Rel *rel, ElfW(Sym) *sym, + const char *symname) { unsigned int r_type = ELF64_R_TYPE(rel->r_info); ElfW(Addr) offset = rel->r_offset; @@ -279,6 +405,22 @@ static int do_reloc(struct section *sec, Elf_Rel *rel) case R_390_GOTPCDBL: case R_390_GOTOFF64: break; + case R_390_32: { + static const char kcfipfx[] = "__kcfi_typeid_"; + + if (sym->st_shndx != SHN_ABS) + die("Unsupported relocation type: %d\n", r_type); + /* + * Symbols with __kcfi_typeid_ prefix have constant values, + * which do not change if bzImage is loaded at a different + * physical address than the address for which it has been + * compiled. + */ + if (!strncmp(kcfipfx, symname, sizeof(kcfipfx) - 1)) + break; + die("Invalid absolute R_390_32 relocation: %s\n", symname); + break; + } case R_390_64: add_reloc(&relocs64, offset); break; @@ -296,21 +438,29 @@ static void walk_relocs(void) /* Walk through the relocations */ for (i = 0; i < shnum; i++) { - struct section *sec_applies; + char *sym_strtab; + Elf_Sym *sh_symtab; + struct section *sec_applies, *sec_symtab; int j; struct section *sec = &secs[i]; if (sec->shdr.sh_type != SHT_REL_TYPE) continue; + sec_symtab = sec->link; sec_applies = &secs[sec->shdr.sh_info]; if (!(sec_applies->shdr.sh_flags & SHF_ALLOC)) continue; + sh_symtab = sec_symtab->symtab; + sym_strtab = sec_symtab->link->strtab; + for (j = 0; j < sec->shdr.sh_size / sizeof(Elf_Rel); j++) { Elf_Rel *rel = &sec->reltab[j]; + Elf_Sym *sym = &sh_symtab[ELF_R_SYM(rel->r_info)]; + const char *symname = sym_name(sym_strtab, sym); - do_reloc(sec, rel); + do_reloc(sec, rel, sym, symname); } } } @@ -349,6 +499,8 @@ static void process(FILE *fp) { read_ehdr(fp); read_shdrs(fp); + read_strtabs(fp); + read_symtabs(fp); read_relocs(fp); emit_relocs(); } |