diff options
Diffstat (limited to 'arch/s390/include')
36 files changed, 614 insertions, 388 deletions
diff --git a/arch/s390/include/asm/Kbuild b/arch/s390/include/asm/Kbuild index 80bad7de7a04..0c1fc47c3ba0 100644 --- a/arch/s390/include/asm/Kbuild +++ b/arch/s390/include/asm/Kbuild @@ -7,3 +7,4 @@ generated-y += unistd_nr.h generic-y += asm-offsets.h generic-y += mcs_spinlock.h generic-y += mmzone.h +generic-y += ring_buffer.h diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h index d23ea0c94e4e..99748c20e767 100644 --- a/arch/s390/include/asm/asm-extable.h +++ b/arch/s390/include/asm/asm-extable.h @@ -12,7 +12,6 @@ #define EX_TYPE_UA_FAULT 3 #define EX_TYPE_UA_LOAD_REG 5 #define EX_TYPE_UA_LOAD_REGPAIR 6 -#define EX_TYPE_ZEROPAD 7 #define EX_TYPE_FPC 8 #define EX_TYPE_UA_MVCOS_TO 9 #define EX_TYPE_UA_MVCOS_FROM 10 @@ -80,9 +79,6 @@ #define EX_TABLE_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0) -#define EX_TABLE_ZEROPAD(_fault, _target, _regdata, _regaddr) \ - __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_ZEROPAD, _regdata, _regaddr, 0) - #define EX_TABLE_FPC(_fault, _target) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FPC, __stringify(%%r0), __stringify(%%r0), 0) diff --git a/arch/s390/include/asm/asm-prototypes.h b/arch/s390/include/asm/asm-prototypes.h index 7bd1801cf241..d4da4436d02b 100644 --- a/arch/s390/include/asm/asm-prototypes.h +++ b/arch/s390/include/asm/asm-prototypes.h @@ -8,8 +8,4 @@ #include <asm/nospec-branch.h> #include <asm-generic/asm-prototypes.h> -__int128_t __ashlti3(__int128_t a, int b); -__int128_t __ashrti3(__int128_t a, int b); -__int128_t __lshrti3(__int128_t a, int b); - #endif /* _ASM_S390_PROTOTYPES_H */ diff --git a/arch/s390/include/asm/barrier.h b/arch/s390/include/asm/barrier.h index dad02f5b3c8d..d1e4201f3850 100644 --- a/arch/s390/include/asm/barrier.h +++ b/arch/s390/include/asm/barrier.h @@ -8,6 +8,7 @@ #ifndef __ASM_BARRIER_H #define __ASM_BARRIER_H +#include <asm/alternative.h> #include <asm/march.h> /* @@ -16,16 +17,11 @@ * to devices. */ -#ifdef MARCH_HAS_Z196_FEATURES -/* Fast-BCR without checkpoint synchronization */ -#define __ASM_BCR_SERIALIZE "bcr 14,0" -#else -#define __ASM_BCR_SERIALIZE "bcr 15,0" -#endif - static __always_inline void bcr_serialize(void) { - asm volatile(__ASM_BCR_SERIALIZE : : : "memory"); + asm_inline volatile( + ALTERNATIVE("bcr 15,0", "bcr 14,0", ALT_FACILITY(45)) + : : : "memory"); } #define __mb() bcr_serialize() diff --git a/arch/s390/include/asm/bug.h b/arch/s390/include/asm/bug.h index 59017fd3d935..2010342c97b1 100644 --- a/arch/s390/include/asm/bug.h +++ b/arch/s390/include/asm/bug.h @@ -4,6 +4,7 @@ #include <linux/compiler.h> #include <linux/const.h> +#include <linux/stringify.h> #define MONCODE_BUG _AC(0, U) #define MONCODE_BUG_ARG _AC(1, U) @@ -12,12 +13,11 @@ #if defined(CONFIG_BUG) && defined(CONFIG_CC_HAS_ASM_IMMEDIATE_STRINGS) #ifdef CONFIG_DEBUG_BUGVERBOSE -#define __BUG_ENTRY_VERBOSE(format, file, line) \ - " .long " format " - . # bug_entry::format\n" \ +#define __BUG_ENTRY_VERBOSE(file, line) \ " .long " file " - . # bug_entry::file\n" \ " .short " line " # bug_entry::line\n" #else -#define __BUG_ENTRY_VERBOSE(format, file, line) +#define __BUG_ENTRY_VERBOSE(file, line) #endif #ifdef CONFIG_DEBUG_BUGVERBOSE_DETAILED @@ -28,9 +28,10 @@ #define __BUG_ENTRY(format, file, line, flags, size) \ " .section __bug_table,\"aw\"\n" \ - "1: .long 0b - . # bug_entry::bug_addr\n" \ - __BUG_ENTRY_VERBOSE(format, file, line) \ - " .short "flags" # bug_entry::flags\n" \ + "1: .long 0b - . # bug_entry::bug_addr\n"\ + " .long " format " - . # bug_entry::format\n" \ + __BUG_ENTRY_VERBOSE(file, line) \ + " .short "flags" # bug_entry::flags\n" \ " .org 1b+"size"\n" \ " .previous" @@ -121,6 +122,17 @@ do { \ #define HAVE_ARCH_BUG_FORMAT #define HAVE_ARCH_BUG_FORMAT_ARGS +#define ARCH_WARN_ASM(file, line, flags, size) \ + ".section .rodata.str,\"aMS\",@progbits,1\n" \ + "9:\n" \ + ".asciz \"\"\n" /* Empty string for compatibility */ \ + ".previous\n" \ + "0:\n" \ + __stringify(mc 0(%r0),0) "\n" \ + __BUG_ENTRY("9b", file, line, flags, size) + +#define ARCH_WARN_REACHABLE + #endif /* CONFIG_BUG && CONFIG_CC_HAS_ASM_IMMEDIATE_STRINGS */ #endif /* __ASSEMBLER__ */ diff --git a/arch/s390/include/asm/ccwdev.h b/arch/s390/include/asm/ccwdev.h index e3afcece375e..d77f26390c07 100644 --- a/arch/s390/include/asm/ccwdev.h +++ b/arch/s390/include/asm/ccwdev.h @@ -10,7 +10,7 @@ #define _S390_CCWDEV_H_ #include <linux/device.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/ccw.h> #include <asm/chsc.h> #include <asm/fcx.h> #include <asm/irq.h> diff --git a/arch/s390/include/asm/cmpxchg.h b/arch/s390/include/asm/cmpxchg.h index 008357996262..e6ac55cf3c17 100644 --- a/arch/s390/include/asm/cmpxchg.h +++ b/arch/s390/include/asm/cmpxchg.h @@ -35,7 +35,7 @@ static __always_inline u64 __csg_asm(u64 ptr, u64 old, u64 new) return old; } -static inline u8 __arch_cmpxchg1(u64 ptr, u8 old, u8 new) +static __no_sanitize_or_inline u8 __arch_cmpxchg1(u64 ptr, u8 old, u8 new) { union { u8 b[4]; @@ -58,7 +58,7 @@ static inline u8 __arch_cmpxchg1(u64 ptr, u8 old, u8 new) return old; } -static inline u16 __arch_cmpxchg2(u64 ptr, u16 old, u16 new) +static __no_sanitize_or_inline u16 __arch_cmpxchg2(u64 ptr, u16 old, u16 new) { union { u16 b[2]; @@ -173,7 +173,7 @@ static __always_inline u64 __arch_cmpxchg(u64 ptr, u64 old, u64 new, int size) void __xchg_called_with_bad_pointer(void); -static inline u8 __arch_xchg1(u64 ptr, u8 x) +static __no_sanitize_or_inline u8 __arch_xchg1(u64 ptr, u8 x) { int shift = (3 ^ (ptr & 3)) << 3; u32 mask, old, new; @@ -188,7 +188,7 @@ static inline u8 __arch_xchg1(u64 ptr, u8 x) return old >> shift; } -static inline u16 __arch_xchg2(u64 ptr, u16 x) +static __no_sanitize_or_inline u16 __arch_xchg2(u64 ptr, u16 x) { int shift = (2 ^ (ptr & 2)) << 3; u32 mask, old, new; diff --git a/arch/s390/include/asm/debug.h b/arch/s390/include/asm/debug.h index c5440b3ee53d..39d484c59774 100644 --- a/arch/s390/include/asm/debug.h +++ b/arch/s390/include/asm/debug.h @@ -490,6 +490,7 @@ arch_initcall(VNAME(var, reg)) __DEFINE_STATIC_AREA(var); \ static debug_info_t __refdata var = \ __DEBUG_INFO_INIT(var, (name), (buf_size)); \ +static debug_info_t __used __section(".s390dbf_info") *VNAME(var, info) = &var; \ __REGISTER_STATIC_DEBUG_INFO(var, name, pages, nr_areas, view) void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas); diff --git a/arch/s390/include/asm/entry-percpu.h b/arch/s390/include/asm/entry-percpu.h new file mode 100644 index 000000000000..4ef47265cfce --- /dev/null +++ b/arch/s390/include/asm/entry-percpu.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ARCH_S390_ENTRY_PERCPU_H +#define ARCH_S390_ENTRY_PERCPU_H + +#include <linux/kprobes.h> +#include <linux/percpu.h> +#include <asm/lowcore.h> +#include <asm/ptrace.h> +#include <asm/asm-offsets.h> + +static __always_inline void percpu_entry(struct pt_regs *regs) +{ + struct lowcore *lc = get_lowcore(); + + if (user_mode(regs)) + return; + regs->cpu = lc->cpu_nr; + regs->percpu_register = lc->percpu_register; + lc->percpu_register = 0; +} + +static __always_inline bool percpu_code_check(struct pt_regs *regs) +{ + unsigned int insn, disp; + struct kprobe *p; + + if (likely(user_mode(regs) || !regs->percpu_register)) + return false; + /* + * Within a percpu code section - check if the percpu base register + * needs to be updated. This is the case if the PSW does not point to + * the ADD instruction within the section. + * - AG %rx,percpu_offset_in_lowcore(%r0,%r0) + * which adds the percpu offset to the percpu base register. + */ + lockdep_assert_preemption_disabled(); +again: + insn = READ_ONCE(*(u16 *)psw_bits(regs->psw).ia); + if (unlikely(insn == BREAKPOINT_INSTRUCTION)) { + p = get_kprobe((void *)psw_bits(regs->psw).ia); + /* + * If the kprobe is concurrently removed on a different CPU + * it might not be found anymore. However text must have + * been restored - try again. + */ + if (!p) + goto again; + insn = p->opcode; + } + if ((insn & 0xff0f) != 0xe300) + return true; + disp = offsetof(struct lowcore, percpu_offset); + if (machine_has_relocated_lowcore()) + disp += LOWCORE_ALT_ADDRESS; + insn = (disp & 0xff000) >> 4 | (disp & 0x00fff) << 16 | 0x8; + if (*(u32 *)(psw_bits(regs->psw).ia + 2) != insn) + return true; + return false; +} + +static __always_inline void percpu_exit(struct pt_regs *regs, bool needs_fixup) +{ + struct lowcore *lc = get_lowcore(); + unsigned char reg; + + if (user_mode(regs)) + return; + reg = regs->percpu_register; + lc->percpu_register = reg; + if (likely(!needs_fixup)) + return; + /* Check if process has been migrated to a different CPU. */ + if (regs->cpu == lc->cpu_nr) + return; + /* Fixup percpu base register */ + regs->gprs[reg] -= __per_cpu_offset[regs->cpu]; + regs->gprs[reg] += lc->percpu_offset; +} + +#endif diff --git a/arch/s390/include/asm/fpu-insn-asm.h b/arch/s390/include/asm/fpu-insn-asm.h index cc0468fdf2d0..4a1bfc0f578e 100644 --- a/arch/s390/include/asm/fpu-insn-asm.h +++ b/arch/s390/include/asm/fpu-insn-asm.h @@ -16,181 +16,9 @@ #error only <asm/fpu-insn.h> can be included directly #endif -#ifdef __ASSEMBLER__ - -/* Macros to generate vector instruction byte code */ +#include <asm/insn-common-asm.h> -/* GR_NUM - Retrieve general-purpose register number - * - * @opd: Operand to store register number - * @r64: String designation register in the format "%rN" - */ -.macro GR_NUM opd gr - \opd = 255 - .ifc \gr,%r0 - \opd = 0 - .endif - .ifc \gr,%r1 - \opd = 1 - .endif - .ifc \gr,%r2 - \opd = 2 - .endif - .ifc \gr,%r3 - \opd = 3 - .endif - .ifc \gr,%r4 - \opd = 4 - .endif - .ifc \gr,%r5 - \opd = 5 - .endif - .ifc \gr,%r6 - \opd = 6 - .endif - .ifc \gr,%r7 - \opd = 7 - .endif - .ifc \gr,%r8 - \opd = 8 - .endif - .ifc \gr,%r9 - \opd = 9 - .endif - .ifc \gr,%r10 - \opd = 10 - .endif - .ifc \gr,%r11 - \opd = 11 - .endif - .ifc \gr,%r12 - \opd = 12 - .endif - .ifc \gr,%r13 - \opd = 13 - .endif - .ifc \gr,%r14 - \opd = 14 - .endif - .ifc \gr,%r15 - \opd = 15 - .endif - .if \opd == 255 - \opd = \gr - .endif -.endm - -/* VX_NUM - Retrieve vector register number - * - * @opd: Operand to store register number - * @vxr: String designation register in the format "%vN" - * - * The vector register number is used for as input number to the - * instruction and, as well as, to compute the RXB field of the - * instruction. - */ -.macro VX_NUM opd vxr - \opd = 255 - .ifc \vxr,%v0 - \opd = 0 - .endif - .ifc \vxr,%v1 - \opd = 1 - .endif - .ifc \vxr,%v2 - \opd = 2 - .endif - .ifc \vxr,%v3 - \opd = 3 - .endif - .ifc \vxr,%v4 - \opd = 4 - .endif - .ifc \vxr,%v5 - \opd = 5 - .endif - .ifc \vxr,%v6 - \opd = 6 - .endif - .ifc \vxr,%v7 - \opd = 7 - .endif - .ifc \vxr,%v8 - \opd = 8 - .endif - .ifc \vxr,%v9 - \opd = 9 - .endif - .ifc \vxr,%v10 - \opd = 10 - .endif - .ifc \vxr,%v11 - \opd = 11 - .endif - .ifc \vxr,%v12 - \opd = 12 - .endif - .ifc \vxr,%v13 - \opd = 13 - .endif - .ifc \vxr,%v14 - \opd = 14 - .endif - .ifc \vxr,%v15 - \opd = 15 - .endif - .ifc \vxr,%v16 - \opd = 16 - .endif - .ifc \vxr,%v17 - \opd = 17 - .endif - .ifc \vxr,%v18 - \opd = 18 - .endif - .ifc \vxr,%v19 - \opd = 19 - .endif - .ifc \vxr,%v20 - \opd = 20 - .endif - .ifc \vxr,%v21 - \opd = 21 - .endif - .ifc \vxr,%v22 - \opd = 22 - .endif - .ifc \vxr,%v23 - \opd = 23 - .endif - .ifc \vxr,%v24 - \opd = 24 - .endif - .ifc \vxr,%v25 - \opd = 25 - .endif - .ifc \vxr,%v26 - \opd = 26 - .endif - .ifc \vxr,%v27 - \opd = 27 - .endif - .ifc \vxr,%v28 - \opd = 28 - .endif - .ifc \vxr,%v29 - \opd = 29 - .endif - .ifc \vxr,%v30 - \opd = 30 - .endif - .ifc \vxr,%v31 - \opd = 31 - .endif - .if \opd == 255 - \opd = \vxr - .endif -.endm +#ifdef __ASSEMBLER__ /* RXB - Compute most significant bit used vector registers * diff --git a/arch/s390/include/asm/fpu-insn.h b/arch/s390/include/asm/fpu-insn.h index 96727f3bd0dc..ae8b7033cfd2 100644 --- a/arch/s390/include/asm/fpu-insn.h +++ b/arch/s390/include/asm/fpu-insn.h @@ -15,6 +15,7 @@ #include <linux/kmsan.h> #include <asm/asm-extable.h> +asm(".include \"asm/insn-common-asm.h\"\n"); asm(".include \"asm/fpu-insn-asm.h\"\n"); /* diff --git a/arch/s390/include/asm/gmap_helpers.h b/arch/s390/include/asm/gmap_helpers.h index 2d3ae421077e..d2b616604a46 100644 --- a/arch/s390/include/asm/gmap_helpers.h +++ b/arch/s390/include/asm/gmap_helpers.h @@ -12,5 +12,6 @@ void gmap_helper_zap_one_page(struct mm_struct *mm, unsigned long vmaddr); void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned long end); int gmap_helper_disable_cow_sharing(void); void gmap_helper_try_set_pte_unused(struct mm_struct *mm, unsigned long vmaddr); +pte_t *try_get_locked_pte(struct mm_struct *mm, unsigned long addr, spinlock_t **ptl); #endif /* _ASM_S390_GMAP_HELPERS_H */ diff --git a/arch/s390/include/asm/hugetlb.h b/arch/s390/include/asm/hugetlb.h index 6983e52eaf81..e33a5b587ee4 100644 --- a/arch/s390/include/asm/hugetlb.h +++ b/arch/s390/include/asm/hugetlb.h @@ -41,7 +41,7 @@ static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm, static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned long sz) { - if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) + if ((pte_val(ptep_get(ptep)) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) set_pte(ptep, __pte(_REGION3_ENTRY_EMPTY)); else set_pte(ptep, __pte(_SEGMENT_ENTRY_EMPTY)); diff --git a/arch/s390/include/asm/idle.h b/arch/s390/include/asm/idle.h index 32536ee34aa0..07819f11987c 100644 --- a/arch/s390/include/asm/idle.h +++ b/arch/s390/include/asm/idle.h @@ -8,20 +8,20 @@ #ifndef _S390_IDLE_H #define _S390_IDLE_H +#include <linux/percpu-defs.h> #include <linux/types.h> -#include <linux/device.h> +#include <asm/tod_types.h> struct s390_idle_data { - unsigned long idle_count; - unsigned long idle_time; - unsigned long clock_idle_enter; +#ifdef CONFIG_NO_HZ_COMMON + bool in_idle; +#endif unsigned long timer_idle_enter; unsigned long mt_cycles_enter[8]; + union tod_clock clock_idle_enter; + union tod_clock clock_idle_exit; }; DECLARE_PER_CPU(struct s390_idle_data, s390_idle); -extern struct device_attribute dev_attr_idle_count; -extern struct device_attribute dev_attr_idle_time_us; - #endif /* _S390_IDLE_H */ diff --git a/arch/s390/include/asm/insn-common-asm.h b/arch/s390/include/asm/insn-common-asm.h new file mode 100644 index 000000000000..fd9b3cacb7c5 --- /dev/null +++ b/arch/s390/include/asm/insn-common-asm.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Assembler helper macros to generate .byte/.word code for instructions + * that are unknown to older binutils versions. + */ + +#ifndef __ASM_S390_INSN_COMMON_ASM_H +#define __ASM_S390_INSN_COMMON_ASM_H + +#ifdef __ASSEMBLER__ + +/* + * GR_NUM - Retrieve general-purpose register number + * + * @opd: Operand to store register number + * @gr: String designation register in the format "%rN" + */ +.macro GR_NUM opd gr + \opd = 255 + .irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 + .ifc \gr,%r\rs + \opd = \rs + .endif + .endr + .if \opd == 255 + \opd = \gr + .endif +.endm + +/* + * VX_NUM - Retrieve vector register number + * + * @opd: Operand to store register number + * @vxr: String designation register in the format "%vN" + * + * The vector register number is used for as input number to the + * instruction and, as well as, to compute the RXB field of the + * instruction. + */ +.macro VX_NUM opd vxr + \opd = 255 + .irp vs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 + .ifc \vxr,%v\vs + \opd = \vs + .endif + .endr + .if \opd == 255 + \opd = \vxr + .endif +.endm + +#endif /* __ASSEMBLER__ */ +#endif /* __ASM_S390_INSN_COMMON_ASM_H */ diff --git a/arch/s390/include/asm/jump_label.h b/arch/s390/include/asm/jump_label.h index d9cbc18f6b2e..0e28c90dc242 100644 --- a/arch/s390/include/asm/jump_label.h +++ b/arch/s390/include/asm/jump_label.h @@ -19,19 +19,29 @@ #define JUMP_LABEL_STATIC_KEY_CONSTRAINT "jdd" #endif +#define ARCH_JUMP_TABLE_ENTRY(key, label, local_label) \ + ".pushsection __jump_table,\"aw\"\n" \ + ".balign 8\n" \ + ".long " local_label "-.," label "-.\n" \ + ".quad " key "-.\n" \ + ".popsection\n" + /* * We use a brcl 0,<offset> instruction for jump labels so it * can be easily distinguished from a hotpatch generated instruction. */ +#define ARCH_STATIC_BRANCH_ASM(key, label) \ + "0: brcl 0," label "\n" \ + ARCH_JUMP_TABLE_ENTRY(key, label, "0b") + +#define ARCH_STATIC_BRANCH_JUMP_ASM(key, label) \ + "0: brcl 15," label "\n" \ + ARCH_JUMP_TABLE_ENTRY(key, label, "0b") + static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { - asm goto("0: brcl 0,%l[label]\n" - ".pushsection __jump_table,\"aw\"\n" - ".balign 8\n" - ".long 0b-.,%l[label]-.\n" - ".quad %0+%1-.\n" - ".popsection\n" - : : JUMP_LABEL_STATIC_KEY_CONSTRAINT (key), "i" (branch) : : label); + asm goto(ARCH_STATIC_BRANCH_ASM("%0+%1", "%l[label]") + : : JUMP_LABEL_STATIC_KEY_CONSTRAINT (key), "i" (branch) : : label); return false; label: return true; @@ -39,13 +49,8 @@ label: static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) { - asm goto("0: brcl 15,%l[label]\n" - ".pushsection __jump_table,\"aw\"\n" - ".balign 8\n" - ".long 0b-.,%l[label]-.\n" - ".quad %0+%1-.\n" - ".popsection\n" - : : JUMP_LABEL_STATIC_KEY_CONSTRAINT (key), "i" (branch) : : label); + asm goto(ARCH_STATIC_BRANCH_JUMP_ASM("%0+%1", "%l[label]") + : : JUMP_LABEL_STATIC_KEY_CONSTRAINT (key), "i" (branch) : : label); return false; label: return true; diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index 8a4f4a39f7a2..b4182ca4435f 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -359,7 +359,7 @@ struct kvm_s390_float_interrupt { struct kvm_s390_mchk_info mchk; struct kvm_s390_ext_info srv_signal; int last_sleep_cpu; - struct mutex ais_lock; + spinlock_t ais_lock; u8 simm; u8 nimm; }; @@ -440,6 +440,7 @@ struct kvm_vcpu_arch { bool skey_enabled; /* Indicator if the access registers have been loaded from guest */ bool acrs_loaded; + bool initialized; struct kvm_s390_pv_vcpu pv; union diag318_info diag318_info; struct kvm_s390_mmu_cache *mc; @@ -448,6 +449,12 @@ struct kvm_vcpu_arch { struct kvm_vm_stat { struct kvm_vm_stat_generic generic; u64 inject_io; + u64 io_390_adapter_map; + u64 io_390_adapter_unmap; + u64 io_390_inatomic; + u64 io_flic_inject_airq; + u64 io_set_adapter_int; + u64 io_390_inatomic_no_inject; u64 inject_float_mchk; u64 inject_pfault_done; u64 inject_service_signal; @@ -470,6 +477,11 @@ struct s390_map_info { __u64 guest_addr; __u64 addr; struct page *page; + /* + * True if the page is long-term pinned. False if long-term pinning + * failed and this entry exists only to preserve MAP/UNMAP symmetry. + */ + bool pinned; }; struct s390_io_adapter { @@ -479,6 +491,9 @@ struct s390_io_adapter { bool masked; bool swap; bool suppressible; + spinlock_t maps_lock; + struct list_head maps; + unsigned int nr_maps; }; #define MAX_S390_IO_ADAPTERS ((MAX_ISC + 1) * 8) @@ -504,6 +519,18 @@ struct kvm_s390_cpu_model { struct kvm_s390_vm_cpu_uv_feat uv_feat_guest; }; +#define S390_ARCH_FAC_FORMAT_2 2 +struct kvm_s390_flcb2 { + union { + struct { + u8 reserved0[7]; + u8 length; + }; + u64 header_val; + }; + u64 facilities[S390_ARCH_FAC_LIST_SIZE_U64]; +}; + typedef int (*crypto_hook)(struct kvm_vcpu *vcpu); struct kvm_s390_crypto { diff --git a/arch/s390/include/asm/linkage.h b/arch/s390/include/asm/linkage.h index df3fb7d8227b..1b3ac553a642 100644 --- a/arch/s390/include/asm/linkage.h +++ b/arch/s390/include/asm/linkage.h @@ -7,4 +7,6 @@ #define __ALIGN .balign CONFIG_FUNCTION_ALIGNMENT, 0x07 #define __ALIGN_STR __stringify(__ALIGN) +#define _THIS_IP_ ({ unsigned long __ip; asm volatile("larl %0, ." : "=d" (__ip)); __ip; }) + #endif diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 50ffe75adeb4..5cef215d30e7 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -10,6 +10,7 @@ #define _ASM_S390_LOWCORE_H #include <linux/types.h> +#include <asm/tod_types.h> #include <asm/machine.h> #include <asm/ptrace.h> #include <asm/ctlreg.h> @@ -125,8 +126,7 @@ struct lowcore { __u64 avg_steal_timer; /* 0x0300 */ __u64 last_update_timer; /* 0x0308 */ __u64 last_update_clock; /* 0x0310 */ - __u64 int_clock; /* 0x0318 */ - __u8 pad_0x0320[0x0328-0x0320]; /* 0x0320 */ + union tod_clock int_clock; /* 0x0318 */ __u64 clock_comparator; /* 0x0328 */ __u8 pad_0x0330[0x0340-0x0330]; /* 0x0330 */ @@ -160,12 +160,18 @@ struct lowcore { /* SMP info area */ __u32 cpu_nr; /* 0x03a0 */ __u32 softirq_pending; /* 0x03a4 */ - __s32 preempt_count; /* 0x03a8 */ - __u32 spinlock_lockval; /* 0x03ac */ - __u32 spinlock_index; /* 0x03b0 */ - __u8 pad_0x03b4[0x03b8-0x03b4]; /* 0x03b4 */ + union { + struct { + __u32 need_resched; /* 0x03a8 */ + __u32 count; /* 0x03ac */ + } preempt; + __u64 preempt_count; /* 0x03a8 */ + }; + __u32 spinlock_lockval; /* 0x03b0 */ + __u32 spinlock_index; /* 0x03b4 */ __u64 percpu_offset; /* 0x03b8 */ - __u8 pad_0x03c0[0x0400-0x03c0]; /* 0x03c0 */ + __u8 percpu_register; /* 0x03c0 */ + __u8 pad_0x03c1[0x0400-0x03c1]; /* 0x03c1 */ __u32 return_lpswe; /* 0x0400 */ __u32 return_mcck_lpswe; /* 0x0404 */ diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index b18a96f3a334..1d955dd0defa 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -60,6 +60,68 @@ #define this_cpu_or_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) #define this_cpu_or_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) +/* + * Macros to be used for percpu code section based on atomic instructions. + * + * Avoid the need to use preempt_disable() / preempt_disable() pairs and the + * conditional preempt_schedule_notrace() function calls which come with + * this. The idea is that this_cpu operations based on atomic instructions are + * guarded with mviy instructions: + * + * - The first mviy instruction writes the register number, which contains the + * percpu address variable to lowcore. This also indicates that a percpu + * code section is executed. + * + * - The first mviy instruction following the mviy instruction must be the ag + * instruction which adds the percpu offset to the percpu address register. + * + * - Afterwards the atomic percpu operation follows. + * + * - Then a second mviy instruction writes a zero to lowcore, which indicates + * the end of the percpu code section. + * + * - In case of an interrupt/exception/nmi the register number which was + * written to lowcore is copied to the exception frame (pt_regs), and a zero + * is written to lowcore. + * + * - On return to the previous context it is checked if a percpu code section + * was executed (saved register number not zero), and if the process was + * migrated to a different cpu. If the percpu offset was already added to + * the percpu address register (instruction address does _not_ point to the + * ag instruction) the content of the percpu address register is adjusted so + * it points to percpu variable of the new cpu. + * + * Inline assemblies making use of this typically have a code sequence like: + * + * MVIY_PERCPU(...) <- start of percpu code section + * AG_ALT(...) <- add percpu offset; must be the second instruction + * atomic_op <- atomic op + * MVIY_ALT(...) <- end of percpu code section + */ + +#define MVIY_PERCPU(disp, dispalt, reg) \ + ".macro GEN_MVIY disp reg\n" \ + ".irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\n" \ + " .ifc \\reg,%%r\\rs\n" \ + " mviy \\disp(%%r0),\\rs\n" \ + " .endif\n" \ + ".endr\n" \ + ".endm\n" \ + ALTERNATIVE("GEN_MVIY " __stringify(disp) " " __stringify(reg) "\n", \ + "GEN_MVIY " __stringify(dispalt) " " __stringify(reg) "\n", \ + ALT_FEATURE(MFEATURE_LOWCORE)) \ + ".purgem GEN_MVIY\n" + +#define MVIY_ALT(disp, dispalt) \ + ALTERNATIVE(" mviy " disp "(%%r0),0\n", \ + " mviy " dispalt "(%%r0),0\n", \ + ALT_FEATURE(MFEATURE_LOWCORE)) + +#define AG_ALT(disp, dispalt, reg) \ + ALTERNATIVE(" ag " reg ", " disp "(%%r0)\n", \ + " ag " reg ", " dispalt "(%%r0)\n", \ + ALT_FEATURE(MFEATURE_LOWCORE)) + #ifndef MARCH_HAS_Z196_FEATURES #define this_cpu_add_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) @@ -73,46 +135,79 @@ #else /* MARCH_HAS_Z196_FEATURES */ -#define arch_this_cpu_add(pcp, val, op1, op2, szcast) \ -{ \ - typedef typeof(pcp) pcp_op_T__; \ - pcp_op_T__ val__ = (val); \ - pcp_op_T__ old__, *ptr__; \ - preempt_disable_notrace(); \ - ptr__ = raw_cpu_ptr(&(pcp)); \ - if (__builtin_constant_p(val__) && \ - ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ - asm volatile( \ - op2 " %[ptr__],%[val__]" \ - : [ptr__] "+Q" (*ptr__) \ - : [val__] "i" ((szcast)val__) \ - : "cc"); \ - } else { \ - asm volatile( \ - op1 " %[old__],%[val__],%[ptr__]" \ - : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ - : [val__] "d" (val__) \ - : "cc"); \ - } \ - preempt_enable_notrace(); \ -} +#define arch_this_cpu_add(pcp, val, op1, op2, szcast) \ +do { \ + unsigned long lc_pcpr, lc_pcpo; \ + typedef typeof(pcp) pcp_op_T__; \ + pcp_op_T__ val__ = (val); \ + pcp_op_T__ old__, *ptr__; \ + \ + lc_pcpr = offsetof(struct lowcore, percpu_register); \ + lc_pcpo = offsetof(struct lowcore, percpu_offset); \ + ptr__ = PERCPU_PTR(&(pcp)); \ + if (__builtin_constant_p(val__) && \ + ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ + asm volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op2 " 0(%[ptr__]),%[val__]\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [val__] "i" ((szcast)val__), \ + [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "cc"); \ + } else { \ + asm volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op1 " %[old__],%[val__],0(%[ptr__])\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [old__] "=&d" (old__), \ + [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [val__] "d" (val__), \ + [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "cc"); \ + } \ +} while (0) #define this_cpu_add_4(pcp, val) arch_this_cpu_add(pcp, val, "laa", "asi", int) #define this_cpu_add_8(pcp, val) arch_this_cpu_add(pcp, val, "laag", "agsi", long) #define arch_this_cpu_add_return(pcp, val, op) \ ({ \ + unsigned long lc_pcpr, lc_pcpo; \ typedef typeof(pcp) pcp_op_T__; \ pcp_op_T__ val__ = (val); \ pcp_op_T__ old__, *ptr__; \ - preempt_disable_notrace(); \ - ptr__ = raw_cpu_ptr(&(pcp)); \ - asm volatile( \ - op " %[old__],%[val__],%[ptr__]" \ - : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ - : [val__] "d" (val__) \ + \ + lc_pcpr = offsetof(struct lowcore, percpu_register); \ + lc_pcpo = offsetof(struct lowcore, percpu_offset); \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op " %[old__],%[val__],0(%[ptr__])\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [old__] "=&d" (old__), \ + [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [val__] "d" (val__), \ + [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ - preempt_enable_notrace(); \ old__ + val__; \ }) @@ -120,19 +215,31 @@ #define this_cpu_add_return_8(pcp, val) arch_this_cpu_add_return(pcp, val, "laag") #define arch_this_cpu_to_op(pcp, val, op) \ -{ \ +do { \ + unsigned long lc_pcpr, lc_pcpo; \ typedef typeof(pcp) pcp_op_T__; \ pcp_op_T__ val__ = (val); \ pcp_op_T__ old__, *ptr__; \ - preempt_disable_notrace(); \ - ptr__ = raw_cpu_ptr(&(pcp)); \ - asm volatile( \ - op " %[old__],%[val__],%[ptr__]" \ - : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ - : [val__] "d" (val__) \ + \ + lc_pcpr = offsetof(struct lowcore, percpu_register); \ + lc_pcpo = offsetof(struct lowcore, percpu_offset); \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op " %[old__],%[val__],0(%[ptr__])\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [old__] "=&d" (old__), \ + [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [val__] "d" (val__), \ + [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ - preempt_enable_notrace(); \ -} +} while (0) #define this_cpu_and_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lan") #define this_cpu_and_8(pcp, val) arch_this_cpu_to_op(pcp, val, "lang") @@ -141,6 +248,67 @@ #endif /* MARCH_HAS_Z196_FEATURES */ +#define arch_this_cpu_read(pcp, op) \ +({ \ + unsigned long lc_pcpr, lc_pcpo, res__; \ + typedef typeof(pcp) pcp_op_T__; \ + pcp_op_T__ *ptr__; \ + \ + lc_pcpr = offsetof(struct lowcore, percpu_register); \ + lc_pcpo = offsetof(struct lowcore, percpu_offset); \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op " %[res__],0(%[ptr__])\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [res__] "=&d" (res__), [ptr__] "+&a" (ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (*ptr__), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "cc"); \ + (pcp_op_T__)res__; \ +}) + +#define this_cpu_read_1(pcp) arch_this_cpu_read(pcp, "llgc") +#define this_cpu_read_2(pcp) arch_this_cpu_read(pcp, "llgh") +#define this_cpu_read_4(pcp) arch_this_cpu_read(pcp, "llgf") +#define this_cpu_read_8(pcp) arch_this_cpu_read(pcp, "lg") + +#define arch_this_cpu_write(pcp, val, op) \ +do { \ + unsigned long lc_pcpr, lc_pcpo; \ + typedef typeof(pcp) pcp_op_T__; \ + pcp_op_T__ *ptr__, val__ = (val); \ + \ + lc_pcpr = offsetof(struct lowcore, percpu_register); \ + lc_pcpo = offsetof(struct lowcore, percpu_offset); \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op " %[val__],0(%[ptr__])\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [ptr__] "+&a" (ptr__), "=m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [val__] "d" (val__), \ + [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "cc"); \ +} while (0) + +#define this_cpu_write_1(pcp, val) arch_this_cpu_write(pcp, val, "stc") +#define this_cpu_write_2(pcp, val) arch_this_cpu_write(pcp, val, "sth") +#define this_cpu_write_4(pcp, val) arch_this_cpu_write(pcp, val, "st") +#define this_cpu_write_8(pcp, val) arch_this_cpu_write(pcp, val, "stg") + #define arch_this_cpu_cmpxchg(pcp, oval, nval) \ ({ \ typedef typeof(pcp) pcp_op_T__; \ diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 2c6cee8241e0..859ce7c7d454 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -980,30 +980,39 @@ static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) static inline void set_pte(pte_t *ptep, pte_t pte) { + if (pte_present(pte)) + pte = clear_pte_bit(pte, __pgprot(_PAGE_UNUSED)); WRITE_ONCE(*ptep, pte); } -static inline void pgd_clear(pgd_t *pgd) +#define ptep_get ptep_get +static inline pte_t ptep_get(pte_t *ptep) { - if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R1) - set_pgd(pgd, __pgd(_REGION1_ENTRY_EMPTY)); + return READ_ONCE(*ptep); } -static inline void p4d_clear(p4d_t *p4d) +#define pmdp_get pmdp_get +static inline pmd_t pmdp_get(pmd_t *pmdp) { - if ((p4d_val(*p4d) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2) - set_p4d(p4d, __p4d(_REGION2_ENTRY_EMPTY)); + return READ_ONCE(*pmdp); } -static inline void pud_clear(pud_t *pud) +#define pudp_get pudp_get +static inline pud_t pudp_get(pud_t *pudp) { - if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) - set_pud(pud, __pud(_REGION3_ENTRY_EMPTY)); + return READ_ONCE(*pudp); } -static inline void pmd_clear(pmd_t *pmdp) +#define p4dp_get p4dp_get +static inline p4d_t p4dp_get(p4d_t *p4dp) { - set_pmd(pmdp, __pmd(_SEGMENT_ENTRY_EMPTY)); + return READ_ONCE(*p4dp); +} + +#define pgdp_get pgdp_get +static inline pgd_t pgdp_get(pgd_t *pgdp) +{ + return READ_ONCE(*pgdp); } static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) @@ -1011,6 +1020,29 @@ static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *pt set_pte(ptep, __pte(_PAGE_INVALID)); } +static inline void pmd_clear(pmd_t *pmdp) +{ + set_pmd(pmdp, __pmd(_SEGMENT_ENTRY_EMPTY)); +} + +static inline void pud_clear(pud_t *pud) +{ + if ((pud_val(pudp_get(pud)) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) + set_pud(pud, __pud(_REGION3_ENTRY_EMPTY)); +} + +static inline void p4d_clear(p4d_t *p4d) +{ + if ((p4d_val(p4dp_get(p4d)) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2) + set_p4d(p4d, __p4d(_REGION2_ENTRY_EMPTY)); +} + +static inline void pgd_clear(pgd_t *pgd) +{ + if ((pgd_val(pgdp_get(pgd)) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R1) + set_pgd(pgd, __pgd(_REGION1_ENTRY_EMPTY)); +} + /* * The following pte modification functions only work if * pte_present() is true. Undefined behaviour if not.. @@ -1169,7 +1201,7 @@ pte_t ptep_xchg_lazy(struct mm_struct *, unsigned long, pte_t *, pte_t); static inline bool ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { - pte_t pte = *ptep; + pte_t pte = ptep_get(ptep); pte = ptep_xchg_direct(vma->vm_mm, addr, ptep, pte_mkold(pte)); return pte_young(pte); @@ -1189,10 +1221,10 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, pte_t res; res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID)); + page_table_check_pte_clear(mm, addr, res); /* At this point the reference through the mapping is still present */ if (mm_is_protected(mm) && pte_present(res)) WARN_ON_ONCE(uv_convert_from_secure_pte(res)); - page_table_check_pte_clear(mm, addr, res); return res; } @@ -1208,10 +1240,10 @@ static inline pte_t ptep_clear_flush(struct vm_area_struct *vma, pte_t res; res = ptep_xchg_direct(vma->vm_mm, addr, ptep, __pte(_PAGE_INVALID)); + page_table_check_pte_clear(vma->vm_mm, addr, res); /* At this point the reference through the mapping is still present */ if (mm_is_protected(vma->vm_mm) && pte_present(res)) WARN_ON_ONCE(uv_convert_from_secure_pte(res)); - page_table_check_pte_clear(vma->vm_mm, addr, res); return res; } @@ -1230,31 +1262,28 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, pte_t res; if (full) { - res = *ptep; + res = ptep_get(ptep); set_pte(ptep, __pte(_PAGE_INVALID)); } else { res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID)); } - page_table_check_pte_clear(mm, addr, res); - - /* Nothing to do */ - if (!mm_is_protected(mm) || !pte_present(res)) - return res; - /* - * At this point the reference through the mapping is still present. - * The notifier should have destroyed all protected vCPUs at this - * point, so the destroy should be successful. - */ - if (full && !uv_destroy_pte(res)) - return res; - /* - * If something went wrong and the page could not be destroyed, or - * if this is not a mm teardown, the slower export is used as - * fallback instead. If even that fails, print a warning and leak - * the page, to avoid crashing the whole system. - */ - WARN_ON_ONCE(uv_convert_from_secure_pte(res)); + /* At this point the reference through the mapping is still present */ + if (mm_is_protected(mm) && pte_present(res)) { + /* + * The notifier should have destroyed all protected vCPUs at + * this point, so the destroy should be successful. + */ + if (full && !uv_destroy_pte(res)) + return res; + /* + * If something went wrong and the page could not be destroyed, + * or if this is not a mm teardown, the slower export is used + * as fallback instead. If even that fails, print a warning and + * leak the page, to avoid crashing the whole system. + */ + WARN_ON_ONCE(uv_convert_from_secure_pte(res)); + } return res; } @@ -1262,7 +1291,7 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - pte_t pte = *ptep; + pte_t pte = ptep_get(ptep); if (pte_write(pte)) ptep_xchg_lazy(mm, addr, ptep, pte_wrprotect(pte)); @@ -1298,7 +1327,7 @@ static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma, * PTE does not have _PAGE_PROTECT set, to avoid unnecessary overhead. * A local RDP can be used to do the flush. */ - if (cpu_has_rdp() && !(pte_val(*ptep) & _PAGE_PROTECT)) + if (cpu_has_rdp() && !(pte_val(ptep_get(ptep)) & _PAGE_PROTECT)) __ptep_rdp(address, ptep, 1); } #define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault @@ -1332,8 +1361,6 @@ pgprot_t pgprot_writecombine(pgprot_t prot); static inline void set_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t entry, unsigned int nr) { - if (pte_present(entry)) - entry = clear_pte_bit(entry, __pgprot(_PAGE_UNUSED)); page_table_check_ptes_set(mm, addr, ptep, entry, nr); for (;;) { set_pte(ptep, entry); diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preempt.h index 6e5821bb047e..5560d5fca2a3 100644 --- a/arch/s390/include/asm/preempt.h +++ b/arch/s390/include/asm/preempt.h @@ -8,11 +8,8 @@ #include <asm/cmpxchg.h> #include <asm/march.h> -/* - * Use MSB so it is possible to read preempt_count with LLGT which - * reads the least significant 31 bits with a single instruction. - */ -#define PREEMPT_NEED_RESCHED 0x80000000 +/* Use MSB for PREEMPT_NEED_RESCHED mostly because it is available. */ +#define PREEMPT_NEED_RESCHED 0x8000000000000000UL /* * We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such @@ -26,25 +23,25 @@ */ static __always_inline int preempt_count(void) { - unsigned long lc_preempt, count; + unsigned long lc_preempt; + int count; - BUILD_BUG_ON(sizeof_field(struct lowcore, preempt_count) != sizeof(int)); - lc_preempt = offsetof(struct lowcore, preempt_count); - /* READ_ONCE(get_lowcore()->preempt_count) & ~PREEMPT_NEED_RESCHED */ + lc_preempt = offsetof(struct lowcore, preempt.count); + /* READ_ONCE(get_lowcore()->preempt.count) (without PREEMPT_NEED_RESCHED) */ asm_inline( - ALTERNATIVE("llgt %[count],%[offzero](%%r0)\n", - "llgt %[count],%[offalt](%%r0)\n", + ALTERNATIVE("ly %[count],%[offzero](%%r0)\n", + "ly %[count],%[offalt](%%r0)\n", ALT_FEATURE(MFEATURE_LOWCORE)) : [count] "=d" (count) : [offzero] "i" (lc_preempt), [offalt] "i" (lc_preempt + LOWCORE_ALT_ADDRESS), - "m" (((struct lowcore *)0)->preempt_count)); + "m" (((struct lowcore *)0)->preempt.count)); return count; } -static __always_inline void preempt_count_set(int pc) +static __always_inline void preempt_count_set(unsigned long pc) { - int old, new; + unsigned long old, new; old = READ_ONCE(get_lowcore()->preempt_count); do { @@ -63,12 +60,12 @@ static __always_inline void preempt_count_set(int pc) static __always_inline void set_preempt_need_resched(void) { - __atomic_and(~PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count); + __atomic64_and(~PREEMPT_NEED_RESCHED, (long *)&get_lowcore()->preempt_count); } static __always_inline void clear_preempt_need_resched(void) { - __atomic_or(PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count); + __atomic64_or(PREEMPT_NEED_RESCHED, (long *)&get_lowcore()->preempt_count); } static __always_inline bool test_preempt_need_resched(void) @@ -88,8 +85,8 @@ static __always_inline void __preempt_count_add(int val) lc_preempt = offsetof(struct lowcore, preempt_count); asm_inline( - ALTERNATIVE("asi %[offzero](%%r0),%[val]\n", - "asi %[offalt](%%r0),%[val]\n", + ALTERNATIVE("agsi %[offzero](%%r0),%[val]\n", + "agsi %[offalt](%%r0),%[val]\n", ALT_FEATURE(MFEATURE_LOWCORE)) : "+m" (((struct lowcore *)0)->preempt_count) : [offzero] "i" (lc_preempt), [val] "i" (val), @@ -98,7 +95,7 @@ static __always_inline void __preempt_count_add(int val) return; } } - __atomic_add(val, &get_lowcore()->preempt_count); + __atomic64_add(val, (long *)&get_lowcore()->preempt_count); } static __always_inline void __preempt_count_sub(int val) @@ -119,15 +116,15 @@ static __always_inline bool __preempt_count_dec_and_test(void) lc_preempt = offsetof(struct lowcore, preempt_count); asm_inline( - ALTERNATIVE("alsi %[offzero](%%r0),%[val]\n", - "alsi %[offalt](%%r0),%[val]\n", + ALTERNATIVE("algsi %[offzero](%%r0),%[val]\n", + "algsi %[offalt](%%r0),%[val]\n", ALT_FEATURE(MFEATURE_LOWCORE)) : "=@cc" (cc), "+m" (((struct lowcore *)0)->preempt_count) : [offzero] "i" (lc_preempt), [val] "i" (-1), [offalt] "i" (lc_preempt + LOWCORE_ALT_ADDRESS)); return (cc == 0) || (cc == 2); #else - return __atomic_add_const_and_test(-1, &get_lowcore()->preempt_count); + return __atomic64_add_const_and_test(-1, (long *)&get_lowcore()->preempt_count); #endif } @@ -139,6 +136,16 @@ static __always_inline bool should_resched(int preempt_offset) return unlikely(READ_ONCE(get_lowcore()->preempt_count) == preempt_offset); } +static __always_inline int __preempt_count_add_return(int val) +{ + return val + __atomic64_add(val, (long *)&get_lowcore()->preempt_count); +} + +static __always_inline int __preempt_count_sub_return(int val) +{ + return __preempt_count_add_return(-val); +} + #define init_task_preempt_count(p) do { } while (0) /* Deferred to CPU bringup time */ #define init_idle_preempt_count(p, cpu) do { } while (0) diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 78195ee5e99f..ecd3341686eb 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -33,6 +33,7 @@ #include <linux/irqflags.h> #include <linux/instruction_pointer.h> #include <linux/bitops.h> +#include <asm/vdso/processor.h> #include <asm/fpu-types.h> #include <asm/cpu.h> #include <asm/page.h> @@ -282,8 +283,6 @@ static __always_inline unsigned short stap(void) return cpu_address; } -#define cpu_relax() barrier() - #define ECAG_CACHE_ATTRIBUTE 0 #define ECAG_CPU_ATTRIBUTE 1 diff --git a/arch/s390/include/asm/ptrace.h b/arch/s390/include/asm/ptrace.h index aaceb1d9110a..495e310c3d6d 100644 --- a/arch/s390/include/asm/ptrace.h +++ b/arch/s390/include/asm/ptrace.h @@ -134,6 +134,8 @@ struct pt_regs { }; unsigned long flags; unsigned long last_break; + unsigned int cpu; + unsigned char percpu_register; }; /* diff --git a/arch/s390/include/asm/runtime-const.h b/arch/s390/include/asm/runtime-const.h index 17878b1d048c..7b71156031ec 100644 --- a/arch/s390/include/asm/runtime-const.h +++ b/arch/s390/include/asm/runtime-const.h @@ -33,6 +33,20 @@ __ret; \ }) +#define runtime_const_mask_32(val, sym) \ +({ \ + unsigned int __ret = (val); \ + \ + asm_inline( \ + "0: nilf %[__ret],12\n" \ + ".pushsection runtime_mask_" #sym ",\"a\"\n" \ + ".long 0b - .\n" \ + ".popsection" \ + : [__ret] "+d" (__ret) \ + : : "cc"); \ + __ret; \ +}) + #define runtime_const_init(type, sym) do { \ extern s32 __start_runtime_##type##_##sym[]; \ extern s32 __stop_runtime_##type##_##sym[]; \ @@ -43,12 +57,12 @@ __stop_runtime_##type##_##sym); \ } while (0) -/* 32-bit immediate for iihf and iilf in bits in I2 field */ static inline void __runtime_fixup_32(u32 *p, unsigned int val) { s390_kernel_write(p, &val, sizeof(val)); } +/* 32-bit immediate for iihf and iilf in bits in I2 field */ static inline void __runtime_fixup_ptr(void *where, unsigned long val) { __runtime_fixup_32(where + 2, val >> 32); @@ -65,6 +79,12 @@ static inline void __runtime_fixup_shift(void *where, unsigned long val) s390_kernel_write(where, &insn, sizeof(insn)); } +/* 32-bit immediate for nilf in bits in I2 field */ +static inline void __runtime_fixup_mask(void *where, unsigned long val) +{ + __runtime_fixup_32(where + 2, val); +} + static inline void runtime_const_fixup(void (*fn)(void *, unsigned long), unsigned long val, s32 *start, s32 *end) { diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h index 0f184dbdbe5e..f2d490558054 100644 --- a/arch/s390/include/asm/sclp.h +++ b/arch/s390/include/asm/sclp.h @@ -20,6 +20,7 @@ #define SCLP_ERRNOTIFY_AQ_REPAIR 1 #define SCLP_ERRNOTIFY_AQ_INFO_LOG 2 #define SCLP_ERRNOTIFY_AQ_OPTICS_DATA 3 +#define SCLP_ERRNOTIFY_AQ_NVME_SMART_LOG 4 #ifndef __ASSEMBLER__ #include <linux/uio.h> @@ -104,6 +105,7 @@ struct sclp_info { unsigned char has_aisii : 1; unsigned char has_aeni : 1; unsigned char has_aisi : 1; + unsigned char has_astfleie2 : 1; unsigned int ibc; unsigned int mtid; unsigned int mtid_cp; diff --git a/arch/s390/include/asm/string.h b/arch/s390/include/asm/string.h index 238e721e5a22..378f85304ef5 100644 --- a/arch/s390/include/asm/string.h +++ b/arch/s390/include/asm/string.h @@ -26,7 +26,6 @@ void *memmove(void *dest, const void *src, size_t n); #define __HAVE_ARCH_MEMSCAN /* inline & arch function */ #define __HAVE_ARCH_STRCAT /* inline & arch function */ #define __HAVE_ARCH_STRCMP /* arch function */ -#define __HAVE_ARCH_STRLCAT /* arch function */ #define __HAVE_ARCH_STRLEN /* inline & arch function */ #define __HAVE_ARCH_STRNCAT /* arch function */ #define __HAVE_ARCH_STRNLEN /* inline & arch function */ @@ -38,7 +37,6 @@ void *memmove(void *dest, const void *src, size_t n); /* Prototypes for non-inlined arch strings functions. */ int memcmp(const void *s1, const void *s2, size_t n); int strcmp(const char *s1, const char *s2); -size_t strlcat(char *dest, const char *src, size_t n); char *strncat(char *dest, const char *src, size_t n); char *strstr(const char *s1, const char *s2); #endif /* !defined(CONFIG_KASAN) && !defined(CONFIG_KMSAN) */ diff --git a/arch/s390/include/asm/syscall.h b/arch/s390/include/asm/syscall.h index 4271e4169f45..5f310caad1fc 100644 --- a/arch/s390/include/asm/syscall.h +++ b/arch/s390/include/asm/syscall.h @@ -89,11 +89,6 @@ static inline int syscall_get_arch(struct task_struct *task) return AUDIT_ARCH_S390X; } -static inline bool arch_syscall_is_vdso_sigreturn(struct pt_regs *regs) -{ - return false; -} - #define SYSCALL_FMT_0 #define SYSCALL_FMT_1 , "0" (r2) #define SYSCALL_FMT_2 , "d" (r3) SYSCALL_FMT_1 diff --git a/arch/s390/include/asm/timex.h b/arch/s390/include/asm/timex.h index 49447b40f038..23421b6eb225 100644 --- a/arch/s390/include/asm/timex.h +++ b/arch/s390/include/asm/timex.h @@ -12,6 +12,7 @@ #include <linux/preempt.h> #include <linux/time64.h> +#include <asm/tod_types.h> #include <asm/lowcore.h> #include <asm/machine.h> #include <asm/asm.h> @@ -21,25 +22,6 @@ extern u64 clock_comparator_max; -union tod_clock { - __uint128_t val; - struct { - __uint128_t ei : 8; /* epoch index */ - __uint128_t tod : 64; /* bits 0-63 of tod clock */ - __uint128_t : 40; - __uint128_t pf : 16; /* programmable field */ - }; - struct { - __uint128_t eitod : 72; /* epoch index + bits 0-63 tod clock */ - __uint128_t : 56; - }; - struct { - __uint128_t us : 60; /* micro-seconds */ - __uint128_t sus : 12; /* sub-microseconds */ - __uint128_t : 56; - }; -} __packed; - /* Inline functions for clock register access. */ static inline int set_tod_clock(__u64 time) { @@ -177,8 +159,6 @@ static inline void local_tick_enable(unsigned long comp) set_clock_comparator(get_lowcore()->clock_comparator); } -#define CLOCK_TICK_RATE 1193180 /* Underlying HZ */ - typedef unsigned long cycles_t; static __always_inline unsigned long get_tod_clock(void) diff --git a/arch/s390/include/asm/tod_types.h b/arch/s390/include/asm/tod_types.h new file mode 100644 index 000000000000..976fa0a1e895 --- /dev/null +++ b/arch/s390/include/asm/tod_types.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_S390_TOD_TYPES_H +#define _ASM_S390_TOD_TYPES_H + +#include <linux/types.h> + +#ifndef __ASSEMBLER__ + +union tod_clock { + __uint128_t val; + struct { + __uint128_t ei : 8; /* epoch index */ + __uint128_t tod : 64; /* bits 0-63 of tod clock */ + __uint128_t : 40; + __uint128_t pf : 16; /* programmable field */ + }; + struct { + __uint128_t eitod : 72; /* epoch index + bits 0-63 tod clock */ + __uint128_t : 56; + }; + struct { + __uint128_t us : 60; /* micro-seconds */ + __uint128_t sus : 12; /* sub-microseconds */ + __uint128_t : 56; + }; +} __packed; + +#endif +#endif diff --git a/arch/s390/include/asm/trace_clock.h b/arch/s390/include/asm/trace_clock.h new file mode 100644 index 000000000000..273e05cbdae0 --- /dev/null +++ b/arch/s390/include/asm/trace_clock.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_S390_TRACE_CLOCK_H +#define _ASM_S390_TRACE_CLOCK_H + +#include <linux/compiler.h> +#include <linux/types.h> + +u64 notrace trace_clock_s390_tod(void); + +#define ARCH_TRACE_CLOCKS \ + { trace_clock_s390_tod, "s390-tod", .in_ns = 0 }, + +#endif /* _ASM_S390_TRACE_CLOCK_H */ diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index dff035372601..a9f32c53f699 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -30,8 +30,7 @@ void debug_user_asce(int exit); #define uaccess_kmsan_or_inline __always_inline #endif -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER static uaccess_kmsan_or_inline __must_check unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long size) diff --git a/arch/s390/include/asm/vdso/processor.h b/arch/s390/include/asm/vdso/processor.h index cfcc3e117c4c..6775621e5a5a 100644 --- a/arch/s390/include/asm/vdso/processor.h +++ b/arch/s390/include/asm/vdso/processor.h @@ -2,6 +2,8 @@ #ifndef __ASM_VDSO_PROCESSOR_H #define __ASM_VDSO_PROCESSOR_H -#define cpu_relax() barrier() +#include <asm/barrier.h> + +#define cpu_relax() bcr_serialize() #endif /* __ASM_VDSO_PROCESSOR_H */ diff --git a/arch/s390/include/asm/vtime.h b/arch/s390/include/asm/vtime.h index b1db75d14e9d..da116a93d3b6 100644 --- a/arch/s390/include/asm/vtime.h +++ b/arch/s390/include/asm/vtime.h @@ -48,8 +48,8 @@ static inline void update_timer_idle(void) * The accounted CPU times will be subtracted again from steal_timer * when accumulated steal time is calculated in do_account_vtime(). */ - lc->steal_timer += idle->clock_idle_enter - lc->last_update_clock; - lc->last_update_clock = lc->int_clock; + lc->steal_timer += idle->clock_idle_enter.tod - lc->last_update_clock; + lc->last_update_clock = lc->int_clock.tod; lc->system_timer += lc->last_update_timer - idle->timer_idle_enter; lc->last_update_timer = lc->sys_enter_timer; } diff --git a/arch/s390/include/asm/word-at-a-time.h b/arch/s390/include/asm/word-at-a-time.h index eaa19dee7699..e9287036392d 100644 --- a/arch/s390/include/asm/word-at-a-time.h +++ b/arch/s390/include/asm/word-at-a-time.h @@ -4,7 +4,6 @@ #include <linux/bitops.h> #include <linux/wordpart.h> -#include <asm/asm-extable.h> #include <asm/bitsperlong.h> struct word_at_a_time { @@ -41,25 +40,4 @@ static inline unsigned long zero_bytemask(unsigned long data) return ~1UL << data; } -/* - * Load an unaligned word from kernel space. - * - * In the (very unlikely) case of the word being a page-crosser - * and the next page not being mapped, take the exception and - * return zeroes in the non-existing part. - */ -static inline unsigned long load_unaligned_zeropad(const void *addr) -{ - unsigned long data; - - asm_inline volatile( - "0: lg %[data],0(%[addr])\n" - "1: nopr %%r7\n" - EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr]) - EX_TABLE_ZEROPAD(1b, 1b, %[data], %[addr]) - : [data] "=d" (data) - : [addr] "a" (addr), "m" (*(unsigned long *)addr)); - return data; -} - #endif /* _ASM_WORD_AT_A_TIME_H */ diff --git a/arch/s390/include/uapi/asm/kvm.h b/arch/s390/include/uapi/asm/kvm.h index 60345dd2cba2..4192769b5ce0 100644 --- a/arch/s390/include/uapi/asm/kvm.h +++ b/arch/s390/include/uapi/asm/kvm.h @@ -444,6 +444,7 @@ struct kvm_s390_vm_cpu_machine { #define KVM_S390_VM_CPU_FEAT_PFMFI 11 #define KVM_S390_VM_CPU_FEAT_SIGPIF 12 #define KVM_S390_VM_CPU_FEAT_KSS 13 +#define KVM_S390_VM_CPU_FEAT_ASTFLEIE2 14 struct kvm_s390_vm_cpu_feat { __u64 feat[16]; }; |