diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2026-10-20 14:28:32 -1000 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2026-10-20 14:28:32 -1000 |
| commit | 375d5d57545f53d2bc3e15ada0e9da360fc68bea (patch) | |
| tree | 5c8a03a7fc1eeace002f03a31ee4a70f9990a8ea | |
| parent | ae3af2d105b224e91fd5abefd96af5f9760e1744 (diff) | |
| parent | 3e30b96fab780110f62156d5a8cd5240152b3c59 (diff) | |
Merge tag 's390-7.4-1' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux
Pull s390 updates from Heiko Carstens:
- Add support for sclp-vt220 terminal resize events e.g. for QEMU
- Warn if kernel command line contains non-printable EBCDIC characters
- Improve show_code() disassembly output by using reliable markers for
the instruction address and faulting instruction
- Rework exception handling to require only a single exception table
entry per faulting instruction: convert all inline assemblies from
EX_TABLE to the new EX_TABLE_INSN macros. The previous exception
handling was error prone since it usually required two EX_TABLE
entries: one pointing at the instruction, and one pointing behind it
- Disallow kprobes on instructions with exception table entries to
avoid exception handlers decoding a breakpoint instead of the
original instruction
- Preemptible percpu ops changes:
- Simplify percpu_entry() and percpu_exit() by encoding the percpu
base, offset, and pointer registers into the lowcore marker,
allowing percpu_exit() to update the percpu pointer on CPU
migration without comparing the interrupted instruction
- Convert this_cpu_cmpxchg(), this_cpu_xchg(), and
this_cpu_cmpxchg128() to preemptible percpu ops; provide
this_cpu_try_cmpxchg()
- Emulate virtual timer in appldata via delayed work and remove the
s390 specific vtimer infrastructure
- Add lock contention tracepoints to rwlock slowpath
- Fix incorrect return address in signal frame setup
- Fix cpum_cf CPU hotplug handling
- Convert various __get_free_pages() / get_zeroed_page() users to
kmalloc() / kzalloc()
- PCI:
- Improve RPCIT instruction tracing
- Fix double-free, use-after-free, and resource leaks in zPCI MSI
and IRQ setup / cleanup paths
- Crypto and zcrypt:
- Improve reply message and length verification checks for CCA,
EP11, and ICA messages to prevent out-of-bounds reads
- Add missing array_index_nospec() for domain index uses in zcrypt
to prevent speculative execution bypass
- Fix out-of-bounds read in cca_query_crypto_facility()
- Various other small fixes and improvements
* tag 's390-7.4-1' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux: (102 commits)
s390/signal: Cleanup coding style
s390/signal: Fix incorrect return address in signal frame
s390/probes: Fix wrong mnemonic comment for clgib
s390/probes: Add missing fixup for bctgr and bic
s390/probes: Fix wrong opcode for cgrb fixup
s390: Select ARCH_MEMORY_ORDER_TSO
s390/mm: Cleanup pXXp_flush_lazy() routines
s390/pci: Add function handle to RPCIT insn trace
s390/pci: Use 32-bit fh outside of inline asm
s390/pci: Adjust alignment in insn trace
s390/sclp: Remove redundant memset after memblock_alloc_low()
s390/entry: Save lowcore on invalid kernel stack
s390/entry: Last breaking event vs invalid kernel stack
s390/percpu: ImplemenHeiko Carstenst this_cpu_try_cmpxchg()
s390/percpu: Drop unused output register from arch_this_cpu_to_op()
s390/percpu: Drop unused output register from arch_this_cpu_add()
s390/percpu: Use asm_inline for arch_this_cpu_add()
s390/extable: Remove EX_TABLE macros and fallback extable search
s390/bpf: Simplify exception table entry handling
s390/sclp: Convert inline assembly to EX_TABLE_INSN
...
90 files changed, 1680 insertions, 1451 deletions
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 2682a2bacc58..10ada552d870 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -145,6 +145,7 @@ config S390 select ARCH_INLINE_WRITE_UNLOCK_BH select ARCH_INLINE_WRITE_UNLOCK_IRQ select ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE + select ARCH_MEMORY_ORDER_TSO select ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE select ARCH_STACKWALK select ARCH_SUPPORTS_ATOMIC_RMW diff --git a/arch/s390/appldata/appldata_base.c b/arch/s390/appldata/appldata_base.c index 9cba4633c3f3..836c51da18f4 100644 --- a/arch/s390/appldata/appldata_base.c +++ b/arch/s390/appldata/appldata_base.c @@ -28,19 +28,14 @@ #include <linux/workqueue.h> #include <linux/uaccess.h> #include <linux/io.h> +#include <linux/kernel_stat.h> #include <asm/appldata.h> -#include <asm/vtimer.h> #include <asm/smp.h> #include "appldata.h" - -#define APPLDATA_CPU_INTERVAL 10000 /* default (CPU) time for - sampling interval in - milliseconds */ - -#define TOD_MICRO 0x01000 /* nr. of TOD clock units - for 1 microsecond */ +/* Default CPU time for sampling interval */ +#define APPLDATA_CPU_INTERVAL (10 * NSEC_PER_SEC) /* * /proc entries (sysctl) @@ -64,22 +59,14 @@ static const struct ctl_table appldata_table[] = { }, }; -/* - * Timer - */ -static struct vtimer_list appldata_timer; +static void appldata_work_fn(struct work_struct *work); +static DECLARE_DELAYED_WORK(appldata_work, appldata_work_fn); -static DEFINE_SPINLOCK(appldata_timer_lock); -static int appldata_interval = APPLDATA_CPU_INTERVAL; +static DEFINE_MUTEX(appldata_timer_lock); +static u64 appldata_interval = APPLDATA_CPU_INTERVAL; static int appldata_timer_active; -/* - * Work queue - */ -static struct workqueue_struct *appldata_wq; -static void appldata_work_fn(struct work_struct *work); -static DECLARE_WORK(appldata_work, appldata_work_fn); - +static u64 appldata_cputime_start; /* * Ops list @@ -89,14 +76,34 @@ static LIST_HEAD(appldata_ops_list); /*************************** timer, work, DIAG *******************************/ -/* - * appldata_timer_function() - * - * schedule work and reschedule timer - */ -static void appldata_timer_function(unsigned long data) +static u64 appldata_total_cpu_time_ns(void) { - queue_work(appldata_wq, (struct work_struct *) data); + u64 total = 0; + int cpu; + + for_each_possible_cpu(cpu) { + total += kcpustat_cpu(cpu).cpustat[CPUTIME_USER]; + total += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE]; + total += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM]; + total += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ]; + total += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ]; + } + return total; +} + +static void appldata_schedule_work(u64 remaining) +{ + unsigned int ncpus = num_online_cpus(); + unsigned long delay = HZ / 10; + + /* + * At most ncpus CPUs consume CPU time simultaneously, so the + * minimum wall-clock time until the remaining CPU time elapses + * is remaining / ncpus. + * Make sure the work is not scheduled more than once per 100ms. + */ + delay = max(delay, nsecs_to_jiffies(remaining / ncpus)); + mod_delayed_work(system_percpu_wq, &appldata_work, delay); } /* @@ -106,9 +113,27 @@ static void appldata_timer_function(unsigned long data) */ static void appldata_work_fn(struct work_struct *work) { - struct list_head *lh; + u64 now, elapsed, interval; struct appldata_ops *ops; - + struct list_head *lh; + bool expired = false; + + now = appldata_total_cpu_time_ns(); + mutex_lock(&appldata_timer_lock); + if (appldata_timer_active) { + interval = appldata_interval; + elapsed = now - appldata_cputime_start; + if (elapsed < interval) { + appldata_schedule_work(interval - elapsed); + } else { + appldata_cputime_start = now; + appldata_schedule_work(interval); + expired = true; + } + } + mutex_unlock(&appldata_timer_lock); + if (!expired) + return; mutex_lock(&appldata_ops_mutex); list_for_each(lh, &appldata_ops_list) { ops = list_entry(lh, struct appldata_ops, list); @@ -162,33 +187,52 @@ int appldata_diag(char record_nr, u16 function, unsigned long buffer, #define APPLDATA_MOD_TIMER 2 /* - * __appldata_vtimer_setup() + * __appldata_timer_setup() * - * Add, delete or modify virtual timers on all online cpus. - * The caller needs to get the appldata_timer_lock spinlock. + * Add, delete or modify the appldata delayed work. */ -static void __appldata_vtimer_setup(int cmd) +static void __appldata_timer_setup(int cmd) { - u64 timer_interval = (u64) appldata_interval * 1000 * TOD_MICRO; + static DEFINE_MUTEX(appldata_cancel_lock); + u64 now, elapsed, remaining; switch (cmd) { case APPLDATA_ADD_TIMER: - if (appldata_timer_active) - break; - appldata_timer.expires = timer_interval; - add_virt_timer_periodic(&appldata_timer); - appldata_timer_active = 1; + mutex_lock(&appldata_cancel_lock); + mutex_lock(&appldata_timer_lock); + if (!appldata_timer_active) { + appldata_cputime_start = appldata_total_cpu_time_ns(); + appldata_schedule_work(appldata_interval); + appldata_timer_active = 1; + } + mutex_unlock(&appldata_timer_lock); + mutex_unlock(&appldata_cancel_lock); break; case APPLDATA_DEL_TIMER: - del_virt_timer(&appldata_timer); - if (!appldata_timer_active) + mutex_lock(&appldata_cancel_lock); + mutex_lock(&appldata_timer_lock); + if (!appldata_timer_active) { + mutex_unlock(&appldata_timer_lock); + mutex_unlock(&appldata_cancel_lock); break; + } appldata_timer_active = 0; + mutex_unlock(&appldata_timer_lock); + cancel_delayed_work_sync(&appldata_work); + mutex_unlock(&appldata_cancel_lock); break; case APPLDATA_MOD_TIMER: - if (!appldata_timer_active) - break; - mod_virt_timer_periodic(&appldata_timer, timer_interval); + mutex_lock(&appldata_timer_lock); + if (appldata_timer_active) { + now = appldata_total_cpu_time_ns(); + elapsed = now - appldata_cputime_start; + remaining = appldata_interval - elapsed; + if (elapsed >= appldata_interval) + remaining = 1; + appldata_schedule_work(remaining); + } + mutex_unlock(&appldata_timer_lock); + break; } } @@ -215,12 +259,10 @@ appldata_timer_handler(const struct ctl_table *ctl, int write, if (rc < 0 || !write) return rc; - spin_lock(&appldata_timer_lock); if (timer_active) - __appldata_vtimer_setup(APPLDATA_ADD_TIMER); + __appldata_timer_setup(APPLDATA_ADD_TIMER); else - __appldata_vtimer_setup(APPLDATA_DEL_TIMER); - spin_unlock(&appldata_timer_lock); + __appldata_timer_setup(APPLDATA_DEL_TIMER); return 0; } @@ -234,7 +276,7 @@ static int appldata_interval_handler(const struct ctl_table *ctl, int write, void *buffer, size_t *lenp, loff_t *ppos) { - int interval = appldata_interval; + int interval = appldata_interval / NSEC_PER_MSEC; int rc; struct ctl_table ctl_entry = { .procname = ctl->procname, @@ -247,10 +289,10 @@ appldata_interval_handler(const struct ctl_table *ctl, int write, if (rc < 0 || !write) return rc; - spin_lock(&appldata_timer_lock); - appldata_interval = interval; - __appldata_vtimer_setup(APPLDATA_MOD_TIMER); - spin_unlock(&appldata_timer_lock); + mutex_lock(&appldata_timer_lock); + appldata_interval = interval * NSEC_PER_MSEC; + mutex_unlock(&appldata_timer_lock); + __appldata_timer_setup(APPLDATA_MOD_TIMER); return 0; } @@ -392,19 +434,20 @@ void appldata_unregister_ops(struct appldata_ops *ops) /******************************* init / exit *********************************/ -/* - * appldata_init() - * - * init timer, register /proc entries - */ +static int appldata_cpu_online(unsigned int cpu) +{ + __appldata_timer_setup(APPLDATA_MOD_TIMER); + return 0; +} + static int __init appldata_init(void) { - init_virt_timer(&appldata_timer); - appldata_timer.function = appldata_timer_function; - appldata_timer.data = (unsigned long) &appldata_work; - appldata_wq = alloc_ordered_workqueue("appldata", 0); - if (!appldata_wq) - return -ENOMEM; + int rc; + + rc = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "s390/appldata:online", + appldata_cpu_online, NULL); + if (rc < 0) + return rc; register_sysctl(appldata_proc_name, appldata_table); return 0; } diff --git a/arch/s390/boot/ipl_parm.c b/arch/s390/boot/ipl_parm.c index c1b43e5e688a..8bbd102fa50f 100644 --- a/arch/s390/boot/ipl_parm.c +++ b/arch/s390/boot/ipl_parm.c @@ -40,9 +40,9 @@ static inline int __diag308(unsigned long subcode, void *addr) union register_pair r1 = { .even = (unsigned long)addr, .odd = 0 }; asm_inline volatile( - " diag %[r1],%[subcode],0x308\n" - "0:\n" - EX_TABLE(0b, 0b) + "0: diag %[r1],%[subcode],0x308\n" + "1:\n" + EX_TABLE_INSN(0b, 1b) : [r1] "+d" (r1.pair) : [subcode] "d" (subcode) : "cc", "memory"); @@ -172,12 +172,23 @@ static inline int has_ebcdic_char(const char *str) return 0; } +static inline bool has_nonprintable_ebcdic_char(const char *str) +{ + for (; *str; str++) + if (!isprint_ebc_asc(*str) && !isspace_ebc_asc(*str)) + return true; + return false; +} + void setup_boot_command_line(void) { parmarea.command_line[COMMAND_LINE_SIZE - 1] = 0; /* convert arch command line to ascii if necessary */ - if (has_ebcdic_char(parmarea.command_line)) + if (has_ebcdic_char(parmarea.command_line)) { + if (has_nonprintable_ebcdic_char(parmarea.command_line)) + boot_warn("Kernel command line was treated as EBCDIC, but contains non-printable characters\n"); EBCASC(parmarea.command_line, COMMAND_LINE_SIZE); + } /* copy arch command line */ strscpy(early_command_line, strim(parmarea.command_line)); diff --git a/arch/s390/boot/pgm_check.c b/arch/s390/boot/pgm_check.c index fa621fa5bc02..e5a454b21669 100644 --- a/arch/s390/boot/pgm_check.c +++ b/arch/s390/boot/pgm_check.c @@ -3,9 +3,11 @@ #include <linux/stdarg.h> #include <linux/string.h> #include <linux/ctype.h> +#include <asm/pgmcheck-defs.h> #include <asm/stacktrace.h> #include <asm/boot_data.h> #include <asm/lowcore.h> +#include <asm/processor.h> #include <asm/setup.h> #include <asm/sclp.h> #include <asm/uv.h> @@ -40,12 +42,22 @@ static inline unsigned long extable_insn(const struct exception_table_entry *x) return (unsigned long)&x->insn + x->insn; } +static unsigned long boot_get_trap_ip(struct pt_regs *regs) +{ + static const unsigned long nullified[2] = { PGM_CHECKS_NULLIFIED }; + + if (test_bit(regs->int_code & PGM_INT_CODE_MASK, nullified)) + return regs->psw.addr; + return __rewind_psw(regs->psw, regs->int_code >> 16); +} + static bool ex_handler(struct pt_regs *regs) { const struct exception_table_entry *ex; + unsigned long ip = boot_get_trap_ip(regs); for (ex = __start___ex_table; ex < __stop___ex_table; ex++) { - if (extable_insn(ex) != regs->psw.addr) + if (extable_insn(ex) != ip) continue; if (ex->type != EX_TYPE_FIXUP) return false; diff --git a/arch/s390/boot/physmem_info.c b/arch/s390/boot/physmem_info.c index 0ebb2174713f..e5248b637326 100644 --- a/arch/s390/boot/physmem_info.c +++ b/arch/s390/boot/physmem_info.c @@ -68,11 +68,11 @@ static int __diag260(unsigned long rx1, unsigned long rx2) ry = 0x10; /* storage configuration */ exception = 1; asm_inline volatile( - " diag %[rx],%[ry],0x260\n" - "0: lhi %[exc],0\n" + "0: diag %[rx],%[ry],0x260\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [exc] "+d" (exception), [ry] "+d" (ry) : [rx] "d" (rx.pair) : CC_CLOBBER_LIST("memory")); @@ -108,9 +108,9 @@ static int diag500_storage_limit(unsigned long *max_physmem_end) asm_inline volatile( " lghi %%r1,%[subcode]\n" " lghi %%r2,0\n" - " diag %%r2,%%r4,0x500\n" - "0: lgr %[slimit],%%r2\n" - EX_TABLE(0b, 0b) + "0: diag %%r2,%%r4,0x500\n" + "1: lgr %[slimit],%%r2\n" + EX_TABLE_INSN(0b, 1b) : [slimit] "=d" (storage_limit) : [subcode] "i" (DIAG500_SC_STOR_LIMIT) : "memory", "1", "2"); @@ -127,11 +127,11 @@ static int tprot(unsigned long addr) exception = 1; asm_inline volatile( - " tprot 0(%[addr]),0\n" - "0: lhi %[exc],0\n" + "0: tprot 0(%[addr]),0\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [exc] "+d" (exception) : [addr] "a" (addr) : CC_CLOBBER_LIST("memory")); diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index c1dcf8b1b579..5eb7a3e1f9f1 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -89,10 +89,10 @@ static void detect_diag9c(void) cpu = stap(); asm_inline volatile( - " diag %[cpu],%%r0,0x9c\n" - "0: lhi %[rc],0\n" + "0: diag %[cpu],%%r0,0x9c\n" + " lhi %[rc],0\n" "1:\n" - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : [rc] "+d" (rc) : [cpu] "d" (cpu) : "cc", "memory"); @@ -150,10 +150,10 @@ static int cmma_test_essa(void) /* Test ESSA_GET_STATE */ asm_inline volatile( - " .insn rrf,0xb9ab0000,%[tmp],%[tmp],%[cmd],0\n" - "0: lhi %[rc],0\n" + "0: .insn rrf,0xb9ab0000,%[tmp],%[tmp],%[cmd],0\n" + " lhi %[rc],0\n" "1:\n" - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : [rc] "+d" (rc), [tmp] "+d" (tmp) : [cmd] "i" (ESSA_GET_STATE) : "cc", "memory"); diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index 3dae71474333..3892d9dd75d9 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -451,6 +451,20 @@ CONFIG_BLK_DEV_RAM_SIZE=32768 CONFIG_VIRTIO_BLK=y CONFIG_BLK_DEV_RBD=m CONFIG_BLK_DEV_NVME=m +CONFIG_NVME_MULTIPATH=y +CONFIG_NVME_VERBOSE_ERRORS=y +CONFIG_NVME_RDMA=m +CONFIG_NVME_TCP=m +CONFIG_NVME_TCP_TLS=y +CONFIG_NVME_HOST_AUTH=y +CONFIG_NVME_TARGET=m +CONFIG_NVME_TARGET_DEBUGFS=y +CONFIG_NVME_TARGET_PASSTHRU=y +CONFIG_NVME_TARGET_LOOP=m +CONFIG_NVME_TARGET_RDMA=m +CONFIG_NVME_TARGET_TCP=m +CONFIG_NVME_TARGET_TCP_TLS=y +CONFIG_NVME_TARGET_AUTH=y CONFIG_BLK_DEV_NULL_BLK=m CONFIG_ENCLOSURE_SERVICES=m CONFIG_GENWQE=m diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index 6f5722634b4d..95720e8ad80d 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -441,6 +441,20 @@ CONFIG_BLK_DEV_RAM_SIZE=32768 CONFIG_VIRTIO_BLK=y CONFIG_BLK_DEV_RBD=m CONFIG_BLK_DEV_NVME=m +CONFIG_NVME_MULTIPATH=y +CONFIG_NVME_VERBOSE_ERRORS=y +CONFIG_NVME_RDMA=m +CONFIG_NVME_TCP=m +CONFIG_NVME_TCP_TLS=y +CONFIG_NVME_HOST_AUTH=y +CONFIG_NVME_TARGET=m +CONFIG_NVME_TARGET_DEBUGFS=y +CONFIG_NVME_TARGET_PASSTHRU=y +CONFIG_NVME_TARGET_LOOP=m +CONFIG_NVME_TARGET_RDMA=m +CONFIG_NVME_TARGET_TCP=m +CONFIG_NVME_TARGET_TCP_TLS=y +CONFIG_NVME_TARGET_AUTH=y CONFIG_BLK_DEV_NULL_BLK=m CONFIG_ENCLOSURE_SERVICES=m CONFIG_GENWQE=m diff --git a/arch/s390/hypfs/hypfs_vm.c b/arch/s390/hypfs/hypfs_vm.c index 4db2895e4da3..bd47fce59731 100644 --- a/arch/s390/hypfs/hypfs_vm.c +++ b/arch/s390/hypfs/hypfs_vm.c @@ -42,9 +42,9 @@ static int diag2fc(int size, char* query, void *addr) diag_stat_inc(DIAG_STAT_X2FC); asm volatile( - " diag %0,%1,0x2fc\n" - "0: nopr %%r7\n" - EX_TABLE(0b,0b) + "0: diag %0,%1,0x2fc\n" + "1:\n" + EX_TABLE_INSN(0b, 1b) : "=d" (residual_cnt), "+d" (rc) : "0" (&parm_list) : "memory"); if ((rc != 0 ) && (rc != -2)) diff --git a/arch/s390/include/asm/Kbuild b/arch/s390/include/asm/Kbuild index 0c1fc47c3ba0..d13dd7e3dc01 100644 --- a/arch/s390/include/asm/Kbuild +++ b/arch/s390/include/asm/Kbuild @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 generated-y += dis-defs.h generated-y += facility-defs.h +generated-y += pgmcheck-defs.h generated-y += syscall_table.h generated-y += unistd_nr.h diff --git a/arch/s390/include/asm/ap.h b/arch/s390/include/asm/ap.h index c91b6ace199d..b0f8e3946a43 100644 --- a/arch/s390/include/asm/ap.h +++ b/arch/s390/include/asm/ap.h @@ -91,10 +91,10 @@ static inline bool ap_instructions_available(void) " lgr 0,%[reg0]\n" /* qid into gr0 */ " lghi 1,0\n" /* 0 into gr1 */ " lghi 2,0\n" /* 0 into gr2 */ - " .insn rre,0xb2af0000,0,0\n" /* PQAP(TAPQ) */ - "0: la %[reg1],1\n" /* 1 into reg1 */ + "0: .insn rre,0xb2af0000,0,0\n" /* PQAP(TAPQ) */ + " la %[reg1],1\n" /* 1 into reg1 */ "1:\n" - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : [reg1] "+&d" (reg1) : [reg0] "d" (reg0) : "cc", "0", "1", "2"); @@ -272,10 +272,10 @@ static inline int ap_qci(struct ap_config_info *config) asm volatile( " lgr 0,%[reg0]\n" /* QCI fc into gr0 */ " lgr 2,%[reg2]\n" /* ptr to config into gr2 */ - " .insn rre,0xb2af0000,0,0\n" /* PQAP(QCI) */ - "0: la %[reg1],0\n" /* good case, QCI fc available */ + "0: .insn rre,0xb2af0000,0,0\n" /* PQAP(QCI) */ + " la %[reg1],0\n" /* good case, QCI fc available */ "1:\n" - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : [reg1] "+&d" (reg1) : [reg0] "d" (reg0), [reg2] "d" (reg2) : "cc", "memory", "0", "2"); diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h index d23ea0c94e4e..fd43ecd3e491 100644 --- a/arch/s390/include/asm/asm-extable.h +++ b/arch/s390/include/asm/asm-extable.h @@ -65,31 +65,41 @@ stringify_in_c(.purgem extable_reg;) \ stringify_in_c(.previous) -#define EX_TABLE(_fault, _target) \ +/* + * EX_TABLE_INSN usage rules: + * + * _fault must point to the faulting instruction, never behind it. + * + * This differs from the former EX_TABLE convention, where usually two entries + * were needed. One that pointed to the faulting instruction (for nullifying + * exceptions), and a second one that pointed behind the instruction (for + * suppressing, completing, and terminating exceptions). + */ +#define EX_TABLE_INSN(_fault, _target) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FIXUP, __stringify(%%r0), __stringify(%%r0), 0) -#define EX_TABLE_AMODE31(_fault, _target) \ +#define EX_TABLE_INSN_AMODE31(_fault, _target) \ __EX_TABLE(.amode31.ex_table, _fault, _target, EX_TYPE_FIXUP, __stringify(%%r0), __stringify(%%r0), 0) -#define EX_TABLE_UA_FAULT(_fault, _target, _regerr) \ +#define EX_TABLE_INSN_UA_FAULT(_fault, _target, _regerr) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_FAULT, _regerr, _regerr, 0) -#define EX_TABLE_UA_LOAD_REG(_fault, _target, _regerr, _regzero) \ +#define EX_TABLE_INSN_UA_LOAD_REG(_fault, _target, _regerr, _regzero) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REG, _regerr, _regzero, 0) -#define EX_TABLE_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero) \ +#define EX_TABLE_INSN_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0) -#define EX_TABLE_ZEROPAD(_fault, _target, _regdata, _regaddr) \ +#define EX_TABLE_INSN_ZEROPAD(_fault, _target, _regdata, _regaddr) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_ZEROPAD, _regdata, _regaddr, 0) -#define EX_TABLE_FPC(_fault, _target) \ +#define EX_TABLE_INSN_FPC(_fault, _target) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FPC, __stringify(%%r0), __stringify(%%r0), 0) -#define EX_TABLE_UA_MVCOS_TO(_fault, _target) \ +#define EX_TABLE_INSN_UA_MVCOS_TO(_fault, _target) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_MVCOS_TO, __stringify(%%r0), __stringify(%%r0), 0) -#define EX_TABLE_UA_MVCOS_FROM(_fault, _target) \ +#define EX_TABLE_INSN_UA_MVCOS_FROM(_fault, _target) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_MVCOS_FROM, __stringify(%%r0), __stringify(%%r0), 0) #endif /* __ASM_EXTABLE_H */ diff --git a/arch/s390/include/asm/cpu_mf.h b/arch/s390/include/asm/cpu_mf.h index 1798fbd59068..dd2991622e82 100644 --- a/arch/s390/include/asm/cpu_mf.h +++ b/arch/s390/include/asm/cpu_mf.h @@ -173,9 +173,9 @@ static inline int qctri(struct cpumf_ctr_info *info) asm_inline volatile ( "0: qctri %1\n" - "1: lhi %0,0\n" - "2:\n" - EX_TABLE(1b, 2b) + " lhi %0,0\n" + "1:\n" + EX_TABLE_INSN(0b, 1b) : "+d" (rc), "=Q" (*info)); return rc; } @@ -257,9 +257,9 @@ static inline int qsi(struct hws_qsi_info_block *info) asm volatile( "0: qsi %1\n" - "1: lhi %0,0\n" - "2:\n" - EX_TABLE(0b, 2b) EX_TABLE(1b, 2b) + " lhi %0,0\n" + "1:\n" + EX_TABLE_INSN(0b, 1b) : "+d" (cc), "+Q" (*info)); return cc ? -EINVAL : 0; } @@ -272,10 +272,10 @@ static inline int lsctl(struct hws_lsctl_request_block *req) exception = 1; asm volatile( "0: lsctl %[req]\n" - "1: lhi %[exc],0\n" - "2:\n" + " lhi %[exc],0\n" + "1:\n" CC_IPM(cc) - EX_TABLE(0b, 2b) EX_TABLE(1b, 2b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [exc] "+d" (exception) : [req] "Q" (*req) : CC_CLOBBER); diff --git a/arch/s390/include/asm/diag.h b/arch/s390/include/asm/diag.h index 8db8db3b1018..1416ee1ad323 100644 --- a/arch/s390/include/asm/diag.h +++ b/arch/s390/include/asm/diag.h @@ -68,9 +68,8 @@ static inline void diag10_range(unsigned long start_pfn, unsigned long num_pfn) diag_stat_inc(DIAG_STAT_X010); asm_inline volatile( "0: diag %0,%1,0x10\n" - "1: nopr %%r7\n" - EX_TABLE(0b, 1b) - EX_TABLE(1b, 1b) + "1:\n" + EX_TABLE_INSN(0b, 1b) : : "a" (start_addr), "a" (end_addr)); } diff --git a/arch/s390/include/asm/diag288.h b/arch/s390/include/asm/diag288.h index 5e1b43cea9d6..f0a37a80d672 100644 --- a/arch/s390/include/asm/diag288.h +++ b/arch/s390/include/asm/diag288.h @@ -28,10 +28,10 @@ static inline int __diag288(unsigned int func, unsigned int timeout, int rc = -EINVAL; asm volatile( - " diag %[r1],%[r3],0x288\n" - "0: lhi %[rc],0\n" + "0: diag %[r1],%[r3],0x288\n" + " lhi %[rc],0\n" "1:" - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : [rc] "+d" (rc) : [r1] "d" (r1.pair), [r3] "d" (r3.pair) : "cc", "memory"); diff --git a/arch/s390/include/asm/ebcdic.h b/arch/s390/include/asm/ebcdic.h index 7164cb658435..78d2e37e985d 100644 --- a/arch/s390/include/asm/ebcdic.h +++ b/arch/s390/include/asm/ebcdic.h @@ -10,6 +10,7 @@ #ifndef _EBCDIC_H #define _EBCDIC_H +#include <linux/ctype.h> #include <linux/types.h> extern __u8 _ascebc_500[256]; /* ASCII -> EBCDIC 500 conversion table */ @@ -18,6 +19,11 @@ extern __u8 _ascebc[256]; /* ASCII -> EBCDIC conversion table */ extern __u8 _ebcasc[256]; /* EBCDIC -> ASCII conversion table */ extern __u8 _ebc_tolower[256]; /* EBCDIC -> lowercase */ extern __u8 _ebc_toupper[256]; /* EBCDIC -> uppercase */ +extern const __u8 _ebctype_asc[256]; /* EBCDIC -> character classes */ + +#define __ismask_ebc_asc(x) (_ebctype_asc[(int)(unsigned char)(x)]) +#define isprint_ebc_asc(c) ((__ismask_ebc_asc(c)&(_P|_U|_L|_D|_SP)) != 0) +#define isspace_ebc_asc(c) ((__ismask_ebc_asc(c)&(_S)) != 0) static inline void codepage_convert(const __u8 *codepage, volatile char *addr, unsigned long nr) diff --git a/arch/s390/include/asm/entry-percpu.h b/arch/s390/include/asm/entry-percpu.h index 4ef47265cfce..f6f79ec4bee3 100644 --- a/arch/s390/include/asm/entry-percpu.h +++ b/arch/s390/include/asm/entry-percpu.h @@ -2,79 +2,42 @@ #ifndef ARCH_S390_ENTRY_PERCPU_H #define ARCH_S390_ENTRY_PERCPU_H -#include <linux/kprobes.h> +#include <linux/bitfield.h> #include <linux/percpu.h> #include <asm/lowcore.h> #include <asm/ptrace.h> -#include <asm/asm-offsets.h> static __always_inline void percpu_entry(struct pt_regs *regs) { struct lowcore *lc = get_lowcore(); - if (user_mode(regs)) - return; regs->cpu = lc->cpu_nr; regs->percpu_register = lc->percpu_register; lc->percpu_register = 0; } -static __always_inline bool percpu_code_check(struct pt_regs *regs) -{ - unsigned int insn, disp; - struct kprobe *p; - - if (likely(user_mode(regs) || !regs->percpu_register)) - return false; - /* - * Within a percpu code section - check if the percpu base register - * needs to be updated. This is the case if the PSW does not point to - * the ADD instruction within the section. - * - AG %rx,percpu_offset_in_lowcore(%r0,%r0) - * which adds the percpu offset to the percpu base register. - */ - lockdep_assert_preemption_disabled(); -again: - insn = READ_ONCE(*(u16 *)psw_bits(regs->psw).ia); - if (unlikely(insn == BREAKPOINT_INSTRUCTION)) { - p = get_kprobe((void *)psw_bits(regs->psw).ia); - /* - * If the kprobe is concurrently removed on a different CPU - * it might not be found anymore. However text must have - * been restored - try again. - */ - if (!p) - goto again; - insn = p->opcode; - } - if ((insn & 0xff0f) != 0xe300) - return true; - disp = offsetof(struct lowcore, percpu_offset); - if (machine_has_relocated_lowcore()) - disp += LOWCORE_ALT_ADDRESS; - insn = (disp & 0xff000) >> 4 | (disp & 0x00fff) << 16 | 0x8; - if (*(u32 *)(psw_bits(regs->psw).ia + 2) != insn) - return true; - return false; -} - -static __always_inline void percpu_exit(struct pt_regs *regs, bool needs_fixup) +static __always_inline void percpu_exit(struct pt_regs *regs) { + unsigned char regval, regpcp, regoff, regptr; struct lowcore *lc = get_lowcore(); - unsigned char reg; - if (user_mode(regs)) - return; - reg = regs->percpu_register; - lc->percpu_register = reg; - if (likely(!needs_fixup)) + if (!regs->percpu_register) return; - /* Check if process has been migrated to a different CPU. */ + regval = regs->percpu_register; + lc->percpu_register = regval; + /* Migrated to a different CPU? */ if (regs->cpu == lc->cpu_nr) return; - /* Fixup percpu base register */ - regs->gprs[reg] -= __per_cpu_offset[regs->cpu]; - regs->gprs[reg] += lc->percpu_offset; + regpcp = FIELD_GET(PCPU_REG_PCP, regval); + regoff = FIELD_GET(PCPU_REG_OFF, regval); + regptr = regoff + 1; + /* + * Update register 'regoff' with the current CPU's percpu offset, + * and recalculate and update current CPU's percpu variable address + * contained in register 'regptr'. + */ + regs->gprs[regoff] = lc->percpu_offset; + regs->gprs[regptr] = regs->gprs[regpcp] + regs->gprs[regoff]; } #endif diff --git a/arch/s390/include/asm/extable.h b/arch/s390/include/asm/extable.h index af6ba52743e9..5032c648781b 100644 --- a/arch/s390/include/asm/extable.h +++ b/arch/s390/include/asm/extable.h @@ -5,25 +5,14 @@ #include <asm/ptrace.h> #include <linux/compiler.h> -/* - * The exception table consists of three addresses: - * - * - Address of an instruction that is allowed to fault. - * - Address at which the program should continue. - * - Optional address of handler that takes pt_regs * argument and runs in - * interrupt context. - * - * No registers are modified, so it is entirely up to the continuation code - * to figure out what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. +/** + * struct exception_table_entry - entry in exception table + * @insn: Relative pointer to faulting instruction + * @fixup: Relative pointer to instruction where execution resumes + * @type: Exception type + * @data: handler-specific data */ - -struct exception_table_entry -{ +struct exception_table_entry { int insn, fixup; short type, data; }; diff --git a/arch/s390/include/asm/fpu-insn.h b/arch/s390/include/asm/fpu-insn.h index ae8b7033cfd2..eeb0a233d5c7 100644 --- a/arch/s390/include/asm/fpu-insn.h +++ b/arch/s390/include/asm/fpu-insn.h @@ -104,9 +104,9 @@ static inline void fpu_lfpc_safe(unsigned int *fpc) { instrument_read(fpc, sizeof(*fpc)); asm_inline volatile( - " lfpc %[fpc]\n" - "0: nopr %%r7\n" - EX_TABLE_FPC(0b, 0b) + "0: lfpc %[fpc]\n" + "1:\n" + EX_TABLE_INSN_FPC(0b, 1b) : : [fpc] "Q" (*fpc) : "memory"); diff --git a/arch/s390/include/asm/futex.h b/arch/s390/include/asm/futex.h index 942f21c39697..71e8060637be 100644 --- a/arch/s390/include/asm/futex.h +++ b/arch/s390/include/asm/futex.h @@ -23,13 +23,11 @@ __futex_atomic_##name(int oparg, int *old, u32 __user *uaddr) \ "0: l %[old],%[uaddr]\n" \ "1:"insn \ "2: cs %[old],%[new],%[uaddr]\n" \ - "3: jl 1b\n" \ + " jl 1b\n" \ " lhi %[rc],0\n" \ - "4: sacf 768\n" \ - EX_TABLE_UA_FAULT(0b, 4b, %[rc]) \ - EX_TABLE_UA_FAULT(1b, 4b, %[rc]) \ - EX_TABLE_UA_FAULT(2b, 4b, %[rc]) \ - EX_TABLE_UA_FAULT(3b, 4b, %[rc]) \ + "3: sacf 768\n" \ + EX_TABLE_INSN_UA_FAULT(0b, 3b, %[rc]) \ + EX_TABLE_INSN_UA_FAULT(2b, 3b, %[rc]) \ : [rc] "=d" (rc), [old] "=&d" (*old), \ [new] "=&d" (new), [uaddr] "+Q" (*uaddr) \ : [oparg] "d" (oparg) \ @@ -86,10 +84,9 @@ int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 oldval, u32 asm_inline volatile( " sacf 256\n" "0: cs %[old],%[new],%[uaddr]\n" - "1: lhi %[rc],0\n" - "2: sacf 768\n" - EX_TABLE_UA_FAULT(0b, 2b, %[rc]) - EX_TABLE_UA_FAULT(1b, 2b, %[rc]) + " lhi %[rc],0\n" + "1: sacf 768\n" + EX_TABLE_INSN_UA_FAULT(0b, 1b, %[rc]) : [rc] "=d" (rc), [old] "+d" (oldval), [uaddr] "+Q" (*uaddr) : [new] "d" (newval) : "cc", "memory"); diff --git a/arch/s390/include/asm/idals.h b/arch/s390/include/asm/idals.h index 06e1ec2afd5a..213f430a8d33 100644 --- a/arch/s390/include/asm/idals.h +++ b/arch/s390/include/asm/idals.h @@ -18,6 +18,7 @@ #include <linux/types.h> #include <linux/slab.h> #include <linux/uaccess.h> +#include <linux/kmemleak.h> #include <asm/dma-types.h> #include <asm/cio.h> @@ -147,9 +148,11 @@ static inline struct idal_buffer *idal_buffer_alloc(size_t size, int page_order) ib->data[i] = dma64_add(ib->data[i - 1], IDA_BLOCK_SIZE); continue; } - vaddr = (void *)__get_free_pages(GFP_KERNEL, page_order); + vaddr = kmalloc(PAGE_SIZE << page_order, GFP_KERNEL); if (!vaddr) goto error; + /* Only DMA addresses are retained in ib->data. */ + kmemleak_ignore(vaddr); ib->data[i] = virt_to_dma64(vaddr); } return ib; @@ -157,7 +160,7 @@ error: while (i >= nr_chunks) { i -= nr_chunks; vaddr = dma64_to_virt(ib->data[i]); - free_pages((unsigned long)vaddr, ib->page_order); + kfree(vaddr); } kfree(ib); return ERR_PTR(-ENOMEM); @@ -175,7 +178,7 @@ static inline void idal_buffer_free(struct idal_buffer *ib) nr_chunks = (PAGE_SIZE << ib->page_order) >> IDA_SIZE_SHIFT; for (i = 0; i < nr_ptrs; i += nr_chunks) { vaddr = dma64_to_virt(ib->data[i]); - free_pages((unsigned long)vaddr, ib->page_order); + kfree(vaddr); } kfree(ib); } diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 5cef215d30e7..f4517bbe405b 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -21,7 +21,7 @@ #define LC_ORDER 1 #define LC_PAGES 2 -#define LOWCORE_ALT_ADDRESS _AC(0x70000, UL) +#define LOWCORE_ALT_ADDRESS 0x70000 #ifndef __ASSEMBLER__ @@ -222,6 +222,9 @@ struct lowcore { __u8 pad_0x1900[0x2000-0x1900]; /* 0x1900 */ } __packed __aligned(8192); +#define LC_PERCPU_OFFSET offsetof(struct lowcore, percpu_offset) +#define LC_PERCPU_REGISTER offsetof(struct lowcore, percpu_register) + static __always_inline struct lowcore *get_lowcore(void) { struct lowcore *lc; diff --git a/arch/s390/include/asm/pci_insn.h b/arch/s390/include/asm/pci_insn.h index 025c6dcbf893..e6613b037f93 100644 --- a/arch/s390/include/asm/pci_insn.h +++ b/arch/s390/include/asm/pci_insn.h @@ -147,7 +147,7 @@ union zpci_sic_iib { DECLARE_STATIC_KEY_FALSE(have_mio); u8 zpci_mod_fc(u64 req, struct zpci_fib *fib, u8 *status); -int zpci_refresh_trans(u64 fn, u64 addr, u64 range); +int zpci_refresh_trans(u32 fh, u64 addr, u64 range); int __zpci_load(u64 *data, u64 req, u64 offset); int zpci_load(u64 *data, const volatile void __iomem *addr, unsigned long len); int __zpci_store(u64 data, u64 req, u64 offset); diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index feb58acf87d6..7b7a8cb51daa 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -5,6 +5,7 @@ #include <linux/preempt.h> #include <asm/cmpxchg.h> #include <asm/march.h> +#include <asm/asm.h> /* * s390 uses its own implementation for per cpu data, the offset of @@ -14,17 +15,16 @@ #define arch_raw_cpu_ptr(_ptr) \ ({ \ - unsigned long lc_percpu, tcp_ptr__; \ + unsigned long tcp_ptr__; \ \ tcp_ptr__ = (__force unsigned long)(_ptr); \ - lc_percpu = offsetof(struct lowcore, percpu_offset); \ asm_inline volatile( \ ALTERNATIVE("ag %[__ptr__],%[offzero](%%r0)\n", \ "ag %[__ptr__],%[offalt](%%r0)\n", \ ALT_FEATURE(MFEATURE_LOWCORE)) \ : [__ptr__] "+d" (tcp_ptr__) \ - : [offzero] "i" (lc_percpu), \ - [offalt] "i" (lc_percpu + LOWCORE_ALT_ADDRESS), \ + : [offzero] "i" (LC_PERCPU_OFFSET), \ + [offalt] "i" (LC_PERCPU_OFFSET + LOWCORE_ALT_ADDRESS), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ (TYPEOF_UNQUAL(*(_ptr)) __force __kernel *)tcp_ptr__; \ @@ -63,65 +63,114 @@ /* * Macros to be used for percpu code section based on atomic instructions. * - * Avoid the need to use preempt_disable() / preempt_disable() pairs and the + * Avoid the need to use preempt_disable() / preempt_enable() pairs and the * conditional preempt_schedule_notrace() function calls which come with * this. The idea is that this_cpu operations based on atomic instructions are * guarded with mviy instructions: * - * - The first mviy instruction writes the register number, which contains the - * percpu address variable to lowcore. This also indicates that a percpu - * code section is executed. + * - The first mviy instruction writes the register number of the percpu address + * variable and the even register number of a register pair (which encodes two + * registers: the even register for the percpu offset and the odd register for + * the percpu pointer) to lowcore. This also indicates that a percpu code + * section is executed. * - * - The first mviy instruction following the mviy instruction must be the ag - * instruction which adds the percpu offset to the percpu address register. + * - The mviy instruction is followed by the lg instruction which loads the + * percpu offset into the even register of the pair and the la instruction + * which adds the percpu offset and the percpu address into the odd register + * of the pair (the percpu pointer register). * * - Afterwards the atomic percpu operation follows. * * - Then a second mviy instruction writes a zero to lowcore, which indicates * the end of the percpu code section. * - * - In case of an interrupt/exception/nmi the register number which was - * written to lowcore is copied to the exception frame (pt_regs), and a zero - * is written to lowcore. + * - In case of an interrupt/exception/nmi the encoded register numbers which + * were written to lowcore are copied to the exception frame (pt_regs), and a + * zero is written to lowcore. * * - On return to the previous context it is checked if a percpu code section - * was executed (saved register number not zero), and if the process was - * migrated to a different cpu. If the percpu offset was already added to - * the percpu address register (instruction address does _not_ point to the - * ag instruction) the content of the percpu address register is adjusted so - * it points to percpu variable of the new cpu. + * was executed (saved register value not zero), and if the process was + * migrated to a different cpu. The content of the percpu offset register + * (even register of the pair) is reloaded with the current cpu's percpu + * offset and the percpu pointer register (odd register of the pair) is + * recalculated so it points to the percpu variable of the new cpu. * * Inline assemblies making use of this typically have a code sequence like: * - * MVIY_PERCPU(...) <- start of percpu code section - * AG_ALT(...) <- add percpu offset; must be the second instruction - * atomic_op <- atomic op - * MVIY_ALT(...) <- end of percpu code section + * __PCPU_BEGIN(...) <- start of percpu code section + * atomic_op <- atomic op + * __PCPU_END(...) <- end of percpu code section */ -#define MVIY_PERCPU(disp, dispalt, reg) \ - ".macro GEN_MVIY disp reg\n" \ - ".irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\n" \ - " .ifc \\reg,%%r\\rs\n" \ - " mviy \\disp(%%r0),\\rs\n" \ +#define LC_ALT_ADDR __stringify(LOWCORE_ALT_ADDRESS) + +#define DEFINE_GR_NUM \ + ".macro _GR_NUM opd, gr\n" \ + ".set \\opd,255\n" \ + ".irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\n" \ + " .ifc \\gr,%%r\\rs\n" \ + " \\opd = \\rs\n" \ " .endif\n" \ ".endr\n" \ - ".endm\n" \ - ALTERNATIVE("GEN_MVIY " disp ", " reg "\n", \ - "GEN_MVIY " dispalt ", " reg "\n", \ - ALT_FEATURE(MFEATURE_LOWCORE)) \ - ".purgem GEN_MVIY\n" - -#define MVIY_ALT(disp, dispalt) \ - ALTERNATIVE(" mviy " disp "(%%r0),0\n", \ - " mviy " dispalt "(%%r0),0\n", \ + ".if \\opd == 255\n" \ + " .error \"Illegal register number\"\n" \ + ".endif\n" \ + ".endm\n" + +#define UNDEF_GR_NUM \ + ".purgem _GR_NUM\n" + +#define PCPU_REG_PCP_SHIFT 0 +#define PCPU_REG_PCP GENMASK(3, 0) +#define PCPU_REG_OFF_SHIFT 4 +#define PCPU_REG_OFF GENMASK(7, 4) + +#define __PCPU_MVIY(lcreg, imm) \ + ALTERNATIVE(" mviy " lcreg "(%%r0)," imm "\n", \ + " mviy " lcreg "+" LC_ALT_ADDR "(%%r0)," imm "\n", \ ALT_FEATURE(MFEATURE_LOWCORE)) -#define AG_ALT(disp, dispalt, reg) \ - ALTERNATIVE(" ag " reg ", " disp "(%%r0)\n", \ - " ag " reg ", " dispalt "(%%r0)\n", \ +#define __PCPU_LG(regoff, lcoff) \ + ALTERNATIVE(" lg " regoff ", " lcoff "(%%r0)\n", \ + " lg " regoff ", " lcoff "+" LC_ALT_ADDR "(%%r0)\n", \ ALT_FEATURE(MFEATURE_LOWCORE)) +#define __PCPU_LA(regptr, regoff, regpcp) \ + " la " regptr ",0(" regoff "," regpcp ")\n" + +#define __PCPU_CALC_REGVAL(regpcp, regoff) \ + "(" regpcp " << " __stringify(PCPU_REG_PCP_SHIFT) ") |" \ + "(" regoff " << " __stringify(PCPU_REG_OFF_SHIFT) ")" + +#define __PCPU_CHECK_REGS(regpcp, regoff, regptr) \ + ".if " regoff " & 1\n" \ + " .error \"Percpu offset register must be even\"\n" \ + ".endif\n" \ + ".if (" regpcp " == 0) || (" regoff " == 0)\n" \ + " .error \"Percpu address and offset register must be non-zero\"\n"\ + ".endif\n" \ + ".if " regptr " != " regoff " + 1\n" \ + " .error \"Percpu pointer and offset register must be a pair\"\n" \ + ".endif\n" \ + ".if (" regpcp " == " regoff ") || (" regpcp " == " regptr ")\n" \ + " .error \"Percpu registers must be distinct\"\n" \ + ".endif\n" + +#define __PCPU_BEGIN(lcreg, lcoff, regpcp, regoff, regptr) \ + DEFINE_GR_NUM \ + "_GR_NUM .Lregpcp, " regpcp "\n" \ + "_GR_NUM .Lregoff, " regoff "\n" \ + "_GR_NUM .Lregptr, " regptr "\n" \ + UNDEF_GR_NUM \ + __PCPU_CHECK_REGS(".Lregpcp", ".Lregoff", ".Lregptr") \ + ".set .Lregval, " __PCPU_CALC_REGVAL(".Lregpcp", ".Lregoff") "\n" \ + __PCPU_MVIY(lcreg, ".Lregval") \ + __PCPU_LG(regoff, lcoff) \ + __PCPU_LA(regptr, regoff, regpcp) + +#define __PCPU_END(lcreg) \ + __PCPU_MVIY(lcreg, "0") + #ifndef MARCH_HAS_Z196_FEATURES #define this_cpu_add_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) @@ -137,44 +186,38 @@ #define arch_this_cpu_add(pcp, val, op1, op2, szcast) \ do { \ - unsigned long lc_pcpr, lc_pcpo; \ typedef typeof(pcp) pcp_op_T__; \ + union register_pair rp__; \ pcp_op_T__ val__ = (val); \ - pcp_op_T__ old__, *ptr__; \ + pcp_op_T__ *ptr__; \ \ - lc_pcpr = offsetof(struct lowcore, percpu_register); \ - lc_pcpo = offsetof(struct lowcore, percpu_offset); \ ptr__ = PERCPU_PTR(&(pcp)); \ if (__builtin_constant_p(val__) && \ ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ - asm volatile( \ - MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ - AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ - op2 " 0(%[ptr__]),%[val__]\n" \ - MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ - : [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + asm_inline volatile( \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ + "%[pair__]","%N[pair__]") \ + op2 " 0(%N[pair__]),%[val__]\n" \ + __PCPU_END("%[lcreg]") \ + : [pair__] "=&a" (rp__.pair), "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ - : [val__] "i" ((szcast)val__), \ - [disppcpr] "i" (lc_pcpr), \ - [disppcpo] "i" (lc_pcpo), \ - [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ - [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + : [val__] "i" ((szcast)val__), [ptr__] "a" (ptr__), \ + [lcreg] "i" (LC_PERCPU_REGISTER), \ + [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ } else { \ - asm volatile( \ - MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ - AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ - op1 " %[old__],%[val__],0(%[ptr__])\n" \ - MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ - : [old__] "=&d" (old__), \ - [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + asm_inline volatile( \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ + "%[pair__]","%N[pair__]") \ + op1 " %[val__],%[val__],0(%N[pair__])\n" \ + __PCPU_END("%[lcreg]") \ + : [val__] "+&d" (val__), [pair__] "=&a" (rp__.pair), \ + "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ - : [val__] "d" (val__), \ - [disppcpr] "i" (lc_pcpr), \ - [disppcpo] "i" (lc_pcpo), \ - [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ - [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + : [ptr__] "a" (ptr__), \ + [lcreg] "i" (LC_PERCPU_REGISTER), \ + [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ } \ @@ -185,27 +228,23 @@ do { \ #define arch_this_cpu_add_return(pcp, val, op) \ ({ \ - unsigned long lc_pcpr, lc_pcpo; \ typedef typeof(pcp) pcp_op_T__; \ + union register_pair rp__; \ pcp_op_T__ val__ = (val); \ pcp_op_T__ old__, *ptr__; \ \ - lc_pcpr = offsetof(struct lowcore, percpu_register); \ - lc_pcpo = offsetof(struct lowcore, percpu_offset); \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ - AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ - op " %[old__],%[val__],0(%[ptr__])\n" \ - MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ - : [old__] "=&d" (old__), \ - [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ + "%[pair__]","%N[pair__]") \ + op " %[old__],%[val__],0(%N[pair__])\n" \ + __PCPU_END("%[lcreg]") \ + : [old__] "=&d" (old__), [pair__] "=&a" (rp__.pair), \ + "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ - : [val__] "d" (val__), \ - [disppcpr] "i" (lc_pcpr), \ - [disppcpo] "i" (lc_pcpo), \ - [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ - [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + : [val__] "d" (val__), [ptr__] "a" (ptr__), \ + [lcreg] "i" (LC_PERCPU_REGISTER), \ + [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ old__ + val__; \ @@ -216,27 +255,23 @@ do { \ #define arch_this_cpu_to_op(pcp, val, op) \ do { \ - unsigned long lc_pcpr, lc_pcpo; \ typedef typeof(pcp) pcp_op_T__; \ + union register_pair rp__; \ pcp_op_T__ val__ = (val); \ - pcp_op_T__ old__, *ptr__; \ + pcp_op_T__ *ptr__; \ \ - lc_pcpr = offsetof(struct lowcore, percpu_register); \ - lc_pcpo = offsetof(struct lowcore, percpu_offset); \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ - AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ - op " %[old__],%[val__],0(%[ptr__])\n" \ - MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ - : [old__] "=&d" (old__), \ - [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ + "%[pair__]","%N[pair__]") \ + op " %[val__],%[val__],0(%N[pair__])\n" \ + __PCPU_END("%[lcreg]") \ + : [val__] "+&d" (val__), [pair__] "=&a" (rp__.pair), \ + "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ - : [val__] "d" (val__), \ - [disppcpr] "i" (lc_pcpr), \ - [disppcpo] "i" (lc_pcpo), \ - [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ - [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + : [ptr__] "a" (ptr__), \ + [lcreg] "i" (LC_PERCPU_REGISTER), \ + [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ } while (0) @@ -250,24 +285,22 @@ do { \ #define arch_this_cpu_read(pcp, op) \ ({ \ - unsigned long lc_pcpr, lc_pcpo, res__; \ typedef typeof(pcp) pcp_op_T__; \ + union register_pair rp__; \ + unsigned long res__; \ pcp_op_T__ *ptr__; \ \ - lc_pcpr = offsetof(struct lowcore, percpu_register); \ - lc_pcpo = offsetof(struct lowcore, percpu_offset); \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ - AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ - op " %[res__],0(%[ptr__])\n" \ - MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ - : [res__] "=&d" (res__), [ptr__] "+&a" (ptr__), \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ + "%[pair__]","%N[pair__]") \ + op " %[res__],0(%N[pair__])\n" \ + __PCPU_END("%[lcreg]") \ + : [res__] "=&d" (res__), [pair__] "=&a" (rp__.pair), \ "=m" (((struct lowcore *)0)->percpu_register) \ - : [disppcpr] "i" (lc_pcpr), \ - [disppcpo] "i" (lc_pcpo), \ - [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ - [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + : [ptr__] "a" (ptr__), \ + [lcreg] "i" (LC_PERCPU_REGISTER), \ + [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (*ptr__), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ @@ -281,25 +314,21 @@ do { \ #define arch_this_cpu_write(pcp, val, op) \ do { \ - unsigned long lc_pcpr, lc_pcpo; \ typedef typeof(pcp) pcp_op_T__; \ + union register_pair rp__; \ pcp_op_T__ *ptr__, val__ = (val); \ \ - lc_pcpr = offsetof(struct lowcore, percpu_register); \ - lc_pcpo = offsetof(struct lowcore, percpu_offset); \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ - AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ - op " %[val__],0(%[ptr__])\n" \ - MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ - : [ptr__] "+&a" (ptr__), "=m" (*ptr__), \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ + "%[pair__]","%N[pair__]") \ + op " %[val__],0(%N[pair__])\n" \ + __PCPU_END("%[lcreg]") \ + : [pair__] "=&a" (rp__.pair), "=m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ - : [val__] "d" (val__), \ - [disppcpr] "i" (lc_pcpr), \ - [disppcpo] "i" (lc_pcpo), \ - [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ - [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + : [val__] "d" (val__), [ptr__] "a" (ptr__), \ + [lcreg] "i" (LC_PERCPU_REGISTER), \ + [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ } while (0) @@ -309,7 +338,7 @@ do { \ #define this_cpu_write_4(pcp, val) arch_this_cpu_write(pcp, val, "st") #define this_cpu_write_8(pcp, val) arch_this_cpu_write(pcp, val, "stg") -#define arch_this_cpu_cmpxchg(pcp, oval, nval) \ +#define arch_this_cpu_cmpxchg_simple(pcp, oval, nval) \ ({ \ typedef typeof(pcp) pcp_op_T__; \ pcp_op_T__ ret__; \ @@ -321,28 +350,100 @@ do { \ ret__; \ }) -#define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) -#define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) -#define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) -#define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) +#define arch_this_cpu_cmpxchg(pcp, oval, nval, op) \ +({ \ + typedef typeof(pcp) pcp_op_T__; \ + pcp_op_T__ old__ = (oval), new__ = (nval); \ + union register_pair rp__; \ + pcp_op_T__ *ptr__; \ + \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ + "%[pair__]","%N[pair__]") \ + op " %[old__],%[new__],0(%N[pair__])\n" \ + __PCPU_END("%[lcreg]") \ + : [old__] "+&d" (old__), [pair__] "=&a" (rp__.pair), \ + "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [new__] "d" (new__), [ptr__] "a" (ptr__), \ + [lcreg] "i" (LC_PERCPU_REGISTER), \ + [lcoff] "i" (LC_PERCPU_OFFSET), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "memory", "cc"); \ + old__; \ +}) + +#define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg_simple(pcp, oval, nval) +#define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg_simple(pcp, oval, nval) +#define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval, "cs") +#define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval, "csg") #define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n) #define this_cpu_cmpxchg128(pcp, oval, nval) \ ({ \ typedef typeof(pcp) pcp_op_T__; \ - u128 old__, new__, ret__; \ + u128 old__ = (oval), new__ = (nval); \ + union register_pair rp__; \ pcp_op_T__ *ptr__; \ - old__ = oval; \ - new__ = nval; \ - preempt_disable_notrace(); \ - ptr__ = raw_cpu_ptr(&(pcp)); \ - ret__ = cmpxchg128((void *)ptr__, old__, new__); \ - preempt_enable_notrace(); \ - ret__; \ + \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ + "%[pair__]","%N[pair__]") \ + " cdsg %[old__],%[new__],0(%N[pair__])\n" \ + __PCPU_END("%[lcreg]") \ + : [old__] "+&d" (old__), [pair__] "=&a" (rp__.pair), \ + "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [new__] "d" (new__), [ptr__] "a" (ptr__), \ + [lcreg] "i" (LC_PERCPU_REGISTER), \ + [lcoff] "i" (LC_PERCPU_OFFSET), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "memory", "cc"); \ + old__; \ }) -#define arch_this_cpu_xchg(pcp, nval) \ +#ifdef __HAVE_ASM_FLAG_OUTPUTS__ + +#define arch_this_cpu_try_cmpxchg(pcp, ovalp, nval, op) \ +({ \ + typedef typeof(pcp) pcp_op_T__; \ + typeof(ovalp) ovalp__ = (ovalp); \ + pcp_op_T__ old__ = *ovalp__; \ + pcp_op_T__ new__ = (nval); \ + union register_pair rp__; \ + pcp_op_T__ *ptr__; \ + int cc__; \ + \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ + "%[pair__]","%N[pair__]") \ + op " %[old__],%[new__],0(%N[pair__])\n" \ + __PCPU_END("%[lcreg]") \ + : [old__] "+&d" (old__), [pair__] "=&a" (rp__.pair), \ + "+m" (*ptr__), "=@cc" (cc__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [new__] "d" (new__), [ptr__] "a" (ptr__), \ + [lcreg] "i" (LC_PERCPU_REGISTER), \ + [lcoff] "i" (LC_PERCPU_OFFSET), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "memory"); \ + if (unlikely(cc__)) \ + *ovalp__ = old__; \ + likely(cc__ == 0); \ +}) + +#define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) arch_this_cpu_try_cmpxchg(pcp, ovalp, nval, "cs") +#define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) arch_this_cpu_try_cmpxchg(pcp, ovalp, nval, "csg") + +#define this_cpu_try_cmpxchg64(pcp, ovalp, nval) this_cpu_try_cmpxchg_8(pcp, ovalp, nval) + +#endif /* __HAVE_ASM_FLAG_OUTPUTS__ */ + +#define arch_this_cpu_xchg_simple(pcp, nval) \ ({ \ typeof(pcp) *ptr__; \ typeof(pcp) ret__; \ @@ -353,10 +454,36 @@ do { \ ret__; \ }) -#define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg(pcp, nval) -#define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg(pcp, nval) -#define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval) -#define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval) +#define arch_this_cpu_xchg(pcp, nval, ldop, csop) \ +({ \ + typedef typeof(pcp) pcp_op_T__; \ + pcp_op_T__ old__, new__ = (nval); \ + union register_pair rp__; \ + pcp_op_T__ *ptr__; \ + \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ + "%[pair__]","%N[pair__]") \ + " " ldop " %[old__],0(%N[pair__])\n" \ + "0: " csop " %[old__],%[new__],0(%N[pair__])\n"\ + " jnz 0b\n" \ + __PCPU_END("%[lcreg]") \ + : [old__] "=&d" (old__), [pair__] "=&a" (rp__.pair), \ + "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [new__] "d" (new__), [ptr__] "a" (ptr__), \ + [lcreg] "i" (LC_PERCPU_REGISTER), \ + [lcoff] "i" (LC_PERCPU_OFFSET), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "memory", "cc"); \ + old__; \ +}) + +#define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg_simple(pcp, nval) +#define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg_simple(pcp, nval) +#define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval, "l", "cs") +#define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval, "lg", "csg") #include <asm-generic/percpu.h> diff --git a/arch/s390/include/asm/ptrace.h b/arch/s390/include/asm/ptrace.h index 6411e3584283..3e56ef8c268b 100644 --- a/arch/s390/include/asm/ptrace.h +++ b/arch/s390/include/asm/ptrace.h @@ -17,11 +17,15 @@ #define PIF_PSW_ADDR_ADJUSTED 1 /* psw address has been adjusted */ #define PIF_SYSCALL_RET_SET 2 /* return value was set via ptrace */ #define PIF_FTRACE_FULL_REGS 3 /* all register contents valid (ftrace) */ +#define PIF_WARN_TRAP_SKIPPED 4 /* __WARN_trap() was skipped */ +#define PIF_PGM_CHECK 5 /* inside a program check */ #define _PIF_SYSCALL BIT(PIF_SYSCALL) #define _PIF_ADDR_PSW_ADJUSTED BIT(PIF_PSW_ADDR_ADJUSTED) #define _PIF_SYSCALL_RET_SET BIT(PIF_SYSCALL_RET_SET) #define _PIF_FTRACE_FULL_REGS BIT(PIF_FTRACE_FULL_REGS) +#define _PIF_WARN_TRAP_SKIPPED BIT(PIF_WARN_TRAP_SKIPPED) +#define _PIF_PGM_CHECK BIT(PIF_PGM_CHECK) #define PSW32_MASK_PER _AC(0x40000000, UL) #define PSW32_MASK_DAT _AC(0x04000000, UL) @@ -240,6 +244,8 @@ static inline void instruction_pointer_set(struct pt_regs *regs, regs->psw.addr = val; } +unsigned long get_trap_ip(struct pt_regs *regs); + int regs_query_register_offset(const char *name); const char *regs_query_register_name(unsigned int offset); diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index a9f32c53f699..ac99b32c6ceb 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -43,10 +43,8 @@ raw_copy_from_user(void *to, const void __user *from, unsigned long size) asm_inline volatile( " lhi %%r0,%[spec]\n" "0: mvcos %[to],%[from],%[size]\n" - "1: nopr %%r7\n" CC_IPM(cc) - EX_TABLE_UA_MVCOS_FROM(0b, 0b) - EX_TABLE_UA_MVCOS_FROM(1b, 0b) + EX_TABLE_INSN_UA_MVCOS_FROM(0b, 0b) : CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char *)to) : [spec] "I" (0x81), [from] "Q" (*(const char __user *)from) : CC_CLOBBER_LIST("memory", "0")); @@ -71,10 +69,8 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long size) asm_inline volatile( " llilh %%r0,%[spec]\n" "0: mvcos %[to],%[from],%[size]\n" - "1: nopr %%r7\n" CC_IPM(cc) - EX_TABLE_UA_MVCOS_TO(0b, 0b) - EX_TABLE_UA_MVCOS_TO(1b, 0b) + EX_TABLE_INSN_UA_MVCOS_TO(0b, 0b) : CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char __user *)to) : [spec] "I" (0x81), [from] "Q" (*(const char *)from) : CC_CLOBBER_LIST("memory", "0")); @@ -123,9 +119,7 @@ __put_user_##type##_noinstr(unsigned type __user *to, \ asm goto( \ " llilh %%r0,%[spec]\n" \ "0: mvcos %[to],%[from],%[size]\n" \ - "1: nopr %%r7\n" \ - EX_TABLE(0b, %l[Efault]) \ - EX_TABLE(1b, %l[Efault]) \ + EX_TABLE_INSN(0b, %l[Efault]) \ : [to] "+Q" (*to) \ : [size] "d" (size), [from] "Q" (*from), \ [spec] "I" (0x81) \ @@ -150,10 +144,9 @@ __put_user_##type##_noinstr(unsigned type __user *to, \ asm_inline volatile( \ " llilh %%r0,%[spec]\n" \ "0: mvcos %[to],%[from],%[size]\n" \ - "1: lhi %[rc],0\n" \ - "2:\n" \ - EX_TABLE_UA_FAULT(0b, 2b, %[rc]) \ - EX_TABLE_UA_FAULT(1b, 2b, %[rc]) \ + " lhi %[rc],0\n" \ + "1:\n" \ + EX_TABLE_INSN_UA_FAULT(0b, 1b, %[rc]) \ : [rc] "=d" (rc), [to] "+Q" (*to) \ : [size] "d" (size), [from] "Q" (*from), \ [spec] "I" (0x81) \ @@ -238,9 +231,7 @@ __get_user_##type##_noinstr(unsigned type *to, \ asm goto( \ " lhi %%r0,%[spec]\n" \ "0: mvcos %[to],%[from],%[size]\n" \ - "1: nopr %%r7\n" \ - EX_TABLE(0b, %l[Efault]) \ - EX_TABLE(1b, %l[Efault]) \ + EX_TABLE_INSN(0b, %l[Efault]) \ : [to] "=Q" (*to) \ : [size] "d" (size), [from] "Q" (*from), \ [spec] "I" (0x81) \ @@ -266,10 +257,9 @@ __get_user_##type##_noinstr(unsigned type *to, \ asm_inline volatile( \ " lhi %%r0,%[spec]\n" \ "0: mvcos %[to],%[from],%[size]\n" \ - "1: lhi %[rc],0\n" \ - "2:\n" \ - EX_TABLE_UA_FAULT(0b, 2b, %[rc]) \ - EX_TABLE_UA_FAULT(1b, 2b, %[rc]) \ + " lhi %[rc],0\n" \ + "1:\n" \ + EX_TABLE_INSN_UA_FAULT(0b, 1b, %[rc]) \ : [rc] "=d" (rc), [to] "=Q" (*to) \ : [size] "d" (size), [from] "Q" (*from), \ [spec] "I" (0x81) \ @@ -374,10 +364,8 @@ __clear_user(void __user *to, unsigned long size) asm_inline volatile( " llilh %%r0,%[spec]\n" "0: mvcos %[to],%[from],%[size]\n" - "1: nopr %%r7\n" CC_IPM(cc) - EX_TABLE_UA_MVCOS_TO(0b, 0b) - EX_TABLE_UA_MVCOS_TO(1b, 0b) + EX_TABLE_INSN_UA_MVCOS_TO(0b, 0b) : CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char __user *)to) : [spec] "I" (0x81), [from] "Q" (*(const char *)empty_zero_page) : CC_CLOBBER_LIST("memory", "0")); @@ -418,9 +406,7 @@ do { \ case 8: \ asm goto( \ "0: mvc %O[_dst](%[_len],%R[_dst]),%[_src]\n" \ - "1: nopr %%r7\n" \ - EX_TABLE(0b, %l[err_label]) \ - EX_TABLE(1b, %l[err_label]) \ + EX_TABLE_INSN(0b, %l[err_label]) \ : [_dst] "=Q" (*(type *)dst) \ : [_src] "Q" (*(type *)(src)), \ [_len] "I" (sizeof(type)) \ @@ -447,10 +433,9 @@ do { \ case 8: \ asm_inline volatile( \ "0: mvc 0(%[_len],%[_dst]),%[_src]\n" \ - "1: lhi %[_rc],0\n" \ - "2:\n" \ - EX_TABLE_UA_FAULT(0b, 2b, %[_rc]) \ - EX_TABLE_UA_FAULT(1b, 2b, %[_rc]) \ + " lhi %[_rc],0\n" \ + "1:\n" \ + EX_TABLE_INSN_UA_FAULT(0b, 1b, %[_rc]) \ : [_rc] "=d" (__rc), \ "=m" (*__dst) \ : [_src] "Q" (*(type *)(src)), \ diff --git a/arch/s390/include/asm/vtime.h b/arch/s390/include/asm/vtime.h index da116a93d3b6..22e494b22192 100644 --- a/arch/s390/include/asm/vtime.h +++ b/arch/s390/include/asm/vtime.h @@ -8,6 +8,8 @@ DECLARE_PER_CPU(u64, mt_cycles[8]); +void vtime_init(void); + static inline void update_timer_sys(void) { struct lowcore *lc = get_lowcore(); diff --git a/arch/s390/include/asm/vtimer.h b/arch/s390/include/asm/vtimer.h deleted file mode 100644 index e601adaa6320..000000000000 --- a/arch/s390/include/asm/vtimer.h +++ /dev/null @@ -1,30 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright IBM Corp. 2003, 2012 - * Virtual CPU timer - * - * Author(s): Jan Glauber <jan.glauber@de.ibm.com> - */ - -#ifndef _ASM_S390_TIMER_H -#define _ASM_S390_TIMER_H - -#define VTIMER_MAX_SLICE (0x7fffffffffffffffULL) - -struct vtimer_list { - struct list_head entry; - u64 expires; - u64 interval; - void (*function)(unsigned long); - unsigned long data; -}; - -extern void init_virt_timer(struct vtimer_list *timer); -extern void add_virt_timer(struct vtimer_list *timer); -extern void add_virt_timer_periodic(struct vtimer_list *timer); -extern int mod_virt_timer(struct vtimer_list *timer, u64 expires); -extern int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires); -extern int del_virt_timer(struct vtimer_list *timer); -extern void vtime_init(void); - -#endif /* _ASM_S390_TIMER_H */ diff --git a/arch/s390/include/asm/word-at-a-time.h b/arch/s390/include/asm/word-at-a-time.h index eaa19dee7699..0b7779c8d5a4 100644 --- a/arch/s390/include/asm/word-at-a-time.h +++ b/arch/s390/include/asm/word-at-a-time.h @@ -54,9 +54,8 @@ static inline unsigned long load_unaligned_zeropad(const void *addr) asm_inline volatile( "0: lg %[data],0(%[addr])\n" - "1: nopr %%r7\n" - EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr]) - EX_TABLE_ZEROPAD(1b, 1b, %[data], %[addr]) + "1:\n" + EX_TABLE_INSN_ZEROPAD(0b, 1b, %[data], %[addr]) : [data] "=d" (data) : [addr] "a" (addr), "m" (*(unsigned long *)addr)); return data; diff --git a/arch/s390/kernel/cert_store.c b/arch/s390/kernel/cert_store.c index dc1992a675de..998f2da56175 100644 --- a/arch/s390/kernel/cert_store.c +++ b/arch/s390/kernel/cert_store.c @@ -236,9 +236,9 @@ static int __diag320(unsigned long subcode, void *addr) union register_pair rp = { .even = (unsigned long)addr, }; asm_inline volatile( - " diag %[rp],%[subcode],0x320\n" - "0: nopr %%r7\n" - EX_TABLE(0b, 0b) + "0: diag %[rp],%[subcode],0x320\n" + "1:\n" + EX_TABLE_INSN(0b, 1b) : [rp] "+d" (rp.pair) : [subcode] "d" (subcode) : "cc", "memory"); diff --git a/arch/s390/kernel/diag/diag.c b/arch/s390/kernel/diag/diag.c index 56b862ba9be8..f23b3e2b070e 100644 --- a/arch/s390/kernel/diag/diag.c +++ b/arch/s390/kernel/diag/diag.c @@ -196,9 +196,9 @@ static inline int __diag204(unsigned long *subcode, unsigned long size, void *ad union register_pair rp = { .even = *subcode, .odd = size }; asm_inline volatile( - " diag %[addr],%[rp],0x204\n" - "0: nopr %%r7\n" - EX_TABLE(0b,0b) + "0: diag %[addr],%[rp],0x204\n" + "1:\n" + EX_TABLE_INSN(0b, 1b) : [rp] "+&d" (rp.pair) : [addr] "d" (addr) : "memory"); *subcode = rp.even; return rp.odd; @@ -287,10 +287,10 @@ int diag224(void *ptr) diag_stat_inc(DIAG_STAT_X224); asm_inline volatile("\n" - " diag %[type],%[addr],0x224\n" - "0: lhi %[rc],0\n" + "0: diag %[type],%[addr],0x224\n" + " lhi %[rc],0\n" "1:\n" - EX_TABLE(0b,1b) + EX_TABLE_INSN(0b, 1b) : [rc] "+d" (rc) , "=m" (*(struct { char buf[PAGE_SIZE]; } *)ptr) : [type] "d" (0), [addr] "d" (addr)); diff --git a/arch/s390/kernel/dis.c b/arch/s390/kernel/dis.c index 1cec93895b3a..d2d299ba8b86 100644 --- a/arch/s390/kernel/dis.c +++ b/arch/s390/kernel/dis.c @@ -51,7 +51,7 @@ struct s390_operand { struct s390_insn { union { - const char name[5]; + const char name[5] __nonstring; struct { unsigned char zero; unsigned int offset; @@ -340,7 +340,7 @@ static const unsigned char formats[][6] = { [INSTR_VSI_URDV] = { V_32, D_20, B_16, U8_8, 0, 0 }, }; -static char long_insn_name[][7] = LONG_INSN_INITIALIZER; +static char long_insn_name[][7] __nonstring_array = LONG_INSN_INITIALIZER; static struct s390_insn opcode[] = OPCODE_TABLE_INITIALIZER; static struct s390_opcode_offset opcode_offset[] = OPCODE_OFFSET_INITIALIZER; @@ -503,7 +503,8 @@ static int copy_from_regs(struct pt_regs *regs, void *dst, void *src, int len) void show_code(struct pt_regs *regs) { char *mode = user_mode(regs) ? "User" : "Krnl"; - unsigned long addr, pswaddr; + unsigned long addr, pswaddr, trapaddr; + bool pgm_check; unsigned char code[64]; char buffer[128], *ptr; int start, end, opsize, hops, i; @@ -511,6 +512,9 @@ void show_code(struct pt_regs *regs) pswaddr = regs->psw.addr; if (test_pt_regs_flag(regs, PIF_PSW_ADDR_ADJUSTED)) pswaddr = __forward_psw(regs->psw, regs->int_code >> 16); + pgm_check = test_pt_regs_flag(regs, PIF_PGM_CHECK); + if (pgm_check) + trapaddr = get_trap_ip(regs); /* Get a snapshot of the 64 bytes surrounding the fault address. */ for (start = 32; start && pswaddr >= 34 - start; start -= 2) { addr = pswaddr - 34 + start; @@ -540,34 +544,33 @@ void show_code(struct pt_regs *regs) start += 2; } /* Decode the instructions. */ - ptr = buffer; - ptr += sprintf(ptr, "%s Code:", mode); + printk("%s Code:\n", mode); hops = 0; while (start < end && hops < 8) { + ptr = buffer; opsize = insn_length(code[start]); - if (start + opsize == 32) - *ptr++ = '*'; + addr = pswaddr + start - 32; + if (start == 32 && test_pt_regs_flag(regs, PIF_WARN_TRAP_SKIPPED)) + ptr += sprintf(ptr, " WARN RET >"); + else if (pgm_check && addr == trapaddr && start == 32) + ptr += sprintf(ptr, " PGM+PSW >"); + else if (pgm_check && addr == trapaddr) + ptr += sprintf(ptr, " PGM >"); else if (start == 32) - *ptr++ = '>'; + ptr += sprintf(ptr, " PSW >"); else - *ptr++ = ' '; - addr = pswaddr + start - 32; + ptr += sprintf(ptr, " "); ptr += sprintf(ptr, "%px: ", (void *)addr); if (start + opsize >= end) break; for (i = 0; i < opsize; i++) ptr += sprintf(ptr, "%02x", code[start + i]); - *ptr++ = '\t'; - if (i < 6) - *ptr++ = '\t'; + ptr += sprintf(ptr, "%*s", 13 - 2 * opsize, " "); ptr += print_insn(ptr, code + start, addr); start += opsize; - pr_cont("%s", buffer); - ptr = buffer; - ptr += sprintf(ptr, "\n "); + printk("%s\n", buffer); hops++; } - pr_cont("\n"); } void print_fn_code(unsigned char *code, unsigned long len) diff --git a/arch/s390/kernel/dumpstack.c b/arch/s390/kernel/dumpstack.c index f9d52e05e01e..4ec08264b3ef 100644 --- a/arch/s390/kernel/dumpstack.c +++ b/arch/s390/kernel/dumpstack.c @@ -203,8 +203,11 @@ void __noreturn die(struct pt_regs *regs, const char *str) console_verbose(); spin_lock_irq(&die_lock); bust_spinlocks(1); - printk("%s: %04x ilc:%d [#%d]", str, regs->int_code & 0xffff, - regs->int_code >> 17, ++die_counter); + if (test_pt_regs_flag(regs, PIF_WARN_TRAP_SKIPPED)) + printk("%s: [#%d]", str, ++die_counter); + else + printk("%s: %04x ilc:%d [#%d]", str, regs->int_code & 0xffff, + regs->int_code >> 17, ++die_counter); pr_cont("SMP "); if (debug_pagealloc_enabled()) pr_cont("DEBUG_PAGEALLOC"); diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index b27239c03d79..a603e034fa82 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -153,16 +153,28 @@ static __init void setup_topology(void) void __init __do_early_pgm_check(struct pt_regs *regs) { struct lowcore *lc = get_lowcore(); - unsigned long ip; + regs->flags = 0; regs->int_code = lc->pgm_int_code; regs->int_parm_long = lc->trans_exc_code; regs->last_break = lc->pgm_last_break; - ip = __rewind_psw(regs->psw, regs->int_code >> 16); + regs->monitor_code = lc->monitor_code; + set_pt_regs_flag(regs, PIF_PGM_CHECK); /* Monitor Event? Might be a warning */ if ((regs->int_code & PGM_INT_CODE_MASK) == 0x40) { - if (report_bug(ip, regs) == BUG_TRAP_TYPE_WARN) + enum bug_trap_type btt; + unsigned long ip; + + if (regs->monitor_code == MONCODE_BUG_ARG) { + regs->psw.addr = regs->gprs[14]; + set_pt_regs_flag(regs, PIF_WARN_TRAP_SKIPPED); + btt = report_bug_entry((struct bug_entry *)regs->gprs[2], regs); + } else { + ip = __rewind_psw(regs->psw, regs->int_code >> 16); + btt = report_bug(ip, regs); + } + if (btt == BUG_TRAP_TYPE_WARN) return; } if (fixup_exception(regs)) @@ -173,8 +185,12 @@ void __init __do_early_pgm_check(struct pt_regs *regs) * some basic information in case it is too early for printk(). */ register_early_console(); - early_printk("PANIC: early exception %04x PSW: %016lx %016lx\n", - regs->int_code & 0xffff, regs->psw.mask, regs->psw.addr); + if (test_pt_regs_flag(regs, PIF_WARN_TRAP_SKIPPED)) + early_printk("PANIC: early exception PSW: %016lx %016lx\n", + regs->psw.mask, regs->psw.addr); + else + early_printk("PANIC: early exception %04x PSW: %016lx %016lx\n", + regs->int_code & 0xffff, regs->psw.mask, regs->psw.addr); show_regs(regs); disabled_wait(); } diff --git a/arch/s390/kernel/ebcdic.c b/arch/s390/kernel/ebcdic.c index 0e51fa537262..970aace8719d 100644 --- a/arch/s390/kernel/ebcdic.c +++ b/arch/s390/kernel/ebcdic.c @@ -392,10 +392,48 @@ __u8 _ebc_toupper[256] = 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF }; +/* + * ASCII subset of EBCDIC -> character classes + */ +const __u8 _ebctype_asc[256] = { + _C, _C, _C, _C, _C, _C|_S, _C, _C, /* 0x00-0x07 */ + _C, _C, _C, _C|_S, _C|_S, _C|_S, _C, _C, /* 0x08-0x0f */ + _C, _C, _C, _C, _C, _C|_S, _C, _C, /* 0x10-0x17 */ + _C, _C, _C, _C, _C, _C, _C, _C, /* 0x18-0x1f */ + _C, _C, _C, _C, _C, _C|_S, _C, _C, /* 0x20-0x27 */ + _C, _C, _C, _C, _C, _C, _C, _C, /* 0x28-0x2f */ + 0, 0, _C, _C, _C, _C, _C, _C, /* 0x30-0x37 */ + _C, _C, _C, _C, _C, _C, 0, _C, /* 0x38-0x3f */ + _S|_SP, 0, 0, 0, 0, 0, 0, 0, /* 0x40-0x47 */ + 0, 0, 0, _P, _P, _P, _P, _P, /* 0x48-0x4f */ + _P, 0, 0, 0, 0, 0, 0, 0, /* 0x50-0x57 */ + 0, 0, _P, _P, _P, _P, _P, 0, /* 0x58-0x5f */ + _P, _P, 0, 0, 0, 0, 0, 0, /* 0x60-0x67 */ + 0, 0, 0, _P, _P, _P, _P, _P, /* 0x68-0x6f */ + 0, 0, 0, 0, 0, 0, 0, 0, /* 0x70-0x77 */ + 0, _P, _P, _P, _P, _P, _P, _P, /* 0x78-0x7f */ + 0, _L|_X, _L|_X, _L|_X, _L|_X, _L|_X, _L|_X, _L, /* 0x80-0x87 */ + _L, _L, 0, 0, 0, 0, 0, 0, /* 0x88-0x8f */ + 0, _L, _L, _L, _L, _L, _L, _L, /* 0x90-0x97 */ + _L, _L, 0, 0, 0, 0, 0, 0, /* 0x98-0x9f */ + 0, _P, _L, _L, _L, _L, _L, _L, /* 0xa0-0xa7 */ + _L, _L, 0, 0, 0, 0, 0, 0, /* 0xa8-0xaf */ + _P, 0, 0, 0, 0, 0, 0, 0, /* 0xb0-0xb7 */ + 0, 0, _P, _P, 0, 0, 0, 0, /* 0xb8-0xbf */ + _P, _U|_X, _U|_X, _U|_X, _U|_X, _U|_X, _U|_X, _U, /* 0xc0-0xc7 */ + _U, _U, 0, 0, 0, 0, 0, 0, /* 0xc8-0xcf */ + _P, _U, _U, _U, _U, _U, _U, _U, /* 0xd0-0xd7 */ + _U, _U, 0, 0, 0, 0, 0, 0, /* 0xd8-0xdf */ + _P, 0, _U, _U, _U, _U, _U, _U, /* 0xe0-0xe7 */ + _U, _U, 0, 0, 0, 0, 0, 0, /* 0xe8-0xef */ + _D, _D, _D, _D, _D, _D, _D, _D, /* 0xf0-0xf7 */ + _D, _D, 0, 0, 0, 0, 0, _C, /* 0xf8-0xff */ +}; + EXPORT_SYMBOL(_ascebc_500); EXPORT_SYMBOL(_ebcasc_500); EXPORT_SYMBOL(_ascebc); EXPORT_SYMBOL(_ebcasc); EXPORT_SYMBOL(_ebc_tolower); EXPORT_SYMBOL(_ebc_toupper); - +EXPORT_SYMBOL(_ebctype_asc); diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 10dd9bbdf985..4242cd7ab2fe 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -613,6 +613,8 @@ SYM_CODE_END(early_pgm_check_handler) * The synchronous or the asynchronous stack pointer is invalid. We are dead. * No need to properly save the registers, we are going to panic anyway. * Setup a pt_regs so that show_trace can provide a good call trace. + * Copy lowcore of the current CPU to stack_invalid_lowcore before it + * is modified by printing debug messages and crash handling. */ SYM_CODE_START(stack_invalid) GET_LC %r15 @@ -621,8 +623,21 @@ SYM_CODE_START(stack_invalid) stmg %r0,%r7,__PT_R0(%r11) stmg %r8,%r9,__PT_PSW(%r11) mvc __PT_R8(64,%r11),0(%r14) + xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) GET_LC %r2 - mvc __PT_ORIG_GPR2(8,%r11),__LC_PGM_LAST_BREAK(%r2) + mvc __PT_LAST_BREAK(8,%r11),__LC_PGM_LAST_BREAK(%r2) + /* copy lowcore; skip if used by another CPU */ + larl %r4,stack_invalid_lowcore_used + lhi %r1,0 + lhi %r3,1 + cs %r1,%r3,0(%r4) + jne .Lskip + larl %r0,stack_invalid_lowcore + lghi %r1,LC_PAGES * PAGE_SIZE + lgr %r4,%r2 + lgr %r5,%r1 + mvcl %r0,%r4 +.Lskip: xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) lgr %r2,%r11 # pass pointer to pt_regs jg kernel_stack_invalid diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h index dc84ab497251..5d6685e5bf5f 100644 --- a/arch/s390/kernel/entry.h +++ b/arch/s390/kernel/entry.h @@ -11,6 +11,7 @@ #include <linux/types.h> #include <linux/signal.h> #include <asm/extable.h> +#include <asm/lowcore.h> #include <asm/ptrace.h> #include <asm/idle.h> @@ -64,6 +65,9 @@ long sys_s390_sthyi(unsigned long function_code, void __user *buffer, u64 __user unsigned long stack_alloc(void); void stack_free(unsigned long stack); +extern int stack_invalid_lowcore_used; +extern struct lowcore stack_invalid_lowcore; + extern char kprobes_insn_page[]; extern char _samode31[], _eamode31[]; diff --git a/arch/s390/kernel/hiperdispatch.c b/arch/s390/kernel/hiperdispatch.c index c21496f0a141..1397dab8a677 100644 --- a/arch/s390/kernel/hiperdispatch.c +++ b/arch/s390/kernel/hiperdispatch.c @@ -261,7 +261,7 @@ static void hd_capacity_work_fn(struct work_struct *work) } trace_s390_hd_work_fn(steal_percentage, hd_entitled_cores, hd_high_capacity_cores); mutex_unlock(&smp_cpu_state_mutex); - schedule_delayed_work(&hd_capacity_work, HD_DELAY_INTERVAL); + queue_delayed_work(system_dfl_wq, &hd_capacity_work, HD_DELAY_INTERVAL); } static int hiperdispatch_ctl_handler(const struct ctl_table *ctl, int write, diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index b1e798f8e1dd..c441b1357492 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -188,9 +188,9 @@ static inline int __diag308(unsigned long subcode, unsigned long addr) r1.even = addr; r1.odd = 0; asm_inline volatile( - " diag %[r1],%[subcode],0x308\n" - "0: nopr %%r7\n" - EX_TABLE(0b,0b) + "0: diag %[r1],%[subcode],0x308\n" + "1:\n" + EX_TABLE_INSN(0b, 1b) : [r1] "+&d" (r1.pair) : [subcode] "d" (subcode) : "cc", "memory"); diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index c923496aa7b4..8f21bea349a6 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -144,9 +144,9 @@ static int irq_pending(struct pt_regs *regs) void noinstr do_io_irq(struct pt_regs *regs) { - bool from_idle, percpu_needs_fixup; struct pt_regs *old_regs; irqentry_state_t state; + bool from_idle; percpu_entry(regs); state = irqentry_enter(regs); @@ -174,21 +174,20 @@ void noinstr do_io_irq(struct pt_regs *regs) do_irq_async(regs, IO_INTERRUPT); } while (machine_is_lpar() && irq_pending(regs)); - percpu_needs_fixup = percpu_code_check(regs); irq_exit_rcu(); set_irq_regs(old_regs); irqentry_exit(regs, state); if (from_idle) regs->psw.mask &= ~(PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_WAIT); - percpu_exit(regs, percpu_needs_fixup); + percpu_exit(regs); } void noinstr do_ext_irq(struct pt_regs *regs) { - bool from_idle, percpu_needs_fixup; struct pt_regs *old_regs; irqentry_state_t state; + bool from_idle; percpu_entry(regs); state = irqentry_enter(regs); @@ -214,14 +213,13 @@ void noinstr do_ext_irq(struct pt_regs *regs) do_irq_async(regs, EXT_INTERRUPT); - percpu_needs_fixup = percpu_code_check(regs); irq_exit_rcu(); set_irq_regs(old_regs); irqentry_exit(regs, state); if (from_idle) regs->psw.mask &= ~(PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_WAIT); - percpu_exit(regs, percpu_needs_fixup); + percpu_exit(regs); } static void show_msi_interrupt(struct seq_file *p, int irq) diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index c450120b4474..e806b124e4ba 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -85,6 +85,9 @@ static bool can_probe(unsigned long paddr) if (!kallsyms_lookup_size_offset(paddr, NULL, &offset)) return false; + if (s390_search_extables(paddr)) + return false; + /* Decode instructions */ addr = paddr - offset; while (addr < paddr) { diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 17297a8b63d9..56a15edc65d7 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -361,7 +361,6 @@ NOKPROBE_SYMBOL(s390_backup_mcck_info); */ void notrace s390_do_machine_check(struct pt_regs *regs) { - bool percpu_needs_fixup; static int ipd_count; static DEFINE_SPINLOCK(ipd_lock); static unsigned long long last_ipd; @@ -495,9 +494,8 @@ void notrace s390_do_machine_check(struct pt_regs *regs) if (mcck_pending) schedule_mcck_handler(); - percpu_needs_fixup = percpu_code_check(regs); irqentry_nmi_exit(regs, irq_state); - percpu_exit(regs, percpu_needs_fixup); + percpu_exit(regs); } NOKPROBE_SYMBOL(s390_do_machine_check); diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index 69fa303cd479..a2334af51544 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -1768,8 +1768,9 @@ static struct miscdevice cfset_dev = { .mode = 0666, }; -/* Hotplug add of a CPU. Scan through all active processes and add - * that CPU to the list of CPUs supplied with ioctl(..., START, ...). +/* + * Start counter sets on a newly online CPU for each active hwctr session that + * includes this CPU in its requested mask. */ static int cfset_online_cpu(unsigned int cpu) { @@ -1778,17 +1779,19 @@ static int cfset_online_cpu(unsigned int cpu) if (!list_empty(&cfset_session.head)) { list_for_each_entry(rp, &cfset_session.head, node) { + if (!cpumask_test_cpu(cpu, &rp->mask)) + continue; + p.sets = rp->ctrset; cfset_ioctl_on(&p); - cpumask_set_cpu(cpu, &rp->mask); } } return 0; } -/* Hotplug remove of a CPU. Scan through all active processes and clear - * that CPU from the list of CPUs supplied with ioctl(..., START, ...). - * Adjust reference counts. +/* + * Stop counter sets on an outgoing CPU for each active hwctr session that + * includes this CPU in its requested mask. */ static int cfset_offline_cpu(unsigned int cpu) { @@ -1797,9 +1800,11 @@ static int cfset_offline_cpu(unsigned int cpu) if (!list_empty(&cfset_session.head)) { list_for_each_entry(rp, &cfset_session.head, node) { + if (!cpumask_test_cpu(cpu, &rp->mask)) + continue; + p.sets = rp->ctrset; cfset_ioctl_off(&p); - cpumask_clear_cpu(cpu, &rp->mask); } } return 0; diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 416650ae4871..3092ea0347a1 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -35,7 +35,6 @@ #include <asm/cpu_mf.h> #include <asm/processor.h> #include <asm/ptrace.h> -#include <asm/vtimer.h> #include <asm/exec.h> #include <asm/fpu.h> #include <asm/irq.h> @@ -131,6 +130,12 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) p->thread.hardirq_timer = 0; p->thread.softirq_timer = 0; p->thread.last_break = 1; + /* + * s390 stores the svc return address in arch_data when calling + * sigreturn()/restart_syscall() via vdso. 1 means no valid address + * stored. + */ + p->restart_block.arch_data = 1; frame->sf.back_chain = 0; frame->sf.gprs[11 - 6] = (unsigned long)&frame->childregs; @@ -168,12 +173,6 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) p->thread.acrs[0] = (unsigned int)(tls >> 32); p->thread.acrs[1] = (unsigned int)tls; } - /* - * s390 stores the svc return address in arch_data when calling - * sigreturn()/restart_syscall() via vdso. 1 means no valid address - * stored. - */ - p->restart_block.arch_data = 1; return 0; } diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index 125ca4c4e30c..d8d5e843859c 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -34,6 +34,7 @@ #include <asm/ptrace.h> #include <asm/rwonce.h> #include <asm/fpu.h> +#include <asm/syscall.h> #include "entry.h" @@ -345,11 +346,9 @@ static int __poke_user(struct task_struct *child, addr_t addr, addr_t data) return -EINVAL; } - if (test_pt_regs_flag(regs, PIF_SYSCALL) && - addr == offsetof(struct user, regs.gprs[2])) { - struct pt_regs *regs = task_pt_regs(child); - - regs->int_code = 0x20000 | (data & 0xffff); + if (addr == offsetof(struct user, regs.gprs[2])) { + if (test_pt_regs_flag(regs, PIF_SYSCALL)) + syscall_set_nr(child, regs, data); } *(addr_t *)((addr_t) ®s->psw + addr) = data; } else if (addr < offsetof(struct user, regs.orig_gpr2)) { diff --git a/arch/s390/kernel/signal.c b/arch/s390/kernel/signal.c index 4874de5edea0..deeef053758c 100644 --- a/arch/s390/kernel/signal.c +++ b/arch/s390/kernel/signal.c @@ -27,8 +27,8 @@ #include <linux/personality.h> #include <linux/binfmts.h> #include <linux/syscalls.h> -#include <asm/ucontext.h> #include <linux/uaccess.h> +#include <asm/ucontext.h> #include <asm/vdso-symbols.h> #include <asm/access-regs.h> #include <asm/lowcore.h> @@ -61,8 +61,7 @@ * have a fixed position and moves if gprs_high or vxrs exist. * Future extensions will be added to _sigregs_ext. */ -struct sigframe -{ +struct sigframe { __u8 callee_used_stack[__SIGNAL_FRAMESIZE]; struct sigcontext sc; _sigregs sregs; @@ -96,22 +95,21 @@ struct sigframe * ----------------------------------------- * Future extensions will be added to _sigregs_ext. */ -struct rt_sigframe -{ +struct rt_sigframe { __u8 callee_used_stack[__SIGNAL_FRAMESIZE]; __u16 svc_insn; struct siginfo info; struct ucontext_extended uc; }; -/* Store registers needed to create the signal frame */ +/* Store registers needed to create the signal frame. */ static void store_sigregs(void) { save_access_regs(current->thread.acrs); save_user_fpu_regs(); } -/* Load registers after signal return */ +/* Load registers after signal return. */ static void load_sigregs(void) { restore_access_regs(current->thread.acrs); @@ -122,14 +120,14 @@ static int save_sigregs(struct pt_regs *regs, _sigregs __user *sregs) { _sigregs user_sregs; - /* Copy a 'clean' PSW mask to the user to avoid leaking - information about whether PER is currently on. */ - user_sregs.regs.psw.mask = PSW_USER_BITS | - (regs->psw.mask & (PSW_MASK_USER | PSW_MASK_RI)); + /* + * Copy a 'clean' PSW mask to the user to avoid leaking + * information about whether PER is currently on. + */ + user_sregs.regs.psw.mask = PSW_USER_BITS | (regs->psw.mask & (PSW_MASK_USER | PSW_MASK_RI)); user_sregs.regs.psw.addr = regs->psw.addr; memcpy(&user_sregs.regs.gprs, ®s->gprs, sizeof(sregs->regs.gprs)); - memcpy(&user_sregs.regs.acrs, current->thread.acrs, - sizeof(user_sregs.regs.acrs)); + memcpy(&user_sregs.regs.acrs, current->thread.acrs, sizeof(user_sregs.regs.acrs)); fpregs_store(&user_sregs.fpregs, ¤t->thread.ufpu); if (__copy_to_user(sregs, &user_sregs, sizeof(_sigregs))) return -EFAULT; @@ -140,44 +138,37 @@ static int restore_sigregs(struct pt_regs *regs, _sigregs __user *sregs) { _sigregs user_sregs; - /* Always make any pending restarted system call return -EINTR */ + /* Always make any pending restarted system call return -EINTR. */ current->restart_block.fn = do_no_restart_syscall; - if (__copy_from_user(&user_sregs, sregs, sizeof(user_sregs))) return -EFAULT; - if (!is_ri_task(current) && (user_sregs.regs.psw.mask & PSW_MASK_RI)) return -EINVAL; - /* Use regs->psw.mask instead of PSW_USER_BITS to preserve PER bit. */ - regs->psw.mask = (regs->psw.mask & ~(PSW_MASK_USER | PSW_MASK_RI)) | - (user_sregs.regs.psw.mask & (PSW_MASK_USER | PSW_MASK_RI)); + regs->psw.mask &= ~(PSW_MASK_USER | PSW_MASK_RI); + regs->psw.mask |= user_sregs.regs.psw.mask & (PSW_MASK_USER | PSW_MASK_RI); /* Check for invalid user address space control. */ if ((regs->psw.mask & PSW_MASK_ASC) == PSW_ASC_HOME) - regs->psw.mask = PSW_ASC_PRIMARY | - (regs->psw.mask & ~PSW_MASK_ASC); - /* Check for invalid amode */ + regs->psw.mask = PSW_ASC_PRIMARY | (regs->psw.mask & ~PSW_MASK_ASC); + /* Check for invalid amode. */ if (regs->psw.mask & PSW_MASK_EA) regs->psw.mask |= PSW_MASK_BA; regs->psw.addr = user_sregs.regs.psw.addr; memcpy(®s->gprs, &user_sregs.regs.gprs, sizeof(sregs->regs.gprs)); - memcpy(¤t->thread.acrs, &user_sregs.regs.acrs, - sizeof(current->thread.acrs)); - + memcpy(¤t->thread.acrs, &user_sregs.regs.acrs, sizeof(current->thread.acrs)); fpregs_load(&user_sregs.fpregs, ¤t->thread.ufpu); - - clear_pt_regs_flag(regs, PIF_SYSCALL); /* No longer in a system call */ + /* No longer in a system call. */ + clear_pt_regs_flag(regs, PIF_SYSCALL); return 0; } /* Returns non-zero on fault. */ -static int save_sigregs_ext(struct pt_regs *regs, - _sigregs_ext __user *sregs_ext) +static int save_sigregs_ext(struct pt_regs *regs, _sigregs_ext __user *sregs_ext) { __u64 vxrs[__NUM_VXRS_LOW]; int i; - /* Save vector registers to signal stack */ + /* Save vector registers to signal stack. */ if (cpu_has_vx()) { for (i = 0; i < __NUM_VXRS_LOW; i++) vxrs[i] = current->thread.ufpu.vxrs[i].low; @@ -191,13 +182,12 @@ static int save_sigregs_ext(struct pt_regs *regs, return 0; } -static int restore_sigregs_ext(struct pt_regs *regs, - _sigregs_ext __user *sregs_ext) +static int restore_sigregs_ext(struct pt_regs *regs, _sigregs_ext __user *sregs_ext) { __u64 vxrs[__NUM_VXRS_LOW]; int i; - /* Restore vector registers from signal stack */ + /* Restore vector registers from signal stack. */ if (cpu_has_vx()) { if (__copy_from_user(vxrs, &sregs_ext->vxrs_low, sizeof(sregs_ext->vxrs_low)) || @@ -213,11 +203,12 @@ static int restore_sigregs_ext(struct pt_regs *regs, SYSCALL_DEFINE0(sigreturn) { - struct pt_regs *regs = task_pt_regs(current); - struct sigframe __user *frame = - (struct sigframe __user *) regs->gprs[15]; + struct sigframe __user *frame; + struct pt_regs *regs; sigset_t set; + regs = task_pt_regs(current); + frame = (struct sigframe __user *)regs->gprs[15]; if (__copy_from_user(&set.sig, &frame->sc.oldmask, _SIGMASK_COPY_SIZE)) goto badframe; set_current_blocked(&set); @@ -235,11 +226,12 @@ badframe: SYSCALL_DEFINE0(rt_sigreturn) { - struct pt_regs *regs = task_pt_regs(current); - struct rt_sigframe __user *frame = - (struct rt_sigframe __user *)regs->gprs[15]; + struct rt_sigframe __user *frame; + struct pt_regs *regs; sigset_t set; + regs = task_pt_regs(current); + frame = (struct rt_sigframe __user *)regs->gprs[15]; if (__copy_from_user(&set.sig, &frame->uc.uc_sigmask, sizeof(set))) goto badframe; set_current_blocked(&set); @@ -257,94 +249,76 @@ badframe: return 0; } -/* - * Determine which stack to use.. - */ -static inline void __user * -get_sigframe(struct k_sigaction *ka, struct pt_regs * regs, size_t frame_size) +/* Determine which stack to use. */ +static inline void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size) { unsigned long sp; - /* Default to using normal stack */ + /* Default to using normal stack. */ sp = regs->gprs[15]; - /* Overflow on alternate signal stack gives SIGSEGV. */ if (on_sig_stack(sp) && !on_sig_stack((sp - frame_size) & -8UL)) - return (void __user *) -1UL; - - /* This is the X/Open sanctioned signal stack switching. */ + return (void __user *)-1UL; + /* This is the X/Open sanctioned signal stack switching. */ if (ka->sa.sa_flags & SA_ONSTACK) { - if (! sas_ss_flags(sp)) + if (!sas_ss_flags(sp)) sp = current->sas_ss_sp + current->sas_ss_size; } - - return (void __user *)((sp - frame_size) & -8ul); + return (void __user *)((sp - frame_size) & -8UL); } -static int setup_frame(int sig, struct k_sigaction *ka, - sigset_t *set, struct pt_regs * regs) +static int setup_frame(int sig, struct k_sigaction *ka, sigset_t *set, struct pt_regs *regs) { struct sigframe __user *frame; - struct sigcontext sc; unsigned long restorer; + struct sigcontext sc; size_t frame_size; frame_size = sizeof(*frame) - sizeof(frame->sregs_ext); if (cpu_has_vx()) frame_size += sizeof(frame->sregs_ext); frame = get_sigframe(ka, regs, frame_size); - if (frame == (void __user *) -1UL) + if (frame == (void __user *)-1UL) return -EFAULT; - /* Set up backchain. */ - if (__put_user(regs->gprs[15], (addr_t __user *) frame)) + if (__put_user(regs->gprs[15], (addr_t __user *)frame)) return -EFAULT; - - /* Create struct sigcontext on the signal stack */ + /* Create struct sigcontext on the signal stack. */ memcpy(&sc.oldmask, &set->sig, _SIGMASK_COPY_SIZE); - sc.sregs = (_sigregs __user __force *) &frame->sregs; + sc.sregs = (_sigregs __user __force *)&frame->sregs; if (__copy_to_user(&frame->sc, &sc, sizeof(frame->sc))) return -EFAULT; - - /* Store registers needed to create the signal frame */ + /* Store registers needed to create the signal frame. */ store_sigregs(); - - /* Create _sigregs on the signal stack */ + /* Create _sigregs on the signal stack. */ if (save_sigregs(regs, &frame->sregs)) return -EFAULT; - - /* Place signal number on stack to allow backtrace from handler. */ - if (__put_user(regs->gprs[2], (int __user *) &frame->signo)) + /* Place signal number on stack to allow backtrace from handler. */ + if (__put_user(regs->gprs[2], (int __user *)&frame->signo)) return -EFAULT; - - /* Create _sigregs_ext on the signal stack */ + /* Create _sigregs_ext on the signal stack. */ if (save_sigregs_ext(regs, &frame->sregs_ext)) return -EFAULT; - - /* Set up to return from userspace. If provided, use a stub - already in userspace. */ + /* + * Set up to return from userspace. If provided, use a stub + * already in userspace. + */ if (ka->sa.sa_flags & SA_RESTORER) - restorer = (unsigned long) ka->sa.sa_restorer; + restorer = (unsigned long)ka->sa.sa_restorer; else restorer = VDSO_SYMBOL(current, sigreturn); - - /* Set up registers for signal handler */ + /* Set up registers for signal handler. */ regs->gprs[14] = restorer; - regs->gprs[15] = (unsigned long) frame; + regs->gprs[15] = (unsigned long)frame; /* Force default amode and default user address space control. */ - regs->psw.mask = PSW_MASK_EA | PSW_MASK_BA | - (PSW_USER_BITS & PSW_MASK_ASC) | - (regs->psw.mask & ~PSW_MASK_ASC); - regs->psw.addr = (unsigned long) ka->sa.sa_handler; - + regs->psw.mask &= ~PSW_MASK_ASC; + regs->psw.mask |= (PSW_USER_BITS & PSW_MASK_ASC) | PSW_MASK_EA | PSW_MASK_BA; + regs->psw.addr = (unsigned long)ka->sa.sa_handler; regs->gprs[2] = sig; - regs->gprs[3] = (unsigned long) &frame->sc; - - /* We forgot to include these in the sigcontext. - To avoid breaking binary compatibility, they are passed as args. */ - if (sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || - sig == SIGTRAP || sig == SIGFPE) { - /* set extra registers only for synchronous signals */ + regs->gprs[3] = (unsigned long)&frame->sc; + /* Missing in sigcontext. Keep binary compatibility by passing as args. */ + if (sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || sig == SIGTRAP || sig == SIGFPE) { + /* Set extra registers only for synchronous signals. */ regs->gprs[4] = regs->int_code & 127; regs->gprs[5] = regs->int_parm_long; regs->gprs[6] = current->thread.last_break; @@ -352,8 +326,7 @@ static int setup_frame(int sig, struct k_sigaction *ka, return 0; } -static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, - struct pt_regs *regs) +static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) { struct rt_sigframe __user *frame; unsigned long uc_flags, restorer; @@ -366,27 +339,24 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, uc_flags |= UC_VXRS; } frame = get_sigframe(&ksig->ka, regs, frame_size); - if (frame == (void __user *) -1UL) + if (frame == (void __user *)-1UL) return -EFAULT; - /* Set up backchain. */ - if (__put_user(regs->gprs[15], (addr_t __user *) frame)) + if (__put_user(regs->gprs[15], (addr_t __user *)frame)) return -EFAULT; - - /* Set up to return from userspace. If provided, use a stub - already in userspace. */ + /* + * Set up to return from userspace. If provided, use a stub + * already in userspace. + */ if (ksig->ka.sa.sa_flags & SA_RESTORER) - restorer = (unsigned long) ksig->ka.sa.sa_restorer; + restorer = (unsigned long)ksig->ka.sa.sa_restorer; else restorer = VDSO_SYMBOL(current, rt_sigreturn); - - /* Create siginfo on the signal stack */ + /* Create siginfo on the signal stack. */ if (copy_siginfo_to_user(&frame->info, &ksig->info)) return -EFAULT; - - /* Store registers needed to create the signal frame */ + /* Store registers needed to create the signal frame. */ store_sigregs(); - /* Create ucontext on the signal stack. */ if (__put_user(uc_flags, &frame->uc.uc_flags) || __put_user(NULL, &frame->uc.uc_link) || @@ -395,34 +365,29 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set)) || save_sigregs_ext(regs, &frame->uc.uc_mcontext_ext)) return -EFAULT; - - /* Set up registers for signal handler */ + /* Set up registers for signal handler. */ regs->gprs[14] = restorer; - regs->gprs[15] = (unsigned long) frame; + regs->gprs[15] = (unsigned long)frame; /* Force default amode and default user address space control. */ - regs->psw.mask = PSW_MASK_EA | PSW_MASK_BA | - (PSW_USER_BITS & PSW_MASK_ASC) | - (regs->psw.mask & ~PSW_MASK_ASC); - regs->psw.addr = (unsigned long) ksig->ka.sa.sa_handler; - + regs->psw.mask &= ~PSW_MASK_ASC; + regs->psw.mask |= (PSW_USER_BITS & PSW_MASK_ASC) | PSW_MASK_EA | PSW_MASK_BA; + regs->psw.addr = (unsigned long)ksig->ka.sa.sa_handler; regs->gprs[2] = ksig->sig; - regs->gprs[3] = (unsigned long) &frame->info; - regs->gprs[4] = (unsigned long) &frame->uc; + regs->gprs[3] = (unsigned long)&frame->info; + regs->gprs[4] = (unsigned long)&frame->uc; regs->gprs[5] = current->thread.last_break; return 0; } -static void handle_signal(struct ksignal *ksig, sigset_t *oldset, - struct pt_regs *regs) +static void handle_signal(struct ksignal *ksig, sigset_t *oldset, struct pt_regs *regs) { int ret; - /* Set up the stack frame */ + /* Set up the stack frame. */ if (ksig->ka.sa.sa_flags & SA_SIGINFO) ret = setup_rt_frame(ksig, oldset, regs); else ret = setup_frame(ksig->sig, &ksig->ka, oldset, regs); - signal_setup_done(ret, ksig, test_thread_flag(TIF_SINGLE_STEP)); } @@ -435,22 +400,19 @@ static void handle_signal(struct ksignal *ksig, sigset_t *oldset, * the kernel can handle, and then we build all the user-level signal handling * stack-frames in one go after that. */ - void arch_do_signal_or_restart(struct pt_regs *regs) { - struct ksignal ksig; sigset_t *oldset = sigmask_to_save(); + struct ksignal ksig; /* * Get signal to deliver. When running under ptrace, at this point * the debugger may change all our registers, including the system * call information. */ - current->thread.system_call = - test_pt_regs_flag(regs, PIF_SYSCALL) ? regs->int_code : 0; - + current->thread.system_call = test_pt_regs_flag(regs, PIF_SYSCALL) ? regs->int_code : 0; if (get_signal(&ksig)) { - /* Whee! Actually deliver the signal. */ + /* Whee! Actually deliver the signal. */ if (current->thread.system_call) { regs->int_code = current->thread.system_call; /* Check for system call restarting. */ @@ -467,27 +429,28 @@ void arch_do_signal_or_restart(struct pt_regs *regs) fallthrough; case -ERESTARTNOINTR: regs->gprs[2] = regs->orig_gpr2; - regs->psw.addr = - __rewind_psw(regs->psw, - regs->int_code >> 16); + regs->psw.addr = __rewind_psw(regs->psw, regs->int_code >> 16); break; } } - /* No longer in a system call */ + /* No longer in a system call. */ clear_pt_regs_flag(regs, PIF_SYSCALL); - + if (!(current->restart_block.arch_data & 1)) { + regs->psw.addr = current->restart_block.arch_data; + current->restart_block.arch_data = 1; + regs->gprs[2] = -EINTR; + } rseq_signal_deliver(&ksig, regs); handle_signal(&ksig, oldset, regs); return; } - - /* No handlers present - check for system call restart */ + /* No handlers present - check for system call restart. */ clear_pt_regs_flag(regs, PIF_SYSCALL); if (current->thread.system_call) { regs->int_code = current->thread.system_call; switch (regs->gprs[2]) { case -ERESTART_RESTARTBLOCK: - /* Restart with sys_restart_syscall */ + /* Restart with sys_restart_syscall. */ regs->gprs[2] = regs->orig_gpr2; current->restart_block.arch_data = regs->psw.addr; regs->psw.addr = VDSO_SYMBOL(current, restart_syscall); @@ -504,9 +467,6 @@ void arch_do_signal_or_restart(struct pt_regs *regs) break; } } - - /* - * If there's no signal to deliver, we just put the saved sigmask back. - */ + /* If there's no signal to deliver, put the saved sigmask back. */ restore_saved_sigmask(); } diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 32499cad86f0..f7c262a17c03 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -48,7 +48,7 @@ #include <asm/setup.h> #include <asm/irq.h> #include <asm/tlbflush.h> -#include <asm/vtimer.h> +#include <asm/vtime.h> #include <asm/abs_lowcore.h> #include <asm/sclp.h> #include <asm/debug.h> diff --git a/arch/s390/kernel/text_amode31.S b/arch/s390/kernel/text_amode31.S index f007d892d0c5..7761de491443 100644 --- a/arch/s390/kernel/text_amode31.S +++ b/arch/s390/kernel/text_amode31.S @@ -19,9 +19,9 @@ * affects a few functions that are not performance-relevant. */ .macro BR_EX_AMODE31_r14 - exrl 0,0f + exrl 0,555f j . -0: br %r14 +555: br %r14 .endm /* @@ -33,15 +33,13 @@ SYM_TYPED_FUNC_START(_diag14_amode31) lgr %r3,%r4 lhi %r5,-EIO sam31 - diag %r1,%r2,0x14 -.Ldiag14_ex: +0: diag %r1,%r2,0x14 ipm %r5 srl %r5,28 -.Ldiag14_fault: - sam64 +1: sam64 lgfr %r2,%r5 BR_EX_AMODE31_r14 - EX_TABLE_AMODE31(.Ldiag14_ex, .Ldiag14_fault) + EX_TABLE_INSN_AMODE31(0b, 1b) SYM_FUNC_END(_diag14_amode31) /* @@ -51,15 +49,13 @@ SYM_TYPED_FUNC_START(_diag210_amode31) lgr %r1,%r2 lhi %r2,-1 sam31 - diag %r1,%r0,0x210 -.Ldiag210_ex: +0: diag %r1,%r0,0x210 ipm %r2 srl %r2,28 -.Ldiag210_fault: - sam64 +1: sam64 lgfr %r2,%r2 BR_EX_AMODE31_r14 - EX_TABLE_AMODE31(.Ldiag210_ex, .Ldiag210_fault) + EX_TABLE_INSN_AMODE31(0b, 1b) SYM_FUNC_END(_diag210_amode31) /* @@ -68,12 +64,11 @@ SYM_FUNC_END(_diag210_amode31) SYM_TYPED_FUNC_START(_diag8c_amode31) llgf %r3,0(%r3) sam31 - diag %r2,%r4,0x8c -.Ldiag8c_ex: - sam64 +0: diag %r2,%r4,0x8c +1: sam64 lgfr %r2,%r3 BR_EX_AMODE31_r14 - EX_TABLE_AMODE31(.Ldiag8c_ex, .Ldiag8c_ex) + EX_TABLE_INSN_AMODE31(0b, 1b) SYM_FUNC_END(_diag8c_amode31) /* * int _diag26c_amode31(void *req, void *resp, enum diag26c_sc subcode) @@ -81,12 +76,11 @@ SYM_FUNC_END(_diag8c_amode31) SYM_TYPED_FUNC_START(_diag26c_amode31) lghi %r5,-EOPNOTSUPP sam31 - diag %r2,%r4,0x26c -.Ldiag26c_ex: - sam64 +0: diag %r2,%r4,0x26c +1: sam64 lgfr %r2,%r5 BR_EX_AMODE31_r14 - EX_TABLE_AMODE31(.Ldiag26c_ex, .Ldiag26c_ex) + EX_TABLE_INSN_AMODE31(0b, 1b) SYM_FUNC_END(_diag26c_amode31) /* diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index 2b989bebd220..dd8a88cc9c4b 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -48,9 +48,9 @@ #include <asm/vdso.h> #include <asm/irq.h> #include <asm/irq_regs.h> -#include <asm/vtimer.h> #include <asm/stp.h> #include <asm/cio.h> +#include <asm/vtime.h> #include "entry.h" union tod_clock __bootdata_preserved(tod_clock_base); diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index b6ba4465f59d..6b083b9584eb 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -26,6 +26,7 @@ #include <linux/entry-common.h> #include <linux/kmsan.h> #include <linux/bug.h> +#include <asm/pgmcheck-defs.h> #include <asm/entry-percpu.h> #include <asm/asm-extable.h> #include <asm/irqflags.h> @@ -41,16 +42,21 @@ struct pgm_stat { static DEFINE_PER_CPU_SHARED_ALIGNED(struct pgm_stat, pgm_stat); -static inline void __user *get_trap_ip(struct pt_regs *regs) +unsigned long get_trap_ip(struct pt_regs *regs) { - unsigned long address; + static const unsigned long nullified[2] = { PGM_CHECKS_NULLIFIED }; + psw_t psw = regs->psw; + if (test_pt_regs_flag(regs, PIF_PSW_ADDR_ADJUSTED)) + return psw.addr; if (regs->int_code & 0x200) - address = current->thread.trap_tdb.data[3]; - else - address = regs->psw.addr; - return (void __user *)(address - (regs->int_code >> 16)); + psw.addr = current->thread.trap_tdb.data[3]; + if (test_bit(regs->int_code & PGM_INT_CODE_MASK, nullified)) + return psw.addr; + psw.addr = __rewind_psw(psw, regs->int_code >> 16); + return psw.addr; } +NOKPROBE_SYMBOL(get_trap_ip); #ifdef CONFIG_GENERIC_BUG int is_valid_bugaddr(unsigned long addr) @@ -62,7 +68,7 @@ int is_valid_bugaddr(unsigned long addr) void do_report_trap(struct pt_regs *regs, int si_signo, int si_code, char *str) { if (user_mode(regs)) { - force_sig_fault(si_signo, si_code, get_trap_ip(regs)); + force_sig_fault(si_signo, si_code, (void __user *)get_trap_ip(regs)); report_user_fault(regs, si_signo, 0); } else { if (!fixup_exception(regs)) @@ -148,12 +154,12 @@ static void translation_specification_exception(struct pt_regs *regs) static void illegal_op(struct pt_regs *regs) { int is_uprobe_insn = 0; - u16 __user *location; int signal = 0; u16 opcode; - location = get_trap_ip(regs); if (user_mode(regs)) { + u16 __user *location = (void __user *)get_trap_ip(regs); + if (get_user(opcode, location)) return; if (opcode == S390_BREAKPOINT_U16) { @@ -267,6 +273,7 @@ static void monitor_event_exception(struct pt_regs *regs) return; if (regs->monitor_code == MONCODE_BUG_ARG) { regs->psw.addr = regs->gprs[14]; + set_pt_regs_flag(regs, PIF_WARN_TRAP_SKIPPED); btt = report_bug_entry((struct bug_entry *)regs->gprs[2], regs); } else { btt = report_bug(regs->psw.addr - (regs->int_code >> 16), regs); @@ -283,6 +290,12 @@ static void monitor_event_exception(struct pt_regs *regs) } } +/* + * Content of the faulting CPU's lowcore, copied by stack_invalid(). + */ +struct lowcore stack_invalid_lowcore; +int stack_invalid_lowcore_used; + void kernel_stack_invalid(struct pt_regs *regs) { /* @@ -305,10 +318,10 @@ static void __init test_monitor_call(void) if (!IS_ENABLED(CONFIG_BUG)) return; asm_inline volatile( - " mc %[monc](%%r0),0\n" - "0: lhi %[val],0\n" + "0: mc %[monc](%%r0),0\n" + " lhi %[val],0\n" "1:\n" - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : [val] "+d" (val) : [monc] "i" (MONCODE_BUG)); if (!val) @@ -338,7 +351,6 @@ static void (*pgm_check_table[128])(struct pt_regs *regs); void noinstr __do_pgm_check(struct pt_regs *regs, unsigned long flags) { struct lowcore *lc = get_lowcore(); - bool percpu_needs_fixup; irqentry_state_t state; struct pgm_stat *stat; unsigned int trapnr; @@ -348,6 +360,7 @@ void noinstr __do_pgm_check(struct pt_regs *regs, unsigned long flags) regs->int_code = lc->pgm_int_code; regs->int_parm_long = teid.val; regs->monitor_code = lc->monitor_code; + set_pt_regs_flag(regs, PIF_PGM_CHECK); trapnr = regs->int_code & PGM_INT_CODE_MASK; stat = this_cpu_ptr(&pgm_stat); @@ -400,9 +413,8 @@ void noinstr __do_pgm_check(struct pt_regs *regs, unsigned long flags) pgm_check_table[trapnr](regs); out: local_irq_disable(); - percpu_needs_fixup = percpu_code_check(regs); irqentry_exit(regs, state); - percpu_exit(regs, percpu_needs_fixup); + percpu_exit(regs); } static int pgm_check_stat_show(struct seq_file *p, void *v) diff --git a/arch/s390/kernel/uprobes.c b/arch/s390/kernel/uprobes.c index c624f3361e43..b2b8a377ced6 100644 --- a/arch/s390/kernel/uprobes.c +++ b/arch/s390/kernel/uprobes.c @@ -241,7 +241,7 @@ static void sim_stor_event(struct pt_regs *regs, void *addr, int len) return; if (!(current->thread.per_user.control & PER_EVENT_STORE)) return; - if ((void *)current->thread.per_user.start > (addr + len)) + if ((void *)current->thread.per_user.start > (addr + len - 1)) return; if ((void *)current->thread.per_user.end < addr) return; diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c index efcbf406f03e..d9fd39c5732e 100644 --- a/arch/s390/kernel/vtime.c +++ b/arch/s390/kernel/vtime.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Virtual cpu timer based timer functions. + * Virtual CPU time accounting * * Copyright IBM Corp. 2004, 2012 * Author(s): Jan Glauber <jan.glauber@de.ibm.com> @@ -14,7 +14,6 @@ #include <linux/time.h> #include <asm/alternative.h> #include <asm/cputime.h> -#include <asm/vtimer.h> #include <asm/vtime.h> #include <asm/cpu_mf.h> #include <asm/idle.h> @@ -22,19 +21,14 @@ #include "entry.h" -static void virt_timer_expire(void); - -static LIST_HEAD(virt_timer_list); -static DEFINE_SPINLOCK(virt_timer_lock); -static atomic64_t virt_timer_current; -static atomic64_t virt_timer_elapsed; +#define CPU_TIMER_MAX 0x7fffffffffffffffUL DEFINE_PER_CPU(u64, mt_cycles[8]); static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 }; static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 }; static DEFINE_PER_CPU(unsigned long, mt_scaling_jiffies); -static inline void set_vtimer(u64 expires) +static inline void cpu_timer_init(u64 value) { struct lowcore *lc = get_lowcore(); u64 timer; @@ -42,18 +36,9 @@ static inline void set_vtimer(u64 expires) asm volatile( " stpt %0\n" /* Store current cpu timer value */ " spt %1" /* Set new value imm. afterwards */ - : "=Q" (timer) : "Q" (expires)); + : "=Q" (timer) : "Q" (value)); lc->system_timer += lc->last_update_timer - timer; - lc->last_update_timer = expires; -} - -static inline int virt_timer_forward(u64 elapsed) -{ - lockdep_assert_irqs_disabled(); - if (list_empty(&virt_timer_list)) - return 0; - elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed); - return elapsed >= atomic64_read(&virt_timer_current); + lc->last_update_timer = value; } static void update_mt_scaling(void) @@ -125,7 +110,7 @@ static inline void vtime_reset_last_update(struct lowcore *lc) * Update process times based on virtual cpu times stored by entry.S * to the lowcore fields user_timer, system_timer & steal_clock. */ -static int do_account_vtime(struct task_struct *tsk) +static void do_account_vtime(struct task_struct *tsk) { u64 timer, clock, user, guest, system, hardirq, softirq; struct lowcore *lc = get_lowcore(); @@ -172,8 +157,6 @@ static int do_account_vtime(struct task_struct *tsk) account_system_index_scaled(tsk, hardirq, CPUTIME_IRQ); if (softirq) account_system_index_scaled(tsk, softirq, CPUTIME_SOFTIRQ); - - return virt_timer_forward(user + guest + system + hardirq + softirq); } void vtime_task_switch(struct task_struct *prev) @@ -203,8 +186,7 @@ void vtime_flush(struct task_struct *tsk) struct lowcore *lc = get_lowcore(); u64 steal, avg_steal; - if (do_account_vtime(tsk)) - virt_timer_expire(); + do_account_vtime(tsk); steal = lc->steal_timer; avg_steal = lc->avg_steal_timer; @@ -247,187 +229,9 @@ void vtime_account_hardirq(struct task_struct *tsk) get_lowcore()->hardirq_timer += vtime_delta(); } -/* - * Sorted add to a list. List is linear searched until first bigger - * element is found. - */ -static void list_add_sorted(struct vtimer_list *timer, struct list_head *head) -{ - struct vtimer_list *tmp; - - list_for_each_entry(tmp, head, entry) { - if (tmp->expires > timer->expires) { - list_add_tail(&timer->entry, &tmp->entry); - return; - } - } - list_add_tail(&timer->entry, head); -} - -/* - * Handler for expired virtual CPU timer. - */ -static void virt_timer_expire(void) -{ - struct vtimer_list *timer, *tmp; - unsigned long elapsed; - LIST_HEAD(cb_list); - - /* walk timer list, fire all expired timers */ - spin_lock(&virt_timer_lock); - elapsed = atomic64_read(&virt_timer_elapsed); - list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) { - if (timer->expires < elapsed) - /* move expired timer to the callback queue */ - list_move_tail(&timer->entry, &cb_list); - else - timer->expires -= elapsed; - } - if (!list_empty(&virt_timer_list)) { - timer = list_first_entry(&virt_timer_list, - struct vtimer_list, entry); - atomic64_set(&virt_timer_current, timer->expires); - } - atomic64_sub(elapsed, &virt_timer_elapsed); - spin_unlock(&virt_timer_lock); - - /* Do callbacks and recharge periodic timers */ - list_for_each_entry_safe(timer, tmp, &cb_list, entry) { - list_del_init(&timer->entry); - timer->function(timer->data); - if (timer->interval) { - /* Recharge interval timer */ - timer->expires = timer->interval + - atomic64_read(&virt_timer_elapsed); - spin_lock(&virt_timer_lock); - list_add_sorted(timer, &virt_timer_list); - spin_unlock(&virt_timer_lock); - } - } -} - -void init_virt_timer(struct vtimer_list *timer) -{ - timer->function = NULL; - INIT_LIST_HEAD(&timer->entry); -} -EXPORT_SYMBOL(init_virt_timer); - -static inline int vtimer_pending(struct vtimer_list *timer) -{ - return !list_empty(&timer->entry); -} - -static void internal_add_vtimer(struct vtimer_list *timer) -{ - if (list_empty(&virt_timer_list)) { - /* First timer, just program it. */ - atomic64_set(&virt_timer_current, timer->expires); - atomic64_set(&virt_timer_elapsed, 0); - list_add(&timer->entry, &virt_timer_list); - } else { - /* Update timer against current base. */ - timer->expires += atomic64_read(&virt_timer_elapsed); - if (likely((s64) timer->expires < - (s64) atomic64_read(&virt_timer_current))) - /* The new timer expires before the current timer. */ - atomic64_set(&virt_timer_current, timer->expires); - /* Insert new timer into the list. */ - list_add_sorted(timer, &virt_timer_list); - } -} - -static void __add_vtimer(struct vtimer_list *timer, int periodic) -{ - unsigned long flags; - - timer->interval = periodic ? timer->expires : 0; - spin_lock_irqsave(&virt_timer_lock, flags); - internal_add_vtimer(timer); - spin_unlock_irqrestore(&virt_timer_lock, flags); -} - -/* - * add_virt_timer - add a oneshot virtual CPU timer - */ -void add_virt_timer(struct vtimer_list *timer) -{ - __add_vtimer(timer, 0); -} -EXPORT_SYMBOL(add_virt_timer); - -/* - * add_virt_timer_int - add an interval virtual CPU timer - */ -void add_virt_timer_periodic(struct vtimer_list *timer) -{ - __add_vtimer(timer, 1); -} -EXPORT_SYMBOL(add_virt_timer_periodic); - -static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic) -{ - unsigned long flags; - int rc; - - BUG_ON(!timer->function); - - if (timer->expires == expires && vtimer_pending(timer)) - return 1; - spin_lock_irqsave(&virt_timer_lock, flags); - rc = vtimer_pending(timer); - if (rc) - list_del_init(&timer->entry); - timer->interval = periodic ? expires : 0; - timer->expires = expires; - internal_add_vtimer(timer); - spin_unlock_irqrestore(&virt_timer_lock, flags); - return rc; -} - -/* - * returns whether it has modified a pending timer (1) or not (0) - */ -int mod_virt_timer(struct vtimer_list *timer, u64 expires) -{ - return __mod_vtimer(timer, expires, 0); -} -EXPORT_SYMBOL(mod_virt_timer); - -/* - * returns whether it has modified a pending timer (1) or not (0) - */ -int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires) -{ - return __mod_vtimer(timer, expires, 1); -} -EXPORT_SYMBOL(mod_virt_timer_periodic); - -/* - * Delete a virtual timer. - * - * returns whether the deleted timer was pending (1) or not (0) - */ -int del_virt_timer(struct vtimer_list *timer) -{ - unsigned long flags; - - if (!vtimer_pending(timer)) - return 0; - spin_lock_irqsave(&virt_timer_lock, flags); - list_del_init(&timer->entry); - spin_unlock_irqrestore(&virt_timer_lock, flags); - return 1; -} -EXPORT_SYMBOL(del_virt_timer); - -/* - * Start the virtual CPU timer on the current CPU. - */ void vtime_init(void) { - /* set initial cpu timer */ - set_vtimer(VTIMER_MAX_SLICE); + cpu_timer_init(CPU_TIMER_MAX); /* Setup initial MT scaling values */ if (smp_cpu_mtid) { __this_cpu_write(mt_scaling_jiffies, jiffies); diff --git a/arch/s390/kvm/s390/gaccess.c b/arch/s390/kvm/s390/gaccess.c index 405345ccc4f1..2d27dcb68002 100644 --- a/arch/s390/kvm/s390/gaccess.c +++ b/arch/s390/kvm/s390/gaccess.c @@ -838,12 +838,11 @@ static int mvcos_key(void *to, const void *from, unsigned long size, u8 dst_key, int exception = PGM_PROTECTION; asm_inline volatile( - " lr %%r0,%[spec]\n" - "0: mvcos %[to],%[from],%[size]\n" - "1: lhi %[exc],0\n" - "2:\n" - EX_TABLE(0b, 2b) - EX_TABLE(1b, 2b) + " lr %%r0,%[spec]\n" + "0: mvcos %[to],%[from],%[size]\n" + " lhi %[exc],0\n" + "1:\n" + EX_TABLE_INSN(0b, 1b) : [size] "+d" (size), [to] "=Q" (*(char *)to), [exc] "+d" (exception) : [spec] "d" (spec.val), [from] "Q" (*(const char *)from) : "memory", "cc", "0"); diff --git a/arch/s390/lib/probes.c b/arch/s390/lib/probes.c index 1e184a03442d..66221ec16822 100644 --- a/arch/s390/lib/probes.c +++ b/arch/s390/lib/probes.c @@ -77,6 +77,10 @@ int probe_get_fixup_type(u16 *insn) if ((insn[0] & 0xff) == 0xb2) fixup = FIXUP_NOT_REQUIRED; break; + case 0xb9: /* bctgr */ + if ((insn[0] & 0xff) == 0x46) + fixup = FIXUP_BRANCH_NOT_TAKEN; + break; case 0xa7: /* bras */ if ((insn[0] & 0x0f) == 0x05) fixup |= FIXUP_RETURN_REGISTER; @@ -85,6 +89,14 @@ int probe_get_fixup_type(u16 *insn) if ((insn[0] & 0x0f) == 0x05) /* brasl */ fixup |= FIXUP_RETURN_REGISTER; break; + case 0xe3: + switch (insn[2] & 0xff) { + case 0x46: /* bctg */ + case 0x47: /* bic */ + fixup = FIXUP_BRANCH_NOT_TAKEN; + break; + } + break; case 0xeb: switch (insn[2] & 0xff) { case 0x44: /* bxhg */ @@ -93,18 +105,14 @@ int probe_get_fixup_type(u16 *insn) break; } break; - case 0xe3: /* bctg */ - if ((insn[2] & 0xff) == 0x46) - fixup = FIXUP_BRANCH_NOT_TAKEN; - break; case 0xec: switch (insn[2] & 0xff) { + case 0xe4: /* cgrb */ case 0xe5: /* clgrb */ - case 0xe6: /* cgrb */ case 0xf6: /* crb */ case 0xf7: /* clrb */ case 0xfc: /* cgib */ - case 0xfd: /* cglib */ + case 0xfd: /* clgib */ case 0xfe: /* cib */ case 0xff: /* clib */ fixup = FIXUP_BRANCH_NOT_TAKEN; diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c index dbabca35c008..c84991a1be20 100644 --- a/arch/s390/lib/spinlock.c +++ b/arch/s390/lib/spinlock.c @@ -318,6 +318,7 @@ void arch_read_lock_wait(arch_rwlock_t *rw) /* Remove this reader again to allow recursive read locking */ __atomic_add_const(-1, &rw->cnts); + trace_contention_begin(rw, LCB_F_SPIN | LCB_F_READ); /* Put the reader into the wait queue */ arch_spin_lock(&rw->wait); /* Now add this reader to the count value again */ @@ -326,6 +327,7 @@ void arch_read_lock_wait(arch_rwlock_t *rw) while (READ_ONCE(rw->cnts) & 0x10000) barrier(); arch_spin_unlock(&rw->wait); + trace_contention_end(rw, 0); } EXPORT_SYMBOL(arch_read_lock_wait); @@ -333,6 +335,7 @@ void arch_write_lock_wait(arch_rwlock_t *rw) { int old; + trace_contention_begin(rw, LCB_F_SPIN | LCB_F_WRITE); /* Add this CPU to the write waiters */ __atomic_add(0x20000, &rw->cnts); @@ -349,6 +352,7 @@ void arch_write_lock_wait(arch_rwlock_t *rw) } arch_spin_unlock(&rw->wait); + trace_contention_end(rw, 0); } EXPORT_SYMBOL(arch_write_lock_wait); diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index 0ac2f3998b14..d3f462c12bba 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -49,24 +49,22 @@ static nokprobe_inline int __cmpxchg_key_small(void *address, unsigned int *uval "20: spka 0(%[key])\n" " llill %[count],%[max_loops]\n" "0: l %[prev],%[address]\n" - "1: nr %[prev],%[mask]\n" + " nr %[prev],%[mask]\n" " xilf %[mask],0xffffffff\n" " or %[new],%[prev]\n" " or %[prev],%[tmp]\n" - "2: lr %[tmp],%[prev]\n" - "3: cs %[prev],%[new],%[address]\n" - "4: jnl 5f\n" + "1: lr %[tmp],%[prev]\n" + "2: cs %[prev],%[new],%[address]\n" + " jnl 3f\n" " xr %[tmp],%[prev]\n" " xr %[new],%[tmp]\n" " nr %[tmp],%[mask]\n" - " jnz 5f\n" - " brct %[count],2b\n" - "5: spka %[default_key]\n" + " jnz 3f\n" + " brct %[count],1b\n" + "3: spka %[default_key]\n" "21:\n" - EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev]) - EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev]) - EX_TABLE_UA_LOAD_REG(3b, 5b, %[rc], %[prev]) - EX_TABLE_UA_LOAD_REG(4b, 5b, %[rc], %[prev]) + EX_TABLE_INSN_UA_LOAD_REG(0b, 3b, %[rc], %[prev]) + EX_TABLE_INSN_UA_LOAD_REG(2b, 3b, %[rc], %[prev]) SKEY_REGION(20b, 21b) : [rc] "+&d" (rc), [prev] "=&d" (prev), @@ -133,8 +131,7 @@ int __kprobes __cmpxchg_key4(void *address, unsigned int *uval, unsigned int old "0: cs %[prev],%[new],%[address]\n" "1: spka %[default_key]\n" "21:\n" - EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) - EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev]) + EX_TABLE_INSN_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) SKEY_REGION(20b, 21b) : [rc] "+&d" (rc), [prev] "+&d" (prev), @@ -160,8 +157,7 @@ int __kprobes __cmpxchg_key8(void *address, unsigned long *uval, unsigned long o "0: csg %[prev],%[new],%[address]\n" "1: spka %[default_key]\n" "21:\n" - EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) - EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev]) + EX_TABLE_INSN_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) SKEY_REGION(20b, 21b) : [rc] "+&d" (rc), [prev] "+&d" (prev), @@ -187,8 +183,7 @@ int __kprobes __cmpxchg_key16(void *address, __uint128_t *uval, __uint128_t old, "0: cdsg %[prev],%[new],%[address]\n" "1: spka %[default_key]\n" "21:\n" - EX_TABLE_UA_LOAD_REGPAIR(0b, 1b, %[rc], %[prev]) - EX_TABLE_UA_LOAD_REGPAIR(1b, 1b, %[rc], %[prev]) + EX_TABLE_INSN_UA_LOAD_REGPAIR(0b, 1b, %[rc], %[prev]) SKEY_REGION(20b, 21b) : [rc] "+&d" (rc), [prev] "+&d" (prev), diff --git a/arch/s390/mm/extable.c b/arch/s390/mm/extable.c index 7498e858c401..d0a1ded21520 100644 --- a/arch/s390/mm/extable.c +++ b/arch/s390/mm/extable.c @@ -106,9 +106,9 @@ static bool ex_handler_ua_mvcos(const struct exception_table_entry *ex, regs->psw.addr = extable_fixup(ex); insn = (struct insn_ssf *)regs->psw.addr; if (from) - uaddr = regs->gprs[insn->b2] + insn->d2; + uaddr = (insn->b2 ? regs->gprs[insn->b2] : 0) + insn->d2; else - uaddr = regs->gprs[insn->b1] + insn->d1; + uaddr = (insn->b1 ? regs->gprs[insn->b1] : 0) + insn->d1; remainder = PAGE_SIZE - (uaddr & (PAGE_SIZE - 1)); if (regs->gprs[insn->r3] <= remainder) remainder = 0; @@ -120,7 +120,7 @@ bool fixup_exception(struct pt_regs *regs) { const struct exception_table_entry *ex; - ex = s390_search_extables(instruction_pointer(regs)); + ex = s390_search_extables(get_trap_ip(regs)); if (!ex) return false; switch (ex->type) { diff --git a/arch/s390/mm/pfault.c b/arch/s390/mm/pfault.c index 6ecd6b0a22a8..8f37741471e2 100644 --- a/arch/s390/mm/pfault.c +++ b/arch/s390/mm/pfault.c @@ -58,9 +58,9 @@ int __pfault_init(void) return rc; diag_stat_inc(DIAG_STAT_X258); asm_inline volatile( - " diag %[refbk],%[rc],0x258\n" - "0: nopr %%r7\n" - EX_TABLE(0b, 0b) + "0: diag %[refbk],%[rc],0x258\n" + "1:\n" + EX_TABLE_INSN(0b, 1b) : [rc] "+d" (rc) : [refbk] "a" (virt_to_phys(&pfault_init_refbk)), "m" (pfault_init_refbk) : "cc"); @@ -80,9 +80,9 @@ void __pfault_fini(void) return; diag_stat_inc(DIAG_STAT_X258); asm_inline volatile( - " diag %[refbk],0,0x258\n" - "0: nopr %%r7\n" - EX_TABLE(0b, 0b) + "0: diag %[refbk],0,0x258\n" + "1:\n" + EX_TABLE_INSN(0b, 1b) : : [refbk] "a" (virt_to_phys(&pfault_fini_refbk)), "m" (pfault_fini_refbk) : "cc"); diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c index 4acd8b140c4b..da076928b63b 100644 --- a/arch/s390/mm/pgtable.c +++ b/arch/s390/mm/pgtable.c @@ -107,7 +107,7 @@ static inline pte_t ptep_flush_lazy(struct mm_struct *mm, atomic_inc(&mm->context.flush_count); if (cpumask_equal(&mm->context.cpu_attach_mask, cpumask_of(smp_processor_id()))) { - set_pte(ptep, set_pte_bit(*ptep, __pgprot(_PAGE_INVALID))); + set_pte(ptep, set_pte_bit(old, __pgprot(_PAGE_INVALID))); mm->context.flush_mm = 1; } else ptep_ipte_global(mm, addr, ptep, nodat); @@ -227,7 +227,7 @@ static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, atomic_inc(&mm->context.flush_count); if (cpumask_equal(&mm->context.cpu_attach_mask, cpumask_of(smp_processor_id()))) { - set_pmd(pmdp, set_pmd_bit(*pmdp, __pgprot(_SEGMENT_ENTRY_INVALID))); + set_pmd(pmdp, set_pmd_bit(old, __pgprot(_SEGMENT_ENTRY_INVALID))); mm->context.flush_mm = 1; } else { pmdp_idte_global(mm, addr, pmdp); diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 1b2566012b63..51db92c3329d 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -723,7 +723,6 @@ bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs) */ struct bpf_jit_probe { int prg; /* JITed instruction offset */ - int nop_prg; /* JITed nop offset */ int reg; /* Register to clear on exception */ int arena_reg; /* Register to use for arena addressing */ }; @@ -731,29 +730,10 @@ struct bpf_jit_probe { static void bpf_jit_probe_init(struct bpf_jit_probe *probe) { probe->prg = -1; - probe->nop_prg = -1; probe->reg = -1; probe->arena_reg = REG_0; } -/* - * Handlers of certain exceptions leave psw.addr pointing to the instruction - * directly after the failing one. Therefore, create two exception table - * entries and also add a nop in case two probing instructions come directly - * after each other. - */ -static void bpf_jit_probe_emit_nop(struct bpf_jit *jit, - struct bpf_jit_probe *probe) -{ - if (probe->prg == -1 || probe->nop_prg != -1) - /* The probe is not armed or nop is already emitted. */ - return; - - probe->nop_prg = jit->prg; - /* bcr 0,%0 */ - _EMIT2(0x0700); -} - static void bpf_jit_probe_load_pre(struct bpf_jit *jit, struct bpf_insn *insn, struct bpf_jit_probe *probe) { @@ -815,47 +795,36 @@ static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp, struct bpf_jit_probe *probe) { struct exception_table_entry *ex; - int i, prg; s64 delta; - u8 *insn; if (probe->prg == -1) /* The probe is not armed. */ return 0; - bpf_jit_probe_emit_nop(jit, probe); if (!fp->aux->extable) /* Do nothing during early JIT passes. */ return 0; - insn = jit->prg_buf + probe->prg; - if (WARN_ON_ONCE(probe->prg + insn_length(*insn) != probe->nop_prg)) - /* JIT bug - gap between probe and nop instructions. */ + if (WARN_ON_ONCE(jit->excnt >= fp->aux->num_exentries)) + /* Verifier bug - not enough entries. */ return -1; - for (i = 0; i < 2; i++) { - if (WARN_ON_ONCE(jit->excnt >= fp->aux->num_exentries)) - /* Verifier bug - not enough entries. */ - return -1; - ex = &fp->aux->extable[jit->excnt]; - /* Add extable entries for probe and nop instructions. */ - prg = i == 0 ? probe->prg : probe->nop_prg; - delta = jit->prg_buf + prg - (u8 *)&ex->insn; - if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX)) - /* JIT bug - code and extable must be close. */ - return -1; - ex->insn = delta; - /* - * Land on the current instruction. Note that the extable - * infrastructure ignores the fixup field; it is handled by - * ex_handler_bpf(). - */ - delta = jit->prg_buf + jit->prg - (u8 *)&ex->fixup; - if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX)) - /* JIT bug - landing pad and extable must be close. */ - return -1; - ex->fixup = delta; - ex->type = EX_TYPE_BPF; - ex->data = probe->reg; - jit->excnt++; - } + ex = &fp->aux->extable[jit->excnt]; + delta = jit->prg_buf + probe->prg - (u8 *)&ex->insn; + if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX)) + /* JIT bug - code and extable must be close. */ + return -1; + ex->insn = delta; + /* + * Land on the current instruction. Note that the extable + * infrastructure ignores the fixup field; it is handled by + * ex_handler_bpf(). + */ + delta = jit->prg_buf + jit->prg - (u8 *)&ex->fixup; + if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX)) + /* JIT bug - landing pad and extable must be close. */ + return -1; + ex->fixup = delta; + ex->type = EX_TYPE_BPF; + ex->data = probe->reg; + jit->excnt++; return 0; } @@ -1705,7 +1674,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, EMIT6_DISP_LH(0xe3000000, is32 ? 0x0058 : 0x0004, REG_W0, REG_0, load_probe.arena_reg, off); - bpf_jit_probe_emit_nop(jit, &load_probe); /* Reuse {ly|lg}'s arena_reg for {csy|csg}. */ if (load_probe.prg != -1) { probe.prg = jit->prg; @@ -1716,7 +1684,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, /* 0: {csy|csg} %w0,%src,off(%arena) */ EMIT6_DISP_LH(0xeb000000, is32 ? 0x0014 : 0x0030, REG_W0, src_reg, probe.arena_reg, off); - bpf_jit_probe_emit_nop(jit, &probe); /* brc 4,0b */ EMIT4_PCREL_RIC(0xa7040000, 4, loop_start); /* {llgfr|lgr} %src,%w0 */ @@ -2364,9 +2331,6 @@ static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit, */ fp->aux->num_exentries += 1; } - /* We need two entries per insn. */ - fp->aux->num_exentries *= 2; - code_size = roundup(jit->size, __alignof__(struct exception_table_entry)); extable_size = fp->aux->num_exentries * diff --git a/arch/s390/pci/pci_clp.c b/arch/s390/pci/pci_clp.c index 177aa0214547..2d7eaf4d9d59 100644 --- a/arch/s390/pci/pci_clp.c +++ b/arch/s390/pci/pci_clp.c @@ -55,11 +55,11 @@ static inline int clp_get_ilp(unsigned long *ilp) exception = 1; asm_inline volatile ( - " .insn rrf,0xb9a00000,%[mask],%[cmd],8,0\n" - "0: lhi %[exc],0\n" + "0: .insn rrf,0xb9a00000,%[mask],%[cmd],8,0\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [mask] "=d" (mask), [exc] "+d" (exception) : [cmd] "a" (1) : CC_CLOBBER); @@ -78,11 +78,11 @@ static __always_inline int clp_req(void *data, unsigned int lps) exception = 1; asm_inline volatile ( - " .insn rrf,0xb9a00000,%[ign],%[req],0,%[lps]\n" - "0: lhi %[exc],0\n" + "0: .insn rrf,0xb9a00000,%[ign],%[req],0,%[lps]\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [ign] "=d" (ignored), "+m" (*req), [exc] "+d" (exception) : [req] "a" (req), [lps] "i" (lps) : CC_CLOBBER); diff --git a/arch/s390/pci/pci_insn.c b/arch/s390/pci/pci_insn.c index 35ceb1bea1c6..b5b3ea11ea42 100644 --- a/arch/s390/pci/pci_insn.c +++ b/arch/s390/pci/pci_insn.c @@ -18,22 +18,49 @@ #include <asm/asm.h> #define ZPCI_INSN_BUSY_DELAY 1 /* 1 microsecond */ +#define ZPCI_ERR_INSN_DATA_SIZE 32 struct zpci_err_insn_data { u8 insn; u8 cc; u8 status; + u8 pad0[5]; union { struct { u64 req; - u64 offset; + union { + /* PCILG/PCISTG/MPCIFC */ + struct { + u64 offset; + u64 pad1; + }; + /* RPCIT */ + struct { + u64 iova; + u64 range; + }; + }; }; + /* PCILGI/PCISTGI */ struct { u64 addr; u64 len; + u64 pad2; }; }; -} __packed; +}; + +static_assert(sizeof(struct zpci_err_insn_data) <= ZPCI_ERR_INSN_DATA_SIZE); + +static inline void zpci_err_insn_rpcit(int lvl, u8 insn, u8 cc, u8 status, + u64 req, u64 iova, u64 range) +{ + struct zpci_err_insn_data data = { + .insn = insn, .cc = cc, .status = status, + .req = req, .iova = iova, .range = range}; + + zpci_err_hex_level(lvl, &data, sizeof(data)); +} static inline void zpci_err_insn_req(int lvl, u8 insn, u8 cc, u8 status, u64 req, u64 offset) @@ -111,26 +138,27 @@ static inline u8 __rpcit(u64 fn, u64 addr, u64 range, u8 *status) return CC_TRANSFORM(cc); } -int zpci_refresh_trans(u64 fn, u64 addr, u64 range) +int zpci_refresh_trans(u32 fh, u64 addr, u64 range) { + u64 req = ZPCI_CREATE_REQ(fh, 0, 0); bool retried = false; u8 cc, status; do { - cc = __rpcit(fn, addr, range, &status); + cc = __rpcit(req, addr, range, &status); if (cc == 2) { udelay(ZPCI_INSN_BUSY_DELAY); if (!retried) { - zpci_err_insn_addr(1, 'R', cc, status, addr, range); + zpci_err_insn_rpcit(1, 'R', cc, status, req, addr, range); retried = true; } } } while (cc == 2); if (cc) - zpci_err_insn_addr(0, 'R', cc, status, addr, range); + zpci_err_insn_rpcit(0, 'R', cc, status, req, addr, range); else if (retried) - zpci_err_insn_addr(1, 'R', cc, status, addr, range); + zpci_err_insn_rpcit(1, 'R', cc, status, req, addr, range); if (cc == 1 && (status == 4 || status == 16)) return -ENOMEM; @@ -161,11 +189,11 @@ static inline int ____pcilg(u64 *data, u64 req, u64 offset, u8 *status) exception = 1; asm_inline volatile ( - " .insn rre,0xb9d20000,%[data],%[req_off]\n" - "0: lhi %[exc],0\n" + "0: .insn rre,0xb9d20000,%[data],%[req_off]\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [data] "=d" (__data), [req_off] "+d" (req_off.pair), [exc] "+d" (exception) : @@ -230,11 +258,11 @@ static inline int __pcilg_mio(u64 *data, u64 ioaddr, u64 len, u8 *status) exception = 1; asm_inline volatile ( - " .insn rre,0xb9d60000,%[data],%[ioaddr_len]\n" - "0: lhi %[exc],0\n" + "0: .insn rre,0xb9d60000,%[data],%[ioaddr_len]\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [data] "=d" (__data), [ioaddr_len] "+d" (ioaddr_len.pair), [exc] "+d" (exception) : @@ -268,11 +296,11 @@ static inline int __pcistg(u64 data, u64 req, u64 offset, u8 *status) exception = 1; asm_inline volatile ( - " .insn rre,0xb9d00000,%[data],%[req_off]\n" - "0: lhi %[exc],0\n" + "0: .insn rre,0xb9d00000,%[data],%[req_off]\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [req_off] "+d" (req_off.pair), [exc] "+d" (exception) : [data] "d" (data) : CC_CLOBBER); @@ -322,11 +350,11 @@ static inline int __pcistg_mio(u64 data, u64 ioaddr, u64 len, u8 *status) exception = 1; asm_inline volatile ( - " .insn rre,0xb9d40000,%[data],%[ioaddr_len]\n" - "0: lhi %[exc],0\n" + "0: .insn rre,0xb9d40000,%[data],%[ioaddr_len]\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [ioaddr_len] "+d" (ioaddr_len.pair), [exc] "+d" (exception) : [data] "d" (data) : CC_CLOBBER_LIST("memory")); @@ -357,11 +385,11 @@ static inline int __pcistb(const u64 *data, u64 req, u64 offset, u8 *status) exception = 1; asm_inline volatile ( - " .insn rsy,0xeb00000000d0,%[req],%[offset],%[data]\n" - "0: lhi %[exc],0\n" + "0: .insn rsy,0xeb00000000d0,%[req],%[offset],%[data]\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [req] "+d" (req), [exc] "+d" (exception) : [offset] "d" (offset), [data] "Q" (*data) : CC_CLOBBER); @@ -411,11 +439,11 @@ static inline int __pcistb_mio(const u64 *data, u64 ioaddr, u64 len, u8 *status) exception = 1; asm_inline volatile ( - " .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[data]\n" - "0: lhi %[exc],0\n" + "0: .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[data]\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [len] "+d" (len), [exc] "+d" (exception) : [ioaddr] "d" (ioaddr), [data] "Q" (*data) : CC_CLOBBER); diff --git a/arch/s390/pci/pci_irq.c b/arch/s390/pci/pci_irq.c index 9c9ed3d8d959..16ae80ed2d13 100644 --- a/arch/s390/pci/pci_irq.c +++ b/arch/s390/pci/pci_irq.c @@ -22,12 +22,11 @@ static enum {FLOATING, DIRECTED} irq_delivery; */ static struct airq_iv *zpci_sbv; -/* - * interrupt bit vectors - * FLOATING - interrupt bit vector per function - * DIRECTED - interrupt bit vector per cpu - */ -static struct airq_iv **zpci_ibv; +/* FLOATING - interrupt bit vector per function */ +static struct airq_iv __rcu **zpci_ibv; + +/* DIRECTED - interrupt bit vector per cpu */ +static struct airq_iv **zpci_dibv; /* Modify PCI: Register floating adapter interruptions */ static int zpci_set_airq(struct zpci_dev *zdev) @@ -169,7 +168,7 @@ static struct irq_chip zpci_irq_chip = { static void zpci_handle_cpu_local_irq(bool rescan) { - struct airq_iv *dibv = zpci_ibv[smp_processor_id()]; + struct airq_iv *dibv = zpci_dibv[smp_processor_id()]; union zpci_sic_iib iib = {{0}}; struct irq_domain *msi_domain; irq_hw_number_t hwirq; @@ -279,7 +278,9 @@ static void zpci_floating_irq_handler(struct airq_struct *airq, } /* Scan the adapter interrupt vector for this device. */ - aibv = zpci_ibv[si]; + aibv = rcu_dereference(zpci_ibv[si]); + if (!aibv) + continue; for (ai = 0;;) { ai = airq_iv_scan(aibv, ai, airq_iv_end(aibv)); if (ai == -1UL) @@ -299,7 +300,7 @@ static int __alloc_airq(struct zpci_dev *zdev, int msi_vecs, { if (irq_delivery == DIRECTED) { /* Allocate cpu vector bits */ - *bit = airq_iv_alloc(zpci_ibv[0], msi_vecs); + *bit = airq_iv_alloc(zpci_dibv[0], msi_vecs); if (*bit == -1UL) return -EIO; } else { @@ -313,11 +314,14 @@ static int __alloc_airq(struct zpci_dev *zdev, int msi_vecs, zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_PTR | AIRQ_IV_DATA | AIRQ_IV_BITLOCK, NULL); - if (!zdev->aibv) + if (!zdev->aibv) { + airq_iv_free_bit(zpci_sbv, *bit); + zdev->aisb = -1UL; return -ENOMEM; + } /* Wire up shortcut pointer */ - zpci_ibv[*bit] = zdev->aibv; + rcu_assign_pointer(zpci_ibv[*bit], zdev->aibv); /* Each function has its own interrupt vector */ *bit = 0; } @@ -339,13 +343,16 @@ static struct airq_struct zpci_airq = { static void zpci_msi_teardown_directed(struct zpci_dev *zdev) { - airq_iv_free(zpci_ibv[0], zdev->msi_first_bit, zdev->max_msi); + airq_iv_free(zpci_dibv[0], zdev->msi_first_bit, zdev->msi_nr_irqs); zdev->msi_first_bit = -1U; zdev->msi_nr_irqs = 0; } static void zpci_msi_teardown_floating(struct zpci_dev *zdev) { + rcu_assign_pointer(zpci_ibv[zdev->aisb], NULL); + synchronize_rcu(); + airq_iv_release(zdev->aibv); zdev->aibv = NULL; airq_iv_free_bit(zpci_sbv, zdev->aisb); @@ -425,9 +432,9 @@ static int zpci_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, if (irq_delivery == DIRECTED) { for_each_possible_cpu(cpu) { - airq_iv_set_ptr(zpci_ibv[cpu], bit + i, + airq_iv_set_ptr(zpci_dibv[cpu], bit + i, (unsigned long)zbus->msi_parent_domain); - airq_iv_set_data(zpci_ibv[cpu], bit + i, hwirq + i); + airq_iv_set_data(zpci_dibv[cpu], bit + i, hwirq + i); } } else { airq_iv_set_ptr(zdev->aibv, bit + i, @@ -439,7 +446,7 @@ static int zpci_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, return 0; } -static void zpci_msi_clear_airq(struct irq_data *d, int i) +static void zpci_msi_clear_airq(struct irq_data *d) { struct msi_desc *desc = irq_data_get_msi_desc(d); struct zpci_dev *zdev = to_zpci_dev(desc->dev); @@ -452,12 +459,12 @@ static void zpci_msi_clear_airq(struct irq_data *d, int i) if (irq_delivery == DIRECTED) { for_each_possible_cpu(cpu) { - airq_iv_set_ptr(zpci_ibv[cpu], bit + i, 0); - airq_iv_set_data(zpci_ibv[cpu], bit + i, 0); + airq_iv_set_ptr(zpci_dibv[cpu], bit, 0); + airq_iv_set_data(zpci_dibv[cpu], bit, 0); } } else { - airq_iv_set_ptr(zdev->aibv, bit + i, 0); - airq_iv_set_data(zdev->aibv, bit + i, 0); + airq_iv_set_ptr(zdev->aibv, bit, 0); + airq_iv_set_data(zdev->aibv, bit, 0); } } @@ -469,7 +476,7 @@ static void zpci_msi_domain_free(struct irq_domain *domain, unsigned int virq, for (i = 0; i < nr_irqs; i++) { d = irq_domain_get_irq_data(domain, virq + i); - zpci_msi_clear_airq(d, i); + zpci_msi_clear_airq(d); irq_domain_reset_irq_data(d); } } @@ -516,9 +523,6 @@ int zpci_create_parent_msi_domain(struct zpci_bus *zbus) return -ENOMEM; } - if (irq_delivery == FLOATING) - zpci_msi_parent_ops.required_flags |= MSI_FLAG_NO_AFFINITY; - zbus->msi_parent_domain = msi_create_parent_irq_domain(&info, &zpci_msi_parent_ops); if (!zbus->msi_parent_domain) { irq_domain_free_fwnode(info.fwnode); @@ -533,9 +537,13 @@ void zpci_remove_parent_msi_domain(struct zpci_bus *zbus) { struct fwnode_handle *fn; + if (!zbus->msi_parent_domain) + return; + fn = zbus->msi_parent_domain->fwnode; irq_domain_remove(zbus->msi_parent_domain); irq_domain_free_fwnode(fn); + zbus->msi_parent_domain = NULL; } static void __init cpu_enable_directed_irq(void *unused) @@ -543,7 +551,7 @@ static void __init cpu_enable_directed_irq(void *unused) union zpci_sic_iib iib = {{0}}; union zpci_sic_iib ziib = {{0}}; - iib.cdiib.dibv_addr = virt_to_phys(zpci_ibv[smp_processor_id()]->vector); + iib.cdiib.dibv_addr = virt_to_phys(zpci_dibv[smp_processor_id()]->vector); zpci_set_irq_ctrl(SIC_IRQ_MODE_SET_CPU, 0, &iib); zpci_set_irq_ctrl(SIC_IRQ_MODE_D_SINGLE, PCI_ISC, &ziib); @@ -558,33 +566,45 @@ static int __init zpci_directed_irq_init(void) if (!zpci_sbv) return -ENOMEM; - iib.diib.isc = PCI_ISC; - iib.diib.nr_cpus = num_possible_cpus(); - iib.diib.disb_addr = virt_to_phys(zpci_sbv->vector); - zpci_set_irq_ctrl(SIC_IRQ_MODE_DIRECT, 0, &iib); - - zpci_ibv = kzalloc_objs(*zpci_ibv, num_possible_cpus()); - if (!zpci_ibv) - return -ENOMEM; + zpci_dibv = kzalloc_objs(*zpci_dibv, num_possible_cpus()); + if (!zpci_dibv) + goto out_free_sbv; for_each_possible_cpu(cpu) { /* * Per CPU IRQ vectors look the same but bit-allocation * is only done on the first vector. */ - zpci_ibv[cpu] = airq_iv_create(cache_line_size() * BITS_PER_BYTE, - AIRQ_IV_PTR | - AIRQ_IV_DATA | - AIRQ_IV_CACHELINE | - (!cpu ? AIRQ_IV_ALLOC : 0), NULL); - if (!zpci_ibv[cpu]) - return -ENOMEM; + zpci_dibv[cpu] = airq_iv_create(cache_line_size() * BITS_PER_BYTE, + AIRQ_IV_PTR | + AIRQ_IV_DATA | + AIRQ_IV_CACHELINE | + (!cpu ? AIRQ_IV_ALLOC : 0), NULL); + if (!zpci_dibv[cpu]) + goto out_free_dibv; } + + iib.diib.isc = PCI_ISC; + iib.diib.nr_cpus = num_possible_cpus(); + iib.diib.disb_addr = virt_to_phys(zpci_sbv->vector); + zpci_set_irq_ctrl(SIC_IRQ_MODE_DIRECT, 0, &iib); on_each_cpu(cpu_enable_directed_irq, NULL, 1); zpci_irq_chip.irq_set_affinity = zpci_set_irq_affinity; return 0; + +out_free_dibv: + for_each_possible_cpu(cpu) { + if (zpci_dibv[cpu]) + airq_iv_release(zpci_dibv[cpu]); + } + kfree(zpci_dibv); + zpci_dibv = NULL; +out_free_sbv: + airq_iv_release(zpci_sbv); + zpci_sbv = NULL; + return -ENOMEM; } static int __init zpci_floating_irq_init(void) @@ -613,6 +633,9 @@ int __init zpci_irq_init(void) if (s390_pci_force_floating) irq_delivery = FLOATING; + if (irq_delivery == FLOATING) + zpci_msi_parent_ops.required_flags |= MSI_FLAG_NO_AFFINITY; + if (irq_delivery == DIRECTED) zpci_airq.handler = zpci_directed_irq_handler; @@ -651,13 +674,15 @@ void __init zpci_irq_exit(void) { unsigned int cpu; + unregister_adapter_interrupt(&zpci_airq); if (irq_delivery == DIRECTED) { for_each_possible_cpu(cpu) { - airq_iv_release(zpci_ibv[cpu]); + airq_iv_release(zpci_dibv[cpu]); } + kfree(zpci_dibv); + } else { + kfree(zpci_ibv); } - kfree(zpci_ibv); if (zpci_sbv) airq_iv_release(zpci_sbv); - unregister_adapter_interrupt(&zpci_airq); } diff --git a/arch/s390/pci/pci_mmio.c b/arch/s390/pci/pci_mmio.c index f3f79ba78410..8b49175b0031 100644 --- a/arch/s390/pci/pci_mmio.c +++ b/arch/s390/pci/pci_mmio.c @@ -39,10 +39,10 @@ static inline int __pcistb_mio_inuser( asm_inline volatile ( " sacf 256\n" "0: .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n" - "1: lhi %[exc],0\n" - "2: sacf 768\n" + " lhi %[exc],0\n" + "1: sacf 768\n" CC_IPM(cc) - EX_TABLE(0b, 2b) EX_TABLE(1b, 2b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [len] "+d" (len), [exc] "+d" (exception) : [ioaddr] "a" (ioaddr), [src] "Q" (*((u8 __force *)src)) : CC_CLOBBER_LIST("memory")); @@ -72,15 +72,16 @@ static inline int __pcistg_mio_inuser( asm_inline volatile ( " sacf 256\n" "0: llgc %[tmp],0(%[src])\n" - "4: sllg %[val],%[val],8\n" + " sllg %[val],%[val],8\n" " aghi %[src],1\n" " ogr %[val],%[tmp]\n" " brctg %[cnt],0b\n" "1: .insn rre,0xb9d40000,%[val],%[ioaddr_len]\n" - "2: lhi %[exc],0\n" - "3: sacf 768\n" + " lhi %[exc],0\n" + "2: sacf 768\n" CC_IPM(cc) - EX_TABLE(0b, 3b) EX_TABLE(4b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b) + EX_TABLE_INSN(0b, 2b) + EX_TABLE_INSN(1b, 2b) : [src] "+a" (src), [cnt] "+d" (cnt), [val] "+d" (val), [tmp] "=d" (tmp), [exc] "+d" (exception), CC_OUT(cc, cc), [ioaddr_len] "+&d" (ioaddr_len.pair) @@ -227,13 +228,13 @@ static inline int __pcilg_mio_inuser( asm_inline volatile ( " sacf 256\n" "0: .insn rre,0xb9d60000,%[val],%[ioaddr_len]\n" - "1: lhi %[exc],0\n" + " lhi %[exc],0\n" " jne 4f\n" - "2: ahi %[shift],-8\n" + "1: ahi %[shift],-8\n" " srlg %[tmp],%[val],0(%[shift])\n" - "3: stc %[tmp],0(%[dst])\n" - "5: aghi %[dst],1\n" - " brctg %[cnt],2b\n" + "2: stc %[tmp],0(%[dst])\n" + " aghi %[dst],1\n" + " brctg %[cnt],1b\n" /* * Use xr to clear exc and set condition code to zero * to ensure flag output is correct for this branch. @@ -241,7 +242,8 @@ static inline int __pcilg_mio_inuser( " xr %[exc],%[exc]\n" "4: sacf 768\n" CC_IPM(cc) - EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b) EX_TABLE(5b, 4b) + EX_TABLE_INSN(0b, 4b) + EX_TABLE_INSN(2b, 4b) : [ioaddr_len] "+&d" (ioaddr_len.pair), [exc] "+d" (exception), CC_OUT(cc, cc), [val] "=d" (val), [dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp), diff --git a/arch/s390/tools/.gitignore b/arch/s390/tools/.gitignore index e6af51d9d183..e07d27143172 100644 --- a/arch/s390/tools/.gitignore +++ b/arch/s390/tools/.gitignore @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only gen_facilities +gen_bitmaps gen_opcode_table relocs diff --git a/arch/s390/tools/Makefile b/arch/s390/tools/Makefile index f2862364fb42..ba1e42d005e7 100644 --- a/arch/s390/tools/Makefile +++ b/arch/s390/tools/Makefile @@ -4,25 +4,31 @@ # kapi := arch/$(ARCH)/include/generated/asm -kapi-hdrs-y := $(kapi)/facility-defs.h $(kapi)/dis-defs.h +kapi-hdrs-y := $(kapi)/dis-defs.h +kapi-hdrs-y += $(kapi)/facility-defs.h +kapi-hdrs-y += $(kapi)/pgmcheck-defs.h PHONY += kapi kapi: $(kapi-hdrs-y) -hostprogs += gen_facilities +hostprogs += gen_bitmaps hostprogs += gen_opcode_table -HOSTCFLAGS_gen_facilities.o += $(LINUXINCLUDE) +HOSTCFLAGS_gen_bitmaps.o += $(LINUXINCLUDE) -filechk_facility-defs.h = $(obj)/gen_facilities +filechk_facility-defs.h = $(obj)/gen_bitmaps --facility +filechk_pgmcheck-defs.h = $(obj)/gen_bitmaps --pgmcheck filechk_dis-defs.h = \ $(obj)/gen_opcode_table < $(srctree)/arch/$(ARCH)/tools/opcodes.txt -$(kapi)/facility-defs.h: $(obj)/gen_facilities FORCE +$(kapi)/facility-defs.h: $(obj)/gen_bitmaps FORCE $(call filechk,facility-defs.h) +$(kapi)/pgmcheck-defs.h: $(obj)/gen_bitmaps FORCE + $(call filechk,pgmcheck-defs.h) + $(kapi)/dis-defs.h: $(obj)/gen_opcode_table FORCE $(call filechk,dis-defs.h) diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_bitmaps.c index 32dd5a57240d..2e5dc491b19e 100644 --- a/arch/s390/tools/gen_facilities.c +++ b/arch/s390/tools/gen_bitmaps.c @@ -1,24 +1,27 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Simple program to generate defines out of facility lists that use the bit - * numbering scheme from the Princples of Operations: most significant bit - * has bit number 0. + * Simple program to generate bitmaps, for example facility lists. * * Copyright IBM Corp. 2015, 2018 * */ +#include <getopt.h> #include <strings.h> #include <string.h> #include <stdlib.h> #include <stdio.h> -struct facility_def { +#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) + +struct bitmap_def { char *name; int *bits; + int inverted; + int maxbit; }; -static struct facility_def facility_defs[] = { +static struct bitmap_def facility_defs[] = { { /* * FACILITIES_ALS contains the list of facilities that are @@ -26,6 +29,7 @@ static struct facility_def facility_defs[] = { * -march=<machine>. */ .name = "FACILITIES_ALS", + .inverted = 1, .bits = (int[]){ 0, /* N3 instructions */ 1, /* z/Arch mode installed */ @@ -70,6 +74,7 @@ static struct facility_def facility_defs[] = { * guest facilities supported by KVM. */ .name = "FACILITIES_KVM", + .inverted = 1, .bits = (int[]){ 0, /* N3 instructions */ 1, /* z/Arch mode installed */ @@ -115,6 +120,7 @@ static struct facility_def facility_defs[] = { */ .name = "FACILITIES_KVM_CPUMODEL", + .inverted = 1, .bits = (int[]){ 12, /* AP Query Configuration Information */ 15, /* AP Facilities Test */ @@ -124,17 +130,56 @@ static struct facility_def facility_defs[] = { }, }; -static void print_facility_list(struct facility_def *def) +static struct bitmap_def pgm_check_defs[] = { + { + .name = "PGM_CHECKS_NULLIFIED", + .inverted = 0, + .maxbit = 127, + .bits = (int[]){ + 0x10, /* Segment Translation */ + 0x11, /* Page Translation */ + 0x16, /* Trace Table */ + 0x20, /* AFX Translation */ + 0x21, /* ASX Translation */ + 0x22, /* LX Translation */ + 0x23, /* EX Translation */ + 0x24, /* Primary Authority */ + 0x25, /* Secondary Authority */ + 0x26, /* LFX Translation */ + 0x27, /* LSX Translation */ + 0x29, /* ALEN Translation */ + 0x2a, /* ALE Sequence */ + 0x2b, /* ASTE Validity */ + 0x2c, /* ASTE Sequence */ + 0x2d, /* Extended Authority */ + 0x2e, /* LSTE Sequence */ + 0x2f, /* ASTE Instance */ + 0x30, /* Stack Full */ + 0x31, /* Stack Empty */ + 0x32, /* Stack Specification */ + 0x33, /* Stack Type */ + 0x34, /* Stack Operation */ + 0x38, /* ASCE Type */ + 0x39, /* Region First Translation */ + 0x3a, /* Region Second Translation */ + 0x3b, /* Region Third Translation */ + 0x3d, /* Secure Storage Access */ + -1 /* END */ + } + }, +}; + +static void print_bitmap(struct bitmap_def *def) { unsigned int high, bit, dword, i; unsigned long long *array; - array = calloc(1, 8); + high = def->maxbit ? def->maxbit / 64 : 0; + array = calloc(high + 1, 8); if (!array) exit(EXIT_FAILURE); - high = 0; for (i = 0; def->bits[i] != -1; i++) { - bit = 63 - (def->bits[i] & 63); + bit = def->inverted ? 63 - (def->bits[i] & 63) : (def->bits[i] & 63); dword = def->bits[i] / 64; if (dword > high) { array = realloc(array, (dword + 1) * 8); @@ -151,25 +196,45 @@ static void print_facility_list(struct facility_def *def) free(array); } -static void print_facility_lists(void) +static void print_header(const char *guard, struct bitmap_def *defs, size_t num) { unsigned int i; - for (i = 0; i < sizeof(facility_defs) / sizeof(facility_defs[0]); i++) - print_facility_list(&facility_defs[i]); -} - -int main(int argc, char **argv) -{ - printf("#ifndef __ASM_S390_FACILITY_DEFS__\n"); - printf("#define __ASM_S390_FACILITY_DEFS__\n"); + printf("#ifndef __ASM_S390_%s_DEFS__\n", guard); + printf("#define __ASM_S390_%s_DEFS__\n", guard); printf("/*\n"); printf(" * DO NOT MODIFY.\n"); printf(" *\n"); printf(" * This file was generated by %s\n", __FILE__); printf(" */\n\n"); printf("#include <linux/const.h>\n\n"); - print_facility_lists(); + for (i = 0; i < num; i++) + print_bitmap(&defs[i]); printf("\n#endif\n"); +} + +static const struct option long_options[] = { + { "facility", no_argument, NULL, 'f' }, + { "pgmcheck", no_argument, NULL, 'p' }, + { NULL, 0, NULL, 0 } +}; + +int main(int argc, char **argv) +{ + int opt; + + opt = getopt_long(argc, argv, "", long_options, NULL); + if (opt == -1) + return 1; + switch (opt) { + case 'f': + print_header("FACILITY", facility_defs, ARRAY_SIZE(facility_defs)); + break; + case 'p': + print_header("PGMCHECK", pgm_check_defs, ARRAY_SIZE(pgm_check_defs)); + break; + default: + return 1; + } return 0; } diff --git a/arch/s390/tools/gen_opcode_table.c b/arch/s390/tools/gen_opcode_table.c index 7d76c417f83f..820905eca78a 100644 --- a/arch/s390/tools/gen_opcode_table.c +++ b/arch/s390/tools/gen_opcode_table.c @@ -201,17 +201,6 @@ static int cmp_long_insn(const void *a, const void *b) return strcmp(((struct insn *)a)->name, ((struct insn *)b)->name); } -static void print_insn_name(const char *name) -{ - size_t i, len; - - len = strlen(name); - printf("{"); - for (i = 0; i < len; i++) - printf(" \'%c\',", name[i]); - printf(" }"); -} - static void print_long_insn(struct gen_opcode *desc) { struct insn *insn; @@ -234,9 +223,7 @@ static void print_long_insn(struct gen_opcode *desc) insn = &desc->insn[i]; if (insn->name_len < 6) continue; - printf("\t[LONG_INSN_%s] = ", insn->upper); - print_insn_name(insn->name); - printf(", \\\n"); + printf("\t[LONG_INSN_%s] = \"%s\", \\\n", insn->upper, insn->name); } printf("}\n\n"); } @@ -249,12 +236,10 @@ static void print_opcode(struct insn *insn, int nr) if (insn->type->byte != 0) opcode += 2; printf("\t[%4d] = { .opfrag = 0x%s, .format = INSTR_%s, ", nr, opcode, insn->format); - if (insn->name_len < 6) { - printf(".name = "); - print_insn_name(insn->name); - } else { + if (insn->name_len < 6) + printf(".name = \"%s\"", insn->name); + else printf(".offset = LONG_INSN_%s", insn->upper); - } printf(" }, \\\n"); } diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c index 58ca7727b7f2..a280ad4f63b7 100644 --- a/drivers/iommu/s390-iommu.c +++ b/drivers/iommu/s390-iommu.c @@ -788,7 +788,7 @@ static struct iommu_device *s390_iommu_probe_device(struct device *dev) static int zpci_refresh_all(struct zpci_dev *zdev) { - return zpci_refresh_trans((u64)zdev->fh << 32, zdev->start_dma, + return zpci_refresh_trans(zdev->fh, zdev->start_dma, zdev->end_dma - zdev->start_dma + 1); } @@ -819,8 +819,7 @@ static void s390_iommu_iotlb_sync(struct iommu_domain *domain, rcu_read_lock(); list_for_each_entry_rcu(zdev, &s390_domain->devices, iommu_list) { atomic64_inc(&s390_domain->ctrs.sync_rpcits); - zpci_refresh_trans((u64)zdev->fh << 32, gather->start, - size); + zpci_refresh_trans(zdev->fh, gather->start, size); } rcu_read_unlock(); } @@ -837,8 +836,7 @@ static int s390_iommu_iotlb_sync_map(struct iommu_domain *domain, if (!zdev->tlb_refresh) continue; atomic64_inc(&s390_domain->ctrs.sync_map_rpcits); - ret = zpci_refresh_trans((u64)zdev->fh << 32, - iova, size); + ret = zpci_refresh_trans(zdev->fh, iova, size); /* * let the hypervisor discover invalidated entries * allowing it to free IOVAs and unpin pages diff --git a/drivers/s390/block/dasd_diag.c b/drivers/s390/block/dasd_diag.c index af68c2ee659d..ce391d35adf9 100644 --- a/drivers/s390/block/dasd_diag.c +++ b/drivers/s390/block/dasd_diag.c @@ -77,11 +77,11 @@ static inline int __dia250(void *iob, int cmd) exception = 1; asm_inline volatile( - " diag %[rx],%[cmd],0x250\n" - "0: lhi %[exc],0\n" + "0: diag %[rx],%[cmd],0x250\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b,1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [rx] "+d" (rx.pair), "+m" (*(addr_type *)iob), [exc] "+d" (exception) : [cmd] "d" (cmd) diff --git a/drivers/s390/char/diag_ftp.c b/drivers/s390/char/diag_ftp.c index a1e110c96f74..0a506559799e 100644 --- a/drivers/s390/char/diag_ftp.c +++ b/drivers/s390/char/diag_ftp.c @@ -106,11 +106,11 @@ static int diag_ftp_2c4(struct diag_ftp_ldfpl *fpl, diag_stat_inc(DIAG_STAT_X2C4); asm_inline volatile( - " diag %[addr],%[cmd],0x2c4\n" - "0: j 2f\n" + "0: diag %[addr],%[cmd],0x2c4\n" + " j 2f\n" "1: la %[rc],%[err]\n" "2:\n" - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : [rc] "=d" (rc), "+m" (*fpl) : [cmd] "0" (cmd), [addr] "d" (virt_to_phys(fpl)), [err] "i" (DIAG_FTP_RET_EPERM) diff --git a/drivers/s390/char/fs3270.c b/drivers/s390/char/fs3270.c index f8e2cc17ac53..25f0a85d92fa 100644 --- a/drivers/s390/char/fs3270.c +++ b/drivers/s390/char/fs3270.c @@ -355,6 +355,9 @@ static long fs3270_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) if (copy_to_user(argp, &iocb, sizeof(struct raw3270_iocb))) rc = -EFAULT; break; + default: + rc = -ENOTTY; + break; } mutex_unlock(&fs3270_mutex); return rc; diff --git a/drivers/s390/char/keyboard.c b/drivers/s390/char/keyboard.c index ecd9ace2b3aa..63bcb52d6e87 100644 --- a/drivers/s390/char/keyboard.c +++ b/drivers/s390/char/keyboard.c @@ -81,7 +81,8 @@ kbd_alloc(void) { kbd = kzalloc_obj(struct kbd_data); if (!kbd) goto out; - kbd->key_maps = kzalloc(sizeof(ebc_key_maps), GFP_KERNEL); + kbd->key_maps = kcalloc(ARRAY_SIZE(ebc_key_maps), + sizeof(*kbd->key_maps), GFP_KERNEL); if (!kbd->key_maps) goto out_kbd; for (i = 0; i < ARRAY_SIZE(ebc_key_maps); i++) { @@ -93,7 +94,8 @@ kbd_alloc(void) { goto out_maps; } } - kbd->func_table = kzalloc(sizeof(ebc_func_table), GFP_KERNEL); + kbd->func_table = kcalloc(ARRAY_SIZE(ebc_func_table), + sizeof(*kbd->func_table), GFP_KERNEL); if (!kbd->func_table) goto out_maps; for (i = 0; i < ARRAY_SIZE(ebc_func_table); i++) { diff --git a/drivers/s390/char/sclp.h b/drivers/s390/char/sclp.h index b31a680e0871..b91d31f8e0ef 100644 --- a/drivers/s390/char/sclp.h +++ b/drivers/s390/char/sclp.h @@ -320,11 +320,10 @@ static inline int sclp_service_call(sclp_cmdw_t command, void *sccb) exception = 1; asm_inline volatile( "0: .insn rre,0xb2200000,%[cmd],%[sccb]\n" /* servc */ - "1: lhi %[exc],0\n" - "2:\n" + " lhi %[exc],0\n" + "1:\n" CC_IPM(cc) - EX_TABLE(0b, 2b) - EX_TABLE(1b, 2b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [exc] "+d" (exception) : [cmd] "d" (command), [sccb] "a" (__pa(sccb)) : CC_CLOBBER_LIST("memory")); diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c index 22dd797e6229..821b61727d1f 100644 --- a/drivers/s390/char/sclp_early.c +++ b/drivers/s390/char/sclp_early.c @@ -138,7 +138,6 @@ int __init sclp_early_get_core_info(struct sclp_core_info *info) if (!sccb) return -ENOMEM; - memset(sccb, 0, length); sccb->header.length = length; sccb->header.control_mask[2] = 0x80; if (sclp_early_cmd(SCLP_CMDW_READ_CPU_INFO, sccb)) { diff --git a/drivers/s390/char/sclp_vt220.c b/drivers/s390/char/sclp_vt220.c index 4e2bf8186a5d..b95af0974624 100644 --- a/drivers/s390/char/sclp_vt220.c +++ b/drivers/s390/char/sclp_vt220.c @@ -27,6 +27,7 @@ #include <linux/init.h> #include <linux/reboot.h> #include <linux/slab.h> +#include <linux/workqueue.h> #include <linux/uaccess.h> #include "sclp.h" @@ -52,6 +53,17 @@ struct sclp_vt220_sccb { struct evbuf_header evbuf; }; +struct sclp_vt220_evbuf { + struct evbuf_header header; + char type; + char data[]; +} __packed; + +struct sclp_vt220_resize_data_t { + u16 rows; + u16 cols; +} __packed; + #define SCLP_VT220_MAX_CHARS_PER_BUFFER (PAGE_SIZE - \ sizeof(struct sclp_vt220_request) - \ sizeof(struct sclp_vt220_sccb)) @@ -89,6 +101,15 @@ static int __initdata sclp_vt220_init_count; * another buffer */ static int sclp_vt220_flush_later; +/* Work struct required for scheduling resize */ +static struct work_struct sclp_vt220_resize_work; + +/* Current vt220 terminal winsize */ +static struct winsize sclp_vt220_winsize = { + .ws_row = 24, + .ws_col = 80, +}; + static void sclp_vt220_receiver_fn(struct evbuf_header *evbuf); static int __sclp_vt220_emit(struct sclp_vt220_request *request); static void sclp_vt220_emit_current(void); @@ -468,6 +489,7 @@ sclp_vt220_write(struct tty_struct *tty, const u8 *buf, size_t count) #define SCLP_VT220_SESSION_ENDED 0x01 #define SCLP_VT220_SESSION_STARTED 0x80 #define SCLP_VT220_SESSION_DATA 0x00 +#define SCLP_VT220_SESSION_RESIZE 0x08 #ifdef CONFIG_MAGIC_SYSRQ @@ -516,30 +538,65 @@ static void sclp_vt220_handle_input(const char *buffer, unsigned int count) #endif +static void sclp_vt220_resize(struct work_struct *work) +{ + struct tty_struct *tty; + struct winsize ws; + + spin_lock_irq(&sclp_vt220_lock); + ws = sclp_vt220_winsize; + spin_unlock_irq(&sclp_vt220_lock); + + tty = tty_port_tty_get(&sclp_vt220_port); + if (!tty) + return; + + tty_do_resize(tty, &ws); + tty_kref_put(tty); +} + +static void sclp_vt220_resize_sched(void *buffer, unsigned int count) +{ + struct sclp_vt220_resize_data_t *data = buffer; + unsigned long flags; + + if (count < sizeof(*data)) + return; + + spin_lock_irqsave(&sclp_vt220_lock, flags); + sclp_vt220_winsize.ws_row = data->rows; + sclp_vt220_winsize.ws_col = data->cols; + schedule_work(&sclp_vt220_resize_work); + spin_unlock_irqrestore(&sclp_vt220_lock, flags); +} + /* * Called by the SCLP to report incoming event buffers. */ -static void -sclp_vt220_receiver_fn(struct evbuf_header *evbuf) +static void sclp_vt220_receiver_fn(struct evbuf_header *evbuf) { - char *buffer; + struct sclp_vt220_evbuf *buffer; unsigned int count; - buffer = (char *) ((addr_t) evbuf + sizeof(struct evbuf_header)); - count = evbuf->length - sizeof(struct evbuf_header); + if (evbuf->length < offsetof(struct sclp_vt220_evbuf, data)) + return; + + buffer = (struct sclp_vt220_evbuf *)evbuf; + count = evbuf->length - offsetof(struct sclp_vt220_evbuf, data); - switch (*buffer) { + switch (buffer->type) { case SCLP_VT220_SESSION_ENDED: case SCLP_VT220_SESSION_STARTED: sclp_vt220_reset_session(); break; case SCLP_VT220_SESSION_DATA: /* Send input to line discipline */ - buffer++; - count--; - sclp_vt220_handle_input(buffer, count); + sclp_vt220_handle_input(buffer->data, count); tty_flip_buffer_push(&sclp_vt220_port); break; + case SCLP_VT220_SESSION_RESIZE: + sclp_vt220_resize_sched(buffer->data, count); + break; } } @@ -551,10 +608,12 @@ sclp_vt220_open(struct tty_struct *tty, struct file *filp) { if (tty->count == 1) { tty_port_tty_set(&sclp_vt220_port, tty); - if (!tty->winsize.ws_row && !tty->winsize.ws_col) { - tty->winsize.ws_row = 24; - tty->winsize.ws_col = 80; - } + + mutex_lock(&tty->winsize_mutex); + spin_lock_irq(&sclp_vt220_lock); + tty->winsize = sclp_vt220_winsize; + spin_unlock_irq(&sclp_vt220_lock); + mutex_unlock(&tty->winsize_mutex); } return 0; } @@ -747,6 +806,8 @@ static int __init sclp_vt220_tty_init(void) rc = tty_register_driver(driver); if (rc) goto out_init; + + INIT_WORK(&sclp_vt220_resize_work, sclp_vt220_resize); rc = sclp_register(&sclp_vt220_register_input); if (rc) goto out_reg; diff --git a/drivers/s390/char/vmcp.c b/drivers/s390/char/vmcp.c index b277f9d0c321..2cf94f23af2a 100644 --- a/drivers/s390/char/vmcp.c +++ b/drivers/s390/char/vmcp.c @@ -104,13 +104,11 @@ static int vmcp_open(struct inode *inode, struct file *file) if (!capable(CAP_SYS_ADMIN)) return -EPERM; - session = kmalloc_obj(*session); + session = kzalloc_obj(*session); if (!session) return -ENOMEM; session->bufsize = PAGE_SIZE; - session->response = NULL; - session->resp_size = 0; mutex_init(&session->mutex); file->private_data = session; return nonseekable_open(inode, file); diff --git a/drivers/s390/cio/chsc.c b/drivers/s390/cio/chsc.c index 9689f722c863..d98f629e8929 100644 --- a/drivers/s390/cio/chsc.c +++ b/drivers/s390/cio/chsc.c @@ -931,12 +931,12 @@ static int cub_alloc(struct channel_subsystem *css) int i; for (i = 0; i < CSS_NUM_CUB_PAGES; i++) { - css->cub[i] = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + css->cub[i] = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!css->cub[i]) return -ENOMEM; } for (i = 0; i < CSS_NUM_ECUB_PAGES; i++) { - css->ecub[i] = (void *)get_zeroed_page(GFP_KERNEL); + css->ecub[i] = kzalloc(PAGE_SIZE, GFP_KERNEL); if (!css->ecub[i]) return -ENOMEM; } @@ -949,11 +949,11 @@ static void cub_free(struct channel_subsystem *css) int i; for (i = 0; i < CSS_NUM_CUB_PAGES; i++) { - free_page((unsigned long)css->cub[i]); + kfree(css->cub[i]); css->cub[i] = NULL; } for (i = 0; i < CSS_NUM_ECUB_PAGES; i++) { - free_page((unsigned long)css->ecub[i]); + kfree(css->ecub[i]); css->ecub[i] = NULL; } } @@ -1142,8 +1142,8 @@ int __init chsc_init(void) { int ret; - sei_page = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); - chsc_page = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + sei_page = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); + chsc_page = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!sei_page || !chsc_page) { ret = -ENOMEM; goto out_err; @@ -1153,16 +1153,16 @@ int __init chsc_init(void) goto out_err; return ret; out_err: - free_page((unsigned long)chsc_page); - free_page((unsigned long)sei_page); + kfree(chsc_page); + kfree(sei_page); return ret; } void __init chsc_init_cleanup(void) { crw_unregister_handler(CRW_RSC_CSS); - free_page((unsigned long)chsc_page); - free_page((unsigned long)sei_page); + kfree(chsc_page); + kfree(sei_page); } int __chsc_enable_facility(struct chsc_sda_area *sda_area, int operation_code) diff --git a/drivers/s390/cio/chsc_sch.c b/drivers/s390/cio/chsc_sch.c index b6cb8bb8bcc4..225aaac7dd30 100644 --- a/drivers/s390/cio/chsc_sch.c +++ b/drivers/s390/cio/chsc_sch.c @@ -15,6 +15,7 @@ #include <linux/uaccess.h> #include <linux/miscdevice.h> #include <linux/kernel_stat.h> +#include <linux/cleanup.h> #include <asm/cio.h> #include <asm/chsc.h> @@ -292,7 +293,7 @@ static int chsc_ioctl_start(void __user *user_area) if (!css_general_characteristics.dynio) /* It makes no sense to try. */ return -EOPNOTSUPP; - chsc_area = (void *)get_zeroed_page(GFP_DMA | GFP_KERNEL); + chsc_area = kzalloc(PAGE_SIZE, GFP_DMA | GFP_KERNEL); if (!chsc_area) return -ENOMEM; request = kzalloc_obj(*request); @@ -321,7 +322,7 @@ out_free: snprintf(dbf, sizeof(dbf), "ret:%d", ret); CHSC_LOG(0, dbf); kfree(request); - free_page((unsigned long)chsc_area); + kfree(chsc_area); return ret; } @@ -340,7 +341,7 @@ static int chsc_ioctl_on_close_set(void __user *user_area) ret = -ENOMEM; goto out_unlock; } - on_close_chsc_area = (void *)get_zeroed_page(GFP_DMA | GFP_KERNEL); + on_close_chsc_area = kzalloc(PAGE_SIZE, GFP_DMA | GFP_KERNEL); if (!on_close_chsc_area) { ret = -ENOMEM; goto out_free_request; @@ -353,7 +354,7 @@ static int chsc_ioctl_on_close_set(void __user *user_area) goto out_unlock; out_free_chsc: - free_page((unsigned long)on_close_chsc_area); + kfree(on_close_chsc_area); on_close_chsc_area = NULL; out_free_request: kfree(on_close_request); @@ -375,7 +376,7 @@ static int chsc_ioctl_on_close_remove(void) ret = -ENOENT; goto out_unlock; } - free_page((unsigned long)on_close_chsc_area); + kfree(on_close_chsc_area); on_close_chsc_area = NULL; kfree(on_close_request); on_close_request = NULL; @@ -389,39 +390,29 @@ out_unlock: static int chsc_ioctl_start_sync(void __user *user_area) { - struct chsc_sync_area *chsc_area; - int ret, ccode; + struct chsc_sync_area *chsc_area __free(kfree) = NULL; + int ccode; - chsc_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + chsc_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!chsc_area) return -ENOMEM; - if (copy_from_user(chsc_area, user_area, PAGE_SIZE)) { - ret = -EFAULT; - goto out_free; - } - if (chsc_area->header.code & 0x4000) { - ret = -EINVAL; - goto out_free; - } + if (copy_from_user(chsc_area, user_area, PAGE_SIZE)) + return -EFAULT; + if (chsc_area->header.code & 0x4000) + return -EINVAL; chsc_log_command(chsc_area); ccode = chsc(chsc_area); - if (ccode != 0) { - ret = -EIO; - goto out_free; - } + if (ccode != 0) + return -EIO; if (copy_to_user(user_area, chsc_area, PAGE_SIZE)) - ret = -EFAULT; - else - ret = 0; -out_free: - free_page((unsigned long)chsc_area); - return ret; + return -EFAULT; + return 0; } static int chsc_ioctl_info_channel_path(void __user *user_cd) { - struct chsc_chp_cd *cd; - int ret, ccode; + struct chsc_chp_cd *cd __free(kfree) = NULL; + int ccode; struct { struct chsc_header request; u32 : 2; @@ -436,20 +427,16 @@ static int chsc_ioctl_info_channel_path(void __user *user_cd) u32 : 32; struct chsc_header response; u8 data[PAGE_SIZE - 20]; - } __attribute__ ((packed)) *scpcd_area; + } __attribute__ ((packed)) *scpcd_area __free(kfree) = NULL; - scpcd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + scpcd_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!scpcd_area) return -ENOMEM; cd = kzalloc_obj(*cd); - if (!cd) { - ret = -ENOMEM; - goto out_free; - } - if (copy_from_user(cd, user_cd, sizeof(*cd))) { - ret = -EFAULT; - goto out_free; - } + if (!cd) + return -ENOMEM; + if (copy_from_user(cd, user_cd, sizeof(*cd))) + return -EFAULT; scpcd_area->request.length = 0x0010; scpcd_area->request.code = 0x0028; scpcd_area->m = cd->m; @@ -459,31 +446,23 @@ static int chsc_ioctl_info_channel_path(void __user *user_cd) scpcd_area->last_chpid = cd->chpid.id; ccode = chsc(scpcd_area); - if (ccode != 0) { - ret = -EIO; - goto out_free; - } + if (ccode != 0) + return -EIO; if (scpcd_area->response.code != 0x0001) { - ret = -EIO; CHSC_MSG(0, "scpcd: response code=%x\n", scpcd_area->response.code); - goto out_free; + return -EIO; } memcpy(&cd->cpcb, &scpcd_area->response, scpcd_area->response.length); if (copy_to_user(user_cd, cd, sizeof(*cd))) - ret = -EFAULT; - else - ret = 0; -out_free: - kfree(cd); - free_page((unsigned long)scpcd_area); - return ret; + return -EFAULT; + return 0; } static int chsc_ioctl_info_cu(void __user *user_cd) { - struct chsc_cu_cd *cd; - int ret, ccode; + struct chsc_cu_cd *cd __free(kfree) = NULL; + int ccode; struct { struct chsc_header request; u32 : 2; @@ -498,20 +477,16 @@ static int chsc_ioctl_info_cu(void __user *user_cd) u32 : 32; struct chsc_header response; u8 data[PAGE_SIZE - 20]; - } __attribute__ ((packed)) *scucd_area; + } __attribute__ ((packed)) *scucd_area __free(kfree) = NULL; - scucd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + scucd_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!scucd_area) return -ENOMEM; cd = kzalloc_obj(*cd); - if (!cd) { - ret = -ENOMEM; - goto out_free; - } - if (copy_from_user(cd, user_cd, sizeof(*cd))) { - ret = -EFAULT; - goto out_free; - } + if (!cd) + return -ENOMEM; + if (copy_from_user(cd, user_cd, sizeof(*cd))) + return -EFAULT; scucd_area->request.length = 0x0010; scucd_area->request.code = 0x0026; scucd_area->m = cd->m; @@ -521,31 +496,23 @@ static int chsc_ioctl_info_cu(void __user *user_cd) scucd_area->last_cun = cd->cun; ccode = chsc(scucd_area); - if (ccode != 0) { - ret = -EIO; - goto out_free; - } + if (ccode != 0) + return -EIO; if (scucd_area->response.code != 0x0001) { - ret = -EIO; CHSC_MSG(0, "scucd: response code=%x\n", scucd_area->response.code); - goto out_free; + return -EIO; } memcpy(&cd->cucb, &scucd_area->response, scucd_area->response.length); if (copy_to_user(user_cd, cd, sizeof(*cd))) - ret = -EFAULT; - else - ret = 0; -out_free: - kfree(cd); - free_page((unsigned long)scucd_area); - return ret; + return -EFAULT; + return 0; } static int chsc_ioctl_info_sch_cu(void __user *user_cud) { - struct chsc_sch_cud *cud; - int ret, ccode; + struct chsc_sch_cud *cud __free(kfree) = NULL; + int ccode; struct { struct chsc_header request; u32 : 2; @@ -561,20 +528,16 @@ static int chsc_ioctl_info_sch_cu(void __user *user_cud) u32 : 32; struct chsc_header response; u8 data[PAGE_SIZE - 20]; - } __attribute__ ((packed)) *sscud_area; + } __attribute__ ((packed)) *sscud_area __free(kfree) = NULL; - sscud_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + sscud_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!sscud_area) return -ENOMEM; cud = kzalloc_obj(*cud); - if (!cud) { - ret = -ENOMEM; - goto out_free; - } - if (copy_from_user(cud, user_cud, sizeof(*cud))) { - ret = -EFAULT; - goto out_free; - } + if (!cud) + return -ENOMEM; + if (copy_from_user(cud, user_cud, sizeof(*cud))) + return -EFAULT; sscud_area->request.length = 0x0010; sscud_area->request.code = 0x0006; sscud_area->m = cud->schid.m; @@ -585,31 +548,23 @@ static int chsc_ioctl_info_sch_cu(void __user *user_cud) sscud_area->last_sch = cud->schid.sch_no; ccode = chsc(sscud_area); - if (ccode != 0) { - ret = -EIO; - goto out_free; - } + if (ccode != 0) + return -EIO; if (sscud_area->response.code != 0x0001) { - ret = -EIO; CHSC_MSG(0, "sscud: response code=%x\n", sscud_area->response.code); - goto out_free; + return -EIO; } memcpy(&cud->scub, &sscud_area->response, sscud_area->response.length); if (copy_to_user(user_cud, cud, sizeof(*cud))) - ret = -EFAULT; - else - ret = 0; -out_free: - kfree(cud); - free_page((unsigned long)sscud_area); - return ret; + return -EFAULT; + return 0; } static int chsc_ioctl_conf_info(void __user *user_ci) { - struct chsc_conf_info *ci; - int ret, ccode; + struct chsc_conf_info *ci __free(kfree) = NULL; + int ccode; struct { struct chsc_header request; u32 : 2; @@ -623,20 +578,16 @@ static int chsc_ioctl_conf_info(void __user *user_ci) u64 : 64; struct chsc_header response; u8 data[PAGE_SIZE - 20]; - } __attribute__ ((packed)) *sci_area; + } __attribute__ ((packed)) *sci_area __free(kfree) = NULL; - sci_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + sci_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!sci_area) return -ENOMEM; ci = kzalloc_obj(*ci); - if (!ci) { - ret = -ENOMEM; - goto out_free; - } - if (copy_from_user(ci, user_ci, sizeof(*ci))) { - ret = -EFAULT; - goto out_free; - } + if (!ci) + return -ENOMEM; + if (copy_from_user(ci, user_ci, sizeof(*ci))) + return -EFAULT; sci_area->request.length = 0x0010; sci_area->request.code = 0x0012; sci_area->m = ci->id.m; @@ -645,31 +596,23 @@ static int chsc_ioctl_conf_info(void __user *user_ci) sci_area->ssid = ci->id.ssid; ccode = chsc(sci_area); - if (ccode != 0) { - ret = -EIO; - goto out_free; - } + if (ccode != 0) + return -EIO; if (sci_area->response.code != 0x0001) { - ret = -EIO; CHSC_MSG(0, "sci: response code=%x\n", sci_area->response.code); - goto out_free; + return -EIO; } memcpy(&ci->scid, &sci_area->response, sci_area->response.length); if (copy_to_user(user_ci, ci, sizeof(*ci))) - ret = -EFAULT; - else - ret = 0; -out_free: - kfree(ci); - free_page((unsigned long)sci_area); - return ret; + return -EFAULT; + return 0; } static int chsc_ioctl_conf_comp_list(void __user *user_ccl) { - struct chsc_comp_list *ccl; - int ret, ccode; + struct chsc_comp_list *ccl __free(kfree) = NULL; + int ccode; struct { struct chsc_header request; u32 ctype : 8; @@ -681,7 +624,7 @@ static int chsc_ioctl_conf_comp_list(void __user *user_ccl) u64 : 64; struct chsc_header response; u8 data[PAGE_SIZE - 36]; - } __attribute__ ((packed)) *sccl_area; + } __attribute__ ((packed)) *sccl_area __free(kfree) = NULL; struct { u32 m : 1; u32 : 31; @@ -696,18 +639,14 @@ static int chsc_ioctl_conf_comp_list(void __user *user_ccl) u32 res; } __attribute__ ((packed)) *cssids_parm; - sccl_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + sccl_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!sccl_area) return -ENOMEM; ccl = kzalloc_obj(*ccl); - if (!ccl) { - ret = -ENOMEM; - goto out_free; - } - if (copy_from_user(ccl, user_ccl, sizeof(*ccl))) { - ret = -EFAULT; - goto out_free; - } + if (!ccl) + return -ENOMEM; + if (copy_from_user(ccl, user_ccl, sizeof(*ccl))) + return -EFAULT; sccl_area->request.length = 0x0020; sccl_area->request.code = 0x0030; sccl_area->fmt = ccl->req.fmt; @@ -728,61 +667,46 @@ static int chsc_ioctl_conf_comp_list(void __user *user_ccl) break; } ccode = chsc(sccl_area); - if (ccode != 0) { - ret = -EIO; - goto out_free; - } + if (ccode != 0) + return -EIO; if (sccl_area->response.code != 0x0001) { - ret = -EIO; CHSC_MSG(0, "sccl: response code=%x\n", sccl_area->response.code); - goto out_free; + return -EIO; } memcpy(&ccl->sccl, &sccl_area->response, sccl_area->response.length); if (copy_to_user(user_ccl, ccl, sizeof(*ccl))) - ret = -EFAULT; - else - ret = 0; -out_free: - kfree(ccl); - free_page((unsigned long)sccl_area); - return ret; + return -EFAULT; + return 0; } static int chsc_ioctl_chpd(void __user *user_chpd) { - struct chsc_scpd *scpd_area; - struct chsc_cpd_info *chpd; + struct chsc_scpd *scpd_area __free(kfree) = NULL; + struct chsc_cpd_info *chpd __free(kfree) = NULL; int ret; chpd = kzalloc_obj(*chpd); - scpd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); - if (!scpd_area || !chpd) { - ret = -ENOMEM; - goto out_free; - } - if (copy_from_user(chpd, user_chpd, sizeof(*chpd))) { - ret = -EFAULT; - goto out_free; - } + scpd_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); + if (!scpd_area || !chpd) + return -ENOMEM; + if (copy_from_user(chpd, user_chpd, sizeof(*chpd))) + return -EFAULT; ret = chsc_determine_channel_path_desc(chpd->chpid, chpd->fmt, chpd->rfmt, chpd->c, chpd->m, scpd_area); if (ret) - goto out_free; + return ret; memcpy(&chpd->chpdb, &scpd_area->response, scpd_area->response.length); if (copy_to_user(user_chpd, chpd, sizeof(*chpd))) - ret = -EFAULT; -out_free: - kfree(chpd); - free_page((unsigned long)scpd_area); - return ret; + return -EFAULT; + return 0; } static int chsc_ioctl_dcal(void __user *user_dcal) { - struct chsc_dcal *dcal; - int ret, ccode; + struct chsc_dcal *dcal __free(kfree) = NULL; + int ccode; struct { struct chsc_header request; u32 atype : 8; @@ -794,20 +718,16 @@ static int chsc_ioctl_dcal(void __user *user_dcal) u32 res1[2]; struct chsc_header response; u8 data[PAGE_SIZE - 36]; - } __attribute__ ((packed)) *sdcal_area; + } __attribute__ ((packed)) *sdcal_area __free(kfree) = NULL; - sdcal_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + sdcal_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!sdcal_area) return -ENOMEM; dcal = kzalloc_obj(*dcal); - if (!dcal) { - ret = -ENOMEM; - goto out_free; - } - if (copy_from_user(dcal, user_dcal, sizeof(*dcal))) { - ret = -EFAULT; - goto out_free; - } + if (!dcal) + return -ENOMEM; + if (copy_from_user(dcal, user_dcal, sizeof(*dcal))) + return -EFAULT; sdcal_area->request.length = 0x0020; sdcal_area->request.code = 0x0034; sdcal_area->atype = dcal->req.atype; @@ -816,26 +736,18 @@ static int chsc_ioctl_dcal(void __user *user_dcal) sizeof(sdcal_area->list_parm)); ccode = chsc(sdcal_area); - if (ccode != 0) { - ret = -EIO; - goto out_free; - } + if (ccode != 0) + return -EIO; if (sdcal_area->response.code != 0x0001) { - ret = -EIO; CHSC_MSG(0, "sdcal: response code=%x\n", sdcal_area->response.code); - goto out_free; + return -EIO; } memcpy(&dcal->sdcal, &sdcal_area->response, sdcal_area->response.length); if (copy_to_user(user_dcal, dcal, sizeof(*dcal))) - ret = -EFAULT; - else - ret = 0; -out_free: - kfree(dcal); - free_page((unsigned long)sdcal_area); - return ret; + return -EFAULT; + return 0; } static long chsc_ioctl(struct file *filp, unsigned int cmd, @@ -904,7 +816,7 @@ static int chsc_release(struct inode *inode, struct file *filp) } snprintf(dbf, sizeof(dbf), "relret:%d", ret); CHSC_LOG(0, dbf); - free_page((unsigned long)on_close_chsc_area); + kfree(on_close_chsc_area); on_close_chsc_area = NULL; kfree(on_close_request); on_close_request = NULL; diff --git a/drivers/s390/cio/cmf.c b/drivers/s390/cio/cmf.c index 66b14fedbd18..02b0feb9d1fd 100644 --- a/drivers/s390/cio/cmf.c +++ b/drivers/s390/cio/cmf.c @@ -482,6 +482,9 @@ static int alloc_cmb(struct ccw_device *cdev) ssize_t size; struct cmb_data *cmb_data; + if (cmb_area.num_channels <= 0) + return -ENOMEM; + /* Allocate private cmb_data. */ cmb_data = kzalloc_obj(struct cmb_data); if (!cmb_data) @@ -501,12 +504,12 @@ static int alloc_cmb(struct ccw_device *cdev) WARN_ON(!list_empty(&cmb_area.list)); spin_unlock(&cmb_area.lock); - mem = (void *)__get_free_pages(GFP_KERNEL, get_order(size)); + mem = kmalloc(size, GFP_KERNEL); spin_lock(&cmb_area.lock); if (cmb_area.mem) { /* ok, another thread was faster */ - free_pages((unsigned long)mem, get_order(size)); + kfree(mem); } else if (!mem) { /* no luck */ ret = -ENOMEM; @@ -547,10 +550,8 @@ static void free_cmb(struct ccw_device *cdev) list_del_init(&priv->cmb_list); if (list_empty(&cmb_area.list)) { - ssize_t size; - size = sizeof(struct cmb) * cmb_area.num_channels; cmf_activate(NULL, CMF_OFF); - free_pages((unsigned long)cmb_area.mem, get_order(size)); + kfree(cmb_area.mem); cmb_area.mem = NULL; } spin_unlock_irq(cdev->ccwlock); diff --git a/drivers/s390/cio/ioasm.c b/drivers/s390/cio/ioasm.c index 8b06b234e110..4974e80b0568 100644 --- a/drivers/s390/cio/ioasm.c +++ b/drivers/s390/cio/ioasm.c @@ -24,11 +24,11 @@ static inline int __stsch(struct subchannel_id schid, struct schib *addr) exception = 1; asm_inline volatile( " lgr 1,%[r1]\n" - " stsch %[addr]\n" - "0: lhi %[exc],0\n" + "0: stsch %[addr]\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, ccode), [addr] "=Q" (*addr), [exc] "+d" (exception) : [r1] "d" (r1) : CC_CLOBBER_LIST("1")); @@ -54,11 +54,11 @@ static inline int __msch(struct subchannel_id schid, struct schib *addr) exception = 1; asm_inline volatile( " lgr 1,%[r1]\n" - " msch %[addr]\n" - "0: lhi %[exc],0\n" + "0: msch %[addr]\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, ccode), [exc] "+d" (exception) : [r1] "d" (r1), [addr] "Q" (*addr) : CC_CLOBBER_LIST("1")); @@ -108,11 +108,11 @@ static inline int __ssch(struct subchannel_id schid, union orb *addr) exception = 1; asm_inline volatile( " lgr 1,%[r1]\n" - " ssch %[addr]\n" - "0: lhi %[exc],0\n" + "0: ssch %[addr]\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, ccode), [exc] "+d" (exception) : [r1] "d" (r1), [addr] "Q" (*addr) : CC_CLOBBER_LIST("memory", "1")); @@ -179,11 +179,11 @@ int chsc(void *chsc_area) exception = 1; asm_inline volatile( - " .insn rre,0xb25f0000,%[chsc_area],0\n" - "0: lhi %[exc],0\n" + "0: .insn rre,0xb25f0000,%[chsc_area],0\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), "+m" (*(addr_type *)chsc_area), [exc] "+d" (exception) : [chsc_area] "d" (chsc_area) : CC_CLOBBER); diff --git a/drivers/s390/cio/qdio.h b/drivers/s390/cio/qdio.h index 4bd4c00c9c0c..dff5f53a8795 100644 --- a/drivers/s390/cio/qdio.h +++ b/drivers/s390/cio/qdio.h @@ -244,7 +244,7 @@ struct qdio_irq { int perf_stat_enabled; struct qdr *qdr; - unsigned long chsc_page; + void *chsc_page; struct qdio_q *input_qs[QDIO_MAX_QUEUES_PER_IRQ]; struct qdio_q *output_qs[QDIO_MAX_QUEUES_PER_IRQ]; diff --git a/drivers/s390/cio/qdio_main.c b/drivers/s390/cio/qdio_main.c index c1e09fa34e77..889f63f94a4a 100644 --- a/drivers/s390/cio/qdio_main.c +++ b/drivers/s390/cio/qdio_main.c @@ -12,11 +12,11 @@ #include <linux/module.h> #include <linux/init.h> #include <linux/kernel.h> -#include <linux/kmemleak.h> #include <linux/delay.h> #include <linux/gfp.h> #include <linux/io.h> #include <linux/atomic.h> +#include <linux/slab.h> #include <asm/debug.h> #include <asm/qdio.h> #include <asm/asm.h> @@ -933,10 +933,10 @@ int qdio_free(struct ccw_device *cdev) mutex_unlock(&irq_ptr->setup_mutex); qdio_free_queues(irq_ptr); - free_page((unsigned long) irq_ptr->qdr); - free_page(irq_ptr->chsc_page); + kfree(irq_ptr->qdr); + kfree(irq_ptr->chsc_page); kfree(irq_ptr->ccw); - free_page((unsigned long) irq_ptr); + kfree(irq_ptr); return 0; } EXPORT_SYMBOL_GPL(qdio_free); @@ -961,7 +961,7 @@ int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs, no_output_qs > QDIO_MAX_QUEUES_PER_IRQ) return -EINVAL; - irq_ptr = (void *) get_zeroed_page(GFP_KERNEL); + irq_ptr = kzalloc(PAGE_SIZE, GFP_KERNEL); if (!irq_ptr) return -ENOMEM; @@ -969,9 +969,6 @@ int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs, if (!irq_ptr->ccw) goto err_ccw; - /* kmemleak doesn't scan the page-allocated irq_ptr: */ - kmemleak_not_leak(irq_ptr->ccw); - irq_ptr->cdev = cdev; mutex_init(&irq_ptr->setup_mutex); if (qdio_allocate_dbf(irq_ptr)) @@ -986,12 +983,12 @@ int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs, * qdio_establish. In case of low memory and swap on a zfcp disk * we may not be able to allocate memory otherwise. */ - irq_ptr->chsc_page = get_zeroed_page(GFP_KERNEL); + irq_ptr->chsc_page = kzalloc(PAGE_SIZE, GFP_KERNEL); if (!irq_ptr->chsc_page) goto err_chsc; /* qdr is used in ccw1.cda which is u32 */ - irq_ptr->qdr = (struct qdr *) get_zeroed_page(GFP_KERNEL | GFP_DMA); + irq_ptr->qdr = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!irq_ptr->qdr) goto err_qdr; @@ -1004,14 +1001,14 @@ int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs, return 0; err_queues: - free_page((unsigned long) irq_ptr->qdr); + kfree(irq_ptr->qdr); err_qdr: - free_page(irq_ptr->chsc_page); + kfree(irq_ptr->chsc_page); err_chsc: err_dbf: kfree(irq_ptr->ccw); err_ccw: - free_page((unsigned long) irq_ptr); + kfree(irq_ptr); return rc; } EXPORT_SYMBOL_GPL(qdio_allocate); diff --git a/drivers/s390/cio/qdio_setup.c b/drivers/s390/cio/qdio_setup.c index ea09aadaae4e..e1c5c6a8b3b2 100644 --- a/drivers/s390/cio/qdio_setup.c +++ b/drivers/s390/cio/qdio_setup.c @@ -35,7 +35,7 @@ void qdio_free_buffers(struct qdio_buffer **buf, unsigned int count) int pos; for (pos = 0; pos < count; pos += QBUFF_PER_PAGE) - free_page((unsigned long) buf[pos]); + kfree(buf[pos]); } EXPORT_SYMBOL_GPL(qdio_free_buffers); @@ -49,7 +49,7 @@ int qdio_alloc_buffers(struct qdio_buffer **buf, unsigned int count) int pos; for (pos = 0; pos < count; pos += QBUFF_PER_PAGE) { - buf[pos] = (void *) get_zeroed_page(GFP_KERNEL); + buf[pos] = kzalloc(PAGE_SIZE, GFP_KERNEL); if (!buf[pos]) { qdio_free_buffers(buf, count); return -ENOMEM; @@ -83,7 +83,7 @@ static void __qdio_free_queues(struct qdio_q **queues, unsigned int count) for (i = 0; i < count; i++) { q = queues[i]; - free_page((unsigned long)q->sl_page); + kfree(q->sl_page); kmem_cache_free(qdio_q_cache, q); } } @@ -109,7 +109,7 @@ static int __qdio_allocate_qs(struct qdio_q **irq_ptr_qs, int nr_queues) return -ENOMEM; } - q->sl_page = (void *)__get_free_page(GFP_KERNEL); + q->sl_page = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!q->sl_page) { kmem_cache_free(qdio_q_cache, q); __qdio_free_queues(irq_ptr_qs, i); @@ -249,11 +249,11 @@ int qdio_setup_get_ssqd(struct qdio_irq *irq_ptr, DBF_EVENT("getssqd:%4x", schid->sch_no); if (!irq_ptr) { - ssqd = (struct chsc_ssqd_area *)__get_free_page(GFP_KERNEL); + ssqd = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!ssqd) return -ENOMEM; } else { - ssqd = (struct chsc_ssqd_area *)irq_ptr->chsc_page; + ssqd = irq_ptr->chsc_page; } rc = chsc_ssqd(*schid, ssqd); @@ -270,7 +270,7 @@ int qdio_setup_get_ssqd(struct qdio_irq *irq_ptr, out: if (!irq_ptr) - free_page((unsigned long)ssqd); + kfree(ssqd); return rc; } diff --git a/drivers/s390/cio/qdio_thinint.c b/drivers/s390/cio/qdio_thinint.c index e167aa75c3df..a2ff51887537 100644 --- a/drivers/s390/cio/qdio_thinint.c +++ b/drivers/s390/cio/qdio_thinint.c @@ -136,7 +136,7 @@ static struct airq_struct tiqdio_airq = { static int set_subchannel_ind(struct qdio_irq *irq_ptr, int reset) { - struct chsc_scssc_area *scssc = (void *)irq_ptr->chsc_page; + struct chsc_scssc_area *scssc = irq_ptr->chsc_page; dma64_t summary_indicator_addr, subchannel_indicator_addr; int rc; diff --git a/drivers/s390/cio/scm.c b/drivers/s390/cio/scm.c index 171212a6d2d9..b79fe023ab8d 100644 --- a/drivers/s390/cio/scm.c +++ b/drivers/s390/cio/scm.c @@ -13,6 +13,7 @@ #include <linux/slab.h> #include <linux/init.h> #include <linux/err.h> +#include <linux/cleanup.h> #include <asm/eadm.h> #include "chsc.h" @@ -224,12 +225,12 @@ static int scm_add(struct chsc_scm_info *scm_info, size_t num) int scm_update_information(void) { - struct chsc_scm_info *scm_info; + struct chsc_scm_info *scm_info __free(kfree) = NULL; u64 token = 0; size_t num; int ret; - scm_info = (void *)__get_free_page(GFP_KERNEL | GFP_DMA); + scm_info = kmalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!scm_info) return -ENOMEM; @@ -250,8 +251,6 @@ int scm_update_information(void) token = scm_info->restok; } while (token); - free_page((unsigned long)scm_info); - return ret; } diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index ec6a4c2f9f04..80435fd8e6ad 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -854,7 +854,7 @@ static long _zcrypt_send_cprb(u32 xflags, struct ap_perms *perms, struct ica_xcRB *xcrb) { bool userspace = xflags & ZCRYPT_XFLAG_USERSPACE; - unsigned int card, domain, func_code = 0; + unsigned int card, domain, _dom, func_code = 0; unsigned int wgt = 0, pref_wgt = 0; struct zcrypt_queue *zq, *pref_zq; struct zcrypt_card *zc, *pref_zc; @@ -877,10 +877,17 @@ static long _zcrypt_send_cprb(u32 xflags, struct ap_perms *perms, print_hex_dump_debug("ccareq: ", DUMP_PREFIX_ADDRESS, 16, 1, ap_msg.msg, ap_msg.len, false); + /* Check domain for either AUTOSEL_DOM or in range 0...AP_DOMAINS-1 */ + if (domain != AUTOSEL_DOM && domain >= AP_DOMAINS) { + rc = -EINVAL; + goto out; + } + /* Spectre-v1: sanitize domain unconditionally for later use */ + _dom = array_index_nospec(domain, AP_DOMAINS); + if (perms != &ap_perms && domain < AP_DOMAINS) { if (ap_msg.flags & AP_MSG_FLAG_ADMIN) { - domain = array_index_nospec(domain, AP_DOMAINS); - if (!test_bit_inv(domain, perms->adm)) { + if (!test_bit_inv(_dom, perms->adm)) { rc = -ENODEV; goto out; } @@ -893,10 +900,11 @@ static long _zcrypt_send_cprb(u32 xflags, struct ap_perms *perms, * If a valid target domain is set and this domain is NOT a usage * domain but a control only domain, autoselect target domain. */ - if (domain < AP_DOMAINS && - !ap_test_config_usage_domain(domain) && - ap_test_config_ctrl_domain(domain)) - domain = AUTOSEL_DOM; + if (domain < AP_DOMAINS) { + if (!ap_test_config_usage_domain(_dom) && + ap_test_config_ctrl_domain(_dom)) + domain = AUTOSEL_DOM; + } pref_zc = NULL; pref_zq = NULL; @@ -1041,7 +1049,7 @@ static long _zcrypt_send_ep11_cprb(u32 xflags, struct ap_perms *perms, struct ep11_target_dev *targets = NULL; unsigned short target_num; unsigned int wgt = 0, pref_wgt = 0; - unsigned int func_code = 0, domain; + unsigned int func_code = 0, domain, _dom; struct ap_message ap_msg; int cpen, qpen, qid = 0, rc; struct module *mod; @@ -1078,10 +1086,17 @@ static long _zcrypt_send_ep11_cprb(u32 xflags, struct ap_perms *perms, print_hex_dump_debug("ep11req: ", DUMP_PREFIX_ADDRESS, 16, 1, ap_msg.msg, ap_msg.len, false); + /* Check domain for either AUTOSEL_DOM or in range 0...AP_DOMAINS-1 */ + if (domain != AUTOSEL_DOM && domain >= AP_DOMAINS) { + rc = -EINVAL; + goto out; + } + /* Spectre-v1: sanitize domain unconditionally for later use */ + _dom = array_index_nospec(domain, AP_DOMAINS); + if (perms != &ap_perms && domain < AP_DOMAINS) { if (ap_msg.flags & AP_MSG_FLAG_ADMIN) { - domain = array_index_nospec(domain, AP_DOMAINS); - if (!test_bit_inv(domain, perms->adm)) { + if (!test_bit_inv(_dom, perms->adm)) { rc = -ENODEV; goto out; } diff --git a/drivers/s390/crypto/zcrypt_ccamisc.c b/drivers/s390/crypto/zcrypt_ccamisc.c index 19909bf43dc9..862947431164 100644 --- a/drivers/s390/crypto/zcrypt_ccamisc.c +++ b/drivers/s390/crypto/zcrypt_ccamisc.c @@ -1627,6 +1627,7 @@ int cca_query_crypto_facility(u16 cardnr, u16 domain, u8 subfunc_code[2]; u8 lvdata[]; } __packed * prepparm; + size_t datalen; /* get already prepared memory for 2 cprbs with param block each */ rc = alloc_and_prep_cprbmem(parmbsize, &mem, @@ -1673,27 +1674,44 @@ int cca_query_crypto_facility(u16 cardnr, u16 domain, prepcblk->rpl_parmb = (u8 __user *)ptr; prepparm = (struct fqrepparm *)ptr; ptr = prepparm->lvdata; + datalen = parmbsize - 2 * sizeof(u8); /* check and possibly copy reply rule array */ len = *((u16 *)ptr); + if (len > datalen) { + ZCRYPT_DBF_ERR("%s reply rule array len %u exceeds datalen %zu\n", + __func__, len, datalen); + rc = -EIO; + goto out; + } + datalen -= sizeof(u16); + ptr += sizeof(u16); if (len > sizeof(u16)) { - ptr += sizeof(u16); len -= sizeof(u16); if (rarray && rarraylen && *rarraylen > 0) { *rarraylen = (len > *rarraylen ? *rarraylen : len); memcpy(rarray, ptr, *rarraylen); } + datalen -= len; ptr += len; } /* check and possible copy reply var array */ len = *((u16 *)ptr); + if (len > datalen) { + ZCRYPT_DBF_ERR("%s reply var array len %u exceeds datalen %zu\n", + __func__, len, datalen); + rc = -EIO; + goto out; + } + datalen -= sizeof(u16); + ptr += sizeof(u16); if (len > sizeof(u16)) { - ptr += sizeof(u16); len -= sizeof(u16); if (varray && varraylen && *varraylen > 0) { *varraylen = (len > *varraylen ? *varraylen : len); memcpy(varray, ptr, *varraylen); } + datalen -= len; ptr += len; } diff --git a/drivers/s390/crypto/zcrypt_msgtype50.c b/drivers/s390/crypto/zcrypt_msgtype50.c index d6fc2d8e7fad..5907fb2a6920 100644 --- a/drivers/s390/crypto/zcrypt_msgtype50.c +++ b/drivers/s390/crypto/zcrypt_msgtype50.c @@ -152,6 +152,11 @@ struct type80_hdr { unsigned char reserved3[8]; } __packed; +struct type80_reply { + struct type80_hdr hdr; + char data[]; +} __packed; + int get_rsa_modex_fc(struct ica_rsa_modexpo *mex, int *fcode) { if (!mex->inputdatalength) @@ -340,32 +345,43 @@ static int ICACRT_msg_to_type50CRT_msg(struct zcrypt_queue *zq, * @data: pointer to user output data * @length: size of user output data * - * Returns 0 on success or -EFAULT. + * Returns 0 on success or neg. errno value on failure. */ static int convert_type80(struct zcrypt_queue *zq, struct ap_message *reply, char __user *outputdata, unsigned int outputdatalength) { - struct type80_hdr *t80h = reply->msg; - unsigned char *data; + struct type80_reply *msg = reply->msg; + size_t payload_len; + + /* + * reply->len is always >= sizeof(struct type80_reply) here and + * outputdatalength (the modulus size) is guaranteed to be equal to + * inputdatalength and 0 < outputdatalength <= CEX3A_MAX_MOD_SIZE, + * also payload size may be >= outputdatalength. + */ + + payload_len = reply->len - sizeof(msg->hdr); - if (t80h->len < sizeof(*t80h) + outputdatalength) { - /* The result is too short, the CEXxA card may not do that.. */ + if (outputdatalength > payload_len) { + /* We expect at least outputdatalength bytes, broken card ? */ zq->online = 0; pr_err("Crypto dev=%02x.%04x code=0x%02x => online=0 rc=EAGAIN\n", AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), t80h->code); + AP_QID_QUEUE(zq->queue->qid), msg->hdr.code); ZCRYPT_DBF_ERR("%s dev=%02x.%04x code=0x%02x => online=0 rc=EAGAIN\n", __func__, AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), t80h->code); + AP_QID_QUEUE(zq->queue->qid), msg->hdr.code); ap_send_online_uevent(&zq->queue->ap_dev, zq->online); return -EAGAIN; } - BUG_ON(t80h->len > CEX3A_MAX_RESPONSE_SIZE); - data = reply->msg + t80h->len - outputdatalength; - if (copy_to_user(outputdata, data, outputdatalength)) + + if (copy_to_user(outputdata, + msg->data + payload_len - outputdatalength, + outputdatalength)) return -EFAULT; + return 0; } @@ -374,14 +390,19 @@ static int convert_response(struct zcrypt_queue *zq, char __user *outputdata, unsigned int outputdatalength) { - /* Response type byte is the second byte in the response. */ - unsigned char rtype = ((unsigned char *)reply->msg)[1]; + struct type80_reply *msg = reply->msg; + + /* reply->len is always >= sizeof(struct error_hdr) here */ - switch (rtype) { + switch (msg->hdr.type) { case TYPE82_RSP_CODE: case TYPE88_RSP_CODE: return convert_error(zq, reply); case TYPE80_RSP_CODE: + if (msg->hdr.code) + return convert_error(zq, reply); + if (reply->len < sizeof(struct type80_reply)) + return -EINVAL; return convert_type80(zq, reply, outputdata, outputdatalength); default: /* Unknown response type, this should NEVER EVER happen */ @@ -389,11 +410,11 @@ static int convert_response(struct zcrypt_queue *zq, pr_err("Crypto dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n", AP_QID_CARD(zq->queue->qid), AP_QID_QUEUE(zq->queue->qid), - (int)rtype); + (int)msg->hdr.type); ZCRYPT_DBF_ERR( "%s dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n", __func__, AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), (int)rtype); + AP_QID_QUEUE(zq->queue->qid), (int)msg->hdr.type); ap_send_online_uevent(&zq->queue->ap_dev, zq->online); return -EAGAIN; } @@ -416,26 +437,46 @@ static void zcrypt_msgtype50_receive(struct ap_queue *aq, .reply_code = REP82_ERROR_MACHINE_FAILURE, }; struct type80_hdr *t80h; - int len; + size_t len; /* Copy the reply message to the request message buffer. */ if (!reply) goto out; /* ap_msg->rc indicates the error */ + t80h = reply->msg; - if (t80h->type == TYPE80_RSP_CODE) { - len = t80h->len; - if (len > reply->bufsize || len > msg->bufsize || - len != reply->len) { - pr_debug("len mismatch => EMSGSIZE\n"); - msg->rc = -EMSGSIZE; + + if (reply->len < sizeof(*t80h) || + t80h->type != TYPE80_RSP_CODE) { + if (reply->len < sizeof(error_reply)) { + /* total broken reply, use static error reply instead */ + memcpy(msg->msg, &error_reply, sizeof(error_reply)); + msg->len = sizeof(error_reply); goto out; + } else { + /* malformed reply, convert function will handle this */ + len = reply->len; + goto copy_len_and_out; } - memcpy(msg->msg, reply->msg, len); - msg->len = len; - } else { - memcpy(msg->msg, reply->msg, sizeof(error_reply)); - msg->len = sizeof(error_reply); } + + len = t80h->len; + +copy_len_and_out: + if (len != reply->len) { + msg->rc = -EMSGSIZE; + pr_debug("len %zu rpl.len %zu mismatch, msg.rc=%d\n", + len, reply->len, msg->rc); + goto out; + } + if (len > reply->bufsize || len > msg->bufsize) { + msg->rc = -EMSGSIZE; + pr_debug("len %zu exceeds buf %zu/%zu, msg.rc=%d\n", + len, reply->bufsize, msg->bufsize, msg->rc); + goto out; + } + memcpy(msg->msg, reply->msg, len); + msg->len = len; + out: complete(&msg->response.work); } diff --git a/drivers/s390/crypto/zcrypt_msgtype6.c b/drivers/s390/crypto/zcrypt_msgtype6.c index 3df1d676de5d..5962224bfb8b 100644 --- a/drivers/s390/crypto/zcrypt_msgtype6.c +++ b/drivers/s390/crypto/zcrypt_msgtype6.c @@ -766,6 +766,15 @@ static int convert_type86_rng(struct zcrypt_queue *zq, if (msg->cprbx.ccp_rtcode != 0 || msg->cprbx.ccp_rscode != 0) return -EINVAL; + /* + * Note that offset2 and count2 have already been checked in + * zcrypt_msgtype6_receive(). So only check for valid count2 + * and for not exceeding the hard coded rng buffer size. + */ + if (!msg->fmt2.count2) + return -EINVAL; + if (msg->fmt2.count2 > ZCRYPT_RNG_BUFFER_SIZE) + return -EMSGSIZE; memcpy(buffer, data + msg->fmt2.offset2, msg->fmt2.count2); return msg->fmt2.count2; } @@ -777,11 +786,17 @@ static int convert_response_ica(struct zcrypt_queue *zq, { struct type86x_reply *msg = reply->msg; + /* reply->len is always >= sizeof(struct error_hdr) here */ + switch (msg->hdr.type) { case TYPE82_RSP_CODE: case TYPE88_RSP_CODE: return convert_error(zq, reply); case TYPE86_RSP_CODE: + if (msg->hdr.reply_code) + return convert_error(zq, reply); + if (reply->len < sizeof(struct type86x_reply)) + return -EINVAL; if (msg->cprbx.ccp_rtcode && msg->cprbx.ccp_rscode == 0x14f && outputdatalength > 256) { @@ -792,8 +807,6 @@ static int convert_response_ica(struct zcrypt_queue *zq, return -EINVAL; } } - if (msg->hdr.reply_code) - return convert_error(zq, reply); if (msg->cprbx.cprb_ver_id == 0x02) return convert_type86_ica(zq, reply, outputdata, outputdatalength); @@ -820,6 +833,8 @@ static int convert_response_xcrb(bool userspace, struct zcrypt_queue *zq, { struct type86x_reply *msg = reply->msg; + /* reply->len is always >= sizeof(struct error_hdr) here */ + switch (msg->hdr.type) { case TYPE82_RSP_CODE: case TYPE88_RSP_CODE: @@ -827,9 +842,16 @@ static int convert_response_xcrb(bool userspace, struct zcrypt_queue *zq, return convert_error(zq, reply); case TYPE86_RSP_CODE: if (msg->hdr.reply_code) { - xcrb->status = msg->fmt2.apfs; + if (reply->len < sizeof(struct type86_fmt2_msg)) + xcrb->status = 0x0008044DL; + else + xcrb->status = msg->fmt2.apfs; return convert_error(zq, reply); } + if (reply->len < sizeof(struct type86_fmt2_msg) + + offsetof(struct CPRBX, cprb_ver_id) + + sizeof(msg->cprbx.cprb_ver_id)) + return -EINVAL; if (msg->cprbx.cprb_ver_id == 0x02) return convert_type86_xcrb(userspace, zq, reply, xcrb); fallthrough; /* wrong cprb version is an unknown response */ @@ -854,6 +876,8 @@ static int convert_response_ep11_xcrb(bool userspace, struct zcrypt_queue *zq, { struct type86_ep11_reply *msg = reply->msg; + /* reply->len is always >= sizeof(struct error_hdr) here */ + switch (msg->hdr.type) { case TYPE82_RSP_CODE: case TYPE87_RSP_CODE: @@ -861,6 +885,8 @@ static int convert_response_ep11_xcrb(bool userspace, struct zcrypt_queue *zq, case TYPE86_RSP_CODE: if (msg->hdr.reply_code) return convert_error(zq, reply); + if (reply->len < sizeof(struct type86_ep11_reply)) + return -EINVAL; if (msg->cprbx.cprb_ver_id == 0x04) return convert_type86_ep11_xcrb(userspace, zq, reply, xcrb); fallthrough; /* wrong cprb version is an unknown resp */ @@ -885,6 +911,8 @@ static int convert_response_rng(struct zcrypt_queue *zq, { struct type86x_reply *msg = reply->msg; + /* reply->len is always >= sizeof(struct error_hdr) here */ + switch (msg->hdr.type) { case TYPE82_RSP_CODE: case TYPE88_RSP_CODE: @@ -892,6 +920,8 @@ static int convert_response_rng(struct zcrypt_queue *zq, case TYPE86_RSP_CODE: if (msg->hdr.reply_code) return -EINVAL; + if (reply->len < sizeof(struct type86x_reply)) + return -EINVAL; if (msg->cprbx.cprb_ver_id == 0x02) return convert_type86_rng(zq, reply, data); fallthrough; /* wrong cprb version is an unknown response */ @@ -928,48 +958,85 @@ static void zcrypt_msgtype6_receive(struct ap_queue *aq, }; struct ap_response_type *resp_type = &msg->response; struct type86x_reply *t86r; - int len; + size_t minlen, len; /* Copy the reply message to the request message buffer. */ if (!reply) goto out; /* ap_msg->rc indicates the error */ + t86r = reply->msg; - if (t86r->hdr.type == TYPE86_RSP_CODE && - t86r->cprbx.cprb_ver_id == 0x02) { - switch (resp_type->type) { - case CEXXC_RESPONSE_TYPE_ICA: - len = sizeof(struct type86x_reply) + t86r->length; - if (len > reply->bufsize || len > msg->bufsize || - len != reply->len) { - pr_debug("len mismatch => EMSGSIZE\n"); - msg->rc = -EMSGSIZE; - goto out; - } - memcpy(msg->msg, reply->msg, len); - msg->len = len; - break; - case CEXXC_RESPONSE_TYPE_XCRB: - if (t86r->fmt2.count2) - len = t86r->fmt2.offset2 + t86r->fmt2.count2; - else - len = t86r->fmt2.offset1 + t86r->fmt2.count1; - if (len > reply->bufsize || len > msg->bufsize || - len != reply->len) { - pr_debug("len mismatch => EMSGSIZE\n"); + minlen = sizeof(t86r->hdr) + sizeof(t86r->fmt2) + + offsetof(struct CPRBX, cprb_ver_id) + + sizeof(t86r->cprbx.cprb_ver_id); + + if (reply->len < minlen || + t86r->hdr.type != TYPE86_RSP_CODE || + t86r->cprbx.cprb_ver_id != 0x02) { + if (reply->len < sizeof(error_reply)) { + /* total broken reply, use static error reply instead */ + memcpy(msg->msg, &error_reply, sizeof(error_reply)); + msg->len = sizeof(error_reply); + goto out; + } else { + /* malformed reply, convert function will handle this */ + len = reply->len; + goto copy_len_and_out; + } + } + + switch (resp_type->type) { + case CEXXC_RESPONSE_TYPE_ICA: + if (reply->len < sizeof(struct type86x_reply)) { + msg->rc = -EMSGSIZE; + pr_debug("rpl.len %zu < struct type86_reply, msg.rc=%d\n", + reply->len, msg->rc); + goto out; + } + len = sizeof(struct type86x_reply) + (size_t)t86r->length; + break; + case CEXXC_RESPONSE_TYPE_XCRB: + len = (size_t)t86r->fmt2.offset1 + (size_t)t86r->fmt2.count1; + if (len > reply->len) { + msg->rc = -EMSGSIZE; + pr_debug("offset1 %u count1 %u rpl.len %zu mismatch, msg.rc=%d\n", + t86r->fmt2.offset1, t86r->fmt2.count1, + reply->len, msg->rc); + goto out; + } + if (t86r->fmt2.count2) { + len = (size_t)t86r->fmt2.offset2 + + (size_t)t86r->fmt2.count2; + if (len > reply->len) { msg->rc = -EMSGSIZE; + pr_debug("offset2 %u count2 %u rpl.len %zu mismatch, msg.rc=%d\n", + t86r->fmt2.offset2, t86r->fmt2.count2, + reply->len, msg->rc); goto out; } - memcpy(msg->msg, reply->msg, len); - msg->len = len; - break; - default: - memcpy(msg->msg, &error_reply, sizeof(error_reply)); - msg->len = sizeof(error_reply); } - } else { - memcpy(msg->msg, reply->msg, sizeof(error_reply)); + break; + default: + memcpy(msg->msg, &error_reply, sizeof(error_reply)); msg->len = sizeof(error_reply); + goto out; + } + +copy_len_and_out: + if (len != reply->len) { + msg->rc = -EMSGSIZE; + pr_debug("len %zu rpl.len %zu mismatch, msg.rc=%d\n", + len, reply->len, msg->rc); + goto out; + } + if (len > reply->bufsize || len > msg->bufsize) { + msg->rc = -EMSGSIZE; + pr_debug("len %zu exceeds buf %zu/%zu, msg.rc=%d\n", + len, reply->bufsize, msg->bufsize, msg->rc); + goto out; } + memcpy(msg->msg, reply->msg, len); + msg->len = len; + out: complete(&resp_type->work); } @@ -992,34 +1059,65 @@ static void zcrypt_msgtype6_receive_ep11(struct ap_queue *aq, }; struct ap_response_type *resp_type = &msg->response; struct type86_ep11_reply *t86r; - int len; + size_t minlen, len; /* Copy the reply message to the request message buffer. */ if (!reply) goto out; /* ap_msg->rc indicates the error */ + t86r = reply->msg; - if (t86r->hdr.type == TYPE86_RSP_CODE && - t86r->cprbx.cprb_ver_id == 0x04) { - switch (resp_type->type) { - case CEXXC_RESPONSE_TYPE_EP11: - len = t86r->fmt2.offset1 + t86r->fmt2.count1; - if (len > reply->bufsize || len > msg->bufsize || - len != reply->len) { - pr_debug("len mismatch => EMSGSIZE\n"); - msg->rc = -EMSGSIZE; - goto out; - } - memcpy(msg->msg, reply->msg, len); - msg->len = len; - break; - default: + minlen = sizeof(t86r->hdr) + sizeof(t86r->fmt2) + + offsetof(struct ep11_cprb, cprb_ver_id) + + sizeof(t86r->cprbx.cprb_ver_id); + + if (reply->len < minlen || + t86r->hdr.type != TYPE86_RSP_CODE || + t86r->cprbx.cprb_ver_id != 0x04) { + if (reply->len < sizeof(error_reply)) { + /* total broken reply, use static error reply instead */ memcpy(msg->msg, &error_reply, sizeof(error_reply)); msg->len = sizeof(error_reply); + goto out; + } else { + /* malformed reply, convert function will handle this */ + len = reply->len; + goto copy_len_and_out; } - } else { - memcpy(msg->msg, reply->msg, sizeof(error_reply)); + } + + switch (resp_type->type) { + case CEXXC_RESPONSE_TYPE_EP11: + len = (size_t)t86r->fmt2.offset1 + (size_t)t86r->fmt2.count1; + if (len > reply->len) { + msg->rc = -EMSGSIZE; + pr_debug("offset1 %u count1 %u rpl.len %zu mismatch, msg.rc=%d\n", + t86r->fmt2.offset1, t86r->fmt2.count1, + reply->len, msg->rc); + goto out; + } + break; + default: + memcpy(msg->msg, &error_reply, sizeof(error_reply)); msg->len = sizeof(error_reply); + goto out; + } + +copy_len_and_out: + if (len != reply->len) { + msg->rc = -EMSGSIZE; + pr_debug("len %zu rpl.len %zu mismatch, msg.rc=%d\n", + len, reply->len, msg->rc); + goto out; + } + if (len > reply->bufsize || len > msg->bufsize) { + msg->rc = -EMSGSIZE; + pr_debug("len %zu exceeds buf %zu/%zu, msg.rc=%d\n", + len, reply->bufsize, msg->bufsize, msg->rc); + goto out; } + memcpy(msg->msg, reply->msg, len); + msg->len = len; + out: complete(&resp_type->work); } |