From 1dad093b66fdd4fd5d7d2692169dc1bafd794628 Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Mon, 31 Mar 2014 15:24:08 +0200 Subject: s390/irq: Use defines for external interruption codes Use the new defines for external interruption codes to get rid of "magic" numbers in the s390 source code. And while we're at it, also rename the (un-)register_external_interrupt function to something shorter so that this patch does not exceed the 80 columns all over the place. Signed-off-by: Thomas Huth Signed-off-by: Martin Schwidefsky --- arch/s390/kernel/irq.c | 10 +++++----- arch/s390/kernel/perf_cpum_cf.c | 6 ++++-- arch/s390/kernel/perf_cpum_sf.c | 6 ++++-- arch/s390/kernel/runtime_instr.c | 3 ++- arch/s390/kernel/sclp.S | 5 +++-- arch/s390/kernel/smp.c | 4 ++-- arch/s390/kernel/time.c | 6 +++--- 7 files changed, 23 insertions(+), 17 deletions(-) (limited to 'arch/s390/kernel') diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index bb27a262c44a..083617d739d8 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -205,7 +205,7 @@ static inline int ext_hash(u16 code) return (code + (code >> 9)) & (ARRAY_SIZE(ext_int_hash) - 1); } -int register_external_interrupt(u16 code, ext_int_handler_t handler) +int register_external_irq(u16 code, ext_int_handler_t handler) { struct ext_int_info *p; unsigned long flags; @@ -223,9 +223,9 @@ int register_external_interrupt(u16 code, ext_int_handler_t handler) spin_unlock_irqrestore(&ext_int_hash_lock, flags); return 0; } -EXPORT_SYMBOL(register_external_interrupt); +EXPORT_SYMBOL(register_external_irq); -int unregister_external_interrupt(u16 code, ext_int_handler_t handler) +int unregister_external_irq(u16 code, ext_int_handler_t handler) { struct ext_int_info *p; unsigned long flags; @@ -241,7 +241,7 @@ int unregister_external_interrupt(u16 code, ext_int_handler_t handler) spin_unlock_irqrestore(&ext_int_hash_lock, flags); return 0; } -EXPORT_SYMBOL(unregister_external_interrupt); +EXPORT_SYMBOL(unregister_external_irq); static irqreturn_t do_ext_interrupt(int irq, void *dummy) { @@ -251,7 +251,7 @@ static irqreturn_t do_ext_interrupt(int irq, void *dummy) int index; ext_code = *(struct ext_code *) ®s->int_code; - if (ext_code.code != 0x1004) + if (ext_code.code != EXT_IRQ_CLK_COMP) __get_cpu_var(s390_idle).nohz_delay = 1; index = ext_hash(ext_code.code); diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index f51214c04858..ea75d011a6fc 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -673,7 +673,8 @@ static int __init cpumf_pmu_init(void) ctl_clear_bit(0, 48); /* register handler for measurement-alert interruptions */ - rc = register_external_interrupt(0x1407, cpumf_measurement_alert); + rc = register_external_irq(EXT_IRQ_MEASURE_ALERT, + cpumf_measurement_alert); if (rc) { pr_err("Registering for CPU-measurement alerts " "failed with rc=%i\n", rc); @@ -684,7 +685,8 @@ static int __init cpumf_pmu_init(void) rc = perf_pmu_register(&cpumf_pmu, "cpum_cf", PERF_TYPE_RAW); if (rc) { pr_err("Registering the cpum_cf PMU failed with rc=%i\n", rc); - unregister_external_interrupt(0x1407, cpumf_measurement_alert); + unregister_external_irq(EXT_IRQ_MEASURE_ALERT, + cpumf_measurement_alert); goto out; } perf_cpu_notifier(cpumf_pmu_notifier); diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index 6c0d29827cb6..ea0c7b2ef030 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -1621,7 +1621,8 @@ static int __init init_cpum_sampling_pmu(void) pr_err("Registering for s390dbf failed\n"); debug_register_view(sfdbg, &debug_sprintf_view); - err = register_external_interrupt(0x1407, cpumf_measurement_alert); + err = register_external_irq(EXT_IRQ_MEASURE_ALERT, + cpumf_measurement_alert); if (err) { pr_cpumsf_err(RS_INIT_FAILURE_ALRT); goto out; @@ -1630,7 +1631,8 @@ static int __init init_cpum_sampling_pmu(void) err = perf_pmu_register(&cpumf_sampling, "cpum_sf", PERF_TYPE_RAW); if (err) { pr_cpumsf_err(RS_INIT_FAILURE_PERF); - unregister_external_interrupt(0x1407, cpumf_measurement_alert); + unregister_external_irq(EXT_IRQ_MEASURE_ALERT, + cpumf_measurement_alert); goto out; } perf_cpu_notifier(cpumf_pmu_notifier); diff --git a/arch/s390/kernel/runtime_instr.c b/arch/s390/kernel/runtime_instr.c index d817cce7e72d..26b4ae96fdd7 100644 --- a/arch/s390/kernel/runtime_instr.c +++ b/arch/s390/kernel/runtime_instr.c @@ -138,7 +138,8 @@ static int __init runtime_instr_init(void) return 0; irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT); - rc = register_external_interrupt(0x1407, runtime_instr_int_handler); + rc = register_external_irq(EXT_IRQ_MEASURE_ALERT, + runtime_instr_int_handler); if (rc) irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT); else diff --git a/arch/s390/kernel/sclp.S b/arch/s390/kernel/sclp.S index 29bd7bec4176..a41f2c99dcc8 100644 --- a/arch/s390/kernel/sclp.S +++ b/arch/s390/kernel/sclp.S @@ -9,6 +9,7 @@ */ #include +#include LC_EXT_NEW_PSW = 0x58 # addr of ext int handler LC_EXT_NEW_PSW_64 = 0x1b0 # addr of ext int handler 64 bit @@ -73,9 +74,9 @@ _sclp_wait_int: lpsw .LwaitpswS1-.LbaseS1(%r13) # wait until interrupt .LwaitS1: lh %r7,LC_EXT_INT_CODE - chi %r7,0x1004 # timeout? + chi %r7,EXT_IRQ_CLK_COMP # timeout? je .LtimeoutS1 - chi %r7,0x2401 # service int? + chi %r7,EXT_IRQ_SERVICE_SIG # service int? jne .LloopS1 sr %r2,%r2 l %r3,LC_EXT_INT_PARAM diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 8827883310dd..366d14460c2b 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -785,10 +785,10 @@ void __init smp_fill_possible_mask(void) void __init smp_prepare_cpus(unsigned int max_cpus) { /* request the 0x1201 emergency signal external interrupt */ - if (register_external_interrupt(0x1201, do_ext_call_interrupt) != 0) + if (register_external_irq(EXT_IRQ_EMERGENCY_SIG, do_ext_call_interrupt)) panic("Couldn't request external interrupt 0x1201"); /* request the 0x1202 external call external interrupt */ - if (register_external_interrupt(0x1202, do_ext_call_interrupt) != 0) + if (register_external_irq(EXT_IRQ_EXTERNAL_CALL, do_ext_call_interrupt)) panic("Couldn't request external interrupt 0x1202"); smp_detect_cpus(); } diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index dd95f1631621..386d37a228bb 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -262,11 +262,11 @@ void __init time_init(void) stp_reset(); /* request the clock comparator external interrupt */ - if (register_external_interrupt(0x1004, clock_comparator_interrupt)) - panic("Couldn't request external interrupt 0x1004"); + if (register_external_irq(EXT_IRQ_CLK_COMP, clock_comparator_interrupt)) + panic("Couldn't request external interrupt 0x1004"); /* request the timing alert external interrupt */ - if (register_external_interrupt(0x1406, timing_alert_interrupt)) + if (register_external_irq(EXT_IRQ_TIMING_ALERT, timing_alert_interrupt)) panic("Couldn't request external interrupt 0x1406"); if (clocksource_register(&clocksource_tod) != 0) -- cgit v1.2.3 From 1b948d6caec4f28e3524244ca0f77c6ae8ddceef Mon Sep 17 00:00:00 2001 From: Martin Schwidefsky Date: Thu, 3 Apr 2014 13:55:01 +0200 Subject: s390/mm,tlb: optimize TLB flushing for zEC12 The zEC12 machines introduced the local-clearing control for the IDTE and IPTE instruction. If the control is set only the TLB of the local CPU is cleared of entries, either all entries of a single address space for IDTE, or the entry for a single page-table entry for IPTE. Without the local-clearing control the TLB flush is broadcasted to all CPUs in the configuration, which is expensive. The reset of the bit mask of the CPUs that need flushing after a non-local IDTE is tricky. As TLB entries for an address space remain in the TLB even if the address space is detached a new bit field is required to keep track of attached CPUs vs. CPUs in the need of a flush. After a non-local flush with IDTE the bit-field of attached CPUs is copied to the bit-field of CPUs in need of a flush. The ordering of operations on cpu_attach_mask, attach_count and mm_cpumask(mm) is such that an underindication in mm_cpumask(mm) is prevented but an overindication in mm_cpumask(mm) is possible. Signed-off-by: Martin Schwidefsky --- arch/s390/kernel/early.c | 2 ++ arch/s390/kernel/smp.c | 6 ++++++ 2 files changed, 8 insertions(+) (limited to 'arch/s390/kernel') diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 6b594439cca5..a734f3585ceb 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -386,6 +386,8 @@ static __init void detect_machine_facilities(void) S390_lowcore.machine_flags |= MACHINE_FLAG_TE; if (test_facility(66)) S390_lowcore.machine_flags |= MACHINE_FLAG_RRBM; + if (test_facility(51)) + S390_lowcore.machine_flags |= MACHINE_FLAG_TLB_LC; #endif } diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 366d14460c2b..42a501d13a3b 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -236,6 +236,9 @@ static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu) { struct _lowcore *lc = pcpu->lowcore; + if (MACHINE_HAS_TLB_LC) + cpumask_set_cpu(cpu, &init_mm.context.cpu_attach_mask); + cpumask_set_cpu(cpu, mm_cpumask(&init_mm)); atomic_inc(&init_mm.context.attach_count); lc->cpu_nr = cpu; lc->percpu_offset = __per_cpu_offset[cpu]; @@ -760,6 +763,9 @@ void __cpu_die(unsigned int cpu) cpu_relax(); pcpu_free_lowcore(pcpu); atomic_dec(&init_mm.context.attach_count); + cpumask_clear_cpu(cpu, mm_cpumask(&init_mm)); + if (MACHINE_HAS_TLB_LC) + cpumask_clear_cpu(cpu, &init_mm.context.cpu_attach_mask); } void __noreturn cpu_die(void) -- cgit v1.2.3 From 457f2180951cdcbfb4657ddcc83b486e93497f56 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 21 Mar 2014 10:42:25 +0100 Subject: s390/uaccess: rework uaccess code - fix locking issues The current uaccess code uses a page table walk in some circumstances, e.g. in case of the in atomic futex operations or if running on old hardware which doesn't support the mvcos instruction. However it turned out that the page table walk code does not correctly lock page tables when accessing page table entries. In other words: a different cpu may invalidate a page table entry while the current cpu inspects the pte. This may lead to random data corruption. Adding correct locking however isn't trivial for all uaccess operations. Especially copy_in_user() is problematic since that requires to hold at least two locks, but must be protected against ABBA deadlock when a different cpu also performs a copy_in_user() operation. So the solution is a different approach where we change address spaces: User space runs in primary address mode, or access register mode within vdso code, like it currently already does. The kernel usually also runs in home space mode, however when accessing user space the kernel switches to primary or secondary address mode if the mvcos instruction is not available or if a compare-and-swap (futex) instruction on a user space address is performed. KVM however is special, since that requires the kernel to run in home address space while implicitly accessing user space with the sie instruction. So we end up with: User space: - runs in primary or access register mode - cr1 contains the user asce - cr7 contains the user asce - cr13 contains the kernel asce Kernel space: - runs in home space mode - cr1 contains the user or kernel asce -> the kernel asce is loaded when a uaccess requires primary or secondary address mode - cr7 contains the user or kernel asce, (changed with set_fs()) - cr13 contains the kernel asce In case of uaccess the kernel changes to: - primary space mode in case of a uaccess (copy_to_user) and uses e.g. the mvcp instruction to access user space. However the kernel will stay in home space mode if the mvcos instruction is available - secondary space mode in case of futex atomic operations, so that the instructions come from primary address space and data from secondary space In case of kvm the kernel runs in home space mode, but cr1 gets switched to contain the gmap asce before the sie instruction gets executed. When the sie instruction is finished cr1 will be switched back to contain the user asce. A context switch between two processes will always load the kernel asce for the next process in cr1. So the first exit to user space is a bit more expensive (one extra load control register instruction) than before, however keeps the code rather simple. In sum this means there is no need to perform any error prone page table walks anymore when accessing user space. The patch seems to be rather large, however it mainly removes the the page table walk code and restores the previously deleted "standard" uaccess code, with a couple of changes. The uaccess without mvcos mode can be enforced with the "uaccess_primary" kernel parameter. Reported-by: Christian Borntraeger Signed-off-by: Heiko Carstens Signed-off-by: Martin Schwidefsky --- arch/s390/kernel/asm-offsets.c | 1 + arch/s390/kernel/entry.S | 24 ++++++++++++++++++++++-- arch/s390/kernel/entry64.S | 24 ++++++++++++++++++++++-- 3 files changed, 45 insertions(+), 4 deletions(-) (limited to 'arch/s390/kernel') diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c index e4c99a183651..cc10cdd4d6a2 100644 --- a/arch/s390/kernel/asm-offsets.c +++ b/arch/s390/kernel/asm-offsets.c @@ -136,6 +136,7 @@ int main(void) DEFINE(__LC_RESTART_FN, offsetof(struct _lowcore, restart_fn)); DEFINE(__LC_RESTART_DATA, offsetof(struct _lowcore, restart_data)); DEFINE(__LC_RESTART_SOURCE, offsetof(struct _lowcore, restart_source)); + DEFINE(__LC_KERNEL_ASCE, offsetof(struct _lowcore, kernel_asce)); DEFINE(__LC_USER_ASCE, offsetof(struct _lowcore, user_asce)); DEFINE(__LC_INT_CLOCK, offsetof(struct _lowcore, int_clock)); DEFINE(__LC_MCCK_CLOCK, offsetof(struct _lowcore, mcck_clock)); diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 526d3735ed29..1662038516c0 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -38,9 +38,9 @@ __PT_R14 = __PT_GPRS + 56 __PT_R15 = __PT_GPRS + 60 _TIF_WORK_SVC = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ - _TIF_MCCK_PENDING | _TIF_PER_TRAP ) + _TIF_MCCK_PENDING | _TIF_PER_TRAP | _TIF_ASCE) _TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ - _TIF_MCCK_PENDING) + _TIF_MCCK_PENDING | _TIF_ASCE) _TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ _TIF_SYSCALL_TRACEPOINT) _TIF_TRANSFER = (_TIF_MCCK_PENDING | _TIF_TLB_WAIT) @@ -241,6 +241,8 @@ sysc_work: jo sysc_sigpending tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME jo sysc_notify_resume + tm __TI_flags+3(%r12),_TIF_ASCE + jo sysc_uaccess j sysc_return # beware of critical section cleanup # @@ -259,6 +261,14 @@ sysc_mcck_pending: la %r14,BASED(sysc_return) br %r1 # TIF bit will be cleared by handler +# +# _TIF_ASCE is set, load user space asce +# +sysc_uaccess: + ni __TI_flags+3(%r12),255-_TIF_ASCE + lctl %c1,%c1,__LC_USER_ASCE # load primary asce + j sysc_return + # # _TIF_SIGPENDING is set, call do_signal # @@ -522,6 +532,8 @@ io_work_tif: jo io_sigpending tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME jo io_notify_resume + tm __TI_flags+3(%r12),_TIF_ASCE + jo io_uaccess j io_return # beware of critical section cleanup # @@ -534,6 +546,14 @@ io_mcck_pending: TRACE_IRQS_OFF j io_return +# +# _TIF_ASCE is set, load user space asce +# +io_uaccess: + ni __TI_flags+3(%r12),255-_TIF_ASCE + lctl %c1,%c1,__LC_USER_ASCE # load primary asce + j io_return + # # _TIF_NEED_RESCHED is set, call schedule # diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S index e09dbe5f2901..5963e43618bb 100644 --- a/arch/s390/kernel/entry64.S +++ b/arch/s390/kernel/entry64.S @@ -43,9 +43,9 @@ STACK_SIZE = 1 << STACK_SHIFT STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE _TIF_WORK_SVC = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ - _TIF_MCCK_PENDING | _TIF_PER_TRAP ) + _TIF_MCCK_PENDING | _TIF_PER_TRAP | _TIF_ASCE) _TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ - _TIF_MCCK_PENDING) + _TIF_MCCK_PENDING | _TIF_ASCE) _TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ _TIF_SYSCALL_TRACEPOINT) _TIF_TRANSFER = (_TIF_MCCK_PENDING | _TIF_TLB_WAIT) @@ -275,6 +275,8 @@ sysc_work: jo sysc_sigpending tm __TI_flags+7(%r12),_TIF_NOTIFY_RESUME jo sysc_notify_resume + tm __TI_flags+7(%r12),_TIF_ASCE + jo sysc_uaccess j sysc_return # beware of critical section cleanup # @@ -291,6 +293,14 @@ sysc_mcck_pending: larl %r14,sysc_return jg s390_handle_mcck # TIF bit will be cleared by handler +# +# _TIF_ASCE is set, load user space asce +# +sysc_uaccess: + ni __TI_flags+7(%r12),255-_TIF_ASCE + lctlg %c1,%c1,__LC_USER_ASCE # load primary asce + j sysc_return + # # _TIF_SIGPENDING is set, call do_signal # @@ -559,6 +569,8 @@ io_work_tif: jo io_sigpending tm __TI_flags+7(%r12),_TIF_NOTIFY_RESUME jo io_notify_resume + tm __TI_flags+7(%r12),_TIF_ASCE + jo io_uaccess j io_return # beware of critical section cleanup # @@ -570,6 +582,14 @@ io_mcck_pending: TRACE_IRQS_OFF j io_return +# +# _TIF_ASCE is set, load user space asce +# +io_uaccess: + ni __TI_flags+7(%r12),255-_TIF_ASCE + lctlg %c1,%c1,__LC_USER_ASCE # load primary asce + j io_return + # # _TIF_NEED_RESCHED is set, call schedule # -- cgit v1.2.3