From b96bae3ae2cb6337c0a1ad160f4cbb0666e5e38b Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Fri, 12 Mar 2021 12:50:34 +0000 Subject: powerpc/32: Replace ASM exception exit by C exception exit from ppc64 This patch replaces the PPC32 ASM exception exit by C exception exit. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/48f8bae91da899d8e73fc0d75c9af66cc97b4d5b.1615552867.git.christophe.leroy@csgroup.eu --- arch/powerpc/kernel/interrupt.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'arch/powerpc/kernel/interrupt.c') diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c index c475a229a42a..6875b82f613a 100644 --- a/arch/powerpc/kernel/interrupt.c +++ b/arch/powerpc/kernel/interrupt.c @@ -20,6 +20,10 @@ #include #include +#if defined(CONFIG_PPC_ADV_DEBUG_REGS) && defined(CONFIG_PPC32) +unsigned long global_dbcr0[NR_CPUS]; +#endif + typedef long (*syscall_fn)(long, long, long, long, long, long); /* Has to run notrace because it is entered not completely "reconciled" */ @@ -392,7 +396,7 @@ again: ti_flags = READ_ONCE(current_thread_info()->flags); } - if (IS_ENABLED(CONFIG_PPC_BOOK3S) && IS_ENABLED(CONFIG_PPC_FPU)) { + if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && IS_ENABLED(CONFIG_PPC_FPU)) { if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) && unlikely((ti_flags & _TIF_RESTORE_TM))) { restore_tm_state(regs); -- cgit v1.2.3 From b5efec00b671c5d7e9cb9e73a1d4925dd6ce8dcd Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Fri, 12 Mar 2021 12:50:47 +0000 Subject: powerpc/32s: Move KUEP locking/unlocking in C This can be done in C, do it. Unrolling the loop gains approx. 15% performance. From now on, prepare_transfer_to_handler() is only for interrupts from kernel. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/4eadd873927e9a73c3d1dfe2f9497353465514cf.1615552867.git.christophe.leroy@csgroup.eu --- arch/powerpc/kernel/interrupt.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'arch/powerpc/kernel/interrupt.c') diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c index 6875b82f613a..20ace874cd98 100644 --- a/arch/powerpc/kernel/interrupt.c +++ b/arch/powerpc/kernel/interrupt.c @@ -33,6 +33,8 @@ notrace long system_call_exception(long r3, long r4, long r5, { syscall_fn f; + kuep_lock(); + regs->orig_gpr3 = r3; if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) @@ -354,6 +356,8 @@ again: */ kuap_user_restore(regs); #endif + kuep_unlock(); + return ret; } -- cgit v1.2.3 From ad2d2344771dabc5f0f14d85d5e7d2ddc613f385 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Fri, 12 Mar 2021 12:50:48 +0000 Subject: powerpc/64s: Make kuap_check_amr() and kuap_get_and_check_amr() generic In preparation of porting powerpc32 to C syscall entry/exit, rename kuap_check_amr() and kuap_get_and_check_amr() as kuap_assert_locked() and kuap_get_and_assert_locked(), and move in the generic asm/kup.h the stub for when CONFIG_PPC_KUAP is not selected. Signed-off-by: Christophe Leroy Reviewed-by: Nicholas Piggin Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/f82614d9b17b83abd739aa18fc08811815d0c2e3.1615552867.git.christophe.leroy@csgroup.eu --- arch/powerpc/kernel/interrupt.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'arch/powerpc/kernel/interrupt.c') diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c index 20ace874cd98..a7cb511bf945 100644 --- a/arch/powerpc/kernel/interrupt.c +++ b/arch/powerpc/kernel/interrupt.c @@ -76,7 +76,7 @@ notrace long system_call_exception(long r3, long r4, long r5, } else #endif #ifdef CONFIG_PPC64 - kuap_check_amr(); + kuap_assert_locked(); #endif booke_restore_dbcr0(); @@ -254,7 +254,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, CT_WARN_ON(ct_state() == CONTEXT_USER); #ifdef CONFIG_PPC64 - kuap_check_amr(); + kuap_assert_locked(); #endif regs->result = r3; @@ -380,7 +380,7 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned * AMR can only have been unlocked if we interrupted the kernel. */ #ifdef CONFIG_PPC64 - kuap_check_amr(); + kuap_assert_locked(); #endif local_irq_save(flags); @@ -451,7 +451,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign unsigned long flags; unsigned long ret = 0; #ifdef CONFIG_PPC64 - unsigned long amr; + unsigned long kuap; #endif if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x) && @@ -467,7 +467,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign CT_WARN_ON(ct_state() == CONTEXT_USER); #ifdef CONFIG_PPC64 - amr = kuap_get_and_check_amr(); + kuap = kuap_get_and_assert_locked(); #endif if (unlikely(current_thread_info()->flags & _TIF_EMULATE_STACK_STORE)) { @@ -511,7 +511,7 @@ again: * value from the check above. */ #ifdef CONFIG_PPC64 - kuap_kernel_restore(regs, amr); + kuap_kernel_restore(regs, kuap); #endif return ret; -- cgit v1.2.3 From c16728835eec45fa82f4744a52940717ac828f6d Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Fri, 12 Mar 2021 12:50:51 +0000 Subject: powerpc/32: Manage KUAP in C Move all KUAP management in C. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/199365ddb58d579daf724815f2d0acb91cc49d19.1615552867.git.christophe.leroy@csgroup.eu --- arch/powerpc/kernel/interrupt.c | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) (limited to 'arch/powerpc/kernel/interrupt.c') diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c index a7cb511bf945..c4dd4b8f9cfa 100644 --- a/arch/powerpc/kernel/interrupt.c +++ b/arch/powerpc/kernel/interrupt.c @@ -34,6 +34,9 @@ notrace long system_call_exception(long r3, long r4, long r5, syscall_fn f; kuep_lock(); +#ifdef CONFIG_PPC32 + kuap_save_and_lock(regs); +#endif regs->orig_gpr3 = r3; @@ -75,9 +78,7 @@ notrace long system_call_exception(long r3, long r4, long r5, isync(); } else #endif -#ifdef CONFIG_PPC64 kuap_assert_locked(); -#endif booke_restore_dbcr0(); @@ -253,9 +254,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, CT_WARN_ON(ct_state() == CONTEXT_USER); -#ifdef CONFIG_PPC64 kuap_assert_locked(); -#endif regs->result = r3; @@ -350,7 +349,7 @@ again: account_cpu_user_exit(); -#ifdef CONFIG_PPC_BOOK3S_64 /* BOOK3E and ppc32 not using this */ +#ifndef CONFIG_PPC_BOOK3E_64 /* BOOK3E not using this */ /* * We do this at the end so that we do context switch with KERNEL AMR */ @@ -379,9 +378,7 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned * We don't need to restore AMR on the way back to userspace for KUAP. * AMR can only have been unlocked if we interrupted the kernel. */ -#ifdef CONFIG_PPC64 kuap_assert_locked(); -#endif local_irq_save(flags); @@ -438,9 +435,7 @@ again: /* * We do this at the end so that we do context switch with KERNEL AMR */ -#ifdef CONFIG_PPC64 kuap_user_restore(regs); -#endif return ret; } @@ -450,9 +445,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign { unsigned long flags; unsigned long ret = 0; -#ifdef CONFIG_PPC64 unsigned long kuap; -#endif if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x) && unlikely(!(regs->msr & MSR_RI))) @@ -466,9 +459,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign if (TRAP(regs) != 0x700) CT_WARN_ON(ct_state() == CONTEXT_USER); -#ifdef CONFIG_PPC64 kuap = kuap_get_and_assert_locked(); -#endif if (unlikely(current_thread_info()->flags & _TIF_EMULATE_STACK_STORE)) { clear_bits(_TIF_EMULATE_STACK_STORE, ¤t_thread_info()->flags); @@ -510,9 +501,7 @@ again: * which would cause Read-After-Write stalls. Hence, we take the AMR * value from the check above. */ -#ifdef CONFIG_PPC64 kuap_kernel_restore(regs, kuap); -#endif return ret; } -- cgit v1.2.3 From 5a5a893c4ad897b8a36f846602895515b7407a71 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Tue, 16 Mar 2021 20:41:55 +1000 Subject: powerpc/syscall: switch user_exit_irqoff and trace_hardirqs_off order user_exit_irqoff() -> __context_tracking_exit -> vtime_user_exit warns in __seqprop_assert due to lockdep thinking preemption is enabled because trace_hardirqs_off() has not yet been called. Switch the order of these two calls, which matches their ordering in interrupt_enter_prepare. Fixes: 5f0b6ac3905f ("powerpc/64/syscall: Reconcile interrupts") Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20210316104206.407354-2-npiggin@gmail.com --- arch/powerpc/kernel/interrupt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch/powerpc/kernel/interrupt.c') diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c index c4dd4b8f9cfa..fbabb49888d3 100644 --- a/arch/powerpc/kernel/interrupt.c +++ b/arch/powerpc/kernel/interrupt.c @@ -43,11 +43,11 @@ notrace long system_call_exception(long r3, long r4, long r5, if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) BUG_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED); + trace_hardirqs_off(); /* finish reconciling */ + CT_WARN_ON(ct_state() == CONTEXT_KERNEL); user_exit_irqoff(); - trace_hardirqs_off(); /* finish reconciling */ - if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x)) BUG_ON(!(regs->msr & MSR_RI)); BUG_ON(!(regs->msr & MSR_PR)); -- cgit v1.2.3 From dc6231821a148d0392292924fdae5b34679af6b2 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Tue, 16 Mar 2021 20:41:57 +1000 Subject: powerpc/interrupt: update common interrupt code for This makes adjustments to 64-bit asm and common C interrupt return code to be usable by the 64e subarchitecture. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20210316104206.407354-4-npiggin@gmail.com --- arch/powerpc/kernel/interrupt.c | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) (limited to 'arch/powerpc/kernel/interrupt.c') diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c index fbabb49888d3..381a618b5b5b 100644 --- a/arch/powerpc/kernel/interrupt.c +++ b/arch/powerpc/kernel/interrupt.c @@ -235,6 +235,10 @@ static notrace void booke_load_dbcr0(void) #endif } +/* temporary hack for context tracking, removed in later patch */ +#include +asmlinkage __visible void __sched schedule_user(void); + /* * This should be called after a syscall returns, with r3 the return value * from the syscall. If this function returns non-zero, the system call @@ -292,7 +296,11 @@ again: while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) { local_irq_enable(); if (ti_flags & _TIF_NEED_RESCHED) { +#ifdef CONFIG_PPC_BOOK3E_64 + schedule_user(); +#else schedule(); +#endif } else { /* * SIGPENDING must restore signal handler function @@ -349,18 +357,13 @@ again: account_cpu_user_exit(); -#ifndef CONFIG_PPC_BOOK3E_64 /* BOOK3E not using this */ - /* - * We do this at the end so that we do context switch with KERNEL AMR - */ + /* Restore user access locks last */ kuap_user_restore(regs); -#endif kuep_unlock(); return ret; } -#ifndef CONFIG_PPC_BOOK3E_64 /* BOOK3E not yet using this */ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned long msr) { unsigned long ti_flags; @@ -372,7 +375,9 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned BUG_ON(!(regs->msr & MSR_PR)); BUG_ON(!FULL_REGS(regs)); BUG_ON(arch_irq_disabled_regs(regs)); +#ifdef CONFIG_PPC_BOOK3S_64 CT_WARN_ON(ct_state() == CONTEXT_USER); +#endif /* * We don't need to restore AMR on the way back to userspace for KUAP. @@ -387,7 +392,11 @@ again: while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) { local_irq_enable(); /* returning to user: may enable */ if (ti_flags & _TIF_NEED_RESCHED) { +#ifdef CONFIG_PPC_BOOK3E_64 + schedule_user(); +#else schedule(); +#endif } else { if (ti_flags & _TIF_SIGPENDING) ret |= _TIF_RESTOREALL; @@ -432,10 +441,9 @@ again: account_cpu_user_exit(); - /* - * We do this at the end so that we do context switch with KERNEL AMR - */ + /* Restore user access locks last */ kuap_user_restore(regs); + return ret; } @@ -456,7 +464,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign * CT_WARN_ON comes here via program_check_exception, * so avoid recursion. */ - if (TRAP(regs) != 0x700) + if (IS_ENABLED(CONFIG_BOOKS) && TRAP(regs) != 0x700) CT_WARN_ON(ct_state() == CONTEXT_USER); kuap = kuap_get_and_assert_locked(); @@ -497,12 +505,11 @@ again: #endif /* - * Don't want to mfspr(SPRN_AMR) here, because this comes after mtmsr, - * which would cause Read-After-Write stalls. Hence, we take the AMR - * value from the check above. + * 64s does not want to mfspr(SPRN_AMR) here, because this comes after + * mtmsr, which would cause Read-After-Write stalls. Hence, take the + * AMR value from the check above. */ kuap_kernel_restore(regs, kuap); return ret; } -#endif -- cgit v1.2.3 From ceff77efa4f8d9f02d8442171b325d3b7068fe5e Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Tue, 16 Mar 2021 20:42:01 +1000 Subject: powerpc/64e/interrupt: Use new interrupt context tracking scheme With the new interrupt exit code, context tracking can be managed more precisely, so remove the last of the 64e workarounds and switch to the new context tracking code already used by 64s. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20210316104206.407354-8-npiggin@gmail.com --- arch/powerpc/kernel/interrupt.c | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) (limited to 'arch/powerpc/kernel/interrupt.c') diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c index 381a618b5b5b..1b0e1792ac37 100644 --- a/arch/powerpc/kernel/interrupt.c +++ b/arch/powerpc/kernel/interrupt.c @@ -235,10 +235,6 @@ static notrace void booke_load_dbcr0(void) #endif } -/* temporary hack for context tracking, removed in later patch */ -#include -asmlinkage __visible void __sched schedule_user(void); - /* * This should be called after a syscall returns, with r3 the return value * from the syscall. If this function returns non-zero, the system call @@ -296,11 +292,7 @@ again: while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) { local_irq_enable(); if (ti_flags & _TIF_NEED_RESCHED) { -#ifdef CONFIG_PPC_BOOK3E_64 - schedule_user(); -#else schedule(); -#endif } else { /* * SIGPENDING must restore signal handler function @@ -375,9 +367,7 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned BUG_ON(!(regs->msr & MSR_PR)); BUG_ON(!FULL_REGS(regs)); BUG_ON(arch_irq_disabled_regs(regs)); -#ifdef CONFIG_PPC_BOOK3S_64 CT_WARN_ON(ct_state() == CONTEXT_USER); -#endif /* * We don't need to restore AMR on the way back to userspace for KUAP. @@ -392,11 +382,7 @@ again: while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) { local_irq_enable(); /* returning to user: may enable */ if (ti_flags & _TIF_NEED_RESCHED) { -#ifdef CONFIG_PPC_BOOK3E_64 - schedule_user(); -#else schedule(); -#endif } else { if (ti_flags & _TIF_SIGPENDING) ret |= _TIF_RESTOREALL; @@ -464,7 +450,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign * CT_WARN_ON comes here via program_check_exception, * so avoid recursion. */ - if (IS_ENABLED(CONFIG_BOOKS) && TRAP(regs) != 0x700) + if (TRAP(regs) != 0x700) CT_WARN_ON(ct_state() == CONTEXT_USER); kuap = kuap_get_and_assert_locked(); -- cgit v1.2.3 From 8dc7f0229b7892ccb23e19c9f30511c68cc0fdcc Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Tue, 16 Mar 2021 20:42:04 +1000 Subject: powerpc: remove partial register save logic All subarchitectures always save all GPRs to pt_regs interrupt frames now. Remove FULL_REGS and associated bits. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20210316104206.407354-11-npiggin@gmail.com --- arch/powerpc/kernel/interrupt.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'arch/powerpc/kernel/interrupt.c') diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c index 1b0e1792ac37..b953bb5027e6 100644 --- a/arch/powerpc/kernel/interrupt.c +++ b/arch/powerpc/kernel/interrupt.c @@ -51,7 +51,6 @@ notrace long system_call_exception(long r3, long r4, long r5, if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x)) BUG_ON(!(regs->msr & MSR_RI)); BUG_ON(!(regs->msr & MSR_PR)); - BUG_ON(!FULL_REGS(regs)); BUG_ON(arch_irq_disabled_regs(regs)); #ifdef CONFIG_PPC_PKEY @@ -365,7 +364,6 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x)) BUG_ON(!(regs->msr & MSR_RI)); BUG_ON(!(regs->msr & MSR_PR)); - BUG_ON(!FULL_REGS(regs)); BUG_ON(arch_irq_disabled_regs(regs)); CT_WARN_ON(ct_state() == CONTEXT_USER); @@ -445,7 +443,6 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign unlikely(!(regs->msr & MSR_RI))) unrecoverable_exception(regs); BUG_ON(regs->msr & MSR_PR); - BUG_ON(!FULL_REGS(regs)); /* * CT_WARN_ON comes here via program_check_exception, * so avoid recursion. -- cgit v1.2.3 From 7153d4bf0b373428d0393c001019da4d0483fddb Mon Sep 17 00:00:00 2001 From: Xiongwei Song Date: Wed, 14 Apr 2021 19:00:33 +0800 Subject: powerpc/traps: Enhance readability for trap types Define macros to list ppc interrupt types in interttupt.h, replace the reference of the trap hex values with these macros. Referred the hex numbers in arch/powerpc/kernel/exceptions-64e.S, arch/powerpc/kernel/exceptions-64s.S, arch/powerpc/kernel/head_*.S, arch/powerpc/kernel/head_booke.h and arch/powerpc/include/asm/kvm_asm.h. Signed-off-by: Xiongwei Song [mpe: Resolve conflicts in nmi_disables_ftrace(), fix 40x build] Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/1618398033-13025-1-git-send-email-sxwjean@me.com --- arch/powerpc/kernel/interrupt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/powerpc/kernel/interrupt.c') diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c index b953bb5027e6..e4559f8914eb 100644 --- a/arch/powerpc/kernel/interrupt.c +++ b/arch/powerpc/kernel/interrupt.c @@ -447,7 +447,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign * CT_WARN_ON comes here via program_check_exception, * so avoid recursion. */ - if (TRAP(regs) != 0x700) + if (TRAP(regs) != INTERRUPT_PROGRAM) CT_WARN_ON(ct_state() == CONTEXT_USER); kuap = kuap_get_and_assert_locked(); -- cgit v1.2.3