diff options
Diffstat (limited to 'arch/alpha')
56 files changed, 1363 insertions, 1356 deletions
diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig index 80367f2cf821..e53ef2d88463 100644 --- a/arch/alpha/Kconfig +++ b/arch/alpha/Kconfig @@ -31,15 +31,22 @@ config ALPHA select GENERIC_SMP_IDLE_THREAD select HAS_IOPORT select HAVE_ARCH_AUDITSYSCALL + select HAVE_ARCH_SECCOMP + select HAVE_ARCH_SECCOMP_FILTER + select HAVE_ARCH_TRACEHOOK select HAVE_MOD_ARCH_SPECIFIC select LOCK_MM_AND_FIND_VMA select MODULES_USE_ELF_RELA select ODD_RT_SIGACTION select OLD_SIGSUSPEND + select ARCH_STACKWALK select CPU_NO_EFFICIENT_FFS if !ALPHA_EV67 select MMU_GATHER_NO_RANGE + select MMU_GATHER_RCU_TABLE_FREE select SPARSEMEM_EXTREME if SPARSEMEM select ZONE_DMA + select TRACE_IRQFLAGS_SUPPORT + select ARCH_WANT_FRAME_POINTERS help The Alpha is a 64-bit general-purpose processor designed and marketed by the Digital Equipment Corporation of blessed memory, @@ -76,6 +83,12 @@ config PGTABLE_LEVELS config AUDIT_ARCH bool +config STACKTRACE_SUPPORT + def_bool y + +config LOCKDEP_SUPPORT + def_bool y + menu "System setup" choice diff --git a/arch/alpha/Makefile b/arch/alpha/Makefile index 35445ff2e489..7074602dca94 100644 --- a/arch/alpha/Makefile +++ b/arch/alpha/Makefile @@ -27,10 +27,16 @@ cpuflags-$(CONFIG_ALPHA_GENERIC) := -mcpu=ev56 -mtune=ev6 cflags-y += $(cpuflags-y) +KBUILD_CFLAGS += $(cflags-y) + # For TSUNAMI, we must have the assembler not emulate our instructions. # The same is true for IRONGATE, POLARIS, PYXIS. # BWX is most important, but we don't really want any emulation ever. -KBUILD_CFLAGS += $(cflags-y) -Wa,-mev6 +# Only gas emulates instructions the target does not implement, and only gas +# understands -mev6. LLVM's integrated assembler never emulates. +ifdef CONFIG_AS_IS_GNU +KBUILD_CFLAGS += -Wa,-mev6 +endif libs-y += arch/alpha/lib/ diff --git a/arch/alpha/configs/defconfig b/arch/alpha/configs/defconfig index 3280bd9e6578..29748bbd94f3 100644 --- a/arch/alpha/configs/defconfig +++ b/arch/alpha/configs/defconfig @@ -45,7 +45,6 @@ CONFIG_NET_TULIP=y CONFIG_DE2104X=m CONFIG_TULIP=y CONFIG_TULIP_MMIO=y -CONFIG_YELLOWFIN=y CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y CONFIG_RTC_CLASS=y diff --git a/arch/alpha/include/asm/Kbuild b/arch/alpha/include/asm/Kbuild index 483965c5a4de..b154b4e3dfa8 100644 --- a/arch/alpha/include/asm/Kbuild +++ b/arch/alpha/include/asm/Kbuild @@ -5,4 +5,5 @@ generic-y += agp.h generic-y += asm-offsets.h generic-y += kvm_para.h generic-y += mcs_spinlock.h +generic-y += ring_buffer.h generic-y += text-patching.h diff --git a/arch/alpha/include/asm/device.h b/arch/alpha/include/asm/device.h deleted file mode 100644 index 9ca75a7db23e..000000000000 --- a/arch/alpha/include/asm/device.h +++ /dev/null @@ -1,6 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Arch specific extensions to struct device - */ -#include <asm-generic/device.h> - diff --git a/arch/alpha/include/asm/elf.h b/arch/alpha/include/asm/elf.h index 50c82187e60e..b15946621d57 100644 --- a/arch/alpha/include/asm/elf.h +++ b/arch/alpha/include/asm/elf.h @@ -53,6 +53,7 @@ #define EF_ALPHA_32BIT 1 /* All addresses are below 2GB */ +#define CORE_DUMP_USE_REGSET 1 /* * ELF register definitions.. */ diff --git a/arch/alpha/include/asm/ftrace.h b/arch/alpha/include/asm/ftrace.h index 40a8c178f10d..7ec44134c804 100644 --- a/arch/alpha/include/asm/ftrace.h +++ b/arch/alpha/include/asm/ftrace.h @@ -1 +1,29 @@ -/* empty */ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_ALPHA_FTRACE_H +#define _ASM_ALPHA_FTRACE_H + +#ifdef CONFIG_FRAME_POINTER + +static void *alpha_ftrace_return_address0(void) + noinline notrace; +static void *alpha_ftrace_return_address0(void) +{ + return __builtin_return_address(0); +} + +#define ftrace_return_address0 alpha_ftrace_return_address0() + +/* + * __builtin_return_address() requires a constant integer argument. + * Keep this as a macro so the value is seen at the callsite. + */ +#define ftrace_return_address(n) __builtin_return_address(n) + +#else /* !CONFIG_FRAME_POINTER */ + +#define ftrace_return_address0 0UL +#define ftrace_return_address(n) ((void)(n), 0UL) + +#endif /* CONFIG_FRAME_POINTER */ + +#endif /* _ASM_ALPHA_FTRACE_H */ diff --git a/arch/alpha/include/asm/irqflags.h b/arch/alpha/include/asm/irqflags.h index 9f25d4e0d37e..f207544f52de 100644 --- a/arch/alpha/include/asm/irqflags.h +++ b/arch/alpha/include/asm/irqflags.h @@ -26,7 +26,7 @@ extern int __min_ipl; static inline unsigned long arch_local_save_flags(void) { - return rdps(); + return getipl(); } static inline void arch_local_irq_disable(void) @@ -51,13 +51,13 @@ static inline void arch_local_irq_enable(void) static inline void arch_local_irq_restore(unsigned long flags) { barrier(); - setipl(flags); + setipl(flags & 7); barrier(); } static inline bool arch_irqs_disabled_flags(unsigned long flags) { - return flags == IPL_MAX; + return (flags & 7) == IPL_MAX; } static inline bool arch_irqs_disabled(void) diff --git a/arch/alpha/include/asm/page.h b/arch/alpha/include/asm/page.h index d2c6667d73e9..59d01f9b77f6 100644 --- a/arch/alpha/include/asm/page.h +++ b/arch/alpha/include/asm/page.h @@ -11,7 +11,6 @@ #define STRICT_MM_TYPECHECKS extern void clear_page(void *page); -#define clear_user_page(page, vaddr, pg) clear_page(page) #define vma_alloc_zeroed_movable_folio(vma, vaddr) \ vma_alloc_folio(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, 0, vma, vaddr) diff --git a/arch/alpha/include/asm/pci.h b/arch/alpha/include/asm/pci.h index 6c04fcbdc8ed..ad5d1391e1fa 100644 --- a/arch/alpha/include/asm/pci.h +++ b/arch/alpha/include/asm/pci.h @@ -84,8 +84,17 @@ extern int pci_legacy_write(struct pci_bus *bus, loff_t port, u32 val, extern int pci_mmap_legacy_page_range(struct pci_bus *bus, struct vm_area_struct *vma, enum pci_mmap_state mmap_state); -extern void pci_adjust_legacy_attr(struct pci_bus *bus, - enum pci_mmap_state mmap_type); +extern bool pci_legacy_has_sparse(struct pci_bus *bus, + enum pci_mmap_state type); #define HAVE_PCI_LEGACY 1 +extern const struct attribute_group pci_dev_resource_attr_group; +extern const struct attribute_group pci_dev_resource_sparse_attr_group; +extern const struct attribute_group pci_dev_resource_dense_attr_group; + +#define ARCH_PCI_DEV_GROUPS \ + &pci_dev_resource_attr_group, \ + &pci_dev_resource_sparse_attr_group, \ + &pci_dev_resource_dense_attr_group, + #endif /* __ALPHA_PCI_H */ diff --git a/arch/alpha/include/asm/pgtable.h b/arch/alpha/include/asm/pgtable.h index 90e7a9539102..8e00cf9dc39d 100644 --- a/arch/alpha/include/asm/pgtable.h +++ b/arch/alpha/include/asm/pgtable.h @@ -17,6 +17,7 @@ #include <asm/processor.h> /* For TASK_SIZE */ #include <asm/machvec.h> #include <asm/setup.h> +#include <linux/page_table_check.h> struct mm_struct; struct vm_area_struct; @@ -126,10 +127,15 @@ struct vm_area_struct; #define pgprot_noncached(prot) (prot) /* - * ZERO_PAGE is a global shared page that is always zero: used - * for zero-mapped memory areas etc.. + * All caching attribute macros are identity on Alpha, so the generic + * pgprot_modify() degenerates to tautological self-comparisons. + * Override it to just return newprot directly. */ -#define ZERO_PAGE(vaddr) (virt_to_page(ZERO_PGE)) +#define pgprot_modify pgprot_modify +static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) +{ + return newprot; +} /* * On certain platforms whose physical address space can overlap KSEG, @@ -183,6 +189,9 @@ extern inline void pud_set(pud_t * pudp, pmd_t * pmdp) { pud_val(*pudp) = _PAGE_TABLE | ((((unsigned long) pmdp) - PAGE_OFFSET) << (32-PAGE_SHIFT)); } +extern void migrate_flush_tlb_page(struct vm_area_struct *vma, + unsigned long addr); + extern inline unsigned long pmd_page_vaddr(pmd_t pmd) { @@ -202,7 +211,7 @@ extern inline int pte_none(pte_t pte) { return !pte_val(pte); } extern inline int pte_present(pte_t pte) { return pte_val(pte) & _PAGE_VALID; } extern inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - pte_val(*ptep) = 0; + WRITE_ONCE(pte_val(*ptep), 0); } extern inline int pmd_none(pmd_t pmd) { return !pmd_val(pmd); } @@ -264,6 +273,33 @@ extern inline pte_t * pte_offset_kernel(pmd_t * dir, unsigned long address) extern pgd_t swapper_pg_dir[1024]; +#ifdef CONFIG_COMPACTION +#define __HAVE_ARCH_PTEP_GET_AND_CLEAR + +static inline pte_t ptep_get_and_clear(struct mm_struct *mm, + unsigned long address, + pte_t *ptep) +{ + pte_t pte = READ_ONCE(*ptep); + + pte_clear(mm, address, ptep); + return pte; +} + +#define __HAVE_ARCH_PTEP_CLEAR_FLUSH + +static inline pte_t ptep_clear_flush(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + struct mm_struct *mm = vma->vm_mm; + pte_t pte = ptep_get_and_clear(mm, addr, ptep); + + page_table_check_pte_clear(mm, addr, pte); + migrate_flush_tlb_page(vma, addr); + return pte; +} + +#endif /* * The Alpha doesn't have any external MMU info: the kernel page * tables contain all the necessary information. diff --git a/arch/alpha/include/asm/ptrace.h b/arch/alpha/include/asm/ptrace.h index 3557ce64ed21..8e0a589e2d15 100644 --- a/arch/alpha/include/asm/ptrace.h +++ b/arch/alpha/include/asm/ptrace.h @@ -24,4 +24,11 @@ static inline unsigned long regs_return_value(struct pt_regs *regs) return regs->r0; } +/* Helpers for working with the user stack pointer */ +static inline unsigned long user_stack_pointer(struct pt_regs *regs) +{ + /* Valid for user-mode regs */ + return regs->usp; +} + #endif diff --git a/arch/alpha/include/asm/seccomp.h b/arch/alpha/include/asm/seccomp.h new file mode 100644 index 000000000000..311934d20340 --- /dev/null +++ b/arch/alpha/include/asm/seccomp.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_ALPHA_SECCOMP_H +#define _ASM_ALPHA_SECCOMP_H + +#include <asm/unistd.h> +#include <asm-generic/seccomp.h> +#include <uapi/linux/audit.h> + +#define SECCOMP_ARCH_NATIVE AUDIT_ARCH_ALPHA +#define SECCOMP_ARCH_NATIVE_NR NR_syscalls +#define SECCOMP_ARCH_NATIVE_NAME "alpha" + +#endif /* _ASM_ALPHA_SECCOMP_H */ diff --git a/arch/alpha/include/asm/string.h b/arch/alpha/include/asm/string.h index f043f91ff988..7c7545a2319a 100644 --- a/arch/alpha/include/asm/string.h +++ b/arch/alpha/include/asm/string.h @@ -47,8 +47,6 @@ extern inline void *__memset(void *s, int c, size_t n) #define __HAVE_ARCH_STRCPY extern char * strcpy(char *,const char *); -#define __HAVE_ARCH_STRNCPY -extern char * strncpy(char *, const char *, size_t); #define __HAVE_ARCH_STRCAT extern char * strcat(char *, const char *); #define __HAVE_ARCH_STRNCAT diff --git a/arch/alpha/include/asm/syscall.h b/arch/alpha/include/asm/syscall.h index f21babaeed85..1e78cbd46faf 100644 --- a/arch/alpha/include/asm/syscall.h +++ b/arch/alpha/include/asm/syscall.h @@ -3,6 +3,10 @@ #define _ASM_ALPHA_SYSCALL_H #include <uapi/linux/audit.h> +#include <linux/audit.h> +#include <linux/sched.h> +#include <linux/types.h> +#include <asm/ptrace.h> static inline int syscall_get_arch(struct task_struct *task) { @@ -12,7 +16,98 @@ static inline int syscall_get_arch(struct task_struct *task) static inline long syscall_get_return_value(struct task_struct *task, struct pt_regs *regs) { - return regs->r0; + return regs->r19 ? -(long)regs->r0 : (long)regs->r0; +} + +static inline long syscall_get_error(struct task_struct *task, + struct pt_regs *regs) +{ + return regs->r19 ? -(long)regs->r0 : 0; +} + + +/* + * Alpha syscall ABI / kernel conventions: + * - PAL provides syscall number in r0 on entry. + * - The kernel tracks the active syscall number in regs->r1 (mutable) and + * preserves the original syscall number in regs->r2 for rollback/restart. + * - Return value is in regs->r0, with regs->r19 ("a3") as the error flag + * (0=success, 1=error; on error regs->r0 holds positive errno). + */ + +static inline long syscall_get_nr(struct task_struct *task, + struct pt_regs *regs) +{ + return (long)regs->r1; +} + +static inline void syscall_set_nr(struct task_struct *task, + struct pt_regs *regs, + long nr) +{ + regs->r1 = (unsigned long)nr; +} + +/* + * Syscall arguments: + * regs->r16..regs->r21 carry up to 6 syscall arguments on entry. + * Note: regs->r19 is also used as "a3" (error flag) on syscall return. + */ + +static inline void syscall_get_arguments(struct task_struct *task, + struct pt_regs *regs, + unsigned long *args) +{ + args[0] = regs->r16; + args[1] = regs->r17; + args[2] = regs->r18; + args[3] = regs->r19; + args[4] = regs->r20; + args[5] = regs->r21; +} + +static inline void syscall_set_arguments(struct task_struct *task, + struct pt_regs *regs, + const unsigned long *args) +{ + regs->r16 = args[0]; + regs->r17 = args[1]; + regs->r18 = args[2]; + regs->r19 = args[3]; + regs->r20 = args[4]; + regs->r21 = args[5]; +} +/* + * Set return value for a syscall. + * Alpha uses r0 for return value and r19 ("a3") as the error indicator: + * a3 = 0 => success + * a3 = 1 => error, and userspace interprets r0 as errno (positive). + * + * The kernel reports errors to userspace by setting a3=1 and placing a + * positive errno value in r0. Some syscall paths do this in entry.S, + * while others (e.g. seccomp/ptrace helpers) use syscall_set_return_value(). + */ + +static inline void syscall_set_return_value(struct task_struct *task, + struct pt_regs *regs, + int error, long val) +{ + + if (error) { + /* error is negative errno in this tree */ + regs->r0 = (unsigned long)(-error); /* positive errno */ + regs->r19 = 1; /* a3 = error */ + } else { + regs->r0 = (unsigned long)val; + regs->r19 = 0; /* a3 = success */ + } +} + +/* Restore the original syscall nr after seccomp/ptrace modified regs->r1. */ +static inline void syscall_rollback(struct task_struct *task, + struct pt_regs *regs) +{ + regs->r1 = regs->r2; } #endif /* _ASM_ALPHA_SYSCALL_H */ diff --git a/arch/alpha/include/asm/thread_info.h b/arch/alpha/include/asm/thread_info.h index 98ccbca64984..1552ecca8520 100644 --- a/arch/alpha/include/asm/thread_info.h +++ b/arch/alpha/include/asm/thread_info.h @@ -56,7 +56,8 @@ register unsigned long *current_stack_pointer __asm__ ("$30"); * - pending work-to-be-done flags come first and must be assigned to be * within bits 0 to 7 to fit in and immediate operand. * - * TIF_SYSCALL_TRACE is known to be 0 via blbs. + * (Historically TIF_SYSCALL_TRACE was known to be 0 via blbs, but we may + * also test multiple bits via masks now.) */ #define TIF_SYSCALL_TRACE 0 /* syscall trace active */ #define TIF_NOTIFY_RESUME 1 /* callback before returning to user */ @@ -64,6 +65,8 @@ register unsigned long *current_stack_pointer __asm__ ("$30"); #define TIF_NEED_RESCHED 3 /* rescheduling necessary */ #define TIF_SYSCALL_AUDIT 4 /* syscall audit active */ #define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ +#define TIF_SECCOMP 6 /* seccomp syscall filtering active */ +#define TIF_SYSCALL_TRACEPOINT 7 /* syscall tracepoint instrumentation */ #define TIF_DIE_IF_KERNEL 9 /* dik recursion lock */ #define TIF_MEMDIE 13 /* is terminating due to OOM killer */ #define TIF_POLLING_NRFLAG 14 /* idle is polling for TIF_NEED_RESCHED */ @@ -74,7 +77,21 @@ register unsigned long *current_stack_pointer __asm__ ("$30"); #define _TIF_NOTIFY_RESUME (1<<TIF_NOTIFY_RESUME) #define _TIF_SYSCALL_AUDIT (1<<TIF_SYSCALL_AUDIT) #define _TIF_NOTIFY_SIGNAL (1<<TIF_NOTIFY_SIGNAL) +#define _TIF_SECCOMP (1<<TIF_SECCOMP) #define _TIF_POLLING_NRFLAG (1<<TIF_POLLING_NRFLAG) +#define _TIF_SYSCALL_TRACEPOINT (1<<TIF_SYSCALL_TRACEPOINT) + +/* + * Work to do on syscall entry (in entry.S). + * If you want this to exactly mirror what entry.S checks, keep it aligned + * with the mask used before branching to syscall_trace_enter(). + */ +#ifdef CONFIG_AUDITSYSCALL +# define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP \ + | _TIF_SYSCALL_TRACEPOINT) +#else +# define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SECCOMP | _TIF_SYSCALL_TRACEPOINT) +#endif /* Work to do on interrupt/exception return. */ #define _TIF_WORK_MASK (_TIF_SIGPENDING | _TIF_NEED_RESCHED | \ diff --git a/arch/alpha/include/asm/timex.h b/arch/alpha/include/asm/timex.h index f89798da8a14..5a71ba6136b9 100644 --- a/arch/alpha/include/asm/timex.h +++ b/arch/alpha/include/asm/timex.h @@ -7,10 +7,6 @@ #ifndef _ASMALPHA_TIMEX_H #define _ASMALPHA_TIMEX_H -/* With only one or two oddballs, we use the RTC as the ticker, selecting - the 32.768kHz reference clock, which nicely divides down to our HZ. */ -#define CLOCK_TICK_RATE 32768 - /* * Standard way to access the cycle counter. * Currently only used on SMP for scheduling. diff --git a/arch/alpha/include/asm/tlb.h b/arch/alpha/include/asm/tlb.h index 4f79e331af5e..ad586b898fd6 100644 --- a/arch/alpha/include/asm/tlb.h +++ b/arch/alpha/include/asm/tlb.h @@ -4,7 +4,7 @@ #include <asm-generic/tlb.h> -#define __pte_free_tlb(tlb, pte, address) pte_free((tlb)->mm, pte) -#define __pmd_free_tlb(tlb, pmd, address) pmd_free((tlb)->mm, pmd) - +#define __pte_free_tlb(tlb, pte, address) tlb_remove_ptdesc((tlb), page_ptdesc(pte)) +#define __pmd_free_tlb(tlb, pmd, address) tlb_remove_ptdesc((tlb), virt_to_ptdesc(pmd)) + #endif diff --git a/arch/alpha/include/asm/tlbflush.h b/arch/alpha/include/asm/tlbflush.h index ba4b359d6c39..0c8529997f54 100644 --- a/arch/alpha/include/asm/tlbflush.h +++ b/arch/alpha/include/asm/tlbflush.h @@ -58,7 +58,9 @@ flush_tlb_other(struct mm_struct *mm) unsigned long *mmc = &mm->context[smp_processor_id()]; /* Check it's not zero first to avoid cacheline ping pong when possible. */ - if (*mmc) *mmc = 0; + + if (READ_ONCE(*mmc)) + WRITE_ONCE(*mmc, 0); } #ifndef CONFIG_SMP diff --git a/arch/alpha/include/asm/xor.h b/arch/alpha/include/asm/xor.h deleted file mode 100644 index e0de0c233ab9..000000000000 --- a/arch/alpha/include/asm/xor.h +++ /dev/null @@ -1,866 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * include/asm-alpha/xor.h - * - * Optimized RAID-5 checksumming functions for alpha EV5 and EV6 - */ - -extern void -xor_alpha_2(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2); -extern void -xor_alpha_3(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2, - const unsigned long * __restrict p3); -extern void -xor_alpha_4(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2, - const unsigned long * __restrict p3, - const unsigned long * __restrict p4); -extern void -xor_alpha_5(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2, - const unsigned long * __restrict p3, - const unsigned long * __restrict p4, - const unsigned long * __restrict p5); - -extern void -xor_alpha_prefetch_2(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2); -extern void -xor_alpha_prefetch_3(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2, - const unsigned long * __restrict p3); -extern void -xor_alpha_prefetch_4(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2, - const unsigned long * __restrict p3, - const unsigned long * __restrict p4); -extern void -xor_alpha_prefetch_5(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2, - const unsigned long * __restrict p3, - const unsigned long * __restrict p4, - const unsigned long * __restrict p5); - -asm(" \n\ - .text \n\ - .align 3 \n\ - .ent xor_alpha_2 \n\ -xor_alpha_2: \n\ - .prologue 0 \n\ - srl $16, 6, $16 \n\ - .align 4 \n\ -2: \n\ - ldq $0,0($17) \n\ - ldq $1,0($18) \n\ - ldq $2,8($17) \n\ - ldq $3,8($18) \n\ - \n\ - ldq $4,16($17) \n\ - ldq $5,16($18) \n\ - ldq $6,24($17) \n\ - ldq $7,24($18) \n\ - \n\ - ldq $19,32($17) \n\ - ldq $20,32($18) \n\ - ldq $21,40($17) \n\ - ldq $22,40($18) \n\ - \n\ - ldq $23,48($17) \n\ - ldq $24,48($18) \n\ - ldq $25,56($17) \n\ - xor $0,$1,$0 # 7 cycles from $1 load \n\ - \n\ - ldq $27,56($18) \n\ - xor $2,$3,$2 \n\ - stq $0,0($17) \n\ - xor $4,$5,$4 \n\ - \n\ - stq $2,8($17) \n\ - xor $6,$7,$6 \n\ - stq $4,16($17) \n\ - xor $19,$20,$19 \n\ - \n\ - stq $6,24($17) \n\ - xor $21,$22,$21 \n\ - stq $19,32($17) \n\ - xor $23,$24,$23 \n\ - \n\ - stq $21,40($17) \n\ - xor $25,$27,$25 \n\ - stq $23,48($17) \n\ - subq $16,1,$16 \n\ - \n\ - stq $25,56($17) \n\ - addq $17,64,$17 \n\ - addq $18,64,$18 \n\ - bgt $16,2b \n\ - \n\ - ret \n\ - .end xor_alpha_2 \n\ - \n\ - .align 3 \n\ - .ent xor_alpha_3 \n\ -xor_alpha_3: \n\ - .prologue 0 \n\ - srl $16, 6, $16 \n\ - .align 4 \n\ -3: \n\ - ldq $0,0($17) \n\ - ldq $1,0($18) \n\ - ldq $2,0($19) \n\ - ldq $3,8($17) \n\ - \n\ - ldq $4,8($18) \n\ - ldq $6,16($17) \n\ - ldq $7,16($18) \n\ - ldq $21,24($17) \n\ - \n\ - ldq $22,24($18) \n\ - ldq $24,32($17) \n\ - ldq $25,32($18) \n\ - ldq $5,8($19) \n\ - \n\ - ldq $20,16($19) \n\ - ldq $23,24($19) \n\ - ldq $27,32($19) \n\ - nop \n\ - \n\ - xor $0,$1,$1 # 8 cycles from $0 load \n\ - xor $3,$4,$4 # 6 cycles from $4 load \n\ - xor $6,$7,$7 # 6 cycles from $7 load \n\ - xor $21,$22,$22 # 5 cycles from $22 load \n\ - \n\ - xor $1,$2,$2 # 9 cycles from $2 load \n\ - xor $24,$25,$25 # 5 cycles from $25 load \n\ - stq $2,0($17) \n\ - xor $4,$5,$5 # 6 cycles from $5 load \n\ - \n\ - stq $5,8($17) \n\ - xor $7,$20,$20 # 7 cycles from $20 load \n\ - stq $20,16($17) \n\ - xor $22,$23,$23 # 7 cycles from $23 load \n\ - \n\ - stq $23,24($17) \n\ - xor $25,$27,$27 # 7 cycles from $27 load \n\ - stq $27,32($17) \n\ - nop \n\ - \n\ - ldq $0,40($17) \n\ - ldq $1,40($18) \n\ - ldq $3,48($17) \n\ - ldq $4,48($18) \n\ - \n\ - ldq $6,56($17) \n\ - ldq $7,56($18) \n\ - ldq $2,40($19) \n\ - ldq $5,48($19) \n\ - \n\ - ldq $20,56($19) \n\ - xor $0,$1,$1 # 4 cycles from $1 load \n\ - xor $3,$4,$4 # 5 cycles from $4 load \n\ - xor $6,$7,$7 # 5 cycles from $7 load \n\ - \n\ - xor $1,$2,$2 # 4 cycles from $2 load \n\ - xor $4,$5,$5 # 5 cycles from $5 load \n\ - stq $2,40($17) \n\ - xor $7,$20,$20 # 4 cycles from $20 load \n\ - \n\ - stq $5,48($17) \n\ - subq $16,1,$16 \n\ - stq $20,56($17) \n\ - addq $19,64,$19 \n\ - \n\ - addq $18,64,$18 \n\ - addq $17,64,$17 \n\ - bgt $16,3b \n\ - ret \n\ - .end xor_alpha_3 \n\ - \n\ - .align 3 \n\ - .ent xor_alpha_4 \n\ -xor_alpha_4: \n\ - .prologue 0 \n\ - srl $16, 6, $16 \n\ - .align 4 \n\ -4: \n\ - ldq $0,0($17) \n\ - ldq $1,0($18) \n\ - ldq $2,0($19) \n\ - ldq $3,0($20) \n\ - \n\ - ldq $4,8($17) \n\ - ldq $5,8($18) \n\ - ldq $6,8($19) \n\ - ldq $7,8($20) \n\ - \n\ - ldq $21,16($17) \n\ - ldq $22,16($18) \n\ - ldq $23,16($19) \n\ - ldq $24,16($20) \n\ - \n\ - ldq $25,24($17) \n\ - xor $0,$1,$1 # 6 cycles from $1 load \n\ - ldq $27,24($18) \n\ - xor $2,$3,$3 # 6 cycles from $3 load \n\ - \n\ - ldq $0,24($19) \n\ - xor $1,$3,$3 \n\ - ldq $1,24($20) \n\ - xor $4,$5,$5 # 7 cycles from $5 load \n\ - \n\ - stq $3,0($17) \n\ - xor $6,$7,$7 \n\ - xor $21,$22,$22 # 7 cycles from $22 load \n\ - xor $5,$7,$7 \n\ - \n\ - stq $7,8($17) \n\ - xor $23,$24,$24 # 7 cycles from $24 load \n\ - ldq $2,32($17) \n\ - xor $22,$24,$24 \n\ - \n\ - ldq $3,32($18) \n\ - ldq $4,32($19) \n\ - ldq $5,32($20) \n\ - xor $25,$27,$27 # 8 cycles from $27 load \n\ - \n\ - ldq $6,40($17) \n\ - ldq $7,40($18) \n\ - ldq $21,40($19) \n\ - ldq $22,40($20) \n\ - \n\ - stq $24,16($17) \n\ - xor $0,$1,$1 # 9 cycles from $1 load \n\ - xor $2,$3,$3 # 5 cycles from $3 load \n\ - xor $27,$1,$1 \n\ - \n\ - stq $1,24($17) \n\ - xor $4,$5,$5 # 5 cycles from $5 load \n\ - ldq $23,48($17) \n\ - ldq $24,48($18) \n\ - \n\ - ldq $25,48($19) \n\ - xor $3,$5,$5 \n\ - ldq $27,48($20) \n\ - ldq $0,56($17) \n\ - \n\ - ldq $1,56($18) \n\ - ldq $2,56($19) \n\ - xor $6,$7,$7 # 8 cycles from $6 load \n\ - ldq $3,56($20) \n\ - \n\ - stq $5,32($17) \n\ - xor $21,$22,$22 # 8 cycles from $22 load \n\ - xor $7,$22,$22 \n\ - xor $23,$24,$24 # 5 cycles from $24 load \n\ - \n\ - stq $22,40($17) \n\ - xor $25,$27,$27 # 5 cycles from $27 load \n\ - xor $24,$27,$27 \n\ - xor $0,$1,$1 # 5 cycles from $1 load \n\ - \n\ - stq $27,48($17) \n\ - xor $2,$3,$3 # 4 cycles from $3 load \n\ - xor $1,$3,$3 \n\ - subq $16,1,$16 \n\ - \n\ - stq $3,56($17) \n\ - addq $20,64,$20 \n\ - addq $19,64,$19 \n\ - addq $18,64,$18 \n\ - \n\ - addq $17,64,$17 \n\ - bgt $16,4b \n\ - ret \n\ - .end xor_alpha_4 \n\ - \n\ - .align 3 \n\ - .ent xor_alpha_5 \n\ -xor_alpha_5: \n\ - .prologue 0 \n\ - srl $16, 6, $16 \n\ - .align 4 \n\ -5: \n\ - ldq $0,0($17) \n\ - ldq $1,0($18) \n\ - ldq $2,0($19) \n\ - ldq $3,0($20) \n\ - \n\ - ldq $4,0($21) \n\ - ldq $5,8($17) \n\ - ldq $6,8($18) \n\ - ldq $7,8($19) \n\ - \n\ - ldq $22,8($20) \n\ - ldq $23,8($21) \n\ - ldq $24,16($17) \n\ - ldq $25,16($18) \n\ - \n\ - ldq $27,16($19) \n\ - xor $0,$1,$1 # 6 cycles from $1 load \n\ - ldq $28,16($20) \n\ - xor $2,$3,$3 # 6 cycles from $3 load \n\ - \n\ - ldq $0,16($21) \n\ - xor $1,$3,$3 \n\ - ldq $1,24($17) \n\ - xor $3,$4,$4 # 7 cycles from $4 load \n\ - \n\ - stq $4,0($17) \n\ - xor $5,$6,$6 # 7 cycles from $6 load \n\ - xor $7,$22,$22 # 7 cycles from $22 load \n\ - xor $6,$23,$23 # 7 cycles from $23 load \n\ - \n\ - ldq $2,24($18) \n\ - xor $22,$23,$23 \n\ - ldq $3,24($19) \n\ - xor $24,$25,$25 # 8 cycles from $25 load \n\ - \n\ - stq $23,8($17) \n\ - xor $25,$27,$27 # 8 cycles from $27 load \n\ - ldq $4,24($20) \n\ - xor $28,$0,$0 # 7 cycles from $0 load \n\ - \n\ - ldq $5,24($21) \n\ - xor $27,$0,$0 \n\ - ldq $6,32($17) \n\ - ldq $7,32($18) \n\ - \n\ - stq $0,16($17) \n\ - xor $1,$2,$2 # 6 cycles from $2 load \n\ - ldq $22,32($19) \n\ - xor $3,$4,$4 # 4 cycles from $4 load \n\ - \n\ - ldq $23,32($20) \n\ - xor $2,$4,$4 \n\ - ldq $24,32($21) \n\ - ldq $25,40($17) \n\ - \n\ - ldq $27,40($18) \n\ - ldq $28,40($19) \n\ - ldq $0,40($20) \n\ - xor $4,$5,$5 # 7 cycles from $5 load \n\ - \n\ - stq $5,24($17) \n\ - xor $6,$7,$7 # 7 cycles from $7 load \n\ - ldq $1,40($21) \n\ - ldq $2,48($17) \n\ - \n\ - ldq $3,48($18) \n\ - xor $7,$22,$22 # 7 cycles from $22 load \n\ - ldq $4,48($19) \n\ - xor $23,$24,$24 # 6 cycles from $24 load \n\ - \n\ - ldq $5,48($20) \n\ - xor $22,$24,$24 \n\ - ldq $6,48($21) \n\ - xor $25,$27,$27 # 7 cycles from $27 load \n\ - \n\ - stq $24,32($17) \n\ - xor $27,$28,$28 # 8 cycles from $28 load \n\ - ldq $7,56($17) \n\ - xor $0,$1,$1 # 6 cycles from $1 load \n\ - \n\ - ldq $22,56($18) \n\ - ldq $23,56($19) \n\ - ldq $24,56($20) \n\ - ldq $25,56($21) \n\ - \n\ - xor $28,$1,$1 \n\ - xor $2,$3,$3 # 9 cycles from $3 load \n\ - xor $3,$4,$4 # 9 cycles from $4 load \n\ - xor $5,$6,$6 # 8 cycles from $6 load \n\ - \n\ - stq $1,40($17) \n\ - xor $4,$6,$6 \n\ - xor $7,$22,$22 # 7 cycles from $22 load \n\ - xor $23,$24,$24 # 6 cycles from $24 load \n\ - \n\ - stq $6,48($17) \n\ - xor $22,$24,$24 \n\ - subq $16,1,$16 \n\ - xor $24,$25,$25 # 8 cycles from $25 load \n\ - \n\ - stq $25,56($17) \n\ - addq $21,64,$21 \n\ - addq $20,64,$20 \n\ - addq $19,64,$19 \n\ - \n\ - addq $18,64,$18 \n\ - addq $17,64,$17 \n\ - bgt $16,5b \n\ - ret \n\ - .end xor_alpha_5 \n\ - \n\ - .align 3 \n\ - .ent xor_alpha_prefetch_2 \n\ -xor_alpha_prefetch_2: \n\ - .prologue 0 \n\ - srl $16, 6, $16 \n\ - \n\ - ldq $31, 0($17) \n\ - ldq $31, 0($18) \n\ - \n\ - ldq $31, 64($17) \n\ - ldq $31, 64($18) \n\ - \n\ - ldq $31, 128($17) \n\ - ldq $31, 128($18) \n\ - \n\ - ldq $31, 192($17) \n\ - ldq $31, 192($18) \n\ - .align 4 \n\ -2: \n\ - ldq $0,0($17) \n\ - ldq $1,0($18) \n\ - ldq $2,8($17) \n\ - ldq $3,8($18) \n\ - \n\ - ldq $4,16($17) \n\ - ldq $5,16($18) \n\ - ldq $6,24($17) \n\ - ldq $7,24($18) \n\ - \n\ - ldq $19,32($17) \n\ - ldq $20,32($18) \n\ - ldq $21,40($17) \n\ - ldq $22,40($18) \n\ - \n\ - ldq $23,48($17) \n\ - ldq $24,48($18) \n\ - ldq $25,56($17) \n\ - ldq $27,56($18) \n\ - \n\ - ldq $31,256($17) \n\ - xor $0,$1,$0 # 8 cycles from $1 load \n\ - ldq $31,256($18) \n\ - xor $2,$3,$2 \n\ - \n\ - stq $0,0($17) \n\ - xor $4,$5,$4 \n\ - stq $2,8($17) \n\ - xor $6,$7,$6 \n\ - \n\ - stq $4,16($17) \n\ - xor $19,$20,$19 \n\ - stq $6,24($17) \n\ - xor $21,$22,$21 \n\ - \n\ - stq $19,32($17) \n\ - xor $23,$24,$23 \n\ - stq $21,40($17) \n\ - xor $25,$27,$25 \n\ - \n\ - stq $23,48($17) \n\ - subq $16,1,$16 \n\ - stq $25,56($17) \n\ - addq $17,64,$17 \n\ - \n\ - addq $18,64,$18 \n\ - bgt $16,2b \n\ - ret \n\ - .end xor_alpha_prefetch_2 \n\ - \n\ - .align 3 \n\ - .ent xor_alpha_prefetch_3 \n\ -xor_alpha_prefetch_3: \n\ - .prologue 0 \n\ - srl $16, 6, $16 \n\ - \n\ - ldq $31, 0($17) \n\ - ldq $31, 0($18) \n\ - ldq $31, 0($19) \n\ - \n\ - ldq $31, 64($17) \n\ - ldq $31, 64($18) \n\ - ldq $31, 64($19) \n\ - \n\ - ldq $31, 128($17) \n\ - ldq $31, 128($18) \n\ - ldq $31, 128($19) \n\ - \n\ - ldq $31, 192($17) \n\ - ldq $31, 192($18) \n\ - ldq $31, 192($19) \n\ - .align 4 \n\ -3: \n\ - ldq $0,0($17) \n\ - ldq $1,0($18) \n\ - ldq $2,0($19) \n\ - ldq $3,8($17) \n\ - \n\ - ldq $4,8($18) \n\ - ldq $6,16($17) \n\ - ldq $7,16($18) \n\ - ldq $21,24($17) \n\ - \n\ - ldq $22,24($18) \n\ - ldq $24,32($17) \n\ - ldq $25,32($18) \n\ - ldq $5,8($19) \n\ - \n\ - ldq $20,16($19) \n\ - ldq $23,24($19) \n\ - ldq $27,32($19) \n\ - nop \n\ - \n\ - xor $0,$1,$1 # 8 cycles from $0 load \n\ - xor $3,$4,$4 # 7 cycles from $4 load \n\ - xor $6,$7,$7 # 6 cycles from $7 load \n\ - xor $21,$22,$22 # 5 cycles from $22 load \n\ - \n\ - xor $1,$2,$2 # 9 cycles from $2 load \n\ - xor $24,$25,$25 # 5 cycles from $25 load \n\ - stq $2,0($17) \n\ - xor $4,$5,$5 # 6 cycles from $5 load \n\ - \n\ - stq $5,8($17) \n\ - xor $7,$20,$20 # 7 cycles from $20 load \n\ - stq $20,16($17) \n\ - xor $22,$23,$23 # 7 cycles from $23 load \n\ - \n\ - stq $23,24($17) \n\ - xor $25,$27,$27 # 7 cycles from $27 load \n\ - stq $27,32($17) \n\ - nop \n\ - \n\ - ldq $0,40($17) \n\ - ldq $1,40($18) \n\ - ldq $3,48($17) \n\ - ldq $4,48($18) \n\ - \n\ - ldq $6,56($17) \n\ - ldq $7,56($18) \n\ - ldq $2,40($19) \n\ - ldq $5,48($19) \n\ - \n\ - ldq $20,56($19) \n\ - ldq $31,256($17) \n\ - ldq $31,256($18) \n\ - ldq $31,256($19) \n\ - \n\ - xor $0,$1,$1 # 6 cycles from $1 load \n\ - xor $3,$4,$4 # 5 cycles from $4 load \n\ - xor $6,$7,$7 # 5 cycles from $7 load \n\ - xor $1,$2,$2 # 4 cycles from $2 load \n\ - \n\ - xor $4,$5,$5 # 5 cycles from $5 load \n\ - xor $7,$20,$20 # 4 cycles from $20 load \n\ - stq $2,40($17) \n\ - subq $16,1,$16 \n\ - \n\ - stq $5,48($17) \n\ - addq $19,64,$19 \n\ - stq $20,56($17) \n\ - addq $18,64,$18 \n\ - \n\ - addq $17,64,$17 \n\ - bgt $16,3b \n\ - ret \n\ - .end xor_alpha_prefetch_3 \n\ - \n\ - .align 3 \n\ - .ent xor_alpha_prefetch_4 \n\ -xor_alpha_prefetch_4: \n\ - .prologue 0 \n\ - srl $16, 6, $16 \n\ - \n\ - ldq $31, 0($17) \n\ - ldq $31, 0($18) \n\ - ldq $31, 0($19) \n\ - ldq $31, 0($20) \n\ - \n\ - ldq $31, 64($17) \n\ - ldq $31, 64($18) \n\ - ldq $31, 64($19) \n\ - ldq $31, 64($20) \n\ - \n\ - ldq $31, 128($17) \n\ - ldq $31, 128($18) \n\ - ldq $31, 128($19) \n\ - ldq $31, 128($20) \n\ - \n\ - ldq $31, 192($17) \n\ - ldq $31, 192($18) \n\ - ldq $31, 192($19) \n\ - ldq $31, 192($20) \n\ - .align 4 \n\ -4: \n\ - ldq $0,0($17) \n\ - ldq $1,0($18) \n\ - ldq $2,0($19) \n\ - ldq $3,0($20) \n\ - \n\ - ldq $4,8($17) \n\ - ldq $5,8($18) \n\ - ldq $6,8($19) \n\ - ldq $7,8($20) \n\ - \n\ - ldq $21,16($17) \n\ - ldq $22,16($18) \n\ - ldq $23,16($19) \n\ - ldq $24,16($20) \n\ - \n\ - ldq $25,24($17) \n\ - xor $0,$1,$1 # 6 cycles from $1 load \n\ - ldq $27,24($18) \n\ - xor $2,$3,$3 # 6 cycles from $3 load \n\ - \n\ - ldq $0,24($19) \n\ - xor $1,$3,$3 \n\ - ldq $1,24($20) \n\ - xor $4,$5,$5 # 7 cycles from $5 load \n\ - \n\ - stq $3,0($17) \n\ - xor $6,$7,$7 \n\ - xor $21,$22,$22 # 7 cycles from $22 load \n\ - xor $5,$7,$7 \n\ - \n\ - stq $7,8($17) \n\ - xor $23,$24,$24 # 7 cycles from $24 load \n\ - ldq $2,32($17) \n\ - xor $22,$24,$24 \n\ - \n\ - ldq $3,32($18) \n\ - ldq $4,32($19) \n\ - ldq $5,32($20) \n\ - xor $25,$27,$27 # 8 cycles from $27 load \n\ - \n\ - ldq $6,40($17) \n\ - ldq $7,40($18) \n\ - ldq $21,40($19) \n\ - ldq $22,40($20) \n\ - \n\ - stq $24,16($17) \n\ - xor $0,$1,$1 # 9 cycles from $1 load \n\ - xor $2,$3,$3 # 5 cycles from $3 load \n\ - xor $27,$1,$1 \n\ - \n\ - stq $1,24($17) \n\ - xor $4,$5,$5 # 5 cycles from $5 load \n\ - ldq $23,48($17) \n\ - xor $3,$5,$5 \n\ - \n\ - ldq $24,48($18) \n\ - ldq $25,48($19) \n\ - ldq $27,48($20) \n\ - ldq $0,56($17) \n\ - \n\ - ldq $1,56($18) \n\ - ldq $2,56($19) \n\ - ldq $3,56($20) \n\ - xor $6,$7,$7 # 8 cycles from $6 load \n\ - \n\ - ldq $31,256($17) \n\ - xor $21,$22,$22 # 8 cycles from $22 load \n\ - ldq $31,256($18) \n\ - xor $7,$22,$22 \n\ - \n\ - ldq $31,256($19) \n\ - xor $23,$24,$24 # 6 cycles from $24 load \n\ - ldq $31,256($20) \n\ - xor $25,$27,$27 # 6 cycles from $27 load \n\ - \n\ - stq $5,32($17) \n\ - xor $24,$27,$27 \n\ - xor $0,$1,$1 # 7 cycles from $1 load \n\ - xor $2,$3,$3 # 6 cycles from $3 load \n\ - \n\ - stq $22,40($17) \n\ - xor $1,$3,$3 \n\ - stq $27,48($17) \n\ - subq $16,1,$16 \n\ - \n\ - stq $3,56($17) \n\ - addq $20,64,$20 \n\ - addq $19,64,$19 \n\ - addq $18,64,$18 \n\ - \n\ - addq $17,64,$17 \n\ - bgt $16,4b \n\ - ret \n\ - .end xor_alpha_prefetch_4 \n\ - \n\ - .align 3 \n\ - .ent xor_alpha_prefetch_5 \n\ -xor_alpha_prefetch_5: \n\ - .prologue 0 \n\ - srl $16, 6, $16 \n\ - \n\ - ldq $31, 0($17) \n\ - ldq $31, 0($18) \n\ - ldq $31, 0($19) \n\ - ldq $31, 0($20) \n\ - ldq $31, 0($21) \n\ - \n\ - ldq $31, 64($17) \n\ - ldq $31, 64($18) \n\ - ldq $31, 64($19) \n\ - ldq $31, 64($20) \n\ - ldq $31, 64($21) \n\ - \n\ - ldq $31, 128($17) \n\ - ldq $31, 128($18) \n\ - ldq $31, 128($19) \n\ - ldq $31, 128($20) \n\ - ldq $31, 128($21) \n\ - \n\ - ldq $31, 192($17) \n\ - ldq $31, 192($18) \n\ - ldq $31, 192($19) \n\ - ldq $31, 192($20) \n\ - ldq $31, 192($21) \n\ - .align 4 \n\ -5: \n\ - ldq $0,0($17) \n\ - ldq $1,0($18) \n\ - ldq $2,0($19) \n\ - ldq $3,0($20) \n\ - \n\ - ldq $4,0($21) \n\ - ldq $5,8($17) \n\ - ldq $6,8($18) \n\ - ldq $7,8($19) \n\ - \n\ - ldq $22,8($20) \n\ - ldq $23,8($21) \n\ - ldq $24,16($17) \n\ - ldq $25,16($18) \n\ - \n\ - ldq $27,16($19) \n\ - xor $0,$1,$1 # 6 cycles from $1 load \n\ - ldq $28,16($20) \n\ - xor $2,$3,$3 # 6 cycles from $3 load \n\ - \n\ - ldq $0,16($21) \n\ - xor $1,$3,$3 \n\ - ldq $1,24($17) \n\ - xor $3,$4,$4 # 7 cycles from $4 load \n\ - \n\ - stq $4,0($17) \n\ - xor $5,$6,$6 # 7 cycles from $6 load \n\ - xor $7,$22,$22 # 7 cycles from $22 load \n\ - xor $6,$23,$23 # 7 cycles from $23 load \n\ - \n\ - ldq $2,24($18) \n\ - xor $22,$23,$23 \n\ - ldq $3,24($19) \n\ - xor $24,$25,$25 # 8 cycles from $25 load \n\ - \n\ - stq $23,8($17) \n\ - xor $25,$27,$27 # 8 cycles from $27 load \n\ - ldq $4,24($20) \n\ - xor $28,$0,$0 # 7 cycles from $0 load \n\ - \n\ - ldq $5,24($21) \n\ - xor $27,$0,$0 \n\ - ldq $6,32($17) \n\ - ldq $7,32($18) \n\ - \n\ - stq $0,16($17) \n\ - xor $1,$2,$2 # 6 cycles from $2 load \n\ - ldq $22,32($19) \n\ - xor $3,$4,$4 # 4 cycles from $4 load \n\ - \n\ - ldq $23,32($20) \n\ - xor $2,$4,$4 \n\ - ldq $24,32($21) \n\ - ldq $25,40($17) \n\ - \n\ - ldq $27,40($18) \n\ - ldq $28,40($19) \n\ - ldq $0,40($20) \n\ - xor $4,$5,$5 # 7 cycles from $5 load \n\ - \n\ - stq $5,24($17) \n\ - xor $6,$7,$7 # 7 cycles from $7 load \n\ - ldq $1,40($21) \n\ - ldq $2,48($17) \n\ - \n\ - ldq $3,48($18) \n\ - xor $7,$22,$22 # 7 cycles from $22 load \n\ - ldq $4,48($19) \n\ - xor $23,$24,$24 # 6 cycles from $24 load \n\ - \n\ - ldq $5,48($20) \n\ - xor $22,$24,$24 \n\ - ldq $6,48($21) \n\ - xor $25,$27,$27 # 7 cycles from $27 load \n\ - \n\ - stq $24,32($17) \n\ - xor $27,$28,$28 # 8 cycles from $28 load \n\ - ldq $7,56($17) \n\ - xor $0,$1,$1 # 6 cycles from $1 load \n\ - \n\ - ldq $22,56($18) \n\ - ldq $23,56($19) \n\ - ldq $24,56($20) \n\ - ldq $25,56($21) \n\ - \n\ - ldq $31,256($17) \n\ - xor $28,$1,$1 \n\ - ldq $31,256($18) \n\ - xor $2,$3,$3 # 9 cycles from $3 load \n\ - \n\ - ldq $31,256($19) \n\ - xor $3,$4,$4 # 9 cycles from $4 load \n\ - ldq $31,256($20) \n\ - xor $5,$6,$6 # 8 cycles from $6 load \n\ - \n\ - stq $1,40($17) \n\ - xor $4,$6,$6 \n\ - xor $7,$22,$22 # 7 cycles from $22 load \n\ - xor $23,$24,$24 # 6 cycles from $24 load \n\ - \n\ - stq $6,48($17) \n\ - xor $22,$24,$24 \n\ - ldq $31,256($21) \n\ - xor $24,$25,$25 # 8 cycles from $25 load \n\ - \n\ - stq $25,56($17) \n\ - subq $16,1,$16 \n\ - addq $21,64,$21 \n\ - addq $20,64,$20 \n\ - \n\ - addq $19,64,$19 \n\ - addq $18,64,$18 \n\ - addq $17,64,$17 \n\ - bgt $16,5b \n\ - \n\ - ret \n\ - .end xor_alpha_prefetch_5 \n\ -"); - -static struct xor_block_template xor_block_alpha = { - .name = "alpha", - .do_2 = xor_alpha_2, - .do_3 = xor_alpha_3, - .do_4 = xor_alpha_4, - .do_5 = xor_alpha_5, -}; - -static struct xor_block_template xor_block_alpha_prefetch = { - .name = "alpha prefetch", - .do_2 = xor_alpha_prefetch_2, - .do_3 = xor_alpha_prefetch_3, - .do_4 = xor_alpha_prefetch_4, - .do_5 = xor_alpha_prefetch_5, -}; - -/* For grins, also test the generic routines. */ -#include <asm-generic/xor.h> - -#undef XOR_TRY_TEMPLATES -#define XOR_TRY_TEMPLATES \ - do { \ - xor_speed(&xor_block_8regs); \ - xor_speed(&xor_block_32regs); \ - xor_speed(&xor_block_alpha); \ - xor_speed(&xor_block_alpha_prefetch); \ - } while (0) - -/* Force the use of alpha_prefetch if EV6, as it is significantly - faster in the cold cache case. */ -#define XOR_SELECT_TEMPLATE(FASTEST) \ - (implver() == IMPLVER_EV6 ? &xor_block_alpha_prefetch : FASTEST) diff --git a/arch/alpha/include/uapi/asm/errno.h b/arch/alpha/include/uapi/asm/errno.h index 3d265f6babaf..1a99f38813c7 100644 --- a/arch/alpha/include/uapi/asm/errno.h +++ b/arch/alpha/include/uapi/asm/errno.h @@ -55,6 +55,7 @@ #define ENOSR 82 /* Out of streams resources */ #define ETIME 83 /* Timer expired */ #define EBADMSG 84 /* Not a data message */ +#define EFSBADCRC EBADMSG /* Bad CRC detected */ #define EPROTO 85 /* Protocol error */ #define ENODATA 86 /* No data available */ #define ENOSTR 87 /* Device not a stream */ @@ -96,6 +97,7 @@ #define EREMCHG 115 /* Remote address changed */ #define EUCLEAN 117 /* Structure needs cleaning */ +#define EFSCORRUPTED EUCLEAN /* Filesystem is corrupted */ #define ENOTNAM 118 /* Not a XENIX named type file */ #define ENAVAIL 119 /* No XENIX semaphores available */ #define EISNAM 120 /* Is a named type file */ @@ -125,4 +127,6 @@ #define EHWPOISON 139 /* Memory page has hardware error */ +#define EFTYPE 140 /* Wrong file type for the intended operation */ + #endif diff --git a/arch/alpha/include/uapi/asm/fcntl.h b/arch/alpha/include/uapi/asm/fcntl.h index 50bdc8e8a271..c7e1c5cf646d 100644 --- a/arch/alpha/include/uapi/asm/fcntl.h +++ b/arch/alpha/include/uapi/asm/fcntl.h @@ -2,20 +2,20 @@ #ifndef _ALPHA_FCNTL_H #define _ALPHA_FCNTL_H -#define O_CREAT 01000 /* not fcntl */ -#define O_TRUNC 02000 /* not fcntl */ -#define O_EXCL 04000 /* not fcntl */ -#define O_NOCTTY 010000 /* not fcntl */ - -#define O_NONBLOCK 00004 -#define O_APPEND 00010 -#define O_DSYNC 040000 /* used to be O_SYNC, see below */ -#define O_DIRECTORY 0100000 /* must be a directory */ -#define O_NOFOLLOW 0200000 /* don't follow links */ -#define O_LARGEFILE 0400000 /* will be set by the kernel on every open */ -#define O_DIRECT 02000000 /* direct disk access - should check with OSF/1 */ -#define O_NOATIME 04000000 -#define O_CLOEXEC 010000000 /* set close_on_exec */ +#define O_CREAT (1 << 9) /* not fcntl */ +#define O_TRUNC (1 << 10) /* not fcntl */ +#define O_EXCL (1 << 11) /* not fcntl */ +#define O_NOCTTY (1 << 12) /* not fcntl */ + +#define O_NONBLOCK (1 << 2) +#define O_APPEND (1 << 3) +#define O_DSYNC (1 << 14) /* used to be O_SYNC, see below */ +#define O_DIRECTORY (1 << 15) /* must be a directory */ +#define O_NOFOLLOW (1 << 16) /* don't follow links */ +#define O_LARGEFILE (1 << 17) /* will be set by the kernel on every open */ +#define O_DIRECT (1 << 19) /* direct disk access - should check with OSF/1 */ +#define O_NOATIME (1 << 20) +#define O_CLOEXEC (1 << 21) /* set close_on_exec */ /* * Before Linux 2.6.33 only O_DSYNC semantics were implemented, but using * the O_SYNC flag. We continue to use the existing numerical value @@ -29,11 +29,11 @@ * * Note: __O_SYNC must never be used directly. */ -#define __O_SYNC 020000000 +#define __O_SYNC (1 << 22) #define O_SYNC (__O_SYNC|O_DSYNC) -#define O_PATH 040000000 -#define __O_TMPFILE 0100000000 +#define O_PATH (1 << 23) +#define __O_TMPFILE (1 << 24) #define F_GETLK 7 #define F_SETLK 8 diff --git a/arch/alpha/include/uapi/asm/fpu.h b/arch/alpha/include/uapi/asm/fpu.h index cea9eafa056f..d28dc36786e2 100644 --- a/arch/alpha/include/uapi/asm/fpu.h +++ b/arch/alpha/include/uapi/asm/fpu.h @@ -101,7 +101,12 @@ ieee_swcr_to_fpcr(unsigned long sw) | IEEE_TRAP_ENABLE_OVF)) << 48; fp |= (~sw & (IEEE_TRAP_ENABLE_UNF | IEEE_TRAP_ENABLE_INE)) << 57; fp |= (sw & IEEE_MAP_UMZ ? FPCR_UNDZ | FPCR_UNFD : 0); - fp |= (~sw & IEEE_TRAP_ENABLE_DNO) << 41; + /* + * Disable denormal operand traps only when denormal inputs are to be + * flushed to zero. Otherwise they must keep trapping, so that /S + * instructions reach the kernel emulation handler. + */ + fp |= (sw & IEEE_MAP_DMZ ? FPCR_DNOD : 0); return fp; } @@ -116,7 +121,6 @@ ieee_fpcr_to_swcr(unsigned long fp) | IEEE_TRAP_ENABLE_OVF); sw |= (~fp >> 57) & (IEEE_TRAP_ENABLE_UNF | IEEE_TRAP_ENABLE_INE); sw |= (fp >> 47) & IEEE_MAP_UMZ; - sw |= (~fp >> 41) & IEEE_TRAP_ENABLE_DNO; return sw; } diff --git a/arch/alpha/include/uapi/asm/ptrace.h b/arch/alpha/include/uapi/asm/ptrace.h index 72ed913a910f..9d86b2a1526e 100644 --- a/arch/alpha/include/uapi/asm/ptrace.h +++ b/arch/alpha/include/uapi/asm/ptrace.h @@ -43,7 +43,7 @@ struct pt_regs { unsigned long trap_a1; unsigned long trap_a2; /* This makes the stack 16-byte aligned as GCC expects */ - unsigned long __pad0; + unsigned long usp; /* These are saved by PAL-code: */ unsigned long ps; unsigned long pc; diff --git a/arch/alpha/include/uapi/asm/socket.h b/arch/alpha/include/uapi/asm/socket.h index 5ef57f88df6b..946a5fad2691 100644 --- a/arch/alpha/include/uapi/asm/socket.h +++ b/arch/alpha/include/uapi/asm/socket.h @@ -155,6 +155,8 @@ #define SO_INQ 84 #define SCM_INQ SO_INQ +#define SO_RIGHTS_NOTRUNC 85 + #if !defined(__KERNEL__) #if __BITS_PER_LONG == 64 diff --git a/arch/alpha/kernel/Makefile b/arch/alpha/kernel/Makefile index 187cd8df2faf..4ea5c189e60e 100644 --- a/arch/alpha/kernel/Makefile +++ b/arch/alpha/kernel/Makefile @@ -9,7 +9,8 @@ ccflags-y := -Wno-sign-compare obj-y := head.o entry.o traps.o process.o osf_sys.o irq.o \ irq_alpha.o signal.o setup.o ptrace.o time.o \ - systbls.o err_common.o io.o bugs.o termios.o + systbls.o err_common.o io.o bugs.o termios.o \ + stacktrace.o obj-$(CONFIG_VGA_HOSE) += console.o obj-$(CONFIG_SMP) += smp.o diff --git a/arch/alpha/kernel/asm-offsets.c b/arch/alpha/kernel/asm-offsets.c index 1ebb05890499..1d3bfca319ae 100644 --- a/arch/alpha/kernel/asm-offsets.c +++ b/arch/alpha/kernel/asm-offsets.c @@ -29,4 +29,5 @@ static void __used foo(void) DEFINE(HAE_CACHE, offsetof(struct alpha_machine_vector, hae_cache)); DEFINE(HAE_REG, offsetof(struct alpha_machine_vector, hae_register)); + DEFINE(PT_REGS_USP, offsetof(struct pt_regs, usp)); } diff --git a/arch/alpha/kernel/core_marvel.c b/arch/alpha/kernel/core_marvel.c index d38f4d6759e4..92e3312e786e 100644 --- a/arch/alpha/kernel/core_marvel.c +++ b/arch/alpha/kernel/core_marvel.c @@ -861,7 +861,7 @@ marvel_agp_setup(alpha_agp_info *agp) if (!alpha_agpgart_size) return -ENOMEM; - aper = kmalloc(sizeof(*aper), GFP_KERNEL); + aper = kmalloc_obj(*aper); if (aper == NULL) return -ENOMEM; aper->arena = agp->hose->sg_pci; @@ -1059,7 +1059,7 @@ marvel_agp_info(void) /* * Allocate the info structure. */ - agp = kmalloc(sizeof(*agp), GFP_KERNEL); + agp = kmalloc_obj(*agp); if (!agp) return NULL; diff --git a/arch/alpha/kernel/core_titan.c b/arch/alpha/kernel/core_titan.c index 77f5d68ed04b..6b6b9e6d631f 100644 --- a/arch/alpha/kernel/core_titan.c +++ b/arch/alpha/kernel/core_titan.c @@ -594,7 +594,7 @@ titan_agp_setup(alpha_agp_info *agp) if (!alpha_agpgart_size) return -ENOMEM; - aper = kmalloc(sizeof(struct titan_agp_aperture), GFP_KERNEL); + aper = kmalloc_obj(struct titan_agp_aperture); if (aper == NULL) return -ENOMEM; @@ -760,7 +760,7 @@ titan_agp_info(void) /* * Allocate the info structure. */ - agp = kmalloc(sizeof(*agp), GFP_KERNEL); + agp = kmalloc_obj(*agp); if (!agp) return NULL; diff --git a/arch/alpha/kernel/entry.S b/arch/alpha/kernel/entry.S index f4d41b4538c2..9f2608de2544 100644 --- a/arch/alpha/kernel/entry.S +++ b/arch/alpha/kernel/entry.S @@ -10,6 +10,7 @@ #include <asm/pal.h> #include <asm/errno.h> #include <asm/unistd.h> +#include <linux/errno.h> .text .set noat @@ -36,6 +37,77 @@ .endm /* + * SYSCALL_SKIP_RETURN_RESTART_GATE + * + * Used when syscall dispatch is skipped (seccomp/ptrace injected nr=-1). + * - Ensure we never return r0==-1 with a3==0 (success); convert to ENOSYS. + * - Gate whether syscall restart is allowed by preserving restart context + * only for ERESTART* returns. Result: + * $26 = 0 => restart allowed + * $26 = 1 => restart NOT allowed + * $18 = preserved syscall nr (regs->r2) if restart allowed, else 0 + */ +.macro SYSCALL_SKIP_RETURN_RESTART_GATE + /* Fix up invalid "-1 success" return state. */ + ldq $19, 72($sp) /* a3 */ + bne $19, 1f /* already error => skip fixup */ + + ldq $20, 0($sp) /* r0 */ + lda $21, -1($31) + cmpeq $20, $21, $22 + beq $22, 1f /* r0 != -1 => skip fixup */ + + + lda $20, ENOSYS($31) + stq $20, 0($sp) /* r0 = ENOSYS */ + lda $19, 1($31) + stq $19, 72($sp) /* a3 = 1 */ +1: + /* Restart gating: success is never restartable here. */ + ldq $19, 72($sp) /* a3 */ + beq $19, 3f /* success => not restartable */ + + ldq $20, 0($sp) /* r0 (positive errno if a3==1) */ + lda $21, ERESTARTSYS($31) + cmpeq $20, $21, $22 + bne $22, 2f + lda $21, ERESTARTNOINTR($31) + cmpeq $20, $21, $22 + bne $22, 2f + lda $21, ERESTARTNOHAND($31) + cmpeq $20, $21, $22 + bne $22, 2f + lda $21, ERESTART_RESTARTBLOCK($31) + cmpeq $20, $21, $22 + bne $22, 2f + +3: /* Not a restart code (or success) => restart NOT allowed. */ + addq $31, 1, $26 /* $26=1 => restart NOT allowed */ + mov 0, $18 + br 4f + +2: /* Restart allowed. */ + ldq $18, 16($sp) /* preserved syscall nr (regs->r2) */ + mov $31, $26 /* $26=0 => restart allowed */ + br 4f +4: +.endm + +.macro LOCKDEP_HARDIRQS_ON_RESTORE +#ifdef CONFIG_PROVE_LOCKING + /* a0 = saved PS */ + ldq $16, SP_OFF($sp) + + /* a1 = callsite IP for lockdep */ + lda $17, 1f + + jsr $26, lockdep_on_restore + ldgp $gp, 0($26) +1: +#endif +.endm + +/* * This defines the normal kernel pt-regs layout. * * regs 9-15 preserved by C code @@ -368,6 +440,7 @@ CFI_START_OSF_FRAME entUna .cfi_restore $28 .cfi_restore $29 .cfi_adjust_cfa_offset -256 + LOCKDEP_HARDIRQS_ON_RESTORE call_pal PAL_rti .align 4 @@ -425,7 +498,7 @@ CFI_START_OSF_FRAME entDbg mov $sp, $16 jsr $31, do_entDbg CFI_END_OSF_FRAME entDbg - + /* * The system call entry point is special. Most importantly, it looks * like a function call to userspace as far as clobbered registers. We @@ -435,6 +508,17 @@ CFI_END_OSF_FRAME entDbg * So much for theory. We don't take advantage of this yet. * * Note that a0-a2 are not saved by PALcode as with the other entry points. + * + * Alpha syscall ABI uses: + * - r0 for return value + * - r19 ("a3") as error indicator (0=success, 1=error; r0 holds errno) + * + * For seccomp/ptrace/generic syscall helpers we track the syscall + * number separately: + * - regs->r1: current (mutable) syscall number (may be changed or set to -1) + * - regs->r2: original syscall number for restart/rollback + * + * On entry PAL provides the syscall number in r0; copy it into r1/r2. */ .align 4 @@ -447,6 +531,16 @@ CFI_END_OSF_FRAME entDbg .cfi_rel_offset $gp, 16 entSys: SAVE_ALL + ldq $1, 0($sp) /* syscall nr from saved r0 */ + stq $1, 8($sp) /* regs->r1 = shadow syscall nr */ + stq $1, 16($sp) /* regs->r2 = restart syscall nr */ + /* Syscalls always enter from user mode: snapshot USP into pt_regs->usp */ + mov $0, $8 + call_pal PAL_rdusp + stq $0, PT_REGS_USP($sp) + mov $8, $0 + + lda $8, 0x3fff bic $sp, $8, $8 lda $4, NR_syscalls($31) @@ -461,16 +555,15 @@ entSys: .cfi_rel_offset $16, SP_OFF+24 .cfi_rel_offset $17, SP_OFF+32 .cfi_rel_offset $18, SP_OFF+40 -#ifdef CONFIG_AUDITSYSCALL - lda $6, _TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT + lda $6, _TIF_SYSCALL_WORK and $3, $6, $3 bne $3, strace -#else - blbs $3, strace /* check for SYSCALL_TRACE in disguise */ -#endif + beq $4, 1f ldq $27, 0($5) -1: jsr $26, ($27), sys_ni_syscall +1: ldq $0, 8($sp) /* syscall nr shadow (regs->r1) */ + + jsr $26, ($27), sys_ni_syscall ldgp $gp, 0($26) blt $0, $syscall_error /* the call failed */ $ret_success: @@ -498,6 +591,7 @@ restore_all: bne $3, restore_fpu restore_other: .cfi_remember_state + LOCKDEP_HARDIRQS_ON_RESTORE RESTORE_ALL call_pal PAL_rti @@ -509,15 +603,8 @@ ret_to_kernel: .align 3 $syscall_error: - /* - * Some system calls (e.g., ptrace) can return arbitrary - * values which might normally be mistaken as error numbers. - * Those functions must zero $0 (v0) directly in the stack - * frame to indicate that a negative return value wasn't an - * error number.. - */ - ldq $18, 0($sp) /* old syscall nr (zero if success) */ - beq $18, $ret_success + /* Restart syscall nr comes from saved r2 (preserved even if r0 overwritten). */ + ldq $18, 16($sp) /* old syscall nr for restart */ ldq $19, 72($sp) /* .. and this a3 */ subq $31, $0, $0 /* with error in v0 */ @@ -550,7 +637,7 @@ $work_resched: * or got through work_notifysig already. Either case means no syscall * restarts for us, so let $18 and $19 burn. */ - jsr $26, schedule + jsr $26, alpha_schedule_user_work mov 0, $18 br ret_to_user @@ -581,6 +668,8 @@ strace: jsr $26, syscall_trace_enter /* returns the syscall number */ UNDO_SWITCH_STACK + stq $0, 8($sp) /* regs->r1 = shadow syscall nr */ + /* get the arguments back.. */ ldq $16, SP_OFF+24($sp) ldq $17, SP_OFF+32($sp) @@ -589,6 +678,11 @@ strace: ldq $20, 80($sp) ldq $21, 88($sp) + /* nr == -1: internal skip-dispatch or userspace syscall(-1)? */ + lda $6, -1($31) + cmpeq $0, $6, $6 + bne $6, $strace_skip_call /* nr == -1 => dispatch */ + /* get the system call pointer.. */ lda $1, NR_syscalls($31) lda $2, sys_call_table @@ -607,6 +701,8 @@ $strace_success: stq $31, 72($sp) /* a3=0 => no error */ stq $0, 0($sp) /* save return value */ +$strace_skip_call: + SYSCALL_SKIP_RETURN_RESTART_GATE DO_SWITCH_STACK jsr $26, syscall_trace_leave UNDO_SWITCH_STACK @@ -614,8 +710,7 @@ $strace_success: .align 3 $strace_error: - ldq $18, 0($sp) /* old syscall nr (zero if success) */ - beq $18, $strace_success + ldq $18, 16($sp) /* restart syscall nr */ ldq $19, 72($sp) /* .. and this a3 */ subq $31, $0, $0 /* with error in v0 */ @@ -634,7 +729,7 @@ $strace_error: mov $31, $26 /* tell "ret_from_sys_call" we can restart */ br ret_from_sys_call CFI_END_OSF_FRAME entSys - + /* * Save and restore the switch stack -- aka the balance of the user context. */ diff --git a/arch/alpha/kernel/io.c b/arch/alpha/kernel/io.c index c28035d6d1e6..2bad1b4fb240 100644 --- a/arch/alpha/kernel/io.c +++ b/arch/alpha/kernel/io.c @@ -647,7 +647,7 @@ void _memset_c_io(volatile void __iomem *to, unsigned long c, long count) EXPORT_SYMBOL(_memset_c_io); -#if IS_ENABLED(CONFIG_VGA_CONSOLE) || IS_ENABLED(CONFIG_MDA_CONSOLE) +#if IS_ENABLED(CONFIG_VGA_CONSOLE) #include <asm/vga.h> diff --git a/arch/alpha/kernel/irq.c b/arch/alpha/kernel/irq.c index c67047c5d830..4a6a8b1d5a8b 100644 --- a/arch/alpha/kernel/irq.c +++ b/arch/alpha/kernel/irq.c @@ -72,16 +72,16 @@ int arch_show_interrupts(struct seq_file *p, int prec) int j; #ifdef CONFIG_SMP - seq_puts(p, "IPI: "); + seq_puts(p, " IPI: "); for_each_online_cpu(j) seq_printf(p, "%10lu ", cpu_data[j].ipi_count); seq_putc(p, '\n'); #endif - seq_puts(p, "PMI: "); + seq_puts(p, " PMI: "); for_each_online_cpu(j) seq_printf(p, "%10lu ", per_cpu(irq_pmi_count, j)); - seq_puts(p, " Performance Monitoring\n"); - seq_printf(p, "ERR: %10lu\n", irq_err_count); + seq_puts(p, " Performance Monitoring\n"); + seq_printf(p, " ERR: %10lu\n", irq_err_count); return 0; } diff --git a/arch/alpha/kernel/irq_alpha.c b/arch/alpha/kernel/irq_alpha.c index d17e44c99df9..1b799aafb7dc 100644 --- a/arch/alpha/kernel/irq_alpha.c +++ b/arch/alpha/kernel/irq_alpha.c @@ -41,7 +41,7 @@ EXPORT_SYMBOL(perf_irq); * The main interrupt entry point. */ -asmlinkage void +asmlinkage void do_entInt(unsigned long type, unsigned long vector, unsigned long la_ptr, struct pt_regs *regs) { @@ -52,42 +52,91 @@ do_entInt(unsigned long type, unsigned long vector, * Note that there is no matching local_irq_enable() due to * severe problems with RTI at IPL0 and some MILO PALcode * (namely LX164). + * + * PALcode has already raised PS.IPL to the level of the interrupt + * being delivered. For an IPL 7 entry - a machine check or a system + * event - that is IPL_MAX, which is what arch_irqs_disabled() tests + * for, so local_irq_disable() would decide interrupts were already + * off and skip trace_hardirqs_off(). lockdep would then spend the + * whole handler believing interrupts are enabled. Drive the + * annotation from lockdep's own state rather than the hardware IPL. */ - local_irq_disable(); + raw_local_irq_disable(); + if (lockdep_hardirqs_enabled()) + trace_hardirqs_off(); + + old_regs = set_irq_regs(regs); + switch (type) { case 0: #ifdef CONFIG_SMP + irq_enter(); handle_ipi(regs); - return; + irq_exit(); + break; #else irq_err_count++; - printk(KERN_CRIT "Interprocessor interrupt? " - "You must be kidding!\n"); -#endif + pr_crit("Interprocessor interrupt? You must be kidding!\n"); break; +#endif case 1: - old_regs = set_irq_regs(regs); + /* handle_irq() already does irq_enter()/irq_exit() */ handle_irq(RTC_IRQ); - set_irq_regs(old_regs); - return; + break; case 2: - old_regs = set_irq_regs(regs); + irq_enter(); alpha_mv.machine_check(vector, la_ptr); - set_irq_regs(old_regs); - return; + irq_exit(); + break; case 3: - old_regs = set_irq_regs(regs); + irq_enter(); alpha_mv.device_interrupt(vector); - set_irq_regs(old_regs); - return; + irq_exit(); + break; case 4: + irq_enter(); perf_irq(la_ptr, regs); - return; + irq_exit(); + break; default: - printk(KERN_CRIT "Hardware intr %ld %lx? Huh?\n", - type, vector); + pr_crit("Hardware intr %lu %lx? Huh?\n", type, vector); + pr_crit("PC = %016lx PS=%04lx\n", regs->pc, regs->ps); + break; } - printk(KERN_CRIT "PC = %016lx PS=%04lx\n", regs->pc, regs->ps); + + set_irq_regs(old_regs); + + /* + * Intentionally no local_irq_enable(): Alpha historically avoids + * enabling at IPL0 here due to PAL/RTI issues (LX164/MILO note). + */ +} + +void notrace lockdep_on_restore(unsigned long ps, + unsigned long ip) +{ +#ifdef CONFIG_PROVE_LOCKING + /* Restoring IPL==7 means interrupts remain disabled. */ + if ((ps & 7) == 7) + return; + + /* + * If hardware IRQs are already enabled here, then emitting a + * hardirqs-on transition is redundant. + */ + if (!irqs_disabled()) + return; + + /* + * Only emit the transition if lockdep currently believes + * hardirqs are off. + */ + if (lockdep_hardirqs_enabled()) + return; + + lockdep_hardirqs_on_prepare(); + lockdep_hardirqs_on(ip); +#endif } void __init diff --git a/arch/alpha/kernel/irq_i8259.c b/arch/alpha/kernel/irq_i8259.c index 29c6c477ac35..28f7b0680564 100644 --- a/arch/alpha/kernel/irq_i8259.c +++ b/arch/alpha/kernel/irq_i8259.c @@ -22,7 +22,7 @@ /* Note mask bit is true for DISABLED irqs. */ static unsigned int cached_irq_mask = 0xffff; -static DEFINE_SPINLOCK(i8259_irq_lock); +static DEFINE_RAW_SPINLOCK(i8259_irq_lock); static inline void i8259_update_irq_hw(unsigned int irq, unsigned long mask) @@ -36,9 +36,11 @@ i8259_update_irq_hw(unsigned int irq, unsigned long mask) inline void i8259a_enable_irq(struct irq_data *d) { - spin_lock(&i8259_irq_lock); + unsigned long flags; + + raw_spin_lock_irqsave(&i8259_irq_lock, flags); i8259_update_irq_hw(d->irq, cached_irq_mask &= ~(1 << d->irq)); - spin_unlock(&i8259_irq_lock); + raw_spin_unlock_irqrestore(&i8259_irq_lock, flags); } static inline void @@ -50,17 +52,20 @@ __i8259a_disable_irq(unsigned int irq) void i8259a_disable_irq(struct irq_data *d) { - spin_lock(&i8259_irq_lock); + unsigned long flags; + + raw_spin_lock_irqsave(&i8259_irq_lock, flags); __i8259a_disable_irq(d->irq); - spin_unlock(&i8259_irq_lock); + raw_spin_unlock_irqrestore(&i8259_irq_lock, flags); } void i8259a_mask_and_ack_irq(struct irq_data *d) { unsigned int irq = d->irq; + unsigned long flags; - spin_lock(&i8259_irq_lock); + raw_spin_lock_irqsave(&i8259_irq_lock, flags); __i8259a_disable_irq(irq); /* Ack the interrupt making it the lowest priority. */ @@ -69,7 +74,7 @@ i8259a_mask_and_ack_irq(struct irq_data *d) irq = 2; } outb(0xE0 | irq, 0x20); /* ack the master */ - spin_unlock(&i8259_irq_lock); + raw_spin_unlock_irqrestore(&i8259_irq_lock, flags); } struct irq_chip i8259a_irq_type = { diff --git a/arch/alpha/kernel/module.c b/arch/alpha/kernel/module.c index cbefa5a77384..548d1591cfa9 100644 --- a/arch/alpha/kernel/module.c +++ b/arch/alpha/kernel/module.c @@ -46,7 +46,7 @@ process_reloc_for_got(Elf64_Rela *rela, goto found_entry; } - g = kmalloc (sizeof (*g), GFP_KERNEL); + g = kmalloc_obj(*g); g->next = chains[r_sym].next; g->r_addend = r_addend; g->got_offset = *poffset; @@ -93,7 +93,7 @@ module_frob_arch_sections(Elf64_Ehdr *hdr, Elf64_Shdr *sechdrs, } nsyms = symtab->sh_size / sizeof(Elf64_Sym); - chains = kcalloc(nsyms, sizeof(struct got_entry), GFP_KERNEL); + chains = kzalloc_objs(struct got_entry, nsyms); if (!chains) { printk(KERN_ERR "module %s: no memory for symbol chain buffer\n", diff --git a/arch/alpha/kernel/osf_sys.c b/arch/alpha/kernel/osf_sys.c index a08e8edef1a4..7b6543d2cca3 100644 --- a/arch/alpha/kernel/osf_sys.c +++ b/arch/alpha/kernel/osf_sys.c @@ -454,42 +454,30 @@ static int osf_ufs_mount(const char __user *dirname, struct ufs_args __user *args, int flags) { - int retval; - struct cdfs_args tmp; - struct filename *devname; + struct ufs_args tmp; + char *devname __free(kfree) = NULL; - retval = -EFAULT; if (copy_from_user(&tmp, args, sizeof(tmp))) - goto out; - devname = getname(tmp.devname); - retval = PTR_ERR(devname); + return -EFAULT; + devname = strndup_user(tmp.devname, PATH_MAX); if (IS_ERR(devname)) - goto out; - retval = do_mount(devname->name, dirname, "ext2", flags, NULL); - putname(devname); - out: - return retval; + return PTR_ERR(devname); + return do_mount(devname, dirname, "ext2", flags, NULL); } static int osf_cdfs_mount(const char __user *dirname, struct cdfs_args __user *args, int flags) { - int retval; struct cdfs_args tmp; - struct filename *devname; + char *devname __free(kfree) = NULL; - retval = -EFAULT; if (copy_from_user(&tmp, args, sizeof(tmp))) - goto out; - devname = getname(tmp.devname); - retval = PTR_ERR(devname); + return -EFAULT; + devname = strndup_user(tmp.devname, PATH_MAX); if (IS_ERR(devname)) - goto out; - retval = do_mount(devname->name, dirname, "iso9660", flags, NULL); - putname(devname); - out: - return retval; + return PTR_ERR(devname); + return do_mount(devname, dirname, "iso9660", flags, NULL); } static int diff --git a/arch/alpha/kernel/pci-sysfs.c b/arch/alpha/kernel/pci-sysfs.c index 3048758304b5..67f9822f7626 100644 --- a/arch/alpha/kernel/pci-sysfs.c +++ b/arch/alpha/kernel/pci-sysfs.c @@ -11,8 +11,7 @@ */ #include <linux/sched.h> -#include <linux/stat.h> -#include <linux/slab.h> +#include <linux/security.h> #include <linux/pci.h> static int hose_mmap_page_range(struct pci_controller *hose, @@ -36,20 +35,18 @@ static int hose_mmap_page_range(struct pci_controller *hose, static int __pci_mmap_fits(struct pci_dev *pdev, int num, struct vm_area_struct *vma, int sparse) { + resource_size_t len = pci_resource_len(pdev, num); unsigned long nr, start, size; int shift = sparse ? 5 : 0; + if (!len) + return 0; + nr = vma_pages(vma); start = vma->vm_pgoff; - size = ((pci_resource_len(pdev, num) - 1) >> (PAGE_SHIFT - shift)) + 1; + size = ((len - 1) >> (PAGE_SHIFT - shift)) + 1; - if (start < size && size - start >= nr) - return 1; - WARN(1, "process \"%s\" tried to map%s 0x%08lx-0x%08lx on %s BAR %d " - "(size 0x%08lx)\n", - current->comm, sparse ? " sparse" : "", start, start + nr, - pci_name(pdev), num, size); - return 0; + return start < size && size - start >= nr; } /** @@ -68,26 +65,25 @@ static int pci_mmap_resource(struct kobject *kobj, struct vm_area_struct *vma, int sparse) { struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj)); - struct resource *res = attr->private; + int barno = (unsigned long)attr->private; enum pci_mmap_state mmap_type; struct pci_bus_region bar; - int i; + int ret; - for (i = 0; i < PCI_STD_NUM_BARS; i++) - if (res == &pdev->resource[i]) - break; - if (i >= PCI_STD_NUM_BARS) - return -ENODEV; + ret = security_locked_down(LOCKDOWN_PCI_ACCESS); + if (ret) + return ret; - if (res->flags & IORESOURCE_MEM && iomem_is_exclusive(res->start)) + if (pci_resource_is_mem(pdev, barno) && + iomem_is_exclusive(pci_resource_start(pdev, barno))) return -EINVAL; - if (!__pci_mmap_fits(pdev, i, vma, sparse)) + if (!__pci_mmap_fits(pdev, barno, vma, sparse)) return -EINVAL; - pcibios_resource_to_bus(pdev->bus, &bar, res); + pcibios_resource_to_bus(pdev->bus, &bar, pci_resource_n(pdev, barno)); vma->vm_pgoff += bar.start >> (PAGE_SHIFT - (sparse ? 5 : 0)); - mmap_type = res->flags & IORESOURCE_MEM ? pci_mmap_mem : pci_mmap_io; + mmap_type = pci_resource_is_mem(pdev, barno) ? pci_mmap_mem : pci_mmap_io; return hose_mmap_page_range(pdev->sysdata, vma, mmap_type, sparse); } @@ -106,34 +102,26 @@ static int pci_mmap_resource_dense(struct file *filp, struct kobject *kobj, return pci_mmap_resource(kobj, attr, vma, 0); } -/** - * pci_remove_resource_files - cleanup resource files - * @pdev: pci_dev to cleanup - * - * If we created resource files for @dev, remove them from sysfs and - * free their resources. - */ -void pci_remove_resource_files(struct pci_dev *pdev) -{ - int i; - - for (i = 0; i < PCI_STD_NUM_BARS; i++) { - struct bin_attribute *res_attr; - - res_attr = pdev->res_attr[i]; - if (res_attr) { - sysfs_remove_bin_file(&pdev->dev.kobj, res_attr); - kfree(res_attr); - } - - res_attr = pdev->res_attr_wc[i]; - if (res_attr) { - sysfs_remove_bin_file(&pdev->dev.kobj, res_attr); - kfree(res_attr); - } - } +#define __pci_dev_resource_attr(_suffix, _bar, _name, _mmap) \ +static const struct bin_attribute \ +pci_dev_resource##_bar##_suffix##_attr = { \ + .attr = { .name = __stringify(_name), .mode = 0600 }, \ + .private = (void *)(unsigned long)(_bar), \ + .mmap = (_mmap), \ } +#define pci_dev_resource_attr(_bar) \ + __pci_dev_resource_attr(, _bar, resource##_bar, \ + pci_mmap_resource_dense) + +#define pci_dev_resource_sparse_attr(_bar) \ + __pci_dev_resource_attr(_sparse, _bar, resource##_bar##_sparse, \ + pci_mmap_resource_sparse) + +#define pci_dev_resource_dense_attr(_bar) \ + __pci_dev_resource_attr(_dense, _bar, resource##_bar##_dense, \ + pci_mmap_resource_dense) + static int sparse_mem_mmap_fits(struct pci_dev *pdev, int num) { struct pci_bus_region bar; @@ -141,7 +129,7 @@ static int sparse_mem_mmap_fits(struct pci_dev *pdev, int num) long dense_offset; unsigned long sparse_size; - pcibios_resource_to_bus(pdev->bus, &bar, &pdev->resource[num]); + pcibios_resource_to_bus(pdev->bus, &bar, pci_resource_n(pdev, num)); /* All core logic chips have 4G sparse address space, except CIA which has 16G (see xxx_SPARSE_MEM and xxx_DENSE_MEM @@ -153,109 +141,10 @@ static int sparse_mem_mmap_fits(struct pci_dev *pdev, int num) return bar.end < sparse_size; } -static int pci_create_one_attr(struct pci_dev *pdev, int num, char *name, - char *suffix, struct bin_attribute *res_attr, - unsigned long sparse) -{ - size_t size = pci_resource_len(pdev, num); - - sprintf(name, "resource%d%s", num, suffix); - res_attr->mmap = sparse ? pci_mmap_resource_sparse : - pci_mmap_resource_dense; - res_attr->attr.name = name; - res_attr->attr.mode = S_IRUSR | S_IWUSR; - res_attr->size = sparse ? size << 5 : size; - res_attr->private = &pdev->resource[num]; - return sysfs_create_bin_file(&pdev->dev.kobj, res_attr); -} - -static int pci_create_attr(struct pci_dev *pdev, int num) -{ - /* allocate attribute structure, piggyback attribute name */ - int retval, nlen1, nlen2 = 0, res_count = 1; - unsigned long sparse_base, dense_base; - struct bin_attribute *attr; - struct pci_controller *hose = pdev->sysdata; - char *suffix, *attr_name; - - suffix = ""; /* Assume bwx machine, normal resourceN files. */ - nlen1 = 10; - - if (pdev->resource[num].flags & IORESOURCE_MEM) { - sparse_base = hose->sparse_mem_base; - dense_base = hose->dense_mem_base; - if (sparse_base && !sparse_mem_mmap_fits(pdev, num)) { - sparse_base = 0; - suffix = "_dense"; - nlen1 = 16; /* resourceN_dense */ - } - } else { - sparse_base = hose->sparse_io_base; - dense_base = hose->dense_io_base; - } - - if (sparse_base) { - suffix = "_sparse"; - nlen1 = 17; - if (dense_base) { - nlen2 = 16; /* resourceN_dense */ - res_count = 2; - } - } - - attr = kzalloc(sizeof(*attr) * res_count + nlen1 + nlen2, GFP_ATOMIC); - if (!attr) - return -ENOMEM; - - /* Create bwx, sparse or single dense file */ - attr_name = (char *)(attr + res_count); - pdev->res_attr[num] = attr; - retval = pci_create_one_attr(pdev, num, attr_name, suffix, attr, - sparse_base); - if (retval || res_count == 1) - return retval; - - /* Create dense file */ - attr_name += nlen1; - attr++; - pdev->res_attr_wc[num] = attr; - return pci_create_one_attr(pdev, num, attr_name, "_dense", attr, 0); -} - -/** - * pci_create_resource_files - create resource files in sysfs for @pdev - * @pdev: pci_dev in question - * - * Walk the resources in @dev creating files for each resource available. - * - * Return: %0 on success, or negative error code - */ -int pci_create_resource_files(struct pci_dev *pdev) -{ - int i; - int retval; - - /* Expose the PCI resources from this device as files */ - for (i = 0; i < PCI_STD_NUM_BARS; i++) { - - /* skip empty resources */ - if (!pci_resource_len(pdev, i)) - continue; - - retval = pci_create_attr(pdev, i); - if (retval) { - pci_remove_resource_files(pdev); - return retval; - } - } - return 0; -} - /* Legacy I/O bus mapping stuff. */ -static int __legacy_mmap_fits(struct pci_controller *hose, - struct vm_area_struct *vma, - unsigned long res_size, int sparse) +static int __legacy_mmap_fits(struct vm_area_struct *vma, + unsigned long res_size) { unsigned long nr, start, size; @@ -263,13 +152,7 @@ static int __legacy_mmap_fits(struct pci_controller *hose, start = vma->vm_pgoff; size = ((res_size - 1) >> PAGE_SHIFT) + 1; - if (start < size && size - start >= nr) - return 1; - WARN(1, "process \"%s\" tried to map%s 0x%08lx-0x%08lx on hose %d " - "(size 0x%08lx)\n", - current->comm, sparse ? " sparse" : "", start, start + nr, - hose->index, size); - return 0; + return start < size && size - start >= nr; } static inline int has_sparse(struct pci_controller *hose, @@ -290,36 +173,22 @@ int pci_mmap_legacy_page_range(struct pci_bus *bus, struct vm_area_struct *vma, int sparse = has_sparse(hose, mmap_type); unsigned long res_size; - res_size = (mmap_type == pci_mmap_mem) ? bus->legacy_mem->size : - bus->legacy_io->size; - if (!__legacy_mmap_fits(hose, vma, res_size, sparse)) + res_size = (mmap_type == pci_mmap_mem) ? PCI_LEGACY_MEM_SIZE : + PCI_LEGACY_IO_SIZE; + if (sparse) + res_size <<= 5; + + if (!__legacy_mmap_fits(vma, res_size)) return -EINVAL; return hose_mmap_page_range(hose, vma, mmap_type, sparse); } -/** - * pci_adjust_legacy_attr - adjustment of legacy file attributes - * @bus: bus to create files under - * @mmap_type: I/O port or memory - * - * Adjust file name and size for sparse mappings. - */ -void pci_adjust_legacy_attr(struct pci_bus *bus, enum pci_mmap_state mmap_type) +bool pci_legacy_has_sparse(struct pci_bus *bus, enum pci_mmap_state type) { struct pci_controller *hose = bus->sysdata; - if (!has_sparse(hose, mmap_type)) - return; - - if (mmap_type == pci_mmap_mem) { - bus->legacy_mem->attr.name = "legacy_mem_sparse"; - bus->legacy_mem->size <<= 5; - } else { - bus->legacy_io->attr.name = "legacy_io_sparse"; - bus->legacy_io->size <<= 5; - } - return; + return has_sparse(hose, type); } /* Legacy I/O bus read/write functions */ @@ -355,18 +224,181 @@ int pci_legacy_write(struct pci_bus *bus, loff_t port, u32 val, size_t size) switch(size) { case 1: - outb(port, val); + outb(val, port); return 1; case 2: if (port & 1) return -EINVAL; - outw(port, val); + outw(val, port); return 2; case 4: if (port & 3) return -EINVAL; - outl(port, val); + outl(val, port); return 4; } return -EINVAL; } + +pci_dev_resource_attr(0); +pci_dev_resource_attr(1); +pci_dev_resource_attr(2); +pci_dev_resource_attr(3); +pci_dev_resource_attr(4); +pci_dev_resource_attr(5); + +pci_dev_resource_sparse_attr(0); +pci_dev_resource_sparse_attr(1); +pci_dev_resource_sparse_attr(2); +pci_dev_resource_sparse_attr(3); +pci_dev_resource_sparse_attr(4); +pci_dev_resource_sparse_attr(5); + +pci_dev_resource_dense_attr(0); +pci_dev_resource_dense_attr(1); +pci_dev_resource_dense_attr(2); +pci_dev_resource_dense_attr(3); +pci_dev_resource_dense_attr(4); +pci_dev_resource_dense_attr(5); + +static inline enum pci_mmap_state pci_bar_mmap_type(struct pci_dev *pdev, + int bar) +{ + return pci_resource_is_mem(pdev, bar) ? pci_mmap_mem : pci_mmap_io; +} + +static inline umode_t __pci_resource_attr_is_visible(struct kobject *kobj, + const struct bin_attribute *a, + int bar) +{ + struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj)); + + if (!pci_resource_len(pdev, bar)) + return 0; + + return a->attr.mode; +} + +static umode_t pci_dev_resource_is_visible(struct kobject *kobj, + const struct bin_attribute *a, + int bar) +{ + struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj)); + struct pci_controller *hose = pdev->sysdata; + + if (has_sparse(hose, pci_bar_mmap_type(pdev, bar))) + return 0; + + return __pci_resource_attr_is_visible(kobj, a, bar); +} + +static umode_t pci_dev_resource_sparse_is_visible(struct kobject *kobj, + const struct bin_attribute *a, + int bar) +{ + struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj)); + struct pci_controller *hose = pdev->sysdata; + enum pci_mmap_state type = pci_bar_mmap_type(pdev, bar); + + if (!has_sparse(hose, type)) + return 0; + + if (type == pci_mmap_mem && !sparse_mem_mmap_fits(pdev, bar)) + return 0; + + return __pci_resource_attr_is_visible(kobj, a, bar); +} + +static umode_t pci_dev_resource_dense_is_visible(struct kobject *kobj, + const struct bin_attribute *a, + int bar) +{ + struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj)); + struct pci_controller *hose = pdev->sysdata; + enum pci_mmap_state type = pci_bar_mmap_type(pdev, bar); + unsigned long dense_base; + + if (!has_sparse(hose, type)) + return 0; + + if (type == pci_mmap_mem && !sparse_mem_mmap_fits(pdev, bar)) + return __pci_resource_attr_is_visible(kobj, a, bar); + + dense_base = (type == pci_mmap_mem) ? hose->dense_mem_base : + hose->dense_io_base; + if (!dense_base) + return 0; + + return __pci_resource_attr_is_visible(kobj, a, bar); +} + +static inline size_t __pci_dev_resource_bin_size(struct kobject *kobj, + int bar, bool sparse) +{ + struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj)); + size_t size = pci_resource_len(pdev, bar); + + return sparse ? size << 5 : size; +} + +static size_t pci_dev_resource_bin_size(struct kobject *kobj, + const struct bin_attribute *a, + int bar) +{ + return __pci_dev_resource_bin_size(kobj, bar, false); +} + +static size_t pci_dev_resource_sparse_bin_size(struct kobject *kobj, + const struct bin_attribute *a, + int bar) +{ + return __pci_dev_resource_bin_size(kobj, bar, true); +} + +static const struct bin_attribute *const pci_dev_resource_attrs[] = { + &pci_dev_resource0_attr, + &pci_dev_resource1_attr, + &pci_dev_resource2_attr, + &pci_dev_resource3_attr, + &pci_dev_resource4_attr, + &pci_dev_resource5_attr, + NULL, +}; + +static const struct bin_attribute *const pci_dev_resource_sparse_attrs[] = { + &pci_dev_resource0_sparse_attr, + &pci_dev_resource1_sparse_attr, + &pci_dev_resource2_sparse_attr, + &pci_dev_resource3_sparse_attr, + &pci_dev_resource4_sparse_attr, + &pci_dev_resource5_sparse_attr, + NULL, +}; + +static const struct bin_attribute *const pci_dev_resource_dense_attrs[] = { + &pci_dev_resource0_dense_attr, + &pci_dev_resource1_dense_attr, + &pci_dev_resource2_dense_attr, + &pci_dev_resource3_dense_attr, + &pci_dev_resource4_dense_attr, + &pci_dev_resource5_dense_attr, + NULL, +}; + +const struct attribute_group pci_dev_resource_attr_group = { + .bin_attrs = pci_dev_resource_attrs, + .is_bin_visible = pci_dev_resource_is_visible, + .bin_size = pci_dev_resource_bin_size, +}; + +const struct attribute_group pci_dev_resource_sparse_attr_group = { + .bin_attrs = pci_dev_resource_sparse_attrs, + .is_bin_visible = pci_dev_resource_sparse_is_visible, + .bin_size = pci_dev_resource_sparse_bin_size, +}; + +const struct attribute_group pci_dev_resource_dense_attr_group = { + .bin_attrs = pci_dev_resource_dense_attrs, + .is_bin_visible = pci_dev_resource_dense_is_visible, + .bin_size = pci_dev_resource_bin_size, +}; diff --git a/arch/alpha/kernel/pci.c b/arch/alpha/kernel/pci.c index 8e9b4ac86b7e..11df411b1d18 100644 --- a/arch/alpha/kernel/pci.c +++ b/arch/alpha/kernel/pci.c @@ -125,6 +125,7 @@ DECLARE_PCI_FIXUP_FINAL(PCI_ANY_ID, PCI_ANY_ID, pcibios_fixup_final); resource_size_t pcibios_align_resource(void *data, const struct resource *res, + const struct resource *empty_res, resource_size_t size, resource_size_t align) { struct pci_dev *dev = data; @@ -220,7 +221,7 @@ static void pdev_save_srm_config(struct pci_dev *dev) printed = 1; } - tmp = kmalloc(sizeof(*tmp), GFP_KERNEL); + tmp = kmalloc_obj(*tmp); if (!tmp) { printk(KERN_ERR "%s: kmalloc() failed!\n", __func__); return; diff --git a/arch/alpha/kernel/proto.h b/arch/alpha/kernel/proto.h index a8bc3ead776b..9b262ef09a3a 100644 --- a/arch/alpha/kernel/proto.h +++ b/arch/alpha/kernel/proto.h @@ -173,6 +173,7 @@ extern void do_sigreturn(struct sigcontext __user *); struct rt_sigframe; extern void do_rt_sigreturn(struct rt_sigframe __user *); extern void do_work_pending(struct pt_regs *, unsigned long, unsigned long, unsigned long); +extern void alpha_schedule_user_work(void); /* traps.c */ extern void dik_show_regs(struct pt_regs *regs, unsigned long *r9_15); @@ -185,6 +186,9 @@ struct allregs; extern void do_entUna(void *, unsigned long, unsigned long, struct allregs *); extern void do_entUnaUser(void __user *, unsigned long, unsigned long, struct pt_regs *); +/* irq_alpha.c */ +extern void notrace lockdep_on_restore(unsigned long ps, unsigned long ip); + /* sys_titan.c */ extern void titan_dispatch_irqs(u64); diff --git a/arch/alpha/kernel/ptrace.c b/arch/alpha/kernel/ptrace.c index fde4c68e7a0b..fc8f6cedbb28 100644 --- a/arch/alpha/kernel/ptrace.c +++ b/arch/alpha/kernel/ptrace.c @@ -16,15 +16,23 @@ #include <linux/security.h> #include <linux/signal.h> #include <linux/audit.h> +#include <linux/seccomp.h> +#include <asm/syscall.h> #include <linux/uaccess.h> #include <asm/fpu.h> #include "proto.h" +#include <linux/uio.h> +#include <linux/regset.h> #define DEBUG DBG_MEM #undef DEBUG +#ifndef NT_FPREGSET +#define NT_FPREGSET NT_PRFPREG +#endif + #ifdef DEBUG enum { DBG_MEM = (1<<0), @@ -140,19 +148,163 @@ get_reg(struct task_struct * task, unsigned long regno) return *get_reg_addr(task, regno); } +static void alpha_elf_fpregs_get(struct task_struct *target, + elf_fpreg_t *fpregs) /* points to ELF_NFPREG entries */ +{ + memcpy(fpregs, task_thread_info(target)->fp, sizeof(elf_fpregset_t)); +} + +static void alpha_elf_fpregs_set(struct task_struct *target, + const elf_fpreg_t *fpregs, + size_t nwords) +{ + size_t n = min_t(size_t, nwords, ELF_NFPREG); + + memcpy(task_thread_info(target)->fp, fpregs, n * sizeof(elf_fpreg_t)); +} + +static void alpha_elf_gregs_set(struct task_struct *child, + const elf_greg_t *src, + size_t nwords) +{ + struct pt_regs *pt = task_pt_regs(child); + struct thread_info *ti = task_thread_info(child); + struct switch_stack *sw = ((struct switch_stack *)pt) - 1; + + /* GPRs r0..r8 live in pt_regs */ + if (nwords > 0) + pt->r0 = src[0]; + if (nwords > 1) + pt->r1 = src[1]; + if (nwords > 2) + pt->r2 = src[2]; + if (nwords > 3) + pt->r3 = src[3]; + if (nwords > 4) + pt->r4 = src[4]; + if (nwords > 5) + pt->r5 = src[5]; + if (nwords > 6) + pt->r6 = src[6]; + if (nwords > 7) + pt->r7 = src[7]; + if (nwords > 8) + pt->r8 = src[8]; + + /* r9..r15 live in switch_stack */ + if (nwords > 9) + sw->r9 = src[9]; + if (nwords > 10) + sw->r10 = src[10]; + if (nwords > 11) + sw->r11 = src[11]; + if (nwords > 12) + sw->r12 = src[12]; + if (nwords > 13) + sw->r13 = src[13]; + if (nwords > 14) + sw->r14 = src[14]; + if (nwords > 15) + sw->r15 = src[15]; + + /* r16..r28 live in pt_regs */ + if (nwords > 16) + pt->r16 = src[16]; + if (nwords > 17) + pt->r17 = src[17]; + if (nwords > 18) + pt->r18 = src[18]; + if (nwords > 19) + pt->r19 = src[19]; + if (nwords > 20) + pt->r20 = src[20]; + if (nwords > 21) + pt->r21 = src[21]; + if (nwords > 22) + pt->r22 = src[22]; + if (nwords > 23) + pt->r23 = src[23]; + if (nwords > 24) + pt->r24 = src[24]; + if (nwords > 25) + pt->r25 = src[25]; + if (nwords > 26) + pt->r26 = src[26]; + if (nwords > 27) + pt->r27 = src[27]; + if (nwords > 28) + pt->r28 = src[28]; + + /* gp, usp, pc, unique */ + if (nwords > 29) + pt->gp = src[29]; + + if (nwords > 30) { + ti->pcb.usp = src[30]; + /* + * If someone ever does this to current (rare), keep the + * hardware usp consistent. + */ + if (child == current) + wrusp(src[30]); + } + + if (nwords > 31) + pt->pc = src[31]; + + if (nwords > 32) + ti->pcb.unique = src[32]; + +/* + * PTRACE_SETREGSET can be used at a syscall-entry stop to skip the + * syscall by setting the syscall number to -1. The seccomp/ptrace + * selftests use this to synthesize errno returns. + * + * Alpha uses r19/a3 as the error flag, so a skipped syscall with a + * small positive r0 and a clear r19 must be normalized to an error + * return. + */ + if (pt->r1 == (unsigned long)-1 && + pt->r19 == 0 && + pt->r0 > 0 && + pt->r0 < MAX_ERRNO) + pt->r19 = 1; +} + + /* * Write contents of register REGNO in task TASK. */ static int put_reg(struct task_struct *task, unsigned long regno, unsigned long data) { + struct pt_regs *regs = task_pt_regs(task); + if (regno == 63) { task_thread_info(task)->ieee_state = ((task_thread_info(task)->ieee_state & ~IEEE_SW_MASK) | (data & IEEE_SW_MASK)); data = (data & FPCR_DYN_MASK) | ieee_swcr_to_fpcr(data); } + *get_reg_addr(task, regno) = data; + + /* + * Alpha historically exposes r0/v0 as the syscall number at a + * syscall-entry stop. The generic-entry conversion keeps the + * mutable syscall number in regs->r1, so old ptrace users such + * as strace that skip a syscall by poking r0 to -1 must also + * update the internal shadow syscall number. + * + * Do not mirror other r0 writes. strace later pokes r0 to the + * injected return value, e.g. 42, while r1 must remain -1. + */ + + if (regno == 0 && data == (unsigned long)-1) { + regs->r1 = data; + regs->r19 = 0; + } + return 0; } @@ -321,15 +473,37 @@ long arch_ptrace(struct task_struct *child, long request, asmlinkage unsigned long syscall_trace_enter(void) { - unsigned long ret = 0; struct pt_regs *regs = current_pt_regs(); + if (test_thread_flag(TIF_SYSCALL_TRACE) && - ptrace_report_syscall_entry(current_pt_regs())) - ret = -1UL; - audit_syscall_entry(regs->r0, regs->r16, regs->r17, regs->r18, regs->r19); - return ret ?: current_pt_regs()->r0; + !ptrace_report_syscall_permit_entry(regs)) { + syscall_set_nr(current, regs, -1); + if (regs->r19 == 0 && regs->r0 == (unsigned long)-1) + syscall_set_return_value(current, regs, -ENOSYS, 0); + return -1UL; + } + + /* + * Do the secure computing after ptrace; failures should be fast. + * If this fails, seccomp may already have set up the return value + * (e.g. SECCOMP_RET_ERRNO / TRACE). + */ + if (!seccomp_permit_syscall()) { + if (regs->r19 == 0 && regs->r0 == (unsigned long)-1) + syscall_set_return_value(current, regs, -ENOSYS, 0); + syscall_set_nr(current, regs, -1); + return -1UL; + } + +#ifdef CONFIG_AUDITSYSCALL + audit_syscall_entry(syscall_get_nr(current, regs), + regs->r16, regs->r17, regs->r18, regs->r19); +#endif + return syscall_get_nr(current, regs); } + + asmlinkage void syscall_trace_leave(void) { @@ -337,3 +511,126 @@ syscall_trace_leave(void) if (test_thread_flag(TIF_SYSCALL_TRACE)) ptrace_report_syscall_exit(current_pt_regs(), 0); } + +/* + * Minimal regset support for Alpha. + * + * Alpha-specific notes: + * - Do NOT use ELF_CORE_COPY_REGS(): it uses current_thread_info(), + * which is wrong for non-current tasks. + * - dump_elf_task() returns 1 unconditionally in this tree, while + * regset_get should return 0 on success. So call dump_elf_thread() + * directly and return membuf_write()'s result. + */ + +static int alpha_regset_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, + const void __user *ubuf) +{ + elf_gregset_t gregs; + unsigned int nwords; + + if (pos + count > sizeof(gregs)) + return -EIO; + + /* + * Preserve registers outside the written range. + */ + dump_elf_thread(gregs, task_pt_regs(target), + task_thread_info(target)); + + if (user_regset_copyin(&pos, &count, &kbuf, &ubuf, + gregs, 0, sizeof(gregs))) + return -EFAULT; + + nwords = sizeof(gregs) / sizeof(elf_greg_t); + alpha_elf_gregs_set(target, gregs, nwords); + + return 0; +} + +static int alpha_fpregset_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, + const void __user *ubuf) +{ + elf_fpregset_t fpregs; + unsigned int nwords; + + if (pos + count > sizeof(fpregs)) + return -EIO; + + alpha_elf_fpregs_get(target, fpregs); + + if (user_regset_copyin(&pos, &count, &kbuf, &ubuf, + fpregs, 0, sizeof(fpregs))) + return -EFAULT; + + nwords = sizeof(fpregs) / sizeof(elf_fpreg_t); + alpha_elf_fpregs_set(target, fpregs, nwords); + + return 0; +} + +static int alpha_regset_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + struct pt_regs *pt = task_pt_regs(target); + struct thread_info *ti = task_thread_info(target); + elf_gregset_t gregs; + + dump_elf_thread(gregs, pt, ti); + return membuf_write(&to, gregs, sizeof(gregs)); +} + +static int alpha_fpregset_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + elf_fpregset_t fpregs; + + alpha_elf_fpregs_get(target, fpregs); + return membuf_write(&to, fpregs, sizeof(fpregs)); +} + +enum alpha_regset { + REGSET_GPR, + REGSET_FPR, +}; + +static const struct user_regset alpha_user_regsets[] = { + [REGSET_GPR] = { + .core_note_type = NT_PRSTATUS, + .n = ELF_NGREG, + .size = sizeof(elf_greg_t), + .align = sizeof(elf_greg_t), + .regset_get = alpha_regset_get, + .set = alpha_regset_set, + }, + [REGSET_FPR] = { + .core_note_type = NT_PRFPREG, + .core_note_name = "CORE", + .n = ELF_NFPREG, + .size = sizeof(elf_fpreg_t), + .align = sizeof(elf_fpreg_t), + .regset_get = alpha_fpregset_get, + .set = alpha_fpregset_set, + }, +}; + +static const struct user_regset_view user_alpha_view = { + .name = "alpha", + .e_machine = EM_ALPHA, + .ei_osabi = ELF_OSABI, + .regsets = alpha_user_regsets, + .n = ARRAY_SIZE(alpha_user_regsets), +}; + +const struct user_regset_view *task_user_regset_view(struct task_struct *task) +{ + return &user_alpha_view; +} diff --git a/arch/alpha/kernel/rtc.c b/arch/alpha/kernel/rtc.c index cfdf90bc8b3f..9e7d714ef6f8 100644 --- a/arch/alpha/kernel/rtc.c +++ b/arch/alpha/kernel/rtc.c @@ -15,6 +15,7 @@ #include <linux/bcd.h> #include <linux/rtc.h> #include <linux/platform_device.h> +#include <linux/workqueue.h> #include "proto.h" @@ -142,54 +143,40 @@ static const struct rtc_class_ops alpha_rtc_ops = { }; /* - * Similarly, except do the actual CMOS access on the boot cpu only. - * This requires marshalling the data across an interprocessor call. + * Similarly, except do the actual CMOS access on the boot cpu only. The + * access polls for the RTC update cycle and takes rtc_lock, so run it in a + * worker on that cpu rather than from an interprocessor interrupt. */ #if defined(CONFIG_SMP) && \ (defined(CONFIG_ALPHA_GENERIC) || defined(CONFIG_ALPHA_MARVEL)) # define HAVE_REMOTE_RTC 1 -union remote_data { - struct rtc_time *tm; - long retval; -}; - -static void +static long do_remote_read(void *data) { - union remote_data *x = data; - x->retval = alpha_rtc_read_time(NULL, x->tm); + return alpha_rtc_read_time(NULL, data); } static int remote_read_time(struct device *dev, struct rtc_time *tm) { - union remote_data x; - if (smp_processor_id() != boot_cpuid) { - x.tm = tm; - smp_call_function_single(boot_cpuid, do_remote_read, &x, 1); - return x.retval; - } + if (smp_processor_id() != boot_cpuid) + return work_on_cpu(boot_cpuid, do_remote_read, tm); return alpha_rtc_read_time(NULL, tm); } -static void +static long do_remote_set(void *data) { - union remote_data *x = data; - x->retval = alpha_rtc_set_time(NULL, x->tm); + return alpha_rtc_set_time(NULL, data); } static int remote_set_time(struct device *dev, struct rtc_time *tm) { - union remote_data x; - if (smp_processor_id() != boot_cpuid) { - x.tm = tm; - smp_call_function_single(boot_cpuid, do_remote_set, &x, 1); - return x.retval; - } + if (smp_processor_id() != boot_cpuid) + return work_on_cpu(boot_cpuid, do_remote_set, tm); return alpha_rtc_set_time(NULL, tm); } diff --git a/arch/alpha/kernel/setup.c b/arch/alpha/kernel/setup.c index bebdffafaee8..4cbcfb1852d5 100644 --- a/arch/alpha/kernel/setup.c +++ b/arch/alpha/kernel/setup.c @@ -392,7 +392,7 @@ register_cpus(void) int i; for_each_possible_cpu(i) { - struct cpu *p = kzalloc(sizeof(*p), GFP_KERNEL); + struct cpu *p = kzalloc_obj(*p); if (!p) return -ENOMEM; register_cpu(p, i); @@ -607,7 +607,6 @@ setup_arch(char **cmdline_p) /* Find our memory. */ setup_memory(kernel_end); memblock_set_bottom_up(true); - sparse_init(); /* First guess at cpu cache sizes. Do this before init_arch. */ determine_cpu_caches(cpu->type); diff --git a/arch/alpha/kernel/signal.c b/arch/alpha/kernel/signal.c index e62d1d461b1f..ce40a49b8496 100644 --- a/arch/alpha/kernel/signal.c +++ b/arch/alpha/kernel/signal.c @@ -41,6 +41,14 @@ asmlinkage void ret_from_sys_call(void); * The OSF/1 sigprocmask calling sequence is different from the * C sigprocmask() sequence.. */ + +asmlinkage void alpha_schedule_user_work(void) +{ + local_irq_enable(); + schedule(); + local_irq_disable(); +} + SYSCALL_DEFINE2(osf_sigprocmask, int, how, unsigned long, newmask) { sigset_t oldmask; @@ -525,6 +533,7 @@ do_work_pending(struct pt_regs *regs, unsigned long thread_flags, { do { if (thread_flags & _TIF_NEED_RESCHED) { + local_irq_enable(); schedule(); } else { local_irq_enable(); diff --git a/arch/alpha/kernel/stacktrace.c b/arch/alpha/kernel/stacktrace.c new file mode 100644 index 000000000000..74d95f591039 --- /dev/null +++ b/arch/alpha/kernel/stacktrace.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/sched.h> +#include <linux/sched/task_stack.h> +#include <linux/stacktrace.h> +#include <linux/kallsyms.h> + +#include <asm/thread_info.h> +#include <asm/ptrace.h> + +static __always_inline unsigned long alpha_get_current_ksp(void) +{ + unsigned long sp; + + asm volatile("mov $30, %0" : "=r"(sp)); + return sp; +} + +static void alpha_scan_kernel_stack(unsigned long ksp, + stack_trace_consume_fn consume_entry, + void *cookie) +{ + unsigned long *p = (unsigned long *)ksp; + + if (unlikely(ksp & (sizeof(unsigned long) - 1))) + return; + + while (!kstack_end(p)) { + unsigned long addr = READ_ONCE_NOCHECK(*p++); + + if (!__kernel_text_address(addr)) + continue; + + if (!consume_entry(cookie, addr)) + break; + } +} + +noinline void arch_stack_walk(stack_trace_consume_fn consume_entry, + void *cookie, + struct task_struct *task, + struct pt_regs *regs) +{ + unsigned long ksp; + + if (!task) + task = current; + + if (regs && task == current) { + /* + * pt_regs is stored on the kernel stack; regs+1 matches + * what arch/alpha/kernel/traps.c uses as the trace start. + */ + ksp = (unsigned long)(regs + 1); + } else if (task == current) { + ksp = alpha_get_current_ksp(); + } else { + ksp = task_thread_info(task)->pcb.ksp; + } + + alpha_scan_kernel_stack(ksp, consume_entry, cookie); +} diff --git a/arch/alpha/kernel/sys_dp264.c b/arch/alpha/kernel/sys_dp264.c index 9fb445d7dca5..0a2d319bb1c8 100644 --- a/arch/alpha/kernel/sys_dp264.c +++ b/arch/alpha/kernel/sys_dp264.c @@ -41,7 +41,7 @@ static unsigned long cached_irq_mask; /* dp264 boards handle at max four CPUs */ static unsigned long cpu_irq_affinity[4] = { 0UL, 0UL, 0UL, 0UL }; -DEFINE_SPINLOCK(dp264_irq_lock); +static DEFINE_RAW_SPINLOCK(dp264_irq_lock); static void tsunami_update_irq_hw(unsigned long mask) @@ -99,37 +99,45 @@ tsunami_update_irq_hw(unsigned long mask) static void dp264_enable_irq(struct irq_data *d) { - spin_lock(&dp264_irq_lock); + unsigned long flags; + + raw_spin_lock_irqsave(&dp264_irq_lock, flags); cached_irq_mask |= 1UL << d->irq; tsunami_update_irq_hw(cached_irq_mask); - spin_unlock(&dp264_irq_lock); + raw_spin_unlock_irqrestore(&dp264_irq_lock, flags); } static void dp264_disable_irq(struct irq_data *d) { - spin_lock(&dp264_irq_lock); + unsigned long flags; + + raw_spin_lock_irqsave(&dp264_irq_lock, flags); cached_irq_mask &= ~(1UL << d->irq); tsunami_update_irq_hw(cached_irq_mask); - spin_unlock(&dp264_irq_lock); + raw_spin_unlock_irqrestore(&dp264_irq_lock, flags); } static void clipper_enable_irq(struct irq_data *d) { - spin_lock(&dp264_irq_lock); + unsigned long flags; + + raw_spin_lock_irqsave(&dp264_irq_lock, flags); cached_irq_mask |= 1UL << (d->irq - 16); tsunami_update_irq_hw(cached_irq_mask); - spin_unlock(&dp264_irq_lock); + raw_spin_unlock_irqrestore(&dp264_irq_lock, flags); } static void clipper_disable_irq(struct irq_data *d) { - spin_lock(&dp264_irq_lock); + unsigned long flags; + + raw_spin_lock_irqsave(&dp264_irq_lock, flags); cached_irq_mask &= ~(1UL << (d->irq - 16)); tsunami_update_irq_hw(cached_irq_mask); - spin_unlock(&dp264_irq_lock); + raw_spin_unlock_irqrestore(&dp264_irq_lock, flags); } static void @@ -151,10 +159,12 @@ static int dp264_set_affinity(struct irq_data *d, const struct cpumask *affinity, bool force) { - spin_lock(&dp264_irq_lock); + unsigned long flags; + + raw_spin_lock_irqsave(&dp264_irq_lock, flags); cpu_set_irq_affinity(d->irq, *affinity); tsunami_update_irq_hw(cached_irq_mask); - spin_unlock(&dp264_irq_lock); + raw_spin_unlock_irqrestore(&dp264_irq_lock, flags); return 0; } @@ -163,10 +173,12 @@ static int clipper_set_affinity(struct irq_data *d, const struct cpumask *affinity, bool force) { - spin_lock(&dp264_irq_lock); + unsigned long flags; + + raw_spin_lock_irqsave(&dp264_irq_lock, flags); cpu_set_irq_affinity(d->irq - 16, *affinity); tsunami_update_irq_hw(cached_irq_mask); - spin_unlock(&dp264_irq_lock); + raw_spin_unlock_irqrestore(&dp264_irq_lock, flags); return 0; } diff --git a/arch/alpha/kernel/sys_marvel.c b/arch/alpha/kernel/sys_marvel.c index 1f99b03effc2..a37707e05e34 100644 --- a/arch/alpha/kernel/sys_marvel.c +++ b/arch/alpha/kernel/sys_marvel.c @@ -263,6 +263,18 @@ init_io7_irqs(struct io7 *io7, */ printk(" Interrupts reported to CPU at PE %u\n", boot_cpuid); + /* Set up the lsi irqs. */ + for (i = 0; i < 128; ++i) { + irq_set_chip_and_handler(base + i, lsi_ops, handle_level_irq); + irq_set_status_flags(base + i, IRQ_LEVEL); + } + + /* Set up the msi irqs. */ + for (i = 128; i < (128 + 512); ++i) { + irq_set_chip_and_handler(base + i, msi_ops, handle_level_irq); + irq_set_status_flags(base + i, IRQ_LEVEL); + } + raw_spin_lock(&io7->irq_lock); /* set up the error irqs */ @@ -272,12 +284,6 @@ init_io7_irqs(struct io7 *io7, io7_redirect_irq(io7, &io7->csrs->STV_CTL.csr, boot_cpuid); io7_redirect_irq(io7, &io7->csrs->HEI_CTL.csr, boot_cpuid); - /* Set up the lsi irqs. */ - for (i = 0; i < 128; ++i) { - irq_set_chip_and_handler(base + i, lsi_ops, handle_level_irq); - irq_set_status_flags(i, IRQ_LEVEL); - } - /* Disable the implemented irqs in hardware. */ for (i = 0; i < 0x60; ++i) init_one_io7_lsi(io7, i, boot_cpuid); @@ -285,13 +291,6 @@ init_io7_irqs(struct io7 *io7, init_one_io7_lsi(io7, 0x74, boot_cpuid); init_one_io7_lsi(io7, 0x75, boot_cpuid); - - /* Set up the msi irqs. */ - for (i = 128; i < (128 + 512); ++i) { - irq_set_chip_and_handler(base + i, msi_ops, handle_level_irq); - irq_set_status_flags(i, IRQ_LEVEL); - } - for (i = 0; i < 16; ++i) init_one_io7_msi(io7, i, boot_cpuid); diff --git a/arch/alpha/kernel/sys_rawhide.c b/arch/alpha/kernel/sys_rawhide.c index b5846ffdadce..b4a08890dce9 100644 --- a/arch/alpha/kernel/sys_rawhide.c +++ b/arch/alpha/kernel/sys_rawhide.c @@ -41,7 +41,7 @@ static unsigned int hose_irq_masks[4] = { 0xff0000, 0xfe0000, 0xff0000, 0xff0000 }; static unsigned int cached_irq_masks[4]; -DEFINE_SPINLOCK(rawhide_irq_lock); +DEFINE_RAW_SPINLOCK(rawhide_irq_lock); static inline void rawhide_update_irq_hw(int hose, int mask) @@ -59,6 +59,7 @@ rawhide_enable_irq(struct irq_data *d) { unsigned int mask, hose; unsigned int irq = d->irq; + unsigned long flags; irq -= 16; hose = irq / 24; @@ -68,11 +69,11 @@ rawhide_enable_irq(struct irq_data *d) irq -= hose * 24; mask = 1 << irq; - spin_lock(&rawhide_irq_lock); + raw_spin_lock_irqsave(&rawhide_irq_lock, flags); mask |= cached_irq_masks[hose]; cached_irq_masks[hose] = mask; rawhide_update_irq_hw(hose, mask); - spin_unlock(&rawhide_irq_lock); + raw_spin_unlock_irqrestore(&rawhide_irq_lock, flags); } static void @@ -80,6 +81,7 @@ rawhide_disable_irq(struct irq_data *d) { unsigned int mask, hose; unsigned int irq = d->irq; + unsigned long flags; irq -= 16; hose = irq / 24; @@ -89,11 +91,11 @@ rawhide_disable_irq(struct irq_data *d) irq -= hose * 24; mask = ~(1 << irq) | hose_irq_masks[hose]; - spin_lock(&rawhide_irq_lock); + raw_spin_lock_irqsave(&rawhide_irq_lock, flags); mask &= cached_irq_masks[hose]; cached_irq_masks[hose] = mask; rawhide_update_irq_hw(hose, mask); - spin_unlock(&rawhide_irq_lock); + raw_spin_unlock_irqrestore(&rawhide_irq_lock, flags); } static void @@ -101,6 +103,7 @@ rawhide_mask_and_ack_irq(struct irq_data *d) { unsigned int mask, mask1, hose; unsigned int irq = d->irq; + unsigned long flags; irq -= 16; hose = irq / 24; @@ -111,7 +114,7 @@ rawhide_mask_and_ack_irq(struct irq_data *d) mask1 = 1 << irq; mask = ~mask1 | hose_irq_masks[hose]; - spin_lock(&rawhide_irq_lock); + raw_spin_lock_irqsave(&rawhide_irq_lock, flags); mask &= cached_irq_masks[hose]; cached_irq_masks[hose] = mask; @@ -120,7 +123,7 @@ rawhide_mask_and_ack_irq(struct irq_data *d) /* Clear the interrupt. */ *(vuip)MCPCIA_INT_REQ(MCPCIA_HOSE2MID(hose)) = mask1; - spin_unlock(&rawhide_irq_lock); + raw_spin_unlock_irqrestore(&rawhide_irq_lock, flags); } static struct irq_chip rawhide_irq_type = { diff --git a/arch/alpha/kernel/syscalls/syscall.tbl b/arch/alpha/kernel/syscalls/syscall.tbl index 3fed97478058..52e3538cc7df 100644 --- a/arch/alpha/kernel/syscalls/syscall.tbl +++ b/arch/alpha/kernel/syscalls/syscall.tbl @@ -510,3 +510,5 @@ 578 common file_getattr sys_file_getattr 579 common file_setattr sys_file_setattr 580 common listns sys_listns +581 common rseq_slice_yield sys_rseq_slice_yield +582 common fchroot sys_fchroot diff --git a/arch/alpha/kernel/traps.c b/arch/alpha/kernel/traps.c index 7004397937cf..cfaef96c4ce8 100644 --- a/arch/alpha/kernel/traps.c +++ b/arch/alpha/kernel/traps.c @@ -30,6 +30,12 @@ #include "proto.h" +static __always_inline void alpha_snapshot_usp(struct pt_regs *regs) +{ + if (user_mode(regs)) + regs->usp = rdusp(); +} + void dik_show_regs(struct pt_regs *regs, unsigned long *r9_15) { @@ -166,12 +172,12 @@ static long dummy_emul(void) { return 0; } long (*alpha_fp_emul_imprecise)(struct pt_regs *regs, unsigned long writemask) = (void *)dummy_emul; EXPORT_SYMBOL_GPL(alpha_fp_emul_imprecise); -long (*alpha_fp_emul) (unsigned long pc) +long (*alpha_fp_emul) (unsigned long pc, unsigned long summary) = (void *)dummy_emul; EXPORT_SYMBOL_GPL(alpha_fp_emul); #else long alpha_fp_emul_imprecise(struct pt_regs *regs, unsigned long writemask); -long alpha_fp_emul (unsigned long pc); +long alpha_fp_emul (unsigned long pc, unsigned long summary); #endif asmlinkage void @@ -180,12 +186,13 @@ do_entArith(unsigned long summary, unsigned long write_mask, { long si_code = FPE_FLTINV; + alpha_snapshot_usp(regs); if (summary & 1) { /* Software-completion summary bit is set, so try to emulate the instruction. If the processor supports precise exceptions, we don't have to search. */ if (!amask(AMASK_PRECISE_TRAP)) - si_code = alpha_fp_emul(regs->pc - 4); + si_code = alpha_fp_emul(regs->pc - 4, summary); else si_code = alpha_fp_emul_imprecise(regs, write_mask); if (si_code == 0) @@ -201,6 +208,7 @@ do_entIF(unsigned long type, struct pt_regs *regs) { int signo, code; + alpha_snapshot_usp(regs); if (type == 3) { /* FEN fault */ /* Irritating users can call PAL_clrfen to disable the FPU for the process. The kernel will then trap in @@ -917,7 +925,9 @@ void trap_init(void) { /* Tell PAL-code what global pointer we want in the kernel. */ - register unsigned long gptr __asm__("$29"); + unsigned long gptr; + + __asm__ __volatile__("mov $29, %0" : "=r" (gptr)); wrkgp(gptr); wrent(entArith, 1); diff --git a/arch/alpha/kernel/vmlinux.lds.S b/arch/alpha/kernel/vmlinux.lds.S index 2efa7dfc798a..95704e64b6a6 100644 --- a/arch/alpha/kernel/vmlinux.lds.S +++ b/arch/alpha/kernel/vmlinux.lds.S @@ -28,6 +28,8 @@ SECTIONS TEXT_TEXT SCHED_TEXT LOCK_TEXT + IRQENTRY_TEXT + SOFTIRQENTRY_TEXT *(.fixup) *(.gnu.warning) } :text @@ -71,6 +73,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/alpha/lib/strncpy.S b/arch/alpha/lib/strncpy.S deleted file mode 100644 index cb90cf022df3..000000000000 --- a/arch/alpha/lib/strncpy.S +++ /dev/null @@ -1,83 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * arch/alpha/lib/strncpy.S - * Contributed by Richard Henderson (rth@tamu.edu) - * - * Copy no more than COUNT bytes of the null-terminated string from - * SRC to DST. If SRC does not cover all of COUNT, the balance is - * zeroed. - * - * Or, rather, if the kernel cared about that weird ANSI quirk. This - * version has cropped that bit o' nastiness as well as assuming that - * __stxncpy is in range of a branch. - */ -#include <linux/export.h> - .set noat - .set noreorder - - .text - - .align 4 - .globl strncpy - .ent strncpy -strncpy: - .frame $30, 0, $26 - .prologue 0 - - mov $16, $0 # set return value now - beq $18, $zerolen - unop - bsr $23, __stxncpy # do the work of the copy - - unop - bne $18, $multiword # do we have full words left? - subq $24, 1, $3 # nope - subq $27, 1, $4 - - or $3, $24, $3 # clear the bits between the last - or $4, $27, $4 # written byte and the last byte in COUNT - andnot $3, $4, $4 - zap $1, $4, $1 - - stq_u $1, 0($16) - ret - - .align 4 -$multiword: - subq $27, 1, $2 # clear the final bits in the prev word - or $2, $27, $2 - zapnot $1, $2, $1 - subq $18, 1, $18 - - stq_u $1, 0($16) - addq $16, 8, $16 - unop - beq $18, 1f - - nop - unop - nop - blbc $18, 0f - - stq_u $31, 0($16) # zero one word - subq $18, 1, $18 - addq $16, 8, $16 - beq $18, 1f - -0: stq_u $31, 0($16) # zero two words - subq $18, 2, $18 - stq_u $31, 8($16) - addq $16, 16, $16 - bne $18, 0b - -1: ldq_u $1, 0($16) # clear the leading bits in the final word - subq $24, 1, $2 - or $2, $24, $2 - - zap $1, $2, $1 - stq_u $1, 0($16) -$zerolen: - ret - - .end strncpy - EXPORT_SYMBOL(strncpy) diff --git a/arch/alpha/lib/styncpy.S b/arch/alpha/lib/styncpy.S index 72fc2754eb57..f1078ff12530 100644 --- a/arch/alpha/lib/styncpy.S +++ b/arch/alpha/lib/styncpy.S @@ -1,4 +1,3 @@ -#include "strncpy.S" #ifdef CONFIG_ALPHA_EV67 #include "ev67-strncat.S" #else diff --git a/arch/alpha/math-emu/math.c b/arch/alpha/math-emu/math.c index 68d420bfd3c0..e3f2df3729e3 100644 --- a/arch/alpha/math-emu/math.c +++ b/arch/alpha/math-emu/math.c @@ -52,13 +52,13 @@ MODULE_DESCRIPTION("FP Software completion module"); MODULE_LICENSE("GPL v2"); extern long (*alpha_fp_emul_imprecise)(struct pt_regs *, unsigned long); -extern long (*alpha_fp_emul) (unsigned long pc); +extern long (*alpha_fp_emul) (unsigned long pc, unsigned long summary); static long (*save_emul_imprecise)(struct pt_regs *, unsigned long); -static long (*save_emul) (unsigned long pc); +static long (*save_emul) (unsigned long pc, unsigned long summary); long do_alpha_fp_emul_imprecise(struct pt_regs *, unsigned long); -long do_alpha_fp_emul(unsigned long); +long do_alpha_fp_emul(unsigned long, unsigned long); static int alpha_fp_emul_init_module(void) { @@ -86,7 +86,22 @@ module_exit(alpha_fp_emul_cleanup_module); /* - * Emulate the floating point instruction at address PC. Returns -1 if the + * Exception bits of the exception summary register (EXC_SUM). Bit 0 is the + * software completion bit; bits 1 through 5 report the exceptions the + * hardware attributed to the trapping instruction, and lie at the same + * positions as the corresponding IEEE_TRAP_ENABLE_* bits. + */ +#define EXC_SUM_INV (1UL << 1) +#define EXC_SUM_DZE (1UL << 2) +#define EXC_SUM_OVF (1UL << 3) +#define EXC_SUM_UNF (1UL << 4) +#define EXC_SUM_INE (1UL << 5) +#define EXC_SUM_MASK (EXC_SUM_INV | EXC_SUM_DZE | EXC_SUM_OVF \ + | EXC_SUM_UNF | EXC_SUM_INE) + +/* + * Emulate the floating point instruction at address PC. SUMMARY is the + * exception summary register the trap was delivered with. Returns -1 if the * instruction to be emulated is illegal (such as with the opDEC trap), else * the SI_CODE for a SIGFPE signal, else 0 if everything's ok. * @@ -95,7 +110,7 @@ module_exit(alpha_fp_emul_cleanup_module); * stick the result of the operation into the appropriate register. */ long -alpha_fp_emul (unsigned long pc) +alpha_fp_emul (unsigned long pc, unsigned long summary) { FP_DECL_EX; FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SR); @@ -300,12 +315,56 @@ done: swcr |= (_fex << IEEE_STATUS_TO_EXCSUM_SHIFT); current_thread_info()->ieee_state |= (_fex << IEEE_STATUS_TO_EXCSUM_SHIFT); + } - /* Update hardware control register. */ - fpcr &= (~FPCR_MASK | FPCR_DYN_MASK); - fpcr |= ieee_swcr_to_fpcr(swcr); - wrfpcr(fpcr); + /* + * EV6 records exception status bits in the FPCR before delivering the + * software completion trap, and swcr_update_status() above merged them + * into SWCR. Some can be wrong for the instruction we just emulated: + * a CVTTS of a value exactly representable as a subnormal sets FPCR_UNF + * even though the result is exact. Clear the exceptions the trap + * reported but that soft-fp did not raise. + */ + if (implver() == IMPLVER_EV6) { + unsigned long spurious = summary & EXC_SUM_MASK; + + if (spurious & (EXC_SUM_UNF | EXC_SUM_OVF)) { + /* + * EXC_SUM reports only the underflow or overflow, + * but the hardware sets INE alongside it in the FPCR. + */ + spurious |= EXC_SUM_INE; + } else if (!spurious) { + /* + * No exception reported, so this was a denormal + * operand trap, for which INE and UNF can be + * fabricated as well. + */ + spurious = EXC_SUM_INE | EXC_SUM_UNF; + } + /* + * Never clear an exception software has confirmed. Every + * instruction that genuinely raises one traps for software + * completion and is recorded in ieee_state above, so a bit + * found there -- including one just set from _fex -- belongs + * to this or an earlier instruction and must survive. + */ + spurious &= ~(current_thread_info()->ieee_state + >> IEEE_STATUS_TO_EXCSUM_SHIFT); + + swcr &= ~(spurious << IEEE_STATUS_TO_EXCSUM_SHIFT); + } + + /* + * Update hardware control register. This has to happen even when + * soft-fp raised nothing, to clear any fabricated bits. + */ + fpcr &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr |= ieee_swcr_to_fpcr(swcr); + wrfpcr(fpcr); + + if (_fex) { /* Do we generate a signal? */ _fex = _fex & swcr & IEEE_TRAP_ENABLE_MASK; si_code = 0; @@ -387,9 +446,16 @@ alpha_fp_emul_imprecise (struct pt_regs *regs, unsigned long write_mask) break; } if (!write_mask) { - /* Re-execute insns in the trap-shadow. */ + /* + * Re-execute insns in the trap-shadow. Pass no + * exception summary: it describes the trap, which + * was taken anywhere in the shadow, and so is not + * attribution for this instruction. Nothing is + * lost, since only EV6 -- which traps precisely and + * never comes this way -- needs it. + */ regs->pc = trigger_pc + 4; - si_code = alpha_fp_emul(trigger_pc); + si_code = alpha_fp_emul(trigger_pc, 0); goto egress; } trigger_pc -= 4; diff --git a/arch/alpha/mm/Makefile b/arch/alpha/mm/Makefile index 101dbd06b4ce..2d05664058f6 100644 --- a/arch/alpha/mm/Makefile +++ b/arch/alpha/mm/Makefile @@ -3,4 +3,4 @@ # Makefile for the linux alpha-specific parts of the memory manager. # -obj-y := init.o fault.o +obj-y := init.o fault.o tlbflush.o diff --git a/arch/alpha/mm/init.c b/arch/alpha/mm/init.c index 4c5ab9cd8a0a..f4d65a60c869 100644 --- a/arch/alpha/mm/init.c +++ b/arch/alpha/mm/init.c @@ -63,8 +63,7 @@ pgd_alloc(struct mm_struct *mm) static inline unsigned long load_PCB(struct pcb_struct *pcb) { - register unsigned long sp __asm__("$30"); - pcb->ksp = sp; + pcb->ksp = (unsigned long)current_stack_pointer; return __reload_thread(pcb); } @@ -208,12 +207,8 @@ callback_init(void * kernel_end) return kernel_end; } -/* - * paging_init() sets up the memory map. - */ -void __init paging_init(void) +void __init arch_zone_limits_init(unsigned long *max_zone_pfn) { - unsigned long max_zone_pfn[MAX_NR_ZONES] = {0, }; unsigned long dma_pfn; dma_pfn = virt_to_phys((char *)MAX_DMA_ADDRESS) >> PAGE_SHIFT; @@ -221,11 +216,13 @@ void __init paging_init(void) max_zone_pfn[ZONE_DMA] = dma_pfn; max_zone_pfn[ZONE_NORMAL] = max_pfn; +} - /* Initialize mem_map[]. */ - free_area_init(max_zone_pfn); - - /* Initialize the kernel's ZERO_PGE. */ +/* + * paging_init() initializes the kernel's ZERO_PGE. + */ +void __init paging_init(void) +{ memset(absolute_pointer(ZERO_PGE), 0, PAGE_SIZE); } diff --git a/arch/alpha/mm/tlbflush.c b/arch/alpha/mm/tlbflush.c new file mode 100644 index 000000000000..ccbc317b9a34 --- /dev/null +++ b/arch/alpha/mm/tlbflush.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Alpha TLB shootdown helpers + * + * Copyright (C) 2025 Magnus Lindholm <linmag7@gmail.com> + * + * Alpha-specific TLB flush helpers that cannot be expressed purely + * as inline functions. + * + * These helpers provide combined MM context handling (ASN rollover) + * and immediate TLB invalidation for page migration and memory + * compaction paths, where lazy shootdowns are insufficient. + */ + +#include <linux/mm.h> +#include <linux/smp.h> +#include <linux/sched.h> +#include <asm/tlbflush.h> +#include <asm/pal.h> +#include <asm/mmu_context.h> + +#define asn_locked() (cpu_data[smp_processor_id()].asn_lock) + +/* + * Migration/compaction helper: combine mm context (ASN) handling with an + * immediate per-page TLB invalidate and (for exec) an instruction barrier. + * + * This mirrors the SMP combined IPI handler semantics, but runs locally on UP. + */ +#ifndef CONFIG_SMP +void migrate_flush_tlb_page(struct vm_area_struct *vma, + unsigned long addr) +{ + struct mm_struct *mm = vma->vm_mm; + int tbi_type = (vma->vm_flags & VM_EXEC) ? 3 : 2; + + /* + * First do the mm-context side: + * If we're currently running this mm, reload a fresh context ASN. + * Otherwise, mark context invalid. + * + * On UP, this is mostly about matching the SMP semantics and ensuring + * exec/i-cache tagging assumptions hold when compaction migrates pages. + */ + if (mm == current->active_mm) + flush_tlb_current(mm); + else + flush_tlb_other(mm); + + /* + * Then do the immediate translation kill for this VA. + * For exec mappings, order instruction fetch after invalidation. + */ + tbi(tbi_type, addr); +} + +#else +struct tlb_mm_and_addr { + struct mm_struct *mm; + unsigned long addr; + int tbi_type; /* 2 = DTB, 3 = ITB+DTB */ +}; + +static void ipi_flush_mm_and_page(void *x) +{ + struct tlb_mm_and_addr *d = x; + + /* Part 1: mm context side (Alpha uses ASN/context as a key mechanism). */ + if (d->mm == current->active_mm && !asn_locked()) + __load_new_mm_context(d->mm); + else + flush_tlb_other(d->mm); + + /* Part 2: immediate per-VA invalidation on this CPU. */ + tbi(d->tbi_type, d->addr); +} + +void migrate_flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) +{ + struct mm_struct *mm = vma->vm_mm; + struct tlb_mm_and_addr d = { + .mm = mm, + .addr = addr, + .tbi_type = (vma->vm_flags & VM_EXEC) ? 3 : 2, + }; + + /* + * One synchronous rendezvous: every CPU runs ipi_flush_mm_and_page(). + * This is the "combined" version of flush_tlb_mm + per-page invalidate. + */ + preempt_disable(); + on_each_cpu(ipi_flush_mm_and_page, &d, 1); + + /* + * mimic flush_tlb_mm()'s mm_users<=1 optimization. + */ + if (atomic_read(&mm->mm_users) <= 1) { + + int cpu, this_cpu; + this_cpu = smp_processor_id(); + + for (cpu = 0; cpu < NR_CPUS; cpu++) { + if (!cpu_online(cpu) || cpu == this_cpu) + continue; + if (READ_ONCE(mm->context[cpu])) + WRITE_ONCE(mm->context[cpu], 0); + } + } + preempt_enable(); +} + +#endif |