diff options
Diffstat (limited to 'arch/arc')
48 files changed, 732 insertions, 775 deletions
diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index 68fcbb2d59e2..2ee0c9bfd032 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -20,7 +20,6 @@ config ARC select GENERIC_SMP_IDLE_THREAD select HAVE_ARCH_KGDB select HAVE_ARCH_TRACEHOOK - select HAVE_GENERIC_HARDIRQS select HAVE_IOREMAP_PROT select HAVE_KPROBES select HAVE_KRETPROBES @@ -36,6 +35,12 @@ config ARC select PERF_USE_VMALLOC select HAVE_DEBUG_STACKOVERFLOW +config TRACE_IRQFLAGS_SUPPORT + def_bool y + +config LOCKDEP_SUPPORT + def_bool y + config SCHED_OMIT_FRAME_POINTER def_bool y @@ -120,7 +125,6 @@ config ARC_PLAT_NEEDS_CPU_TO_DMA config SMP bool "Symmetric Multi-Processing (Incomplete)" default n - select USE_GENERIC_SMP_HELPERS help This enables support for systems with more than one CPU. If you have a system with only one CPU, like most personal computers, say N. If @@ -131,17 +135,14 @@ if SMP config ARC_HAS_COH_CACHES def_bool n -config ARC_HAS_COH_RTSC - def_bool n - config ARC_HAS_REENTRANT_IRQ_LV2 def_bool n endif config NR_CPUS - int "Maximum number of CPUs (2-32)" - range 2 32 + int "Maximum number of CPUs (2-4096)" + range 2 4096 depends on SMP default "2" @@ -327,8 +328,7 @@ config ARC_HAS_RTSC bool "Insn: RTSC (64-bit r/o cycle counter)" default y depends on ARC_CPU_REL_4_10 - # if SMP, enable RTSC only if counter is coherent across cores - depends on !SMP || ARC_HAS_COH_RTSC + depends on !SMP endmenu # "ARC CPU Configuration" diff --git a/arch/arc/boot/.gitignore b/arch/arc/boot/.gitignore new file mode 100644 index 000000000000..5d65b54bf17a --- /dev/null +++ b/arch/arc/boot/.gitignore @@ -0,0 +1 @@ +*.dtb* diff --git a/arch/arc/configs/fpga_defconfig b/arch/arc/configs/fpga_defconfig index 4ca50f1f8d05..e283aa586934 100644 --- a/arch/arc/configs/fpga_defconfig +++ b/arch/arc/configs/fpga_defconfig @@ -2,6 +2,8 @@ CONFIG_CROSS_COMPILE="arc-linux-uclibc-" # CONFIG_LOCALVERSION_AUTO is not set CONFIG_DEFAULT_HOSTNAME="ARCLinux" # CONFIG_SWAP is not set +CONFIG_SYSVIPC=y +CONFIG_POSIX_MQUEUE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y @@ -62,4 +64,5 @@ CONFIG_TMPFS=y CONFIG_NFS_FS=y # CONFIG_ENABLE_WARN_DEPRECATED is not set # CONFIG_ENABLE_MUST_CHECK is not set +# CONFIG_DEBUG_PREEMPT is not set CONFIG_XZ_DEC=y diff --git a/arch/arc/include/asm/Kbuild b/arch/arc/include/asm/Kbuild index d8dd660898b9..5943f7f9d325 100644 --- a/arch/arc/include/asm/Kbuild +++ b/arch/arc/include/asm/Kbuild @@ -46,3 +46,4 @@ generic-y += ucontext.h generic-y += user.h generic-y += vga.h generic-y += xor.h +generic-y += preempt.h diff --git a/arch/arc/include/asm/cache.h b/arch/arc/include/asm/cache.h index 5802849a6cae..2fd3162ec4df 100644 --- a/arch/arc/include/asm/cache.h +++ b/arch/arc/include/asm/cache.h @@ -17,13 +17,7 @@ #endif #define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) - -/* For a rare case where customers have differently config I/D */ -#define ARC_ICACHE_LINE_LEN L1_CACHE_BYTES -#define ARC_DCACHE_LINE_LEN L1_CACHE_BYTES - -#define ICACHE_LINE_MASK (~(ARC_ICACHE_LINE_LEN - 1)) -#define DCACHE_LINE_MASK (~(ARC_DCACHE_LINE_LEN - 1)) +#define CACHE_LINE_MASK (~(L1_CACHE_BYTES - 1)) /* * ARC700 doesn't cache any access in top 256M. @@ -57,7 +51,7 @@ extern void arc_cache_init(void); extern char *arc_cache_mumbojumbo(int cpu_id, char *buf, int len); -extern void __init read_decode_cache_bcr(void); +extern void read_decode_cache_bcr(void); #endif /* !__ASSEMBLY__ */ diff --git a/arch/arc/include/asm/delay.h b/arch/arc/include/asm/delay.h index 442ce5d0f709..43de30256981 100644 --- a/arch/arc/include/asm/delay.h +++ b/arch/arc/include/asm/delay.h @@ -53,11 +53,10 @@ static inline void __udelay(unsigned long usecs) { unsigned long loops; - /* (long long) cast ensures 64 bit MPY - real or emulated + /* (u64) cast ensures 64 bit MPY - real or emulated * HZ * 4295 is pre-evaluated by gcc - hence only 2 mpy ops */ - loops = ((long long)(usecs * 4295 * HZ) * - (long long)(loops_per_jiffy)) >> 32; + loops = ((u64) usecs * 4295 * HZ * loops_per_jiffy) >> 32; __delay(loops); } diff --git a/arch/arc/include/asm/entry.h b/arch/arc/include/asm/entry.h index df57611652e5..884081099f80 100644 --- a/arch/arc/include/asm/entry.h +++ b/arch/arc/include/asm/entry.h @@ -365,7 +365,7 @@ * it to memory (non-SMP case) or SCRATCH0 Aux Reg (SMP). * * Before saving the full regfile - this reg is restored back, only - * to be saved again on kernel mode stack, as part of ptregs. + * to be saved again on kernel mode stack, as part of pt_regs. *-------------------------------------------------------------*/ .macro EXCPN_PROLOG_FREEUP_REG reg #ifdef CONFIG_SMP @@ -384,6 +384,28 @@ .endm /*-------------------------------------------------------------- + * Exception Entry prologue + * -Switches stack to K mode (if not already) + * -Saves the register file + * + * After this it is safe to call the "C" handlers + *-------------------------------------------------------------*/ +.macro EXCEPTION_PROLOGUE + + /* Need at least 1 reg to code the early exception prologue */ + EXCPN_PROLOG_FREEUP_REG r9 + + /* U/K mode at time of exception (stack not switched if already K) */ + lr r9, [erstatus] + + /* ARC700 doesn't provide auto-stack switching */ + SWITCH_TO_KERNEL_STK + + /* save the regfile */ + SAVE_ALL_SYS +.endm + +/*-------------------------------------------------------------- * Save all registers used by Exceptions (TLB Miss, Prot-V, Mem err etc) * Requires SP to be already switched to kernel mode Stack * sp points to the next free element on the stack at exit of this macro. diff --git a/arch/arc/include/asm/io.h b/arch/arc/include/asm/io.h index 473424d7528b..334ce7017a18 100644 --- a/arch/arc/include/asm/io.h +++ b/arch/arc/include/asm/io.h @@ -100,6 +100,10 @@ static inline void __raw_writel(u32 w, volatile void __iomem *addr) } +#define readb_relaxed readb +#define readw_relaxed readw +#define readl_relaxed readl + #include <asm-generic/io.h> #endif /* _ASM_ARC_IO_H */ diff --git a/arch/arc/include/asm/irq.h b/arch/arc/include/asm/irq.h index c0a72105ee0b..291a70db68b8 100644 --- a/arch/arc/include/asm/irq.h +++ b/arch/arc/include/asm/irq.h @@ -18,8 +18,8 @@ #include <asm-generic/irq.h> -extern void __init arc_init_IRQ(void); -extern int __init get_hw_config_num_irq(void); +extern void arc_init_IRQ(void); +extern int get_hw_config_num_irq(void); void arc_local_timer_setup(unsigned int cpu); diff --git a/arch/arc/include/asm/irqflags.h b/arch/arc/include/asm/irqflags.h index d99f79bcf865..cb7efc29f16f 100644 --- a/arch/arc/include/asm/irqflags.h +++ b/arch/arc/include/asm/irqflags.h @@ -151,23 +151,38 @@ static inline void arch_unmask_irq(unsigned int irq) #else -.macro IRQ_DISABLE scratch - lr \scratch, [status32] - bic \scratch, \scratch, (STATUS_E1_MASK | STATUS_E2_MASK) - flag \scratch +#ifdef CONFIG_TRACE_IRQFLAGS + +.macro TRACE_ASM_IRQ_DISABLE + bl trace_hardirqs_off +.endm + +.macro TRACE_ASM_IRQ_ENABLE + bl trace_hardirqs_on +.endm + +#else + +.macro TRACE_ASM_IRQ_DISABLE +.endm + +.macro TRACE_ASM_IRQ_ENABLE .endm -.macro IRQ_DISABLE_SAVE scratch, save +#endif + +.macro IRQ_DISABLE scratch lr \scratch, [status32] - mov \save, \scratch /* Make a copy */ bic \scratch, \scratch, (STATUS_E1_MASK | STATUS_E2_MASK) flag \scratch + TRACE_ASM_IRQ_DISABLE .endm .macro IRQ_ENABLE scratch lr \scratch, [status32] or \scratch, \scratch, (STATUS_E1_MASK | STATUS_E2_MASK) flag \scratch + TRACE_ASM_IRQ_ENABLE .endm #endif /* __ASSEMBLY__ */ diff --git a/arch/arc/include/asm/mach_desc.h b/arch/arc/include/asm/mach_desc.h index 9998dc846ebb..e8993a2be6c2 100644 --- a/arch/arc/include/asm/mach_desc.h +++ b/arch/arc/include/asm/mach_desc.h @@ -51,22 +51,12 @@ struct machine_desc { /* * Current machine - only accessible during boot. */ -extern struct machine_desc *machine_desc; +extern const struct machine_desc *machine_desc; /* * Machine type table - also only accessible during boot */ -extern struct machine_desc __arch_info_begin[], __arch_info_end[]; -#define for_each_machine_desc(p) \ - for (p = __arch_info_begin; p < __arch_info_end; p++) - -static inline struct machine_desc *default_machine_desc(void) -{ - /* the default machine is the last one linked in */ - if (__arch_info_end - 1 < __arch_info_begin) - return NULL; - return __arch_info_end - 1; -} +extern const struct machine_desc __arch_info_begin[], __arch_info_end[]; /* * Set of macros to define architecture features. @@ -81,7 +71,6 @@ __attribute__((__section__(".arch.info.init"))) = { \ #define MACHINE_END \ }; -extern struct machine_desc *setup_machine_fdt(void *dt); -extern void __init copy_devtree(void); +extern const struct machine_desc *setup_machine_fdt(void *dt); #endif diff --git a/arch/arc/include/asm/mmu.h b/arch/arc/include/asm/mmu.h index 7c03fe61759c..8c84ae98c337 100644 --- a/arch/arc/include/asm/mmu.h +++ b/arch/arc/include/asm/mmu.h @@ -32,6 +32,8 @@ /* Error code if probe fails */ #define TLB_LKUP_ERR 0x80000000 +#define TLB_DUP_ERR (TLB_LKUP_ERR | 0x00000001) + /* TLB Commands */ #define TLBWrite 0x1 #define TLBRead 0x2 @@ -46,21 +48,18 @@ #ifndef __ASSEMBLY__ typedef struct { - unsigned long asid; /* Pvt Addr-Space ID for mm */ -#ifdef CONFIG_ARC_TLB_DBG - struct task_struct *tsk; -#endif + unsigned long asid[NR_CPUS]; /* 8 bit MMU PID + Generation cycle */ } mm_context_t; #ifdef CONFIG_ARC_DBG_TLB_PARANOIA -void tlb_paranoid_check(unsigned int pid_sw, unsigned long address); +void tlb_paranoid_check(unsigned int mm_asid, unsigned long address); #else #define tlb_paranoid_check(a, b) #endif void arc_mmu_init(void); extern char *arc_mmu_mumbojumbo(int cpu_id, char *buf, int len); -void __init read_decode_mmu_bcr(void); +void read_decode_mmu_bcr(void); #endif /* !__ASSEMBLY__ */ diff --git a/arch/arc/include/asm/mmu_context.h b/arch/arc/include/asm/mmu_context.h index 0d71fb11b57c..1fd467ef658f 100644 --- a/arch/arc/include/asm/mmu_context.h +++ b/arch/arc/include/asm/mmu_context.h @@ -30,99 +30,72 @@ * "Fast Context Switch" i.e. no TLB flush on ctxt-switch * * Linux assigns each task a unique ASID. A simple round-robin allocation - * of H/w ASID is done using software tracker @asid_cache. + * of H/w ASID is done using software tracker @asid_cpu. * When it reaches max 255, the allocation cycle starts afresh by flushing * the entire TLB and wrapping ASID back to zero. * - * For book-keeping, Linux uses a couple of data-structures: - * -mm_struct has an @asid field to keep a note of task's ASID (needed at the - * time of say switch_mm( ) - * -An array of mm structs @asid_mm_map[] for asid->mm the reverse mapping, - * given an ASID, finding the mm struct associated. - * - * The round-robin allocation algorithm allows for ASID stealing. - * If asid tracker is at "x-1", a new req will allocate "x", even if "x" was - * already assigned to another (switched-out) task. Obviously the prev owner - * is marked with an invalid ASID to make it request for a new ASID when it - * gets scheduled next time. However its TLB entries (with ASID "x") could - * exist, which must be cleared before the same ASID is used by the new owner. - * Flushing them would be plausible but costly solution. Instead we force a - * allocation policy quirk, which ensures that a stolen ASID won't have any - * TLB entries associates, alleviating the need to flush. - * The quirk essentially is not allowing ASID allocated in prev cycle - * to be used past a roll-over in the next cycle. - * When this happens (i.e. task ASID > asid tracker), task needs to refresh - * its ASID, aligning it to current value of tracker. If the task doesn't get - * scheduled past a roll-over, hence its ASID is not yet realigned with - * tracker, such ASID is anyways safely reusable because it is - * gauranteed that TLB entries with that ASID wont exist. + * A new allocation cycle, post rollover, could potentially reassign an ASID + * to a different task. Thus the rule is to refresh the ASID in a new cycle. + * The 32 bit @asid_cpu (and mm->asid) have 8 bits MMU PID and rest 24 bits + * serve as cycle/generation indicator and natural 32 bit unsigned math + * automagically increments the generation when lower 8 bits rollover. */ -#define FIRST_ASID 0 -#define MAX_ASID 255 /* 8 bit PID field in PID Aux reg */ -#define NO_ASID (MAX_ASID + 1) /* ASID Not alloc to mmu ctxt */ -#define NUM_ASID ((MAX_ASID - FIRST_ASID) + 1) +#define MM_CTXT_ASID_MASK 0x000000ff /* MMU PID reg :8 bit PID */ +#define MM_CTXT_CYCLE_MASK (~MM_CTXT_ASID_MASK) + +#define MM_CTXT_FIRST_CYCLE (MM_CTXT_ASID_MASK + 1) +#define MM_CTXT_NO_ASID 0UL -/* ASID to mm struct mapping */ -extern struct mm_struct *asid_mm_map[NUM_ASID + 1]; +#define asid_mm(mm, cpu) mm->context.asid[cpu] +#define hw_pid(mm, cpu) (asid_mm(mm, cpu) & MM_CTXT_ASID_MASK) -extern int asid_cache; +DECLARE_PER_CPU(unsigned int, asid_cache); +#define asid_cpu(cpu) per_cpu(asid_cache, cpu) /* - * Assign a new ASID to task. If the task already has an ASID, it is - * relinquished. + * Get a new ASID if task doesn't have a valid one (unalloc or from prev cycle) + * Also set the MMU PID register to existing/updated ASID */ static inline void get_new_mmu_context(struct mm_struct *mm) { - struct mm_struct *prev_owner; + const unsigned int cpu = smp_processor_id(); unsigned long flags; local_irq_save(flags); /* - * Relinquish the currently owned ASID (if any). - * Doing unconditionally saves a cmp-n-branch; for already unused - * ASID slot, the value was/remains NULL + * Move to new ASID if it was not from current alloc-cycle/generation. + * This is done by ensuring that the generation bits in both mm->ASID + * and cpu's ASID counter are exactly same. + * + * Note: Callers needing new ASID unconditionally, independent of + * generation, e.g. local_flush_tlb_mm() for forking parent, + * first need to destroy the context, setting it to invalid + * value. */ - asid_mm_map[mm->context.asid] = (struct mm_struct *)NULL; + if (!((asid_mm(mm, cpu) ^ asid_cpu(cpu)) & MM_CTXT_CYCLE_MASK)) + goto set_hw; - /* move to new ASID */ - if (++asid_cache > MAX_ASID) { /* ASID roll-over */ - asid_cache = FIRST_ASID; - flush_tlb_all(); - } + /* move to new ASID and handle rollover */ + if (unlikely(!(++asid_cpu(cpu) & MM_CTXT_ASID_MASK))) { - /* - * Is next ASID already owned by some-one else (we are stealing it). - * If so, let the orig owner be aware of this, so when it runs, it - * asks for a brand new ASID. This would only happen for a long-lived - * task with ASID from prev allocation cycle (before ASID roll-over). - * - * This might look wrong - if we are re-using some other task's ASID, - * won't we use it's stale TLB entries too. Actually switch_mm( ) takes - * care of such a case: it ensures that task with ASID from prev alloc - * cycle, when scheduled will refresh it's ASID: see switch_mm( ) below - * The stealing scenario described here will only happen if that task - * didn't get a chance to refresh it's ASID - implying stale entries - * won't exist. - */ - prev_owner = asid_mm_map[asid_cache]; - if (prev_owner) - prev_owner->context.asid = NO_ASID; + local_flush_tlb_all(); + + /* + * Above checke for rollover of 8 bit ASID in 32 bit container. + * If the container itself wrapped around, set it to a non zero + * "generation" to distinguish from no context + */ + if (!asid_cpu(cpu)) + asid_cpu(cpu) = MM_CTXT_FIRST_CYCLE; + } /* Assign new ASID to tsk */ - asid_mm_map[asid_cache] = mm; - mm->context.asid = asid_cache; - -#ifdef CONFIG_ARC_TLB_DBG - pr_info("ARC_TLB_DBG: NewMM=0x%x OldMM=0x%x task_struct=0x%x Task: %s," - " pid:%u, assigned asid:%lu\n", - (unsigned int)mm, (unsigned int)prev_owner, - (unsigned int)(mm->context.tsk), (mm->context.tsk)->comm, - (mm->context.tsk)->pid, mm->context.asid); -#endif + asid_mm(mm, cpu) = asid_cpu(cpu); - write_aux_reg(ARC_REG_PID, asid_cache | MMU_ENABLE); +set_hw: + write_aux_reg(ARC_REG_PID, hw_pid(mm, cpu) | MMU_ENABLE); local_irq_restore(flags); } @@ -134,59 +107,61 @@ static inline void get_new_mmu_context(struct mm_struct *mm) static inline int init_new_context(struct task_struct *tsk, struct mm_struct *mm) { - mm->context.asid = NO_ASID; -#ifdef CONFIG_ARC_TLB_DBG - mm->context.tsk = tsk; -#endif + int i; + + for_each_possible_cpu(i) + asid_mm(mm, i) = MM_CTXT_NO_ASID; + return 0; } +static inline void destroy_context(struct mm_struct *mm) +{ + unsigned long flags; + + /* Needed to elide CONFIG_DEBUG_PREEMPT warning */ + local_irq_save(flags); + asid_mm(mm, smp_processor_id()) = MM_CTXT_NO_ASID; + local_irq_restore(flags); +} + /* Prepare the MMU for task: setup PID reg with allocated ASID If task doesn't have an ASID (never alloc or stolen, get a new ASID) */ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk) { + const int cpu = smp_processor_id(); + + /* + * Note that the mm_cpumask is "aggregating" only, we don't clear it + * for the switched-out task, unlike some other arches. + * It is used to enlist cpus for sending TLB flush IPIs and not sending + * it to CPUs where a task once ran-on, could cause stale TLB entry + * re-use, specially for a multi-threaded task. + * e.g. T1 runs on C1, migrates to C3. T2 running on C2 munmaps. + * For a non-aggregating mm_cpumask, IPI not sent C1, and if T1 + * were to re-migrate to C1, it could access the unmapped region + * via any existing stale TLB entries. + */ + cpumask_set_cpu(cpu, mm_cpumask(next)); + #ifndef CONFIG_SMP /* PGD cached in MMU reg to avoid 3 mem lookups: task->mm->pgd */ write_aux_reg(ARC_REG_SCRATCH_DATA0, next->pgd); #endif - /* - * Get a new ASID if task doesn't have a valid one. Possible when - * -task never had an ASID (fresh after fork) - * -it's ASID was stolen - past an ASID roll-over. - * -There's a third obscure scenario (if this task is running for the - * first time afer an ASID rollover), where despite having a valid - * ASID, we force a get for new ASID - see comments at top. - * - * Both the non-alloc scenario and first-use-after-rollover can be - * detected using the single condition below: NO_ASID = 256 - * while asid_cache is always a valid ASID value (0-255). - */ - if (next->context.asid > asid_cache) { - get_new_mmu_context(next); - } else { - /* - * XXX: This will never happen given the chks above - * BUG_ON(next->context.asid > MAX_ASID); - */ - write_aux_reg(ARC_REG_PID, next->context.asid | MMU_ENABLE); - } - + get_new_mmu_context(next); } -static inline void destroy_context(struct mm_struct *mm) -{ - unsigned long flags; - - local_irq_save(flags); - - asid_mm_map[mm->context.asid] = NULL; - mm->context.asid = NO_ASID; - - local_irq_restore(flags); -} +/* + * Called at the time of execve() to get a new ASID + * Note the subtlety here: get_new_mmu_context() behaves differently here + * vs. in switch_mm(). Here it always returns a new ASID, because mm has + * an unallocated "initial" value, while in latter, it moves to a new ASID, + * only if it was unallocated + */ +#define activate_mm(prev, next) switch_mm(prev, next, NULL) /* it seemed that deactivate_mm( ) is a reasonable place to do book-keeping * for retiring-mm. However destroy_context( ) still needs to do that because @@ -197,17 +172,6 @@ static inline void destroy_context(struct mm_struct *mm) */ #define deactivate_mm(tsk, mm) do { } while (0) -static inline void activate_mm(struct mm_struct *prev, struct mm_struct *next) -{ -#ifndef CONFIG_SMP - write_aux_reg(ARC_REG_SCRATCH_DATA0, next->pgd); -#endif - - /* Unconditionally get a new ASID */ - get_new_mmu_context(next); - -} - #define enter_lazy_tlb(mm, tsk) #endif /* __ASM_ARC_MMU_CONTEXT_H */ diff --git a/arch/arc/include/asm/pgalloc.h b/arch/arc/include/asm/pgalloc.h index 36a9f20c21a3..81208bfd9dcb 100644 --- a/arch/arc/include/asm/pgalloc.h +++ b/arch/arc/include/asm/pgalloc.h @@ -105,11 +105,16 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { pgtable_t pte_pg; + struct page *page; pte_pg = __get_free_pages(GFP_KERNEL | __GFP_REPEAT, __get_order_pte()); - if (pte_pg) { - memzero((void *)pte_pg, PTRS_PER_PTE * 4); - pgtable_page_ctor(virt_to_page(pte_pg)); + if (!pte_pg) + return 0; + memzero((void *)pte_pg, PTRS_PER_PTE * 4); + page = virt_to_page(pte_pg); + if (!pgtable_page_ctor(page)) { + __free_page(page); + return 0; } return pte_pg; diff --git a/arch/arc/include/asm/pgtable.h b/arch/arc/include/asm/pgtable.h index 4749a0eee1cf..6b0b7f7ef783 100644 --- a/arch/arc/include/asm/pgtable.h +++ b/arch/arc/include/asm/pgtable.h @@ -57,43 +57,31 @@ #define _PAGE_ACCESSED (1<<1) /* Page is accessed (S) */ #define _PAGE_CACHEABLE (1<<2) /* Page is cached (H) */ -#define _PAGE_U_EXECUTE (1<<3) /* Page has user execute perm (H) */ -#define _PAGE_U_WRITE (1<<4) /* Page has user write perm (H) */ -#define _PAGE_U_READ (1<<5) /* Page has user read perm (H) */ -#define _PAGE_K_EXECUTE (1<<6) /* Page has kernel execute perm (H) */ -#define _PAGE_K_WRITE (1<<7) /* Page has kernel write perm (H) */ -#define _PAGE_K_READ (1<<8) /* Page has kernel perm (H) */ -#define _PAGE_GLOBAL (1<<9) /* Page is global (H) */ -#define _PAGE_MODIFIED (1<<10) /* Page modified (dirty) (S) */ -#define _PAGE_FILE (1<<10) /* page cache/ swap (S) */ -#define _PAGE_PRESENT (1<<11) /* TLB entry is valid (H) */ +#define _PAGE_EXECUTE (1<<3) /* Page has user execute perm (H) */ +#define _PAGE_WRITE (1<<4) /* Page has user write perm (H) */ +#define _PAGE_READ (1<<5) /* Page has user read perm (H) */ +#define _PAGE_MODIFIED (1<<6) /* Page modified (dirty) (S) */ +#define _PAGE_FILE (1<<7) /* page cache/ swap (S) */ +#define _PAGE_GLOBAL (1<<8) /* Page is global (H) */ +#define _PAGE_PRESENT (1<<10) /* TLB entry is valid (H) */ -#else +#else /* MMU v3 onwards */ -/* PD1 */ #define _PAGE_CACHEABLE (1<<0) /* Page is cached (H) */ -#define _PAGE_U_EXECUTE (1<<1) /* Page has user execute perm (H) */ -#define _PAGE_U_WRITE (1<<2) /* Page has user write perm (H) */ -#define _PAGE_U_READ (1<<3) /* Page has user read perm (H) */ -#define _PAGE_K_EXECUTE (1<<4) /* Page has kernel execute perm (H) */ -#define _PAGE_K_WRITE (1<<5) /* Page has kernel write perm (H) */ -#define _PAGE_K_READ (1<<6) /* Page has kernel perm (H) */ -#define _PAGE_ACCESSED (1<<7) /* Page is accessed (S) */ - -/* PD0 */ +#define _PAGE_EXECUTE (1<<1) /* Page has user execute perm (H) */ +#define _PAGE_WRITE (1<<2) /* Page has user write perm (H) */ +#define _PAGE_READ (1<<3) /* Page has user read perm (H) */ +#define _PAGE_ACCESSED (1<<4) /* Page is accessed (S) */ +#define _PAGE_MODIFIED (1<<5) /* Page modified (dirty) (S) */ +#define _PAGE_FILE (1<<6) /* page cache/ swap (S) */ #define _PAGE_GLOBAL (1<<8) /* Page is global (H) */ #define _PAGE_PRESENT (1<<9) /* TLB entry is valid (H) */ -#define _PAGE_SHARED_CODE (1<<10) /* Shared Code page with cmn vaddr +#define _PAGE_SHARED_CODE (1<<11) /* Shared Code page with cmn vaddr usable for shared TLB entries (H) */ - -#define _PAGE_MODIFIED (1<<11) /* Page modified (dirty) (S) */ -#define _PAGE_FILE (1<<12) /* page cache/ swap (S) */ - -#define _PAGE_SHARED_CODE_H (1<<31) /* Hardware counterpart of above */ #endif -/* Kernel allowed all permissions for all pages */ -#define _K_PAGE_PERMS (_PAGE_K_EXECUTE | _PAGE_K_WRITE | _PAGE_K_READ | \ +/* vmalloc permissions */ +#define _K_PAGE_PERMS (_PAGE_EXECUTE | _PAGE_WRITE | _PAGE_READ | \ _PAGE_GLOBAL | _PAGE_PRESENT) #ifdef CONFIG_ARC_CACHE_PAGES @@ -109,10 +97,6 @@ */ #define ___DEF (_PAGE_PRESENT | _PAGE_DEF_CACHEABLE) -#define _PAGE_READ (_PAGE_U_READ | _PAGE_K_READ) -#define _PAGE_WRITE (_PAGE_U_WRITE | _PAGE_K_WRITE) -#define _PAGE_EXECUTE (_PAGE_U_EXECUTE | _PAGE_K_EXECUTE) - /* Set of bits not changed in pte_modify */ #define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_ACCESSED | _PAGE_MODIFIED) @@ -126,8 +110,8 @@ #define PAGE_SHARED PAGE_U_W_R -/* While kernel runs out of unstrslated space, vmalloc/modules use a chunk of - * kernel vaddr space - visible in all addr spaces, but kernel mode only +/* While kernel runs out of unstranslated space, vmalloc/modules use a chunk of + * user vaddr space - visible in all addr spaces, but kernel mode only * Thus Global, all-kernel-access, no-user-access, cached */ #define PAGE_KERNEL __pgprot(_K_PAGE_PERMS | _PAGE_DEF_CACHEABLE) @@ -136,10 +120,9 @@ #define PAGE_KERNEL_NO_CACHE __pgprot(_K_PAGE_PERMS) /* Masks for actual TLB "PD"s */ -#define PTE_BITS_IN_PD0 (_PAGE_GLOBAL | _PAGE_PRESENT) -#define PTE_BITS_IN_PD1 (PAGE_MASK | _PAGE_CACHEABLE | \ - _PAGE_U_EXECUTE | _PAGE_U_WRITE | _PAGE_U_READ | \ - _PAGE_K_EXECUTE | _PAGE_K_WRITE | _PAGE_K_READ) +#define PTE_BITS_IN_PD0 (_PAGE_GLOBAL | _PAGE_PRESENT) +#define PTE_BITS_RWX (_PAGE_EXECUTE | _PAGE_WRITE | _PAGE_READ) +#define PTE_BITS_NON_RWX_IN_PD1 (PAGE_MASK | _PAGE_CACHEABLE) /************************************************************************** * Mapping of vm_flags (Generic VM) to PTE flags (arch specific) diff --git a/arch/arc/include/asm/prom.h b/arch/arc/include/asm/prom.h deleted file mode 100644 index 692d0d0789a7..000000000000 --- a/arch/arc/include/asm/prom.h +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (C) 2012 Synopsys, Inc. (www.synopsys.com) - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ - -#ifndef _ASM_ARC_PROM_H_ -#define _ASM_ARC_PROM_H_ - -#define HAVE_ARCH_DEVTREE_FIXUPS - -#endif diff --git a/arch/arc/include/asm/ptrace.h b/arch/arc/include/asm/ptrace.h index c9938e7a7dbd..1bfeec2c0558 100644 --- a/arch/arc/include/asm/ptrace.h +++ b/arch/arc/include/asm/ptrace.h @@ -20,27 +20,17 @@ struct pt_regs { /* Real registers */ long bta; /* bta_l1, bta_l2, erbta */ - long lp_start; - long lp_end; - long lp_count; + + long lp_start, lp_end, lp_count; + long status32; /* status32_l1, status32_l2, erstatus */ long ret; /* ilink1, ilink2 or eret */ long blink; long fp; long r26; /* gp */ - long r12; - long r11; - long r10; - long r9; - long r8; - long r7; - long r6; - long r5; - long r4; - long r3; - long r2; - long r1; - long r0; + + long r12, r11, r10, r9, r8, r7, r6, r5, r4, r3, r2, r1, r0; + long sp; /* user/kernel sp depending on where we came from */ long orig_r0; @@ -70,19 +60,7 @@ struct pt_regs { /* Callee saved registers - need to be saved only when you are scheduled out */ struct callee_regs { - long r25; - long r24; - long r23; - long r22; - long r21; - long r20; - long r19; - long r18; - long r17; - long r16; - long r15; - long r14; - long r13; + long r25, r24, r23, r22, r21, r20, r19, r18, r17, r16, r15, r14, r13; }; #define instruction_pointer(regs) ((regs)->ret) diff --git a/arch/arc/include/asm/sections.h b/arch/arc/include/asm/sections.h index 6fc1159dfefe..764f1e3ba752 100644 --- a/arch/arc/include/asm/sections.h +++ b/arch/arc/include/asm/sections.h @@ -11,7 +11,6 @@ #include <asm-generic/sections.h> -extern char _int_vec_base_lds[]; extern char __arc_dccm_base[]; extern char __dtb_start[]; diff --git a/arch/arc/include/asm/setup.h b/arch/arc/include/asm/setup.h index 229e50681497..e10f8cef56a8 100644 --- a/arch/arc/include/asm/setup.h +++ b/arch/arc/include/asm/setup.h @@ -31,7 +31,7 @@ struct cpuinfo_data { extern int root_mountflags, end_mem; extern int running_on_hw; -void __init setup_processor(void); +void setup_processor(void); void __init setup_arch_memory(void); #endif /* __ASMARC_SETUP_H */ diff --git a/arch/arc/include/asm/smp.h b/arch/arc/include/asm/smp.h index c4fb211dcd25..eefc29f08cdb 100644 --- a/arch/arc/include/asm/smp.h +++ b/arch/arc/include/asm/smp.h @@ -30,7 +30,7 @@ extern void arch_send_call_function_ipi_mask(const struct cpumask *mask); * APIs provided by arch SMP code to rest of arch code */ extern void __init smp_init_cpus(void); -extern void __init first_lines_of_secondary(void); +extern void first_lines_of_secondary(void); extern const char *arc_platform_smp_cpuinfo(void); /* diff --git a/arch/arc/include/asm/spinlock.h b/arch/arc/include/asm/spinlock.h index f158197ac5b0..b6a8c2dfbe6e 100644 --- a/arch/arc/include/asm/spinlock.h +++ b/arch/arc/include/asm/spinlock.h @@ -45,7 +45,14 @@ static inline int arch_spin_trylock(arch_spinlock_t *lock) static inline void arch_spin_unlock(arch_spinlock_t *lock) { - lock->slock = __ARCH_SPIN_LOCK_UNLOCKED__; + unsigned int tmp = __ARCH_SPIN_LOCK_UNLOCKED__; + + __asm__ __volatile__( + " ex %0, [%1] \n" + : "+r" (tmp) + : "r"(&(lock->slock)) + : "memory"); + smp_mb(); } diff --git a/arch/arc/include/asm/spinlock_types.h b/arch/arc/include/asm/spinlock_types.h index 8276bfd61704..662627ced4f2 100644 --- a/arch/arc/include/asm/spinlock_types.h +++ b/arch/arc/include/asm/spinlock_types.h @@ -20,9 +20,9 @@ typedef struct { #define __ARCH_SPIN_LOCK_LOCKED { __ARCH_SPIN_LOCK_LOCKED__ } /* - * Unlocked: 0x01_00_00_00 - * Read lock(s): 0x00_FF_00_00 to say 0x01 - * Write lock: 0x0, but only possible if prior value "unlocked" 0x0100_0000 + * Unlocked : 0x0100_0000 + * Read lock(s) : 0x00FF_FFFF to 0x01 (Multiple Readers decrement it) + * Write lock : 0x0, but only if prior value is "unlocked" 0x0100_0000 */ typedef struct { volatile unsigned int counter; diff --git a/arch/arc/include/asm/tlbflush.h b/arch/arc/include/asm/tlbflush.h index b2f9bc7f68c8..71c7b2e4b874 100644 --- a/arch/arc/include/asm/tlbflush.h +++ b/arch/arc/include/asm/tlbflush.h @@ -18,11 +18,18 @@ void local_flush_tlb_kernel_range(unsigned long start, unsigned long end); void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); -/* XXX: Revisit for SMP */ +#ifndef CONFIG_SMP #define flush_tlb_range(vma, s, e) local_flush_tlb_range(vma, s, e) #define flush_tlb_page(vma, page) local_flush_tlb_page(vma, page) #define flush_tlb_kernel_range(s, e) local_flush_tlb_kernel_range(s, e) #define flush_tlb_all() local_flush_tlb_all() #define flush_tlb_mm(mm) local_flush_tlb_mm(mm) - +#else +extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end); +extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long page); +extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); +extern void flush_tlb_all(void); +extern void flush_tlb_mm(struct mm_struct *mm); +#endif /* CONFIG_SMP */ #endif diff --git a/arch/arc/include/asm/uaccess.h b/arch/arc/include/asm/uaccess.h index 32420824375b..30c9baffa96f 100644 --- a/arch/arc/include/asm/uaccess.h +++ b/arch/arc/include/asm/uaccess.h @@ -43,7 +43,7 @@ * Because it essentially checks if buffer end is within limit and @len is * non-ngeative, which implies that buffer start will be within limit too. * - * The reason for rewriting being, for majorit yof cases, @len is generally + * The reason for rewriting being, for majority of cases, @len is generally * compile time constant, causing first sub-expression to be compile time * subsumed. * @@ -53,7 +53,7 @@ * */ #define __user_ok(addr, sz) (((sz) <= TASK_SIZE) && \ - (((addr)+(sz)) <= get_fs())) + ((addr) <= (get_fs() - (sz)))) #define __access_ok(addr, sz) (unlikely(__kernel_ok) || \ likely(__user_ok((addr), (sz)))) diff --git a/arch/arc/include/asm/unaligned.h b/arch/arc/include/asm/unaligned.h index 60702f3751d2..3e5f071bc00c 100644 --- a/arch/arc/include/asm/unaligned.h +++ b/arch/arc/include/asm/unaligned.h @@ -22,7 +22,8 @@ static inline int misaligned_fixup(unsigned long address, struct pt_regs *regs, struct callee_regs *cregs) { - return 0; + /* Not fixed */ + return 1; } #endif diff --git a/arch/arc/kernel/.gitignore b/arch/arc/kernel/.gitignore new file mode 100644 index 000000000000..c5f676c3c224 --- /dev/null +++ b/arch/arc/kernel/.gitignore @@ -0,0 +1 @@ +vmlinux.lds diff --git a/arch/arc/kernel/ctx_sw.c b/arch/arc/kernel/ctx_sw.c index 34410eb1a308..c14a5bea0c76 100644 --- a/arch/arc/kernel/ctx_sw.c +++ b/arch/arc/kernel/ctx_sw.c @@ -17,6 +17,8 @@ #include <asm/asm-offsets.h> #include <linux/sched.h> +#define KSP_WORD_OFF ((TASK_THREAD + THREAD_KSP) / 4) + struct task_struct *__sched __switch_to(struct task_struct *prev_task, struct task_struct *next_task) { @@ -45,7 +47,16 @@ __switch_to(struct task_struct *prev_task, struct task_struct *next_task) #endif /* set ksp of outgoing task in tsk->thread.ksp */ +#if KSP_WORD_OFF <= 255 "st.as sp, [%3, %1] \n\t" +#else + /* + * Workaround for NR_CPUS=4k + * %1 is bigger than 255 (S9 offset for st.as) + */ + "add2 r24, %3, %1 \n\t" + "st sp, [r24] \n\t" +#endif "sync \n\t" @@ -97,7 +108,7 @@ __switch_to(struct task_struct *prev_task, struct task_struct *next_task) /* FP/BLINK restore generated by gcc (standard func epilogue */ : "=r"(tmp) - : "n"((TASK_THREAD + THREAD_KSP) / 4), "r"(next), "r"(prev) + : "n"(KSP_WORD_OFF), "r"(next), "r"(prev) : "blink" ); diff --git a/arch/arc/kernel/ctx_sw_asm.S b/arch/arc/kernel/ctx_sw_asm.S index d8972345e4c2..65690e7fcc8c 100644 --- a/arch/arc/kernel/ctx_sw_asm.S +++ b/arch/arc/kernel/ctx_sw_asm.S @@ -14,6 +14,8 @@ #include <asm/asm-offsets.h> #include <asm/linkage.h> +#define KSP_WORD_OFF ((TASK_THREAD + THREAD_KSP) / 4) + ;################### Low Level Context Switch ########################## .section .sched.text,"ax",@progbits @@ -28,8 +30,13 @@ __switch_to: SAVE_CALLEE_SAVED_KERNEL /* Save the now KSP in task->thread.ksp */ - st.as sp, [r0, (TASK_THREAD + THREAD_KSP)/4] - +#if KSP_WORD_OFF <= 255 + st.as sp, [r0, KSP_WORD_OFF] +#else + /* Workaround for NR_CPUS=4k as ST.as can only take s9 offset */ + add2 r24, r0, KSP_WORD_OFF + st sp, [r24] +#endif /* * Return last task in r0 (return reg) * On ARC, Return reg = First Arg reg = r0. diff --git a/arch/arc/kernel/devtree.c b/arch/arc/kernel/devtree.c index bdee3a812052..b6dc4e21fd32 100644 --- a/arch/arc/kernel/devtree.c +++ b/arch/arc/kernel/devtree.c @@ -14,14 +14,20 @@ #include <linux/memblock.h> #include <linux/of.h> #include <linux/of_fdt.h> -#include <asm/prom.h> #include <asm/clk.h> #include <asm/mach_desc.h> -/* called from unflatten_device_tree() to bootstrap devicetree itself */ -void * __init early_init_dt_alloc_memory_arch(u64 size, u64 align) +static const void * __init arch_get_next_mach(const char *const **match) { - return __va(memblock_alloc(size, align)); + static const struct machine_desc *mdesc = __arch_info_begin; + const struct machine_desc *m = mdesc; + + if (m >= __arch_info_end) + return NULL; + + mdesc++; + *match = m->dt_compat; + return m; } /** @@ -31,93 +37,24 @@ void * __init early_init_dt_alloc_memory_arch(u64 size, u64 align) * If a dtb was passed to the kernel, then use it to choose the correct * machine_desc and to setup the system. */ -struct machine_desc * __init setup_machine_fdt(void *dt) +const struct machine_desc * __init setup_machine_fdt(void *dt) { - struct boot_param_header *devtree = dt; - struct machine_desc *mdesc = NULL, *mdesc_best = NULL; - unsigned int score, mdesc_score = ~1; + const struct machine_desc *mdesc; unsigned long dt_root; - const char *model, *compat; void *clk; - char manufacturer[16]; unsigned long len; - /* check device tree validity */ - if (be32_to_cpu(devtree->magic) != OF_DT_HEADER) + if (!early_init_dt_scan(dt)) return NULL; - initial_boot_params = devtree; - dt_root = of_get_flat_dt_root(); - - /* - * The kernel could be multi-platform enabled, thus could have many - * "baked-in" machine descriptors. Search thru all for the best - * "compatible" string match. - */ - for_each_machine_desc(mdesc) { - score = of_flat_dt_match(dt_root, mdesc->dt_compat); - if (score > 0 && score < mdesc_score) { - mdesc_best = mdesc; - mdesc_score = score; - } - } - if (!mdesc_best) { - const char *prop; - long size; - - pr_err("\n unrecognized device tree list:\n[ "); - - prop = of_get_flat_dt_prop(dt_root, "compatible", &size); - if (prop) { - while (size > 0) { - printk("'%s' ", prop); - size -= strlen(prop) + 1; - prop += strlen(prop) + 1; - } - } - printk("]\n\n"); - + mdesc = of_flat_dt_match_machine(NULL, arch_get_next_mach); + if (!mdesc) machine_halt(); - } - - /* compat = "<manufacturer>,<model>" */ - compat = mdesc_best->dt_compat[0]; - - model = strchr(compat, ','); - if (model) - model++; - - strlcpy(manufacturer, compat, model ? model - compat : strlen(compat)); - - pr_info("Board \"%s\" from %s (Manufacturer)\n", model, manufacturer); - - /* Retrieve various information from the /chosen node */ - of_scan_flat_dt(early_init_dt_scan_chosen, boot_command_line); - - /* Initialize {size,address}-cells info */ - of_scan_flat_dt(early_init_dt_scan_root, NULL); - - /* Setup memory, calling early_init_dt_add_memory_arch */ - of_scan_flat_dt(early_init_dt_scan_memory, NULL); + dt_root = of_get_flat_dt_root(); clk = of_get_flat_dt_prop(dt_root, "clock-frequency", &len); if (clk) arc_set_core_freq(of_read_ulong(clk, len/4)); - return mdesc_best; -} - -/* - * Copy the flattened DT out of .init since unflattening doesn't copy strings - * and the normal DT APIs refs them from orig flat DT - */ -void __init copy_devtree(void) -{ - void *alloc = early_init_dt_alloc_memory_arch( - be32_to_cpu(initial_boot_params->totalsize), 64); - if (alloc) { - memcpy(alloc, initial_boot_params, - be32_to_cpu(initial_boot_params->totalsize)); - initial_boot_params = alloc; - } + return mdesc; } diff --git a/arch/arc/kernel/entry.S b/arch/arc/kernel/entry.S index 1d7165156e17..47d09d07f093 100644 --- a/arch/arc/kernel/entry.S +++ b/arch/arc/kernel/entry.S @@ -250,6 +250,14 @@ ARC_ENTRY handle_interrupt_level1 lr r0, [icause1] and r0, r0, 0x1f +#ifdef CONFIG_TRACE_IRQFLAGS + ; icause1 needs to be read early, before calling tracing, which + ; can clobber scratch regs, hence use of stack to stash it + push r0 + TRACE_ASM_IRQ_DISABLE + pop r0 +#endif + bl.d @arch_do_IRQ mov r1, sp @@ -267,12 +275,7 @@ ARC_EXIT handle_interrupt_level1 ARC_ENTRY instr_service - EXCPN_PROLOG_FREEUP_REG r9 - - lr r9, [erstatus] - - SWITCH_TO_KERNEL_STK - SAVE_ALL_SYS + EXCEPTION_PROLOGUE lr r0, [efa] mov r1, sp @@ -289,15 +292,13 @@ ARC_EXIT instr_service ARC_ENTRY mem_service - EXCPN_PROLOG_FREEUP_REG r9 - - lr r9, [erstatus] - - SWITCH_TO_KERNEL_STK - SAVE_ALL_SYS + EXCEPTION_PROLOGUE lr r0, [efa] mov r1, sp + + FAKE_RET_FROM_EXCPN r9 + bl do_memory_error b ret_from_exception ARC_EXIT mem_service @@ -308,11 +309,7 @@ ARC_EXIT mem_service ARC_ENTRY EV_MachineCheck - EXCPN_PROLOG_FREEUP_REG r9 - lr r9, [erstatus] - - SWITCH_TO_KERNEL_STK - SAVE_ALL_SYS + EXCEPTION_PROLOGUE lr r2, [ecr] lr r0, [efa] @@ -342,21 +339,15 @@ ARC_EXIT EV_MachineCheck ARC_ENTRY EV_TLBProtV - EXCPN_PROLOG_FREEUP_REG r9 - - ;Which mode (user/kernel) was the system in when Exception occured - lr r9, [erstatus] - - SWITCH_TO_KERNEL_STK - SAVE_ALL_SYS + EXCEPTION_PROLOGUE ;---------(3) Save some more regs----------------- ; vineetg: Mar 6th: Random Seg Fault issue #1 ; ecr and efa were not saved in case an Intr sneaks in ; after fake rtie - ; + lr r2, [ecr] - lr r1, [efa] ; Faulting Data address + lr r0, [efa] ; Faulting Data address ; --------(4) Return from CPU Exception Mode --------- ; Fake a rtie, but rtie to next label @@ -365,6 +356,8 @@ ARC_ENTRY EV_TLBProtV FAKE_RET_FROM_EXCPN r9 + mov r1, sp + ;------ (5) Type of Protection Violation? ---------- ; ; ProtV Hardware Exception is triggered for Access Faults of 2 types @@ -375,16 +368,12 @@ ARC_ENTRY EV_TLBProtV bbit1 r2, ECR_C_BIT_PROTV_MISALIG_DATA, 4f ;========= (6a) Access Violation Processing ======== - mov r0, sp ; pt_regs bl do_page_fault b ret_from_exception ;========== (6b) Non aligned access ============ 4: - mov r0, r1 - mov r1, sp ; pt_regs -#ifdef CONFIG_ARC_MISALIGN_ACCESS SAVE_CALLEE_SAVED_USER mov r2, sp ; callee_regs @@ -393,9 +382,6 @@ ARC_ENTRY EV_TLBProtV ; TBD: optimize - do this only if a callee reg was involved ; either a dst of emulated LD/ST or src with address-writeback RESTORE_CALLEE_SAVED_USER -#else - bl do_misaligned_error -#endif b ret_from_exception @@ -406,12 +392,7 @@ ARC_EXIT EV_TLBProtV ; --------------------------------------------- ARC_ENTRY EV_PrivilegeV - EXCPN_PROLOG_FREEUP_REG r9 - - lr r9, [erstatus] - - SWITCH_TO_KERNEL_STK - SAVE_ALL_SYS + EXCEPTION_PROLOGUE lr r0, [efa] mov r1, sp @@ -427,14 +408,13 @@ ARC_EXIT EV_PrivilegeV ; --------------------------------------------- ARC_ENTRY EV_Extension - EXCPN_PROLOG_FREEUP_REG r9 - lr r9, [erstatus] - - SWITCH_TO_KERNEL_STK - SAVE_ALL_SYS + EXCEPTION_PROLOGUE lr r0, [efa] mov r1, sp + + FAKE_RET_FROM_EXCPN r9 + bl do_extension_fault b ret_from_exception ARC_EXIT EV_Extension @@ -526,14 +506,7 @@ trap_with_param: ARC_ENTRY EV_Trap - ; Need at least 1 reg to code the early exception prolog - EXCPN_PROLOG_FREEUP_REG r9 - - ;Which mode (user/kernel) was the system in when intr occured - lr r9, [erstatus] - - SWITCH_TO_KERNEL_STK - SAVE_ALL_SYS + EXCEPTION_PROLOGUE ;------- (4) What caused the Trap -------------- lr r12, [ecr] @@ -605,6 +578,7 @@ resume_user_mode_begin: ; --- (Slow Path #2) pending signal --- mov r0, sp ; pt_regs for arg to do_signal()/do_notify_resume() + GET_CURR_THR_INFO_FLAGS r9 bbit0 r9, TIF_SIGPENDING, .Lchk_notify_resume ; Normal Trap/IRQ entry only saves Scratch (caller-saved) regs @@ -642,6 +616,9 @@ resume_kernel_mode: #ifdef CONFIG_PREEMPT + ; This is a must for preempt_schedule_irq() + IRQ_DISABLE r9 + ; Can't preempt if preemption disabled GET_CURR_THR_INFO_FROM_SP r10 ld r8, [r10, THREAD_INFO_PREEMPT_COUNT] @@ -651,8 +628,6 @@ resume_kernel_mode: ld r9, [r10, THREAD_INFO_FLAGS] bbit0 r9, TIF_NEED_RESCHED, restore_regs - IRQ_DISABLE r9 - ; Invoke PREEMPTION bl preempt_schedule_irq @@ -665,12 +640,13 @@ resume_kernel_mode: ; ; Restore the saved sys context (common exit-path for EXCPN/IRQ/Trap) ; IRQ shd definitely not happen between now and rtie +; All 2 entry points to here already disable interrupts restore_regs : - ; Disable Interrupts while restoring reg-file back - ; XXX can this be optimised out - IRQ_DISABLE_SAVE r9, r10 ;@r10 has prisitine (pre-disable) copy + TRACE_ASM_IRQ_ENABLE + + lr r10, [status32] ; Restore REG File. In case multiple Events outstanding, ; use the same priorty as rtie: EXCPN, L2 IRQ, L1 IRQ, None diff --git a/arch/arc/kernel/head.S b/arch/arc/kernel/head.S index 2a913f85a747..2c878e964a64 100644 --- a/arch/arc/kernel/head.S +++ b/arch/arc/kernel/head.S @@ -34,6 +34,9 @@ stext: ; IDENTITY Reg [ 3 2 1 0 ] ; (cpu-id) ^^^ => Zero for UP ARC700 ; => #Core-ID if SMP (Master 0) + ; Note that non-boot CPUs might not land here if halt-on-reset and + ; instead breath life from @first_lines_of_secondary, but we still + ; need to make sure only boot cpu takes this path. GET_CPU_ID r5 cmp r5, 0 jnz arc_platform_smp_wait_to_boot @@ -92,12 +95,14 @@ stext: ;---------------------------------------------------------------- ; First lines of code run by secondary before jumping to 'C' ;---------------------------------------------------------------- - .section .init.text, "ax",@progbits + .section .text, "ax",@progbits .type first_lines_of_secondary, @function .globl first_lines_of_secondary first_lines_of_secondary: + sr @_int_vec_base_lds, [AUX_INTR_VEC_BASE] + ; setup per-cpu idle task as "current" on this CPU ld r0, [@secondary_idle_tsk] SET_CURR_TASK_ON_CPU r0, r1 diff --git a/arch/arc/kernel/irq.c b/arch/arc/kernel/irq.c index 305b3f866aa7..a4b141ee9a6a 100644 --- a/arch/arc/kernel/irq.c +++ b/arch/arc/kernel/irq.c @@ -24,7 +24,6 @@ * -Needed for each CPU (hence not foldable into init_IRQ) * * what it does ? - * -setup Vector Table Base Reg - in case Linux not linked at 0x8000_0000 * -Disable all IRQs (on CPU side) * -Optionally, setup the High priority Interrupts as Level 2 IRQs */ @@ -40,10 +39,14 @@ void arc_init_IRQ(void) level_mask |= IS_ENABLED(CONFIG_ARC_IRQ5_LV2) << 5; level_mask |= IS_ENABLED(CONFIG_ARC_IRQ6_LV2) << 6; - if (level_mask) { + /* + * Write to register, even if no LV2 IRQs configured to reset it + * in case bootloader had mucked with it + */ + write_aux_reg(AUX_IRQ_LEV, level_mask); + + if (level_mask) pr_info("Level-2 interrupts bitset %x\n", level_mask); - write_aux_reg(AUX_IRQ_LEV, level_mask); - } } /* @@ -147,7 +150,7 @@ void arch_do_IRQ(unsigned int irq, struct pt_regs *regs) set_irq_regs(old_regs); } -int __init get_hw_config_num_irq(void) +int get_hw_config_num_irq(void) { uint32_t val = read_aux_reg(ARC_REG_VECBASE_BCR); diff --git a/arch/arc/kernel/kgdb.c b/arch/arc/kernel/kgdb.c index a7698fb14818..a2ff5c5d1450 100644 --- a/arch/arc/kernel/kgdb.c +++ b/arch/arc/kernel/kgdb.c @@ -196,6 +196,18 @@ void kgdb_arch_set_pc(struct pt_regs *regs, unsigned long ip) instruction_pointer(regs) = ip; } +static void kgdb_call_nmi_hook(void *ignored) +{ + kgdb_nmicallback(raw_smp_processor_id(), NULL); +} + +void kgdb_roundup_cpus(unsigned long flags) +{ + local_irq_enable(); + smp_call_function(kgdb_call_nmi_hook, NULL, 0); + local_irq_disable(); +} + struct kgdb_arch arch_kgdb_ops = { /* breakpoint instruction: TRAP_S 0x3 */ #ifdef CONFIG_CPU_BIG_ENDIAN diff --git a/arch/arc/kernel/kprobes.c b/arch/arc/kernel/kprobes.c index 7446c8dcb915..42b05046fad9 100644 --- a/arch/arc/kernel/kprobes.c +++ b/arch/arc/kernel/kprobes.c @@ -87,13 +87,13 @@ static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) { - __get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp; + __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); kcb->kprobe_status = kcb->prev_kprobe.status; } static inline void __kprobes set_current_kprobe(struct kprobe *p) { - __get_cpu_var(current_kprobe) = p; + __this_cpu_write(current_kprobe, p); } static void __kprobes resume_execution(struct kprobe *p, unsigned long addr, @@ -237,7 +237,7 @@ int __kprobes arc_kprobe_handler(unsigned long addr, struct pt_regs *regs) return 1; } else if (kprobe_running()) { - p = __get_cpu_var(current_kprobe); + p = __this_cpu_read(current_kprobe); if (p->break_handler && p->break_handler(p, regs)) { setup_singlestep(p, regs); kcb->kprobe_status = KPROBE_HIT_SS; diff --git a/arch/arc/kernel/ptrace.c b/arch/arc/kernel/ptrace.c index 333238564b67..5d76706139dd 100644 --- a/arch/arc/kernel/ptrace.c +++ b/arch/arc/kernel/ptrace.c @@ -102,7 +102,7 @@ static int genregs_set(struct task_struct *target, REG_IGNORE_ONE(pad2); REG_IN_CHUNK(callee, efa, cregs); /* callee_regs[r25..r13] */ REG_IGNORE_ONE(efa); /* efa update invalid */ - REG_IN_ONE(stop_pc, &ptregs->ret); /* stop_pc: PC update */ + REG_IGNORE_ONE(stop_pc); /* PC updated via @ret */ return ret; } diff --git a/arch/arc/kernel/reset.c b/arch/arc/kernel/reset.c index e227a2b1c943..2768fa1e39b9 100644 --- a/arch/arc/kernel/reset.c +++ b/arch/arc/kernel/reset.c @@ -31,3 +31,4 @@ void machine_power_off(void) } void (*pm_power_off) (void) = NULL; +EXPORT_SYMBOL(pm_power_off); diff --git a/arch/arc/kernel/setup.c b/arch/arc/kernel/setup.c index 6b083454d039..643eae4436e0 100644 --- a/arch/arc/kernel/setup.c +++ b/arch/arc/kernel/setup.c @@ -21,7 +21,6 @@ #include <asm/setup.h> #include <asm/page.h> #include <asm/irq.h> -#include <asm/prom.h> #include <asm/unwind.h> #include <asm/clk.h> #include <asm/mach_desc.h> @@ -31,14 +30,13 @@ int running_on_hw = 1; /* vs. on ISS */ char __initdata command_line[COMMAND_LINE_SIZE]; -struct machine_desc *machine_desc; +const struct machine_desc *machine_desc; struct task_struct *_current_task[NR_CPUS]; /* For stack switching */ struct cpuinfo_arc cpuinfo_arc700[NR_CPUS]; - -void read_arc_build_cfg_regs(void) +static void read_arc_build_cfg_regs(void) { struct bcr_perip uncached_space; struct cpuinfo_arc *cpu = &cpuinfo_arc700[smp_processor_id()]; @@ -47,10 +45,7 @@ void read_arc_build_cfg_regs(void) READ_BCR(AUX_IDENTITY, cpu->core); cpu->timers = read_aux_reg(ARC_REG_TIMERS_BCR); - cpu->vec_base = read_aux_reg(AUX_INTR_VEC_BASE); - if (cpu->vec_base == 0) - cpu->vec_base = (unsigned int)_int_vec_base_lds; READ_BCR(ARC_REG_D_UNCACH_BCR, uncached_space); cpu->uncached_base = uncached_space.start << 24; @@ -109,7 +104,7 @@ static const struct cpuinfo_data arc_cpu_tbl[] = { { {0x00, NULL } } }; -char *arc_cpu_mumbojumbo(int cpu_id, char *buf, int len) +static char *arc_cpu_mumbojumbo(int cpu_id, char *buf, int len) { int n = 0; struct cpuinfo_arc *cpu = &cpuinfo_arc700[cpu_id]; @@ -174,7 +169,7 @@ static const struct id_to_str mac_mul_nm[] = { {0x6, "Dual 16x16 and 32x16"} }; -char *arc_extn_mumbojumbo(int cpu_id, char *buf, int len) +static char *arc_extn_mumbojumbo(int cpu_id, char *buf, int len) { int n = 0; struct cpuinfo_arc *cpu = &cpuinfo_arc700[cpu_id]; @@ -237,7 +232,7 @@ char *arc_extn_mumbojumbo(int cpu_id, char *buf, int len) return buf; } -void arc_chk_ccms(void) +static void arc_chk_ccms(void) { #if defined(CONFIG_ARC_HAS_DCCM) || defined(CONFIG_ARC_HAS_ICCM) struct cpuinfo_arc *cpu = &cpuinfo_arc700[smp_processor_id()]; @@ -272,7 +267,7 @@ void arc_chk_ccms(void) * hardware has dedicated regs which need to be saved/restored on ctx-sw * (Single Precision uses core regs), thus kernel is kind of oblivious to it */ -void arc_chk_fpu(void) +static void arc_chk_fpu(void) { struct cpuinfo_arc *cpu = &cpuinfo_arc700[smp_processor_id()]; @@ -349,16 +344,13 @@ void __init setup_arch(char **cmdline_p) setup_arch_memory(); /* copy flat DT out of .init and then unflatten it */ - copy_devtree(); - unflatten_device_tree(); + unflatten_and_copy_device_tree(); /* Can be issue if someone passes cmd line arg "ro" * But that is unlikely so keeping it as it is */ root_mountflags &= ~MS_RDONLY; - console_verbose(); - #if defined(CONFIG_VT) && defined(CONFIG_DUMMY_CONSOLE) conswitchp = &dummy_con; #endif diff --git a/arch/arc/kernel/signal.c b/arch/arc/kernel/signal.c index ee6ef2f60a28..7e95e1a86510 100644 --- a/arch/arc/kernel/signal.c +++ b/arch/arc/kernel/signal.c @@ -101,7 +101,6 @@ SYSCALL_DEFINE0(rt_sigreturn) { struct rt_sigframe __user *sf; unsigned int magic; - int err; struct pt_regs *regs = current_pt_regs(); /* Always make any pending restarted system calls return -EINTR */ @@ -119,15 +118,16 @@ SYSCALL_DEFINE0(rt_sigreturn) if (!access_ok(VERIFY_READ, sf, sizeof(*sf))) goto badframe; - err = restore_usr_regs(regs, sf); - err |= __get_user(magic, &sf->sigret_magic); - if (err) + if (__get_user(magic, &sf->sigret_magic)) goto badframe; if (unlikely(is_do_ss_needed(magic))) if (restore_altstack(&sf->uc.uc_stack)) goto badframe; + if (restore_usr_regs(regs, sf)) + goto badframe; + /* Don't restart from sigreturn */ syscall_wont_restart(regs); @@ -191,6 +191,15 @@ setup_rt_frame(int signo, struct k_sigaction *ka, siginfo_t *info, return 1; /* + * w/o SA_SIGINFO, struct ucontext is partially populated (only + * uc_mcontext/uc_sigmask) for kernel's normal user state preservation + * during signal handler execution. This works for SA_SIGINFO as well + * although the semantics are now overloaded (the same reg state can be + * inspected by userland: but are they allowed to fiddle with it ? + */ + err |= stash_usr_regs(sf, regs, set); + + /* * SA_SIGINFO requires 3 args to signal handler: * #1: sig-no (common to any handler) * #2: struct siginfo @@ -213,14 +222,6 @@ setup_rt_frame(int signo, struct k_sigaction *ka, siginfo_t *info, magic = MAGIC_SIGALTSTK; } - /* - * w/o SA_SIGINFO, struct ucontext is partially populated (only - * uc_mcontext/uc_sigmask) for kernel's normal user state preservation - * during signal handler execution. This works for SA_SIGINFO as well - * although the semantics are now overloaded (the same reg state can be - * inspected by userland: but are they allowed to fiddle with it ? - */ - err |= stash_usr_regs(sf, regs, set); err |= __put_user(magic, &sf->sigret_magic); if (err) return err; diff --git a/arch/arc/kernel/smp.c b/arch/arc/kernel/smp.c index bca3052c956d..c2f9ebbc38f6 100644 --- a/arch/arc/kernel/smp.c +++ b/arch/arc/kernel/smp.c @@ -95,7 +95,7 @@ void __init smp_cpus_done(unsigned int max_cpus) * If it turns out to be elaborate, it's better to code it in assembly * */ -void __attribute__((weak)) arc_platform_smp_wait_to_boot(int cpu) +void __weak arc_platform_smp_wait_to_boot(int cpu) { /* * As a hack for debugging - since debugger will single-step over the @@ -128,6 +128,7 @@ void start_kernel_secondary(void) atomic_inc(&mm->mm_users); atomic_inc(&mm->mm_count); current->active_mm = mm; + cpumask_set_cpu(cpu, mm_cpumask(mm)); notify_cpu_starting(cpu); set_cpu_online(cpu, true); @@ -210,7 +211,6 @@ enum ipi_msg_type { IPI_NOP = 0, IPI_RESCHEDULE = 1, IPI_CALL_FUNC, - IPI_CALL_FUNC_SINGLE, IPI_CPU_STOP }; @@ -254,7 +254,7 @@ void smp_send_stop(void) void arch_send_call_function_single_ipi(int cpu) { - ipi_send_msg(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE); + ipi_send_msg(cpumask_of(cpu), IPI_CALL_FUNC); } void arch_send_call_function_ipi_mask(const struct cpumask *mask) @@ -286,10 +286,6 @@ static inline void __do_IPI(unsigned long *ops, struct ipi_data *ipi, int cpu) generic_smp_call_function_interrupt(); break; - case IPI_CALL_FUNC_SINGLE: - generic_smp_call_function_single_interrupt(); - break; - case IPI_CPU_STOP: ipi_cpu_stop(cpu); break; diff --git a/arch/arc/kernel/stacktrace.c b/arch/arc/kernel/stacktrace.c index f8b7d880304d..9ce47cfe2303 100644 --- a/arch/arc/kernel/stacktrace.c +++ b/arch/arc/kernel/stacktrace.c @@ -237,11 +237,14 @@ unsigned int get_wchan(struct task_struct *tsk) */ void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) { + /* Assumes @tsk is sleeping so unwinds from __switch_to */ arc_unwind_core(tsk, NULL, __collect_all_but_sched, trace); } void save_stack_trace(struct stack_trace *trace) { - arc_unwind_core(current, NULL, __collect_all, trace); + /* Pass NULL for task so it unwinds the current call frame */ + arc_unwind_core(NULL, NULL, __collect_all, trace); } +EXPORT_SYMBOL_GPL(save_stack_trace); #endif diff --git a/arch/arc/kernel/time.c b/arch/arc/kernel/time.c index 0e51e69cf30d..e5f3a837fb35 100644 --- a/arch/arc/kernel/time.c +++ b/arch/arc/kernel/time.c @@ -63,9 +63,10 @@ int arc_counter_setup(void) { - /* RTSC insn taps into cpu clk, needs no setup */ - - /* For SMP, only allowed if cross-core-sync, hence usable as cs */ + /* + * For SMP this needs to be 0. However Kconfig glue doesn't + * enable this option for SMP configs + */ return 1; } @@ -206,7 +207,7 @@ static DEFINE_PER_CPU(struct clock_event_device, arc_clockevent_device) = { static irqreturn_t timer_irq_handler(int irq, void *dev_id) { - struct clock_event_device *clk = &__get_cpu_var(arc_clockevent_device); + struct clock_event_device *clk = this_cpu_ptr(&arc_clockevent_device); arc_timer_event_ack(clk->mode == CLOCK_EVT_MODE_PERIODIC); clk->event_handler(clk); @@ -223,16 +224,13 @@ static struct irqaction arc_timer_irq = { * Setup the local event timer for @cpu * N.B. weak so that some exotic ARC SoCs can completely override it */ -void __attribute__((weak)) arc_local_timer_setup(unsigned int cpu) +void __weak arc_local_timer_setup(unsigned int cpu) { struct clock_event_device *clk = &per_cpu(arc_clockevent_device, cpu); - clockevents_calc_mult_shift(clk, arc_get_core_freq(), 5); - - clk->max_delta_ns = clockevent_delta2ns(ARC_TIMER_MAX, clk); clk->cpumask = cpumask_of(cpu); - - clockevents_register_device(clk); + clockevents_config_and_register(clk, arc_get_core_freq(), + 0, ARC_TIMER_MAX); /* * setup the per-cpu timer IRQ handler - for all cpus diff --git a/arch/arc/kernel/traps.c b/arch/arc/kernel/traps.c index e21692d2fdab..3eadfdabc322 100644 --- a/arch/arc/kernel/traps.c +++ b/arch/arc/kernel/traps.c @@ -84,19 +84,18 @@ DO_ERROR_INFO(SIGBUS, "Invalid Mem Access", do_memory_error, BUS_ADRERR) DO_ERROR_INFO(SIGTRAP, "Breakpoint Set", trap_is_brkpt, TRAP_BRKPT) DO_ERROR_INFO(SIGBUS, "Misaligned Access", do_misaligned_error, BUS_ADRALN) -#ifdef CONFIG_ARC_MISALIGN_ACCESS /* * Entry Point for Misaligned Data access Exception, for emulating in software */ int do_misaligned_access(unsigned long address, struct pt_regs *regs, struct callee_regs *cregs) { + /* If emulation not enabled, or failed, kill the task */ if (misaligned_fixup(address, regs, cregs) != 0) return do_misaligned_error(address, regs); return 0; } -#endif /* * Entry point for miscll errors such as Nested Exceptions diff --git a/arch/arc/kernel/unaligned.c b/arch/arc/kernel/unaligned.c index c0f832f595d3..7ff5b5c183bb 100644 --- a/arch/arc/kernel/unaligned.c +++ b/arch/arc/kernel/unaligned.c @@ -16,6 +16,16 @@ #include <linux/uaccess.h> #include <asm/disasm.h> +#ifdef CONFIG_CPU_BIG_ENDIAN +#define BE 1 +#define FIRST_BYTE_16 "swap %1, %1\n swape %1, %1\n" +#define FIRST_BYTE_32 "swape %1, %1\n" +#else +#define BE 0 +#define FIRST_BYTE_16 +#define FIRST_BYTE_32 +#endif + #define __get8_unaligned_check(val, addr, err) \ __asm__( \ "1: ldb.ab %1, [%2, 1]\n" \ @@ -36,9 +46,9 @@ do { \ unsigned int err = 0, v, a = addr; \ __get8_unaligned_check(v, a, err); \ - val = v ; \ + val = v << ((BE) ? 8 : 0); \ __get8_unaligned_check(v, a, err); \ - val |= v << 8; \ + val |= v << ((BE) ? 0 : 8); \ if (err) \ goto fault; \ } while (0) @@ -47,13 +57,13 @@ do { \ unsigned int err = 0, v, a = addr; \ __get8_unaligned_check(v, a, err); \ - val = v << 0; \ + val = v << ((BE) ? 24 : 0); \ __get8_unaligned_check(v, a, err); \ - val |= v << 8; \ + val |= v << ((BE) ? 16 : 8); \ __get8_unaligned_check(v, a, err); \ - val |= v << 16; \ + val |= v << ((BE) ? 8 : 16); \ __get8_unaligned_check(v, a, err); \ - val |= v << 24; \ + val |= v << ((BE) ? 0 : 24); \ if (err) \ goto fault; \ } while (0) @@ -63,6 +73,7 @@ unsigned int err = 0, v = val, a = addr;\ \ __asm__( \ + FIRST_BYTE_16 \ "1: stb.ab %1, [%2, 1]\n" \ " lsr %1, %1, 8\n" \ "2: stb %1, [%2]\n" \ @@ -87,8 +98,9 @@ #define put32_unaligned_check(val, addr) \ do { \ unsigned int err = 0, v = val, a = addr;\ - __asm__( \ \ + __asm__( \ + FIRST_BYTE_32 \ "1: stb.ab %1, [%2, 1]\n" \ " lsr %1, %1, 8\n" \ "2: stb.ab %1, [%2, 1]\n" \ @@ -233,6 +245,12 @@ int misaligned_fixup(unsigned long address, struct pt_regs *regs, regs->status32 &= ~STATUS_DE_MASK; } else { regs->ret += state.instr_len; + + /* handle zero-overhead-loop */ + if ((regs->ret == regs->lp_end) && (regs->lp_count)) { + regs->ret = regs->lp_start; + regs->lp_count--; + } } return 0; diff --git a/arch/arc/mm/cache_arc700.c b/arch/arc/mm/cache_arc700.c index f415d851b765..6b58c1de7577 100644 --- a/arch/arc/mm/cache_arc700.c +++ b/arch/arc/mm/cache_arc700.c @@ -182,7 +182,7 @@ void arc_cache_init(void) #ifdef CONFIG_ARC_HAS_ICACHE /* 1. Confirm some of I-cache params which Linux assumes */ - if (ic->line_len != ARC_ICACHE_LINE_LEN) + if (ic->line_len != L1_CACHE_BYTES) panic("Cache H/W doesn't match kernel Config"); if (ic->ver != CONFIG_ARC_MMU_VER) @@ -205,7 +205,7 @@ chk_dc: return; #ifdef CONFIG_ARC_HAS_DCACHE - if (dc->line_len != ARC_DCACHE_LINE_LEN) + if (dc->line_len != L1_CACHE_BYTES) panic("Cache H/W doesn't match kernel Config"); /* check for D-Cache aliasing */ @@ -240,6 +240,67 @@ chk_dc: #define OP_INV 0x1 #define OP_FLUSH 0x2 #define OP_FLUSH_N_INV 0x3 +#define OP_INV_IC 0x4 + +/* + * Common Helper for Line Operations on {I,D}-Cache + */ +static inline void __cache_line_loop(unsigned long paddr, unsigned long vaddr, + unsigned long sz, const int cacheop) +{ + unsigned int aux_cmd, aux_tag; + int num_lines; + const int full_page_op = __builtin_constant_p(sz) && sz == PAGE_SIZE; + + if (cacheop == OP_INV_IC) { + aux_cmd = ARC_REG_IC_IVIL; + aux_tag = ARC_REG_IC_PTAG; + } + else { + /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ + aux_cmd = cacheop & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; + aux_tag = ARC_REG_DC_PTAG; + } + + /* Ensure we properly floor/ceil the non-line aligned/sized requests + * and have @paddr - aligned to cache line and integral @num_lines. + * This however can be avoided for page sized since: + * -@paddr will be cache-line aligned already (being page aligned) + * -@sz will be integral multiple of line size (being page sized). + */ + if (!full_page_op) { + sz += paddr & ~CACHE_LINE_MASK; + paddr &= CACHE_LINE_MASK; + vaddr &= CACHE_LINE_MASK; + } + + num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); + +#if (CONFIG_ARC_MMU_VER <= 2) + /* MMUv2 and before: paddr contains stuffed vaddrs bits */ + paddr |= (vaddr >> PAGE_SHIFT) & 0x1F; +#else + /* if V-P const for loop, PTAG can be written once outside loop */ + if (full_page_op) + write_aux_reg(ARC_REG_DC_PTAG, paddr); +#endif + + while (num_lines-- > 0) { +#if (CONFIG_ARC_MMU_VER > 2) + /* MMUv3, cache ops require paddr seperately */ + if (!full_page_op) { + write_aux_reg(aux_tag, paddr); + paddr += L1_CACHE_BYTES; + } + + write_aux_reg(aux_cmd, vaddr); + vaddr += L1_CACHE_BYTES; +#else + write_aux_reg(aux, paddr); + paddr += L1_CACHE_BYTES; +#endif + } +} #ifdef CONFIG_ARC_HAS_DCACHE @@ -289,53 +350,6 @@ static inline void __dc_entire_op(const int cacheop) write_aux_reg(ARC_REG_DC_CTRL, tmp & ~DC_CTRL_INV_MODE_FLUSH); } -/* - * Per Line Operation on D-Cache - * Doesn't deal with type-of-op/IRQ-disabling/waiting-for-flush-to-complete - * It's sole purpose is to help gcc generate ZOL - * (aliasing VIPT dcache flushing needs both vaddr and paddr) - */ -static inline void __dc_line_loop(unsigned long paddr, unsigned long vaddr, - unsigned long sz, const int aux_reg) -{ - int num_lines; - - /* Ensure we properly floor/ceil the non-line aligned/sized requests - * and have @paddr - aligned to cache line and integral @num_lines. - * This however can be avoided for page sized since: - * -@paddr will be cache-line aligned already (being page aligned) - * -@sz will be integral multiple of line size (being page sized). - */ - if (!(__builtin_constant_p(sz) && sz == PAGE_SIZE)) { - sz += paddr & ~DCACHE_LINE_MASK; - paddr &= DCACHE_LINE_MASK; - vaddr &= DCACHE_LINE_MASK; - } - - num_lines = DIV_ROUND_UP(sz, ARC_DCACHE_LINE_LEN); - -#if (CONFIG_ARC_MMU_VER <= 2) - paddr |= (vaddr >> PAGE_SHIFT) & 0x1F; -#endif - - while (num_lines-- > 0) { -#if (CONFIG_ARC_MMU_VER > 2) - /* - * Just as for I$, in MMU v3, D$ ops also require - * "tag" bits in DC_PTAG, "index" bits in FLDL,IVDL ops - */ - write_aux_reg(ARC_REG_DC_PTAG, paddr); - - write_aux_reg(aux_reg, vaddr); - vaddr += ARC_DCACHE_LINE_LEN; -#else - /* paddr contains stuffed vaddrs bits */ - write_aux_reg(aux_reg, paddr); -#endif - paddr += ARC_DCACHE_LINE_LEN; - } -} - /* For kernel mappings cache operation: index is same as paddr */ #define __dc_line_op_k(p, sz, op) __dc_line_op(p, p, sz, op) @@ -346,7 +360,6 @@ static inline void __dc_line_op(unsigned long paddr, unsigned long vaddr, unsigned long sz, const int cacheop) { unsigned long flags, tmp = tmp; - int aux; local_irq_save(flags); @@ -361,12 +374,7 @@ static inline void __dc_line_op(unsigned long paddr, unsigned long vaddr, write_aux_reg(ARC_REG_DC_CTRL, tmp | DC_CTRL_INV_MODE_FLUSH); } - if (cacheop & OP_INV) /* Inv / flush-n-inv use same cmd reg */ - aux = ARC_REG_DC_IVDL; - else - aux = ARC_REG_DC_FLDL; - - __dc_line_loop(paddr, vaddr, sz, aux); + __cache_line_loop(paddr, vaddr, sz, cacheop); if (cacheop & OP_FLUSH) /* flush / flush-n-inv both wait */ wait_for_flush(); @@ -438,42 +446,9 @@ static void __ic_line_inv_vaddr(unsigned long paddr, unsigned long vaddr, unsigned long sz) { unsigned long flags; - int num_lines; - - /* - * Ensure we properly floor/ceil the non-line aligned/sized requests: - * However page sized flushes can be compile time optimised. - * -@paddr will be cache-line aligned already (being page aligned) - * -@sz will be integral multiple of line size (being page sized). - */ - if (!(__builtin_constant_p(sz) && sz == PAGE_SIZE)) { - sz += paddr & ~ICACHE_LINE_MASK; - paddr &= ICACHE_LINE_MASK; - vaddr &= ICACHE_LINE_MASK; - } - - num_lines = DIV_ROUND_UP(sz, ARC_ICACHE_LINE_LEN); - -#if (CONFIG_ARC_MMU_VER <= 2) - /* bits 17:13 of vaddr go as bits 4:0 of paddr */ - paddr |= (vaddr >> PAGE_SHIFT) & 0x1F; -#endif local_irq_save(flags); - while (num_lines-- > 0) { -#if (CONFIG_ARC_MMU_VER > 2) - /* tag comes from phy addr */ - write_aux_reg(ARC_REG_IC_PTAG, paddr); - - /* index bits come from vaddr */ - write_aux_reg(ARC_REG_IC_IVIL, vaddr); - vaddr += ARC_ICACHE_LINE_LEN; -#else - /* paddr contains stuffed vaddrs bits */ - write_aux_reg(ARC_REG_IC_IVIL, paddr); -#endif - paddr += ARC_ICACHE_LINE_LEN; - } + __cache_line_loop(paddr, vaddr, sz, OP_INV_IC); local_irq_restore(flags); } @@ -622,12 +597,12 @@ void flush_icache_range(unsigned long kstart, unsigned long kend) /* * General purpose helper to make I and D cache lines consistent. * @paddr is phy addr of region - * @vaddr is typically user or kernel vaddr (vmalloc) - * Howver in one instance, flush_icache_range() by kprobe (for a breakpt in + * @vaddr is typically user vaddr (breakpoint) or kernel vaddr (vmalloc) + * However in one instance, when called by kprobe (for a breakpt in * builtin kernel code) @vaddr will be paddr only, meaning CDU operation will * use a paddr to index the cache (despite VIPT). This is fine since since a - * built-in kernel page will not have any virtual mappings (not even kernel) - * kprobe on loadable module is different as it will have kvaddr. + * builtin kernel page will not have any virtual mappings. + * kprobe on loadable module will be kernel vaddr. */ void __sync_icache_dcache(unsigned long paddr, unsigned long vaddr, int len) { diff --git a/arch/arc/mm/fault.c b/arch/arc/mm/fault.c index 0fd1f0d515ff..9c69552350c4 100644 --- a/arch/arc/mm/fault.c +++ b/arch/arc/mm/fault.c @@ -17,7 +17,7 @@ #include <asm/pgalloc.h> #include <asm/mmu.h> -static int handle_vmalloc_fault(struct mm_struct *mm, unsigned long address) +static int handle_vmalloc_fault(unsigned long address) { /* * Synchronize this task's top level page-table @@ -27,7 +27,7 @@ static int handle_vmalloc_fault(struct mm_struct *mm, unsigned long address) pud_t *pud, *pud_k; pmd_t *pmd, *pmd_k; - pgd = pgd_offset_fast(mm, address); + pgd = pgd_offset_fast(current->active_mm, address); pgd_k = pgd_offset_k(address); if (!pgd_present(*pgd_k)) @@ -52,7 +52,7 @@ bad_area: return 1; } -void do_page_fault(struct pt_regs *regs, unsigned long address) +void do_page_fault(unsigned long address, struct pt_regs *regs) { struct vm_area_struct *vma = NULL; struct task_struct *tsk = current; @@ -60,8 +60,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long address) siginfo_t info; int fault, ret; int write = regs->ecr_cause & ECR_C_PROTV_STORE; /* ST/EX */ - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE | - (write ? FAULT_FLAG_WRITE : 0); + unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; /* * We fault-in kernel-space virtual memory on-demand. The @@ -73,7 +72,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long address) * nothing more. */ if (address >= VMALLOC_START && address <= VMALLOC_END) { - ret = handle_vmalloc_fault(mm, address); + ret = handle_vmalloc_fault(address); if (unlikely(ret)) goto bad_area_nosemaphore; else @@ -89,6 +88,8 @@ void do_page_fault(struct pt_regs *regs, unsigned long address) if (in_atomic() || !mm) goto no_context; + if (user_mode(regs)) + flags |= FAULT_FLAG_USER; retry: down_read(&mm->mmap_sem); vma = find_vma(mm, address); @@ -117,12 +118,12 @@ good_area: if (write) { if (!(vma->vm_flags & VM_WRITE)) goto bad_area; + flags |= FAULT_FLAG_WRITE; } else { if (!(vma->vm_flags & (VM_READ | VM_EXEC))) goto bad_area; } -survive: /* * If for any reason at all we couldn't handle the fault, * make sure we exit gracefully rather than endlessly redo @@ -201,10 +202,6 @@ no_context: die("Oops", regs, address); out_of_memory: - if (is_global_init(tsk)) { - yield(); - goto survive; - } up_read(&mm->mmap_sem); if (user_mode(regs)) { diff --git a/arch/arc/mm/init.c b/arch/arc/mm/init.c index a08ce7185423..55e0a85bea78 100644 --- a/arch/arc/mm/init.c +++ b/arch/arc/mm/init.c @@ -125,11 +125,3 @@ void __init free_initrd_mem(unsigned long start, unsigned long end) free_reserved_area((void *)start, (void *)end, -1, "initrd"); } #endif - -#ifdef CONFIG_OF_FLATTREE -void __init early_init_dt_setup_initrd_arch(unsigned long start, - unsigned long end) -{ - pr_err("%s(%lx, %lx)\n", __func__, start, end); -} -#endif /* CONFIG_OF_FLATTREE */ diff --git a/arch/arc/mm/tlb.c b/arch/arc/mm/tlb.c index 7957dc4e4d4a..e1acf0ce5647 100644 --- a/arch/arc/mm/tlb.c +++ b/arch/arc/mm/tlb.c @@ -52,6 +52,7 @@ */ #include <linux/module.h> +#include <linux/bug.h> #include <asm/arcregs.h> #include <asm/setup.h> #include <asm/mmu_context.h> @@ -99,48 +100,45 @@ /* A copy of the ASID from the PID reg is kept in asid_cache */ -int asid_cache = FIRST_ASID; - -/* ASID to mm struct mapping. We have one extra entry corresponding to - * NO_ASID to save us a compare when clearing the mm entry for old asid - * see get_new_mmu_context (asm-arc/mmu_context.h) - */ -struct mm_struct *asid_mm_map[NUM_ASID + 1]; +DEFINE_PER_CPU(unsigned int, asid_cache) = MM_CTXT_FIRST_CYCLE; /* * Utility Routine to erase a J-TLB entry - * The procedure is to look it up in the MMU. If found, ERASE it by - * issuing a TlbWrite CMD with PD0 = PD1 = 0 + * Caller needs to setup Index Reg (manually or via getIndex) */ - -static void __tlb_entry_erase(void) +static inline void __tlb_entry_erase(void) { write_aux_reg(ARC_REG_TLBPD1, 0); write_aux_reg(ARC_REG_TLBPD0, 0); write_aux_reg(ARC_REG_TLBCOMMAND, TLBWrite); } -static void tlb_entry_erase(unsigned int vaddr_n_asid) +static inline unsigned int tlb_entry_lkup(unsigned long vaddr_n_asid) { unsigned int idx; - /* Locate the TLB entry for this vaddr + ASID */ write_aux_reg(ARC_REG_TLBPD0, vaddr_n_asid); + write_aux_reg(ARC_REG_TLBCOMMAND, TLBProbe); idx = read_aux_reg(ARC_REG_TLBINDEX); + return idx; +} + +static void tlb_entry_erase(unsigned int vaddr_n_asid) +{ + unsigned int idx; + + /* Locate the TLB entry for this vaddr + ASID */ + idx = tlb_entry_lkup(vaddr_n_asid); + /* No error means entry found, zero it out */ if (likely(!(idx & TLB_LKUP_ERR))) { __tlb_entry_erase(); - } else { /* Some sort of Error */ - + } else { /* Duplicate entry error */ - if (idx & 0x1) { - /* TODO we need to handle this case too */ - pr_emerg("unhandled Duplicate flush for %x\n", - vaddr_n_asid); - } - /* else entry not found so nothing to do */ + WARN(idx == TLB_DUP_ERR, "Probe returned Dup PD for %x\n", + vaddr_n_asid); } } @@ -159,7 +157,7 @@ static void utlb_invalidate(void) { #if (CONFIG_ARC_MMU_VER >= 2) -#if (CONFIG_ARC_MMU_VER < 3) +#if (CONFIG_ARC_MMU_VER == 2) /* MMU v2 introduced the uTLB Flush command. * There was however an obscure hardware bug, where uTLB flush would * fail when a prior probe for J-TLB (both totally unrelated) would @@ -182,6 +180,36 @@ static void utlb_invalidate(void) } +static void tlb_entry_insert(unsigned int pd0, unsigned int pd1) +{ + unsigned int idx; + + /* + * First verify if entry for this vaddr+ASID already exists + * This also sets up PD0 (vaddr, ASID..) for final commit + */ + idx = tlb_entry_lkup(pd0); + + /* + * If Not already present get a free slot from MMU. + * Otherwise, Probe would have located the entry and set INDEX Reg + * with existing location. This will cause Write CMD to over-write + * existing entry with new PD0 and PD1 + */ + if (likely(idx & TLB_LKUP_ERR)) + write_aux_reg(ARC_REG_TLBCOMMAND, TLBGetIndex); + + /* setup the other half of TLB entry (pfn, rwx..) */ + write_aux_reg(ARC_REG_TLBPD1, pd1); + + /* + * Commit the Entry to MMU + * It doesnt sound safe to use the TLBWriteNI cmd here + * which doesn't flush uTLBs. I'd rather be safe than sorry. + */ + write_aux_reg(ARC_REG_TLBCOMMAND, TLBWrite); +} + /* * Un-conditionally (without lookup) erase the entire MMU contents */ @@ -224,13 +252,14 @@ noinline void local_flush_tlb_mm(struct mm_struct *mm) return; /* - * Workaround for Android weirdism: - * A binder VMA could end up in a task such that vma->mm != tsk->mm - * old code would cause h/w - s/w ASID to get out of sync + * - Move to a new ASID, but only if the mm is still wired in + * (Android Binder ended up calling this for vma->mm != tsk->mm, + * causing h/w - s/w ASID to get out of sync) + * - Also get_new_mmu_context() new implementation allocates a new + * ASID only if it is not allocated already - so unallocate first */ - if (current->mm != mm) - destroy_context(mm); - else + destroy_context(mm); + if (current->mm == mm) get_new_mmu_context(mm); } @@ -245,8 +274,8 @@ noinline void local_flush_tlb_mm(struct mm_struct *mm) void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { + const unsigned int cpu = smp_processor_id(); unsigned long flags; - unsigned int asid; /* If range @start to @end is more than 32 TLB entries deep, * its better to move to a new ASID rather than searching for @@ -268,11 +297,10 @@ void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, start &= PAGE_MASK; local_irq_save(flags); - asid = vma->vm_mm->context.asid; - if (asid != NO_ASID) { + if (asid_mm(vma->vm_mm, cpu) != MM_CTXT_NO_ASID) { while (start < end) { - tlb_entry_erase(start | (asid & 0xff)); + tlb_entry_erase(start | hw_pid(vma->vm_mm, cpu)); start += PAGE_SIZE; } } @@ -319,6 +347,7 @@ void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) { + const unsigned int cpu = smp_processor_id(); unsigned long flags; /* Note that it is critical that interrupts are DISABLED between @@ -326,23 +355,95 @@ void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) */ local_irq_save(flags); - if (vma->vm_mm->context.asid != NO_ASID) { - tlb_entry_erase((page & PAGE_MASK) | - (vma->vm_mm->context.asid & 0xff)); + if (asid_mm(vma->vm_mm, cpu) != MM_CTXT_NO_ASID) { + tlb_entry_erase((page & PAGE_MASK) | hw_pid(vma->vm_mm, cpu)); utlb_invalidate(); } local_irq_restore(flags); } +#ifdef CONFIG_SMP + +struct tlb_args { + struct vm_area_struct *ta_vma; + unsigned long ta_start; + unsigned long ta_end; +}; + +static inline void ipi_flush_tlb_page(void *arg) +{ + struct tlb_args *ta = arg; + + local_flush_tlb_page(ta->ta_vma, ta->ta_start); +} + +static inline void ipi_flush_tlb_range(void *arg) +{ + struct tlb_args *ta = arg; + + local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end); +} + +static inline void ipi_flush_tlb_kernel_range(void *arg) +{ + struct tlb_args *ta = (struct tlb_args *)arg; + + local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end); +} + +void flush_tlb_all(void) +{ + on_each_cpu((smp_call_func_t)local_flush_tlb_all, NULL, 1); +} + +void flush_tlb_mm(struct mm_struct *mm) +{ + on_each_cpu_mask(mm_cpumask(mm), (smp_call_func_t)local_flush_tlb_mm, + mm, 1); +} + +void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) +{ + struct tlb_args ta = { + .ta_vma = vma, + .ta_start = uaddr + }; + + on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_page, &ta, 1); +} + +void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end) +{ + struct tlb_args ta = { + .ta_vma = vma, + .ta_start = start, + .ta_end = end + }; + + on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_range, &ta, 1); +} + +void flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + struct tlb_args ta = { + .ta_start = start, + .ta_end = end + }; + + on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1); +} +#endif + /* * Routine to create a TLB entry */ void create_tlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) { unsigned long flags; - unsigned int idx, asid_or_sasid; - unsigned long pd0_flags; + unsigned int asid_or_sasid, rwx; + unsigned long pd0, pd1; /* * create_tlb() assumes that current->mm == vma->mm, since @@ -374,47 +475,37 @@ void create_tlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) local_irq_save(flags); - tlb_paranoid_check(vma->vm_mm->context.asid, address); + tlb_paranoid_check(asid_mm(vma->vm_mm, smp_processor_id()), address); address &= PAGE_MASK; /* update this PTE credentials */ pte_val(*ptep) |= (_PAGE_PRESENT | _PAGE_ACCESSED); - /* Create HW TLB entry Flags (in PD0) from PTE Flags */ -#if (CONFIG_ARC_MMU_VER <= 2) - pd0_flags = ((pte_val(*ptep) & PTE_BITS_IN_PD0) >> 1); -#else - pd0_flags = ((pte_val(*ptep) & PTE_BITS_IN_PD0)); -#endif + /* Create HW TLB(PD0,PD1) from PTE */ /* ASID for this task */ asid_or_sasid = read_aux_reg(ARC_REG_PID) & 0xff; - write_aux_reg(ARC_REG_TLBPD0, address | pd0_flags | asid_or_sasid); - - /* Load remaining info in PD1 (Page Frame Addr and Kx/Kw/Kr Flags) */ - write_aux_reg(ARC_REG_TLBPD1, (pte_val(*ptep) & PTE_BITS_IN_PD1)); - - /* First verify if entry for this vaddr+ASID already exists */ - write_aux_reg(ARC_REG_TLBCOMMAND, TLBProbe); - idx = read_aux_reg(ARC_REG_TLBINDEX); + pd0 = address | asid_or_sasid | (pte_val(*ptep) & PTE_BITS_IN_PD0); /* - * If Not already present get a free slot from MMU. - * Otherwise, Probe would have located the entry and set INDEX Reg - * with existing location. This will cause Write CMD to over-write - * existing entry with new PD0 and PD1 + * ARC MMU provides fully orthogonal access bits for K/U mode, + * however Linux only saves 1 set to save PTE real-estate + * Here we convert 3 PTE bits into 6 MMU bits: + * -Kernel only entries have Kr Kw Kx 0 0 0 + * -User entries have mirrored K and U bits */ - if (likely(idx & TLB_LKUP_ERR)) - write_aux_reg(ARC_REG_TLBCOMMAND, TLBGetIndex); + rwx = pte_val(*ptep) & PTE_BITS_RWX; - /* - * Commit the Entry to MMU - * It doesnt sound safe to use the TLBWriteNI cmd here - * which doesn't flush uTLBs. I'd rather be safe than sorry. - */ - write_aux_reg(ARC_REG_TLBCOMMAND, TLBWrite); + if (pte_val(*ptep) & _PAGE_GLOBAL) + rwx <<= 3; /* r w x => Kr Kw Kx 0 0 0 */ + else + rwx |= (rwx << 3); /* r w x => Kr Kw Kx Ur Uw Ux */ + + pd1 = rwx | (pte_val(*ptep) & PTE_BITS_NON_RWX_IN_PD1); + + tlb_entry_insert(pd0, pd1); local_irq_restore(flags); } @@ -553,13 +644,6 @@ void arc_mmu_init(void) if (mmu->pg_sz != PAGE_SIZE) panic("MMU pg size != PAGE_SIZE (%luk)\n", TO_KB(PAGE_SIZE)); - /* - * ASID mgmt data structures are compile time init - * asid_cache = FIRST_ASID and asid_mm_map[] all zeroes - */ - - local_flush_tlb_all(); - /* Enable the MMU */ write_aux_reg(ARC_REG_PID, MMU_ENABLE); @@ -601,9 +685,9 @@ void do_tlb_overlap_fault(unsigned long cause, unsigned long address, struct pt_regs *regs) { int set, way, n; - unsigned int pd0[4], pd1[4]; /* assume max 4 ways */ unsigned long flags, is_valid; struct cpuinfo_arc_mmu *mmu = &cpuinfo_arc700[smp_processor_id()].mmu; + unsigned int pd0[mmu->ways], pd1[mmu->ways]; local_irq_save(flags); @@ -628,7 +712,7 @@ void do_tlb_overlap_fault(unsigned long cause, unsigned long address, continue; /* Scan the set for duplicate ways: needs a nested loop */ - for (way = 0; way < mmu->ways; way++) { + for (way = 0; way < mmu->ways - 1; way++) { if (!pd0[way]) continue; @@ -671,25 +755,28 @@ void do_tlb_overlap_fault(unsigned long cause, unsigned long address, * Low Level ASM TLB handler calls this if it finds that HW and SW ASIDS * don't match */ -void print_asid_mismatch(int is_fast_path) +void print_asid_mismatch(int mm_asid, int mmu_asid, int is_fast_path) { - int pid_sw, pid_hw; - pid_sw = current->active_mm->context.asid; - pid_hw = read_aux_reg(ARC_REG_PID) & 0xff; - pr_emerg("ASID Mismatch in %s Path Handler: sw-pid=0x%x hw-pid=0x%x\n", - is_fast_path ? "Fast" : "Slow", pid_sw, pid_hw); + is_fast_path ? "Fast" : "Slow", mm_asid, mmu_asid); __asm__ __volatile__("flag 1"); } -void tlb_paranoid_check(unsigned int pid_sw, unsigned long addr) +void tlb_paranoid_check(unsigned int mm_asid, unsigned long addr) { - unsigned int pid_hw; + unsigned int mmu_asid; - pid_hw = read_aux_reg(ARC_REG_PID) & 0xff; + mmu_asid = read_aux_reg(ARC_REG_PID) & 0xff; - if (addr < 0x70000000 && ((pid_hw != pid_sw) || (pid_sw == NO_ASID))) - print_asid_mismatch(0); + /* + * At the time of a TLB miss/installation + * - HW version needs to match SW version + * - SW needs to have a valid ASID + */ + if (addr < 0x70000000 && + ((mm_asid == MM_CTXT_NO_ASID) || + (mmu_asid != (mm_asid & MM_CTXT_ASID_MASK)))) + print_asid_mismatch(mm_asid, mmu_asid, 0); } #endif diff --git a/arch/arc/mm/tlbex.S b/arch/arc/mm/tlbex.S index 5c5bb23001b0..3fcfdb38d242 100644 --- a/arch/arc/mm/tlbex.S +++ b/arch/arc/mm/tlbex.S @@ -44,17 +44,36 @@ #include <asm/arcregs.h> #include <asm/cache.h> #include <asm/processor.h> -#if (CONFIG_ARC_MMU_VER == 1) #include <asm/tlb-mmu1.h> -#endif -;-------------------------------------------------------------------------- -; scratch memory to save the registers (r0-r3) used to code TLB refill Handler -; For details refer to comments before TLBMISS_FREEUP_REGS below +;----------------------------------------------------------------- +; ARC700 Exception Handling doesn't auto-switch stack and it only provides +; ONE scratch AUX reg "ARC_REG_SCRATCH_DATA0" +; +; For Non-SMP, the scratch AUX reg is repurposed to cache task PGD, so a +; "global" is used to free-up FIRST core reg to be able to code the rest of +; exception prologue (IRQ auto-disabled on Exceptions, so it's IRQ-safe). +; Since the Fast Path TLB Miss handler is coded with 4 regs, the remaining 3 +; need to be saved as well by extending the "global" to be 4 words. Hence +; ".size ex_saved_reg1, 16" +; [All of this dance is to avoid stack switching for each TLB Miss, since we +; only need to save only a handful of regs, as opposed to complete reg file] +; +; For ARC700 SMP, the "global" obviously can't be used for free up the FIRST +; core reg as it will not be SMP safe. +; Thus scratch AUX reg is used (and no longer used to cache task PGD). +; To save the rest of 3 regs - per cpu, the global is made "per-cpu". +; Epilogue thus has to locate the "per-cpu" storage for regs. +; To avoid cache line bouncing the per-cpu global is aligned/sized per +; L1_CACHE_SHIFT, despite fundamentally needing to be 12 bytes only. Hence +; ".size ex_saved_reg1, (CONFIG_NR_CPUS << L1_CACHE_SHIFT)" + +; As simple as that.... ;-------------------------------------------------------------------------- +; scratch memory to save [r0-r3] used to code TLB refill Handler ARCFP_DATA ex_saved_reg1 - .align 1 << L1_CACHE_SHIFT ; IMP: Must be Cache Line aligned + .align 1 << L1_CACHE_SHIFT .type ex_saved_reg1, @object #ifdef CONFIG_SMP .size ex_saved_reg1, (CONFIG_NR_CPUS << L1_CACHE_SHIFT) @@ -66,6 +85,44 @@ ex_saved_reg1: .zero 16 #endif +.macro TLBMISS_FREEUP_REGS +#ifdef CONFIG_SMP + sr r0, [ARC_REG_SCRATCH_DATA0] ; freeup r0 to code with + GET_CPU_ID r0 ; get to per cpu scratch mem, + lsl r0, r0, L1_CACHE_SHIFT ; cache line wide per cpu + add r0, @ex_saved_reg1, r0 +#else + st r0, [@ex_saved_reg1] + mov_s r0, @ex_saved_reg1 +#endif + st_s r1, [r0, 4] + st_s r2, [r0, 8] + st_s r3, [r0, 12] + + ; VERIFY if the ASID in MMU-PID Reg is same as + ; one in Linux data structures + + tlb_paranoid_check_asm +.endm + +.macro TLBMISS_RESTORE_REGS +#ifdef CONFIG_SMP + GET_CPU_ID r0 ; get to per cpu scratch mem + lsl r0, r0, L1_CACHE_SHIFT ; each is cache line wide + add r0, @ex_saved_reg1, r0 + ld_s r3, [r0,12] + ld_s r2, [r0, 8] + ld_s r1, [r0, 4] + lr r0, [ARC_REG_SCRATCH_DATA0] +#else + mov_s r0, @ex_saved_reg1 + ld_s r3, [r0,12] + ld_s r2, [r0, 8] + ld_s r1, [r0, 4] + ld_s r0, [r0] +#endif +.endm + ;============================================================================ ; Troubleshooting Stuff ;============================================================================ @@ -76,34 +133,35 @@ ex_saved_reg1: ; In bizzare scenrios SW and HW ASID can get out-of-sync which is trouble. ; So we try to detect this in TLB Mis shandler - -.macro DBG_ASID_MISMATCH +.macro tlb_paranoid_check_asm #ifdef CONFIG_ARC_DBG_TLB_PARANOIA - ; make sure h/w ASID is same as s/w ASID - GET_CURR_TASK_ON_CPU r3 ld r0, [r3, TASK_ACT_MM] ld r0, [r0, MM_CTXT+MM_CTXT_ASID] + breq r0, 0, 55f ; Error if no ASID allocated lr r1, [ARC_REG_PID] and r1, r1, 0xFF - breq r1, r0, 5f + and r2, r0, 0xFF ; MMU PID bits only for comparison + breq r1, r2, 5f + +55: ; Error if H/w and S/w ASID don't match, but NOT if in kernel mode - lr r0, [erstatus] - bbit0 r0, STATUS_U_BIT, 5f + lr r2, [erstatus] + bbit0 r2, STATUS_U_BIT, 5f ; We sure are in troubled waters, Flag the error, but to do so ; need to switch to kernel mode stack to call error routine GET_TSK_STACK_BASE r3, sp ; Call printk to shoutout aloud - mov r0, 1 + mov r2, 1 j print_asid_mismatch -5: ; ASIDs match so proceed normally +5: ; ASIDs match so proceed normally nop #endif @@ -161,13 +219,17 @@ ex_saved_reg1: ; IN: r0 = PTE, r1 = ptr to PTE .macro CONV_PTE_TO_TLB - and r3, r0, PTE_BITS_IN_PD1 ; Extract permission flags+PFN from PTE - sr r3, [ARC_REG_TLBPD1] ; these go in PD1 + and r3, r0, PTE_BITS_RWX ; r w x + lsl r2, r3, 3 ; r w x 0 0 0 + and.f 0, r0, _PAGE_GLOBAL + or.z r2, r2, r3 ; r w x r w x + + and r3, r0, PTE_BITS_NON_RWX_IN_PD1 ; Extract PFN+cache bits from PTE + or r3, r3, r2 + + sr r3, [ARC_REG_TLBPD1] ; these go in PD1 and r2, r0, PTE_BITS_IN_PD0 ; Extract other PTE flags: (V)alid, (G)lb -#if (CONFIG_ARC_MMU_VER <= 2) /* Neednot be done with v3 onwards */ - lsr r2, r2 ; shift PTE flags to match layout in PD0 -#endif lr r3,[ARC_REG_TLBPD0] ; MMU prepares PD0 with vaddr and asid @@ -191,68 +253,6 @@ ex_saved_reg1: #endif .endm -;----------------------------------------------------------------- -; ARC700 Exception Handling doesn't auto-switch stack and it only provides -; ONE scratch AUX reg "ARC_REG_SCRATCH_DATA0" -; -; For Non-SMP, the scratch AUX reg is repurposed to cache task PGD, so a -; "global" is used to free-up FIRST core reg to be able to code the rest of -; exception prologue (IRQ auto-disabled on Exceptions, so it's IRQ-safe). -; Since the Fast Path TLB Miss handler is coded with 4 regs, the remaining 3 -; need to be saved as well by extending the "global" to be 4 words. Hence -; ".size ex_saved_reg1, 16" -; [All of this dance is to avoid stack switching for each TLB Miss, since we -; only need to save only a handful of regs, as opposed to complete reg file] -; -; For ARC700 SMP, the "global" obviously can't be used for free up the FIRST -; core reg as it will not be SMP safe. -; Thus scratch AUX reg is used (and no longer used to cache task PGD). -; To save the rest of 3 regs - per cpu, the global is made "per-cpu". -; Epilogue thus has to locate the "per-cpu" storage for regs. -; To avoid cache line bouncing the per-cpu global is aligned/sized per -; L1_CACHE_SHIFT, despite fundamentally needing to be 12 bytes only. Hence -; ".size ex_saved_reg1, (CONFIG_NR_CPUS << L1_CACHE_SHIFT)" - -; As simple as that.... - -.macro TLBMISS_FREEUP_REGS -#ifdef CONFIG_SMP - sr r0, [ARC_REG_SCRATCH_DATA0] ; freeup r0 to code with - GET_CPU_ID r0 ; get to per cpu scratch mem, - lsl r0, r0, L1_CACHE_SHIFT ; cache line wide per cpu - add r0, @ex_saved_reg1, r0 -#else - st r0, [@ex_saved_reg1] - mov_s r0, @ex_saved_reg1 -#endif - st_s r1, [r0, 4] - st_s r2, [r0, 8] - st_s r3, [r0, 12] - - ; VERIFY if the ASID in MMU-PID Reg is same as - ; one in Linux data structures - - DBG_ASID_MISMATCH -.endm - -;----------------------------------------------------------------- -.macro TLBMISS_RESTORE_REGS -#ifdef CONFIG_SMP - GET_CPU_ID r0 ; get to per cpu scratch mem - lsl r0, r0, L1_CACHE_SHIFT ; each is cache line wide - add r0, @ex_saved_reg1, r0 - ld_s r3, [r0,12] - ld_s r2, [r0, 8] - ld_s r1, [r0, 4] - lr r0, [ARC_REG_SCRATCH_DATA0] -#else - mov_s r0, @ex_saved_reg1 - ld_s r3, [r0,12] - ld_s r2, [r0, 8] - ld_s r1, [r0, 4] - ld_s r0, [r0] -#endif -.endm ARCFP_CODE ;Fast Path Code, candidate for ICCM @@ -277,8 +277,8 @@ ARC_ENTRY EV_TLBMissI ;---------------------------------------------------------------- ; VERIFY_PTE: Check if PTE permissions approp for executing code cmp_s r2, VMALLOC_START - mov.lo r2, (_PAGE_PRESENT | _PAGE_U_EXECUTE) - mov.hs r2, (_PAGE_PRESENT | _PAGE_K_EXECUTE) + mov_s r2, (_PAGE_PRESENT | _PAGE_EXECUTE) + or.hs r2, r2, _PAGE_GLOBAL and r3, r0, r2 ; Mask out NON Flag bits from PTE xor.f r3, r3, r2 ; check ( ( pte & flags_test ) == flags_test ) @@ -317,26 +317,21 @@ ARC_ENTRY EV_TLBMissD ;---------------------------------------------------------------- ; VERIFY_PTE: Chk if PTE permissions approp for data access (R/W/R+W) - mov_s r2, 0 + cmp_s r2, VMALLOC_START + mov_s r2, _PAGE_PRESENT ; common bit for K/U PTE + or.hs r2, r2, _PAGE_GLOBAL ; kernel PTE only + + ; Linux PTE [RWX] bits are semantically overloaded: + ; -If PAGE_GLOBAL set, they refer to kernel-only flags (vmalloc) + ; -Otherwise they are user-mode permissions, and those are exactly + ; same for kernel mode as well (e.g. copy_(to|from)_user) + lr r3, [ecr] btst_s r3, ECR_C_BIT_DTLB_LD_MISS ; Read Access - or.nz r2, r2, _PAGE_U_READ ; chk for Read flag in PTE + or.nz r2, r2, _PAGE_READ ; chk for Read flag in PTE btst_s r3, ECR_C_BIT_DTLB_ST_MISS ; Write Access - or.nz r2, r2, _PAGE_U_WRITE ; chk for Write flag in PTE - ; Above laddering takes care of XCHG access - ; which is both Read and Write - - ; If kernel mode access, ; make _PAGE_xx flags as _PAGE_K_xx - ; For copy_(to|from)_user, despite exception taken in kernel mode, - ; this code is not hit, because EFA would still be the user mode - ; address (EFA < 0x6000_0000). - ; This code is for legit kernel mode faults, vmalloc specifically - ; (EFA: 0x7000_0000 to 0x7FFF_FFFF) - - lr r3, [efa] - cmp r3, VMALLOC_START - 1 ; If kernel mode access - asl.hi r2, r2, 3 ; make _PAGE_xx flags as _PAGE_K_xx - or r2, r2, _PAGE_PRESENT ; Common flag for K/U mode + or.nz r2, r2, _PAGE_WRITE ; chk for Write flag in PTE + ; Above laddering takes care of XCHG access (both R and W) ; By now, r2 setup with all the Flags we need to check in PTE and r3, r0, r2 ; Mask out NON Flag bits from PTE @@ -371,17 +366,11 @@ do_slow_path_pf: ; Slow path TLB Miss handled as a regular ARC Exception ; (stack switching / save the complete reg-file). - ; That requires freeing up r9 - EXCPN_PROLOG_FREEUP_REG r9 - - lr r9, [erstatus] - - SWITCH_TO_KERNEL_STK - SAVE_ALL_SYS + EXCEPTION_PROLOGUE ; ------- setup args for Linux Page fault Hanlder --------- - mov_s r0, sp - lr r1, [efa] + mov_s r1, sp + lr r0, [efa] ; We don't want exceptions to be disabled while the fault is handled. ; Now that we have saved the context we return from exception hence |