diff options
| -rw-r--r-- | arch/sparc64/kernel/entry.S | 8 | ||||
| -rw-r--r-- | arch/sparc64/kernel/irq.c | 41 | ||||
| -rw-r--r-- | arch/sparc64/kernel/sun4v_ivec.S | 8 | ||||
| -rw-r--r-- | include/asm-sparc64/cpudata.h | 11 | ||||
| -rw-r--r-- | include/asm-sparc64/irq.h | 2 |
5 files changed, 38 insertions, 32 deletions
diff --git a/arch/sparc64/kernel/entry.S b/arch/sparc64/kernel/entry.S index 8059531bf0ac..9a785293203f 100644 --- a/arch/sparc64/kernel/entry.S +++ b/arch/sparc64/kernel/entry.S @@ -430,15 +430,15 @@ do_ivec: membar #Sync sethi %hi(ivector_table), %g2 - sllx %g3, 3, %g3 + sllx %g3, 4, %g3 or %g2, %lo(ivector_table), %g2 add %g2, %g3, %g3 TRAP_LOAD_IRQ_WORK(%g6, %g1) - lduw [%g6], %g5 /* g5 = irq_work(cpu) */ - stw %g5, [%g3 + 0x00] /* bucket->irq_chain = g5 */ - stw %g3, [%g6] /* irq_work(cpu) = bucket */ + ldx [%g6], %g5 /* g5 = irq_work(cpu) */ + stx %g5, [%g3 + 0x00] /* bucket->irq_chain = g5 */ + stx %g3, [%g6] /* irq_work(cpu) = bucket */ wr %g0, 1 << PIL_DEVICE_IRQ, %set_softint retry do_ivec_xcall: diff --git a/arch/sparc64/kernel/irq.c b/arch/sparc64/kernel/irq.c index 045ab27d4271..4db4dd576210 100644 --- a/arch/sparc64/kernel/irq.c +++ b/arch/sparc64/kernel/irq.c @@ -59,23 +59,20 @@ * of the vectored interrupt trap handler(s) in entry.S and sun4v_ivec.S */ struct ino_bucket { - /* Next handler in per-CPU IRQ worklist. We know that - * bucket pointers have the high 32-bits clear, so to - * save space we only store the bits we need. - */ -/*0x00*/unsigned int irq_chain; +/*0x00*/unsigned long irq_chain; /* Virtual interrupt number assigned to this INO. */ -/*0x04*/unsigned int virt_irq; +/*0x08*/unsigned int virt_irq; +/*0x0c*/unsigned int __pad; }; #define NUM_IVECS (IMAP_INR + 1) struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BYTES))); #define __irq_ino(irq) \ - (((struct ino_bucket *)(unsigned long)(irq)) - &ivector_table[0]) -#define __bucket(irq) ((struct ino_bucket *)(unsigned long)(irq)) -#define __irq(bucket) ((unsigned int)(unsigned long)(bucket)) + (((struct ino_bucket *)(irq)) - &ivector_table[0]) +#define __bucket(irq) ((struct ino_bucket *)(irq)) +#define __irq(bucket) ((unsigned long)(bucket)) /* This has to be in the main kernel image, it cannot be * turned into per-cpu data. The reason is that the main @@ -87,13 +84,13 @@ struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BY #define irq_work(__cpu) &(trap_block[(__cpu)].irq_worklist) static struct { - unsigned int irq; + unsigned long irq; unsigned int dev_handle; unsigned int dev_ino; } virt_to_real_irq_table[NR_IRQS]; static DEFINE_SPINLOCK(virt_irq_alloc_lock); -unsigned char virt_irq_alloc(unsigned int real_irq) +unsigned char virt_irq_alloc(unsigned long real_irq) { unsigned long flags; unsigned char ent; @@ -134,7 +131,7 @@ void virt_irq_free(unsigned int virt_irq) } #endif -static unsigned int virt_to_real_irq(unsigned char virt_irq) +static unsigned long virt_to_real_irq(unsigned char virt_irq) { return virt_to_real_irq_table[virt_irq].irq; } @@ -227,7 +224,7 @@ struct irq_handler_data { static inline struct ino_bucket *virt_irq_to_bucket(unsigned int virt_irq) { - unsigned int real_irq = virt_to_real_irq(virt_irq); + unsigned long real_irq = virt_to_real_irq(virt_irq); struct ino_bucket *bucket = NULL; if (likely(real_irq)) @@ -694,18 +691,28 @@ void handler_irq(int irq, struct pt_regs *regs) { struct ino_bucket *bucket; struct pt_regs *old_regs; + unsigned long pstate; clear_softint(1 << irq); old_regs = set_irq_regs(regs); irq_enter(); - /* Sliiiick... */ - bucket = __bucket(xchg32(irq_work(smp_processor_id()), 0)); + /* Grab an atomic snapshot of the pending IVECs. */ + __asm__ __volatile__("rdpr %%pstate, %0\n\t" + "wrpr %0, %3, %%pstate\n\t" + "ldx [%2], %1\n\t" + "stx %%g0, [%2]\n\t" + "wrpr %0, 0x0, %%pstate\n\t" + : "=&r" (pstate), "=&r" (bucket) + : "r" (irq_work(smp_processor_id())), + "i" (PSTATE_IE) + : "memory"); + while (bucket) { struct ino_bucket *next = __bucket(bucket->irq_chain); - bucket->irq_chain = 0; + bucket->irq_chain = 0UL; __do_IRQ(bucket->virt_irq); bucket = next; @@ -808,7 +815,7 @@ void init_irqwork_curcpu(void) { int cpu = hard_smp_processor_id(); - trap_block[cpu].irq_worklist = 0; + trap_block[cpu].irq_worklist = 0UL; } /* Please be very careful with register_one_mondo() and diff --git a/arch/sparc64/kernel/sun4v_ivec.S b/arch/sparc64/kernel/sun4v_ivec.S index 574bc248bca6..e3e9d4c1574b 100644 --- a/arch/sparc64/kernel/sun4v_ivec.S +++ b/arch/sparc64/kernel/sun4v_ivec.S @@ -101,14 +101,14 @@ sun4v_dev_mondo: /* Get &ivector_table[IVEC] into %g4. */ sethi %hi(ivector_table), %g4 - sllx %g3, 3, %g3 + sllx %g3, 4, %g3 or %g4, %lo(ivector_table), %g4 add %g4, %g3, %g4 /* Insert ivector_table[] entry into __irq_work[] queue. */ - lduw [%g1], %g2 /* g2 = irq_work(cpu) */ - stw %g2, [%g4 + 0x00] /* bucket->irq_chain = g2 */ - stw %g4, [%g1] /* irq_work(cpu) = bucket */ + ldx [%g1], %g2 /* g2 = irq_work(cpu) */ + stx %g2, [%g4 + 0x00] /* bucket->irq_chain = g2 */ + stx %g4, [%g1] /* irq_work(cpu) = bucket */ /* Signal the interrupt by setting (1 << pil) in %softint. */ wr %g0, 1 << PIL_DEVICE_IRQ, %set_softint diff --git a/include/asm-sparc64/cpudata.h b/include/asm-sparc64/cpudata.h index 98a6e609163e..379c219c3b7a 100644 --- a/include/asm-sparc64/cpudata.h +++ b/include/asm-sparc64/cpudata.h @@ -75,12 +75,11 @@ struct trap_per_cpu { unsigned long tsb_huge_temp; /* Dcache line 8: IRQ work list, and keep trap_block a power-of-2 in size. */ - unsigned int irq_worklist; + unsigned long irq_worklist; unsigned int cpu_mondo_qmask; unsigned int dev_mondo_qmask; unsigned int resum_qmask; unsigned int nonresum_qmask; - unsigned int __pad2[1]; void *hdesc; } __attribute__((aligned(64))); extern struct trap_per_cpu trap_block[NR_CPUS]; @@ -129,10 +128,10 @@ extern struct sun4v_2insn_patch_entry __sun4v_2insn_patch, #define TRAP_PER_CPU_TSB_HUGE 0xd0 #define TRAP_PER_CPU_TSB_HUGE_TEMP 0xd8 #define TRAP_PER_CPU_IRQ_WORKLIST 0xe0 -#define TRAP_PER_CPU_CPU_MONDO_QMASK 0xe4 -#define TRAP_PER_CPU_DEV_MONDO_QMASK 0xe8 -#define TRAP_PER_CPU_RESUM_QMASK 0xec -#define TRAP_PER_CPU_NONRESUM_QMASK 0xf0 +#define TRAP_PER_CPU_CPU_MONDO_QMASK 0xe8 +#define TRAP_PER_CPU_DEV_MONDO_QMASK 0xec +#define TRAP_PER_CPU_RESUM_QMASK 0xf0 +#define TRAP_PER_CPU_NONRESUM_QMASK 0xf4 #define TRAP_BLOCK_SZ_SHIFT 8 diff --git a/include/asm-sparc64/irq.h b/include/asm-sparc64/irq.h index bad3c28cdee8..24841c22e81b 100644 --- a/include/asm-sparc64/irq.h +++ b/include/asm-sparc64/irq.h @@ -59,7 +59,7 @@ extern unsigned int sun4u_build_msi(u32 portid, unsigned int *virt_irq_p, extern void sun4u_destroy_msi(unsigned int virt_irq); extern unsigned int sbus_build_irq(void *sbus, unsigned int ino); -extern unsigned char virt_irq_alloc(unsigned int real_irq); +extern unsigned char virt_irq_alloc(unsigned long real_irq); #ifdef CONFIG_PCI_MSI extern void virt_irq_free(unsigned int virt_irq); #endif |