summaryrefslogtreecommitdiff
path: root/arch/arc
diff options
context:
space:
mode:
authorVineet Gupta <vgupta@synopsys.com>2013-11-28 13:57:54 +0530
committerVineet Gupta <vgupta@synopsys.com>2013-12-23 12:05:04 +0530
commitd8e8c7dda11f5d5cf90495f2e89d917a83509bc0 (patch)
tree8b8414e2bd4ed6b92bf35a36a3d891fa14399661 /arch/arc
parentf2a4aa5646687f9a51616581221b1d348590d47e (diff)
ARC: [SMP] optimize IPI send and receive
* Don't send an IPI if receiver already has a pending IPI. Atomically piggyback the new msg with pending msg. * IPI receiver looping on xchg() not required References: https://lkml.org/lkml/2013/11/25/232 Suggested-by: Peter Zijlstra <peterz@infradead.org> Signed-off-by: Vineet Gupta <vgupta@synopsys.com>
Diffstat (limited to 'arch/arc')
-rw-r--r--arch/arc/kernel/smp.c68
1 files changed, 40 insertions, 28 deletions
diff --git a/arch/arc/kernel/smp.c b/arch/arc/kernel/smp.c
index c00c612e8dd3..40859e5619f9 100644
--- a/arch/arc/kernel/smp.c
+++ b/arch/arc/kernel/smp.c
@@ -215,16 +215,31 @@ static DEFINE_PER_CPU(unsigned long, ipi_data);
static void ipi_send_msg_one(int cpu, enum ipi_msg_type msg)
{
unsigned long __percpu *ipi_data_ptr = per_cpu_ptr(&ipi_data, cpu);
+ unsigned long old, new;
unsigned long flags;
pr_debug("%d Sending msg [%d] to %d\n", smp_processor_id(), msg, cpu);
local_irq_save(flags);
- set_bit(msg, ipi_data_ptr);
+ /*
+ * Atomically write new msg bit (in case others are writing too),
+ * and read back old value
+ */
+ do {
+ new = old = *ipi_data_ptr;
+ new |= 1U << msg;
+ } while (cmpxchg(ipi_data_ptr, old, new) != old);
- /* Call the platform specific cross-CPU call function */
- if (plat_smp_ops.ipi_send)
+ /*
+ * Call the platform specific IPI kick function, but avoid if possible:
+ * Only do so if there's no pending msg from other concurrent sender(s).
+ * Otherwise, recevier will see this msg as well when it takes the
+ * IPI corresponding to that msg. This is true, even if it is already in
+ * IPI handler, because !@old means it has not yet dequeued the msg(s)
+ * so @new msg can be a free-loader
+ */
+ if (plat_smp_ops.ipi_send && !old)
plat_smp_ops.ipi_send(cpu);
local_irq_restore(flags);
@@ -269,31 +284,23 @@ static void ipi_cpu_stop(void)
machine_halt();
}
-static inline void __do_IPI(unsigned long pending)
+static inline void __do_IPI(unsigned long msg)
{
- while (pending) {
-
- unsigned long msg = __ffs(pending);
-
- switch (msg) {
- case IPI_RESCHEDULE:
- scheduler_ipi();
- break;
+ switch (msg) {
+ case IPI_RESCHEDULE:
+ scheduler_ipi();
+ break;
- case IPI_CALL_FUNC:
- generic_smp_call_function_interrupt();
- break;
-
- case IPI_CPU_STOP:
- ipi_cpu_stop();
- break;
-
- default:
- pr_warn("IPI missing msg\n");
+ case IPI_CALL_FUNC:
+ generic_smp_call_function_interrupt();
+ break;
- }
+ case IPI_CPU_STOP:
+ ipi_cpu_stop();
+ break;
- pending &= ~(1U << msg);
+ default:
+ pr_warn("IPI with unexpected msg %ld\n", msg);
}
}
@@ -312,11 +319,16 @@ irqreturn_t do_IPI(int irq, void *dev_id)
plat_smp_ops.ipi_clear(irq);
/*
- * XXX: is this loop really needed
- * And do we need to move ipi_clean inside
+ * "dequeue" the msg corresponding to this IPI (and possibly other
+ * piggybacked msg from elided IPIs: see ipi_send_msg_one() above)
*/
- while ((pending = xchg(this_cpu_ptr(&ipi_data), 0)) != 0)
- __do_IPI(pending);
+ pending = xchg(this_cpu_ptr(&ipi_data), 0);
+
+ do {
+ unsigned long msg = __ffs(pending);
+ __do_IPI(msg);
+ pending &= ~(1U << msg);
+ } while (pending);
return IRQ_HANDLED;
}