diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/irq/internals.h | 2 | ||||
| -rw-r--r-- | kernel/irq/irqdesc.c | 10 | ||||
| -rw-r--r-- | kernel/irq/manage.c | 31 | ||||
| -rw-r--r-- | kernel/irq/msi.c | 4 | ||||
| -rw-r--r-- | kernel/irq/proc.c | 24 | ||||
| -rw-r--r-- | kernel/softirq.c | 64 |
6 files changed, 101 insertions, 34 deletions
diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index 0ce21dd45404..2c30e7537987 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h @@ -36,6 +36,7 @@ extern struct irqaction chained_action; * IRQTF_AFFINITY - irq thread is requested to adjust affinity * IRQTF_FORCED_THREAD - irq action is force threaded * IRQTF_READY - signals that irq thread is ready + * IRQTF_EXIT_WORK - irq thread exit task_work is pending */ enum { IRQTF_RUNTHREAD, @@ -43,6 +44,7 @@ enum { IRQTF_AFFINITY, IRQTF_FORCED_THREAD, IRQTF_READY, + IRQTF_EXIT_WORK, }; /* diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c index 3a818f07a101..7cbcc3fb9e6d 100644 --- a/kernel/irq/irqdesc.c +++ b/kernel/irq/irqdesc.c @@ -28,7 +28,15 @@ static struct lock_class_key irq_desc_lock_class; static int __init irq_affinity_setup(char *str) { alloc_bootmem_cpumask_var(&irq_default_affinity); - cpulist_parse(str, irq_default_affinity); + if (cpulist_parse(str, irq_default_affinity) < 0) { + /* + * Clear the mask so that init_irq_default_affinity() + * defaults it later. + */ + cpumask_clear(irq_default_affinity); + pr_warn("irqaffinity: incorrect CPU range, using default\n"); + return 1; + } /* * Set at least the boot cpu. We don't want to end up with * bugreports caused by random commandline masks diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 57eff26fa646..3b647d9b458a 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -21,6 +21,7 @@ #include <linux/sched/isolation.h> #include <uapi/linux/sched/types.h> #include <linux/task_work.h> +#include <linux/wait_bit.h> #include "internals.h" @@ -1176,17 +1177,14 @@ void wake_threads_waitq(struct irq_desc *desc) wake_up(&desc->wait_for_threads); } -static void irq_thread_dtor(struct callback_head *unused) +static void __irq_thread_dtor(struct irqaction *action) { struct task_struct *tsk = current; struct irq_desc *desc; - struct irqaction *action; if (WARN_ON_ONCE(!(current->flags & PF_EXITING))) return; - action = kthread_data(tsk); - pr_err("exiting task \"%s\" (%d) is an active IRQ thread (irq %d)\n", tsk->comm, tsk->pid, action->irq); @@ -1203,6 +1201,14 @@ static void irq_thread_dtor(struct callback_head *unused) irq_finalize_oneshot(desc, action); } +static void irq_thread_dtor(struct callback_head *work) +{ + struct irqaction *action = container_of(work, struct irqaction, thread_dtor); + + __irq_thread_dtor(action); + clear_and_wake_up_bit(IRQTF_EXIT_WORK, &action->thread_flags); +} + static void irq_wake_secondary(struct irq_desc *desc, struct irqaction *action) { struct irqaction *secondary = action->secondary; @@ -1244,7 +1250,6 @@ static void wake_up_and_wait_for_irq_thread_ready(struct irq_desc *desc, */ static int irq_thread(void *data) { - struct callback_head on_exit_work; struct irqaction *action = data; struct irq_desc *desc = irq_to_desc(action->irq); irqreturn_t (*handler_fn)(struct irq_desc *desc, @@ -1263,8 +1268,9 @@ static int irq_thread(void *data) else handler_fn = irq_thread_fn; - init_task_work(&on_exit_work, irq_thread_dtor); - task_work_add(current, &on_exit_work, TWA_NONE); + set_bit(IRQTF_EXIT_WORK, &action->thread_flags); + init_task_work(&action->thread_dtor, irq_thread_dtor); + task_work_add(current, &action->thread_dtor, TWA_NONE); while (!irq_wait_for_interrupt(desc, action)) { irqreturn_t action_ret; @@ -1283,6 +1289,7 @@ static int irq_thread(void *data) * oneshot mask bit can be set. */ task_work_cancel_func(current, irq_thread_dtor); + clear_and_wake_up_bit(IRQTF_EXIT_WORK, &action->thread_flags); return 0; } @@ -1853,6 +1860,12 @@ out_mput: return ret; } +static void irq_stop_thread(struct irqaction *action) +{ + kthread_stop_put(action->thread); + wait_on_bit(&action->thread_flags, IRQTF_EXIT_WORK, TASK_UNINTERRUPTIBLE); +} + /* * Internal function to unregister an irqaction - used to free * regular and special interrupts that are part of the architecture. @@ -1958,9 +1971,9 @@ static struct irqaction *__free_irq(struct irq_desc *desc, void *dev_id) * the same bit to a newly requested action. */ if (action->thread) { - kthread_stop_put(action->thread); + irq_stop_thread(action); if (action->secondary && action->secondary->thread) - kthread_stop_put(action->secondary->thread); + irq_stop_thread(action->secondary); } /* Last action releases resources */ diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index fb5f372215bf..b22d74496af1 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -142,7 +142,7 @@ fail: * insert it at @init_desc->msi_index * * @dev: Pointer to the device for which the descriptor is allocated - * @domid: The id of the interrupt domain to which the desriptor is added + * @domid: The id of the interrupt domain to which the descriptor is added * @init_desc: Pointer to an MSI descriptor to initialize the new descriptor * * Return: 0 on success or an appropriate failure code. @@ -416,7 +416,7 @@ EXPORT_SYMBOL_GPL(msi_domain_first_desc); * @domid: The id of the interrupt domain which should be walked. * @filter: Descriptor state filter * - * The first invocation of msi_next_desc() has to be preceeded by a + * The first invocation of msi_next_desc() has to be preceded by a * successful invocation of __msi_first_desc(). Consecutive invocations are * only valid if the previous one was successful. All these operations have * to be done within the same MSI mutex held region. diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index 1b835725f7b1..0c0df16de581 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -462,7 +462,7 @@ int __weak arch_show_interrupts(struct seq_file *p, int prec) static DEFINE_RAW_SPINLOCK(irq_proc_constraints_lock); static struct irq_proc_constraints { - bool print_header; + loff_t first_pos; unsigned int num_prec; unsigned int chip_width; } irq_proc_constraints __read_mostly = { @@ -543,14 +543,13 @@ static int irq_seq_show(struct seq_file *p, void *v) struct irqaction *action; /* Print header for the first interrupt? */ - if (constr->print_header) { + if (p->index == constr->first_pos) { unsigned int cpu; seq_printf(p, "%*s", constr->num_prec + 8, ""); for_each_online_cpu(cpu) seq_printf(p, "CPU%-8d", cpu); seq_putc(p, '\n'); - constr->print_header = false; } if (desc == ARCH_PROC_IRQDESC) @@ -635,14 +634,25 @@ static void *irq_seq_next_desc(loff_t *pos) static void *irq_seq_start(struct seq_file *f, loff_t *pos) { - if (!*pos) { - struct irq_proc_constraints *constr = f->private; + struct irq_proc_constraints *constr = f->private; + bool first = !*pos; + void *ret; + if (first) { constr->num_prec = READ_ONCE(irq_proc_constraints.num_prec); constr->chip_width = READ_ONCE(irq_proc_constraints.chip_width); - constr->print_header = true; } - return irq_seq_next_desc(pos); + + ret = irq_seq_next_desc(pos); + + /* + * Record position of the first output record so that irq_seq_show() + * knows where to write the header. + */ + if (first) + constr->first_pos = *pos; + + return ret; } static void *irq_seq_next(struct seq_file *f, void *v, loff_t *pos) diff --git a/kernel/softirq.c b/kernel/softirq.c index 5d02c36c40e3..7fc2200392cd 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -350,8 +350,8 @@ static inline void ksoftirqd_run_end(void) local_irq_enable(); } -static inline void softirq_handle_begin(void) { } -static inline void softirq_handle_end(void) { } +static inline bool softirq_handle_begin(void) { return false; } +static inline void softirq_handle_end(bool from_irq_exit) { } static inline bool should_wake_ksoftirqd(void) { @@ -481,15 +481,38 @@ void __local_bh_enable_ip(unsigned long ip, unsigned int cnt) } EXPORT_SYMBOL(__local_bh_enable_ip); -static inline void softirq_handle_begin(void) +static inline bool softirq_handle_begin(void) { - __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET); + if (!in_hardirq()) { + __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET); + return false; + } + + /* + * Only reached from irq_exit(), with HARDIRQ_OFFSET still set. + * Replace it with SOFTIRQ_OFFSET before handle_softirqs() enables + * interrupts. Use the raw operation to preserve the preemption + * disable location recorded by irq_enter_rcu(), and update lockdep + * directly. + */ + __preempt_count_sub(HARDIRQ_OFFSET - SOFTIRQ_OFFSET); + lockdep_softirqs_off(_RET_IP_); + WARN_ON_ONCE(irq_count() != SOFTIRQ_OFFSET); + + return true; } -static inline void softirq_handle_end(void) +static inline void softirq_handle_end(bool from_irq_exit) { - __local_bh_enable(SOFTIRQ_OFFSET); - WARN_ON_ONCE(in_interrupt()); + if (!from_irq_exit) { + __local_bh_enable(SOFTIRQ_OFFSET); + WARN_ON_ONCE(in_interrupt()); + return; + } + + lockdep_softirqs_on(_RET_IP_); + __preempt_count_add(HARDIRQ_OFFSET - SOFTIRQ_OFFSET); + WARN_ON_ONCE(irq_count() != HARDIRQ_OFFSET); } static inline void ksoftirqd_run_begin(void) @@ -605,6 +628,7 @@ static void handle_softirqs(bool ksirqd) unsigned long old_flags = current->flags; int max_restart = MAX_SOFTIRQ_RESTART; struct softirq_action *h; + bool from_irq_exit; bool in_hardirq; __u32 pending; int softirq_bit; @@ -618,7 +642,7 @@ static void handle_softirqs(bool ksirqd) pending = local_softirq_pending(); - softirq_handle_begin(); + from_irq_exit = softirq_handle_begin(); in_hardirq = lockdep_softirq_start(); account_softirq_enter(current); @@ -645,8 +669,8 @@ restart: h->action(); trace_softirq_exit(vec_nr); if (unlikely(prev_count != preempt_count())) { - pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n", - vec_nr, softirq_to_name[vec_nr], h->action, + pr_err("huh, entered softirq %u %s with preempt_count %08x, exited with %08x?\n", + vec_nr, softirq_to_name[vec_nr], prev_count, preempt_count()); preempt_count_set(prev_count); } @@ -670,7 +694,7 @@ restart: account_softirq_exit(current); lockdep_softirq_end(in_hardirq); - softirq_handle_end(); + softirq_handle_end(from_irq_exit); current_restore_flags(old_flags, PF_MEMALLOC); } @@ -716,7 +740,7 @@ static inline void tick_irq_exit(void) /* Make sure that timer wheel updates are propagated */ if ((sched_core_idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) { - if (!in_hardirq()) + if (hardirq_count() == HARDIRQ_OFFSET) tick_nohz_irq_exit(); } #endif @@ -748,8 +772,12 @@ static inline void __irq_exit_rcu(void) lockdep_assert_irqs_disabled(); #endif account_hardirq_exit(current); - preempt_count_sub(HARDIRQ_OFFSET); - if (!in_interrupt() && local_softirq_pending()) { + + /* + * HARDIRQ_OFFSET is still set. Only the outermost interrupt handles + * softirqs, and only if it did not hit a softirq or BH disabled section. + */ + if (irq_count() == HARDIRQ_OFFSET && local_softirq_pending()) { /* * If we left hrtimers unarmed, make sure to arm them now, * before enabling interrupts to run softirq. @@ -758,11 +786,17 @@ static inline void __irq_exit_rcu(void) invoke_softirq(); } + /* + * Wake the timer thread even if the interrupt hit a softirq or a + * section with BHs disabled. Only the outermost interrupt can wake it. + */ if (IS_ENABLED(CONFIG_IRQ_FORCED_THREADING) && force_irqthreads() && - local_timers_pending_force_th() && !(in_nmi() | in_hardirq())) + local_timers_pending_force_th() && + hardirq_count() == HARDIRQ_OFFSET) wake_timersd(); tick_irq_exit(); + preempt_count_sub(HARDIRQ_OFFSET); } /** |