Merge tag 'rcu.2026.10.12b' of git://git.kernel.org/pub/scm/linux/kernel/git/rcu/linux

Pull RCU updates from Paul McKenney:
 "Execution contexts and call_rcu() and call_srcu():
   - Make call_rcu() and call_srcu invocable from NMI handlers and while
     holding scheduler locks (using irq-work handlers)
   - Add rcutorture and BPF tests calling these functions from these
     contexts
   - Disable rcutorture overload testing for RCU readers that are
     "fragile", that is, that do not either disable preemption or have
     some form of RCU priority boosting

  SRCU updates:
   - Add an atomic SRCU where readers run with preemption disabled and
     grace periods can be invoked from atomic contexts. David Woodhouse
     has a patch series exploiting this for GPC
   - Add torture and lockdep tests for atomic SRCU
   - Add numerous optimizations for atomic SRCU based on the fact that
     it does not have callbacks, for example, eliminating any transition
     to "big" srcu_struct structures even on systems that configure
     these to happen at initialization time

  Additional torture-test updates:
   - Fix a divide-by-zero bug in forward-progress testing
   - Synchronously wait for all IRQ-queued callbacks to complete
   - Allow alternative "ssh" commands to be specified for use by
     kvm-remote.sh
   - Fix kvm-again.sh re-run failure on PowerPC
   - Add additional lockdep testing for atomic SRCU
   - Fix error double-counting in srcu_lockdep.sh
   - Add atomic SRCU cross-CPU IRQ context mismatch testing, with the
   - trick being that the atomic SRCU readers context must be at least
     as strict as that of the atomic SRCU grace-period wait. So for
     example, if the readers have IRQs disabled, so must the
     grace-period wait

  Miscellaneous updates:
   - Update stallwarn.rst documentation for the re-implementation of RCU
     Tasks Trace in terms of SRCU
   - Fix the lazy-callback shrink scanner budget computation so that it
     honors the reclaim budget
   - Avoid panicking on RCU CPU stalls that end before the warning can
     be printed
   - Add "running" and "boosted" indications to the RCU task stall
     warning output
   - People have been (incorrectly) relying on rcu_barrier() to wait for
     kfree_rcu() callbacks and processing. But it turns out to be easier
     to fix rcu_barrier() to meet their expectations than to fix their
     code to live within the earlier rcu_barrier() specification. So
     rcu_barrier() now waits on kfree_rcu() callbacks and processing
   - Disable RCU Tasks console output that track when this RCU flavor
     switches between per-CPU and global callback queuing. New use cases
     appeared that generated enough console traffic to cause trouble,
     and no one seemed to care about these messages. For those needing
     those messages, there is a new rcupdate.rcu_task_collapse_debug
     kernel boot parameter
   - Fix __rcu pointer attribute handling to eliminate false positives
   - Fix typos and drop redundant #include directives"

* tag 'rcu.2026.10.12b' of git://git.kernel.org/pub/scm/linux/kernel/git/rcu/linux: (46 commits)
  rcutorture: Add atomic SRCU cross-CPU IRQ context mismatch test
  selftests/rcutorture: Fix double nerrs count in srcu_lockdep.sh
  selftests/rcutorture: Wire atomic SRCU into srcu_lockdep.sh
  rcutorture: Add atomic SRCU lockdep support
  rcutorture: Fix kvm-again.sh re-run failure on ppc64
  torture:  Allow specifying alternative ssh command to kvm-remote.sh
  rcutorture: Synchronously wait for all rcu_torture_irq() callbacks to complete
  rcutorture: Fix divide-by-zero with fwd_progress_div=1
  srcutree: Preserve IRQ state in synchronize_srcu_atomic() callchain
  srcutiny: Add lockdep and early-boot checks to synchronize_srcu_atomic()
  srcutree: Add lockdep and early-boot checks to synchronize_srcu_atomic()
  srcutree: Skip torture to-big transition for atomic SRCU
  srcutree: Don't transition atomic SRCU to big in srcu_gp_end()
  srcutree: Make init_srcu_struct_atomic() prevent transition to big
  srcu: Restrict atomic-SRCU non_block annotation to task context
  srcutree: Remove srcu_barrier() sleep for atomic SRCU
  srcutree: Skip callback scheduling for atomic SRCU grace periods
  srcutree: Add reader-free fastpath to synchronize_srcu_atomic()
  rcutorture: Add atomic-SRCU support to torture.sh
  srcutree: Explicitly note DEFINE_SRCU() needs for srcu_barrier()
  ...

1 files changed, 104 insertions, 23 deletions

diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c
index dccccd6be941..656b6a682e31 100644
--- a/kernel/rcu/tiny.c
+++ b/kernel/rcu/tiny.c
@@ -11,6 +11,8 @@
  */
 #include <linux/completion.h>
 #include <linux/interrupt.h>
+#include <linux/irq_work.h>
+#include <linux/llist.h>
 #include <linux/notifier.h>
 #include <linux/rcupdate_wait.h>
 #include <linux/kernel.h>
@@ -42,8 +44,100 @@ static struct rcu_ctrlblk rcu_ctrlblk = {
 	.gp_seq		= 0 - 300UL,
 };
 
+/*
+ * The callback list is only accessed with interrupts disabled, so a call_rcu()
+ * that arrives with interrupts off stages the callback on a lockless list that
+ * an irq_work re-issues later.  One global list and irq_work suffice, as Tiny
+ * RCU is uniprocessor.
+ */
+static void rcu_defer_drain(struct irq_work *iw);
+static LLIST_HEAD(rcu_defer_list);
+static struct irq_work rcu_defer_iw = IRQ_WORK_INIT_HARD(rcu_defer_drain);
+static bool rcu_defer_draining;
+
+/*
+ * Also called by __rcu_defer_drain() to re-issue a deferred callback, so it
+ * must not re-check the deferral condition.
+ */
+static void rcu_do_enqueue(struct rcu_head *head, rcu_callback_t func)
+{
+	static atomic_t doublefrees;
+	unsigned long flags;
+
+	if (debug_rcu_head_queue(head)) {
+		if (atomic_inc_return(&doublefrees) < 4) {
+			pr_err("%s(): Double-freed CB %p->%pS()!!!  ", __func__, head, head->func);
+			mem_dump_obj(head);
+		}
+		return;
+	}
+
+	head->func = func;
+	head->next = NULL;
+
+	local_irq_save(flags);
+	*rcu_ctrlblk.curtail = head;
+	rcu_ctrlblk.curtail = &head->next;
+	local_irq_restore(flags);
+}
+
+/* Force scheduling for rcu_qs() when enqueuing from the idle task. */
+static void rcu_resched_if_idle(void)
+{
+	if (unlikely(is_idle_task(current)))
+		resched_cpu(0);
+}
+
+static void __rcu_defer_drain(void)
+{
+	struct llist_node *node, *next;
+	bool drained = false;
+	unsigned long flags;
+
+	if (!IS_ENABLED(CONFIG_RCU_DEFER))
+		return;
+
+	/* Re-issued newest-first; nothing depends on call_rcu() ordering. */
+	local_irq_save(flags);
+	llist_for_each_safe(node, next, llist_del_all(&rcu_defer_list)) {
+		struct rcu_head *head = (struct rcu_head *)node;
+
+		/* Bounds a node self-linked by a double call_rcu(). */
+		head->next = NULL;
+		rcu_do_enqueue(head, head->func);
+		drained = true;
+	}
+	local_irq_restore(flags);
+
+	if (drained)
+		rcu_resched_if_idle();
+}
+
+/* Only the irq_work drain can be re-fed by its own re-issue; see Tree RCU. */
+static void rcu_defer_drain(struct irq_work *iw)
+{
+	WRITE_ONCE(rcu_defer_draining, true);
+	__rcu_defer_drain();
+	WRITE_ONCE(rcu_defer_draining, false);
+}
+
+static void call_rcu_defer(struct rcu_head *head, rcu_callback_t func)
+{
+	/* A re-entrant call_rcu() during the drain would livelock it; drop it. */
+	if (READ_ONCE(rcu_defer_draining) && !in_nmi()) {
+		WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU),
+			  "call_rcu() re-entered during callback drain; leaking callback\n");
+		return;
+	}
+	head->func = func;
+	if (llist_add((struct llist_node *)head, &rcu_defer_list))
+		irq_work_queue(&rcu_defer_iw);
+}
+
 void rcu_barrier(void)
 {
+	/* Register any deferred callbacks so the wait below covers them. */
+	__rcu_defer_drain();
 	wait_rcu_gp(call_rcu_hurry);
 }
 EXPORT_SYMBOL(rcu_barrier);
@@ -157,29 +251,19 @@ EXPORT_SYMBOL_GPL(synchronize_rcu);
  */
 void call_rcu(struct rcu_head *head, rcu_callback_t func)
 {
-	static atomic_t doublefrees;
-	unsigned long flags;
-
-	if (debug_rcu_head_queue(head)) {
-		if (atomic_inc_return(&doublefrees) < 4) {
-			pr_err("%s(): Double-freed CB %p->%pS()!!!  ", __func__, head, head->func);
-			mem_dump_obj(head);
-		}
+	if (should_rcu_defer()) {
+		call_rcu_defer(head, func);
 		return;
 	}
 
-	head->func = func;
-	head->next = NULL;
+	/*
+	 * Only reachable from an NMI when deferral is off: before the scheduler
+	 * is up, or with CONFIG_RCU_DEFER=n.  The enqueue can then race.
+	 */
+	WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());
 
-	local_irq_save(flags);
-	*rcu_ctrlblk.curtail = head;
-	rcu_ctrlblk.curtail = &head->next;
-	local_irq_restore(flags);
-
-	if (unlikely(is_idle_task(current))) {
-		/* force scheduling for rcu_qs() */
-		resched_cpu(0);
-	}
+	rcu_do_enqueue(head, func);
+	rcu_resched_if_idle();
 }
 EXPORT_SYMBOL_GPL(call_rcu);
 
@@ -211,10 +295,7 @@ unsigned long start_poll_synchronize_rcu(void)
 {
 	unsigned long gp_seq = get_state_synchronize_rcu();
 
-	if (unlikely(is_idle_task(current))) {
-		/* force scheduling for rcu_qs() */
-		resched_cpu(0);
-	}
+	rcu_resched_if_idle();
 	return gp_seq;
 }
 EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu);