From a47dea1baa3b445365ad0757e6dee4ef8fde49e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Onur=20=C3=96zkan?= Date: Sat, 13 Jun 2026 09:40:07 +0300 Subject: srcu: make init_srcu_struct() consistently wrap __init_srcu_struct() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restructure the SRCU initialization functions so it always follows one direction: init_srcu_struct() -> __init_srcu_struct() -> lockdep or generic This uses the same wrapper style as mutex. It avoids the old confusing style where init_srcu_struct() and __init_srcu_struct() called each other in different configs. It also helps Rust side to have simpler helper for SRCU initialization. Signed-off-by: Onur Özkan Reviewed-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Boqun Feng Signed-off-by: Paul E. McKenney --- include/linux/srcu.h | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/srcu.h b/include/linux/srcu.h index a54ce9e808b9..c5ab7df6fe5c 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -25,20 +25,19 @@ context_lock_struct(srcu_struct, __reentrant_ctx_lock); #ifdef CONFIG_DEBUG_LOCK_ALLOC -int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); +int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key); +static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) +{ + return init_srcu_struct_lockdep(ssp, name, key); +} #ifndef CONFIG_TINY_SRCU int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); #endif // #ifndef CONFIG_TINY_SRCU -#define init_srcu_struct(ssp) \ -({ \ - static struct lock_class_key __srcu_key; \ - \ - __init_srcu_struct((ssp), #ssp, &__srcu_key); \ -}) - #define init_srcu_struct_fast(ssp) \ ({ \ static struct lock_class_key __srcu_key; \ @@ -56,7 +55,12 @@ int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, #define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name }, #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ -int init_srcu_struct(struct srcu_struct *ssp); +int init_srcu_struct_generic(struct srcu_struct *ssp); +static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) +{ + return init_srcu_struct_generic(ssp); +} #ifndef CONFIG_TINY_SRCU int init_srcu_struct_fast(struct srcu_struct *ssp); int init_srcu_struct_fast_updown(struct srcu_struct *ssp); @@ -65,6 +69,13 @@ int init_srcu_struct_fast_updown(struct srcu_struct *ssp); #define __SRCU_DEP_MAP_INIT(srcu_name) #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ +#define init_srcu_struct(ssp) \ +({ \ + static struct lock_class_key __srcu_key; \ + \ + __init_srcu_struct((ssp), #ssp, &__srcu_key); \ +}) + /* Values for SRCU Tree srcu_data ->srcu_reader_flavor, but also used by rcutorture. */ #define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock(). #define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe(). -- cgit v1.2.3 From 91d5185187e97611e328d03ee5e00e54b5d6d2b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Onur=20=C3=96zkan?= Date: Sat, 13 Jun 2026 09:40:09 +0300 Subject: srcu: expose srcu_readers_active() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is needed by rust/helpers/srcu.c which now adds rust_helper_srcu_readers_active() as a wrapper around the SRCU helper for Rust callers. To achive this: 1- Move the srcu_readers_active() implementation from "kernel/rcu/srcutree.c" to "include/linux/srcutree.h". 2- Implement a matching srcu_readers_active() in "include/linux/srcutiny.h" and use it on the existing open-coded WARN_ON() check in cleanup_srcu_struct(). Signed-off-by: Onur Özkan Reviewed-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Boqun Feng Signed-off-by: Paul E. McKenney --- include/linux/srcutiny.h | 13 +++++++++++++ include/linux/srcutree.h | 24 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+) (limited to 'include/linux') diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h index 905b629e8fa3..fbcf13bc12d1 100644 --- a/include/linux/srcutiny.h +++ b/include/linux/srcutiny.h @@ -154,4 +154,17 @@ static inline void srcu_torture_stats_print(struct srcu_struct *ssp, data_race(READ_ONCE(ssp->srcu_idx_max))); } +/** + * srcu_readers_active - returns true if there are readers. and false otherwise. + * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). + * + * Note that this is not an atomic primitive, and can therefore suffer + * severe errors when invoked on an active srcu_struct. That said, it + * can be useful as an error check at cleanup time. + */ +static inline bool srcu_readers_active(struct srcu_struct *ssp) +{ + return READ_ONCE(ssp->srcu_lock_nesting[0]) || READ_ONCE(ssp->srcu_lock_nesting[1]); +} + #endif diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h index fd1a9270cb9a..75e54e4f963f 100644 --- a/include/linux/srcutree.h +++ b/include/linux/srcutree.h @@ -374,4 +374,28 @@ static inline void srcu_check_read_flavor(struct srcu_struct *ssp, int read_flav __srcu_check_read_flavor(ssp, read_flavor); } +/** + * srcu_readers_active - returns true if there are readers. and false otherwise. + * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). + * + * Note that this is not an atomic primitive, and can therefore suffer + * severe errors when invoked on an active srcu_struct. That said, it + * can be useful as an error check at cleanup time. + */ +static inline bool srcu_readers_active(struct srcu_struct *ssp) +{ + int cpu; + unsigned long sum = 0; + + for_each_possible_cpu(cpu) { + struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); + + sum += atomic_long_read(&sdp->srcu_ctrs[0].srcu_locks); + sum += atomic_long_read(&sdp->srcu_ctrs[1].srcu_locks); + sum -= atomic_long_read(&sdp->srcu_ctrs[0].srcu_unlocks); + sum -= atomic_long_read(&sdp->srcu_ctrs[1].srcu_unlocks); + } + return sum; +} + #endif -- cgit v1.2.3 From 736084507faa3596caad181a2ba6aa2a96197c52 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Wed, 24 Jun 2026 06:23:43 -0700 Subject: rcu: Rename struct rcu_gp_oldstate to rcu_gp_seq The polled grace-period state structure rcu_gp_oldstate holds a snapshot of the normal (and, on SMP, expedited) grace-period sequence numbers. Upcoming changes store this structure in the callback segment list, where the "oldstate" name reads poorly: there it represents the grace period a segment is waiting on and is also compared against the current grace-period state. Rename struct rcu_gp_oldstate to the more neutral struct rcu_gp_seq, and shorten its members rgos_norm and rgos_exp to norm and exp. Local variables and parameters of this type are renamed from rgosp/rgos to gsp/gs accordingly. While at it, provide a single definition of the structure in rcupdate.h rather than separate Tiny-RCU and Tree-RCU definitions, and give it the ->exp field unconditionally. Tiny RCU does not track expedited grace periods and leaves ->exp unused, but a single definition that always has ->exp lets the shared callback code in rcu_segcblist.c reference it without CONFIG_SMP guards, including on !SMP builds. No functional change. Signed-off-by: Puranjay Mohan Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 13 +++++++++++-- include/linux/rcupdate_wait.h | 2 +- include/linux/rcutiny.h | 36 ++++++++++++++++-------------------- include/linux/rcutree.h | 29 ++++++++++++----------------- 4 files changed, 40 insertions(+), 40 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 5e95acc33989..ce00f1726e95 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -52,9 +52,18 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func); void rcu_barrier_tasks(void); void synchronize_rcu(void); -struct rcu_gp_oldstate; +/* + * Grace-period sequence snapshot for the polled RCU APIs: ->norm for the + * normal grace period and ->exp for the expedited one. ->exp is unused by + * Tiny RCU, but is present unconditionally so that a single definition + * serves both Tiny RCU and Tree RCU. + */ +struct rcu_gp_seq { + unsigned long norm; + unsigned long exp; +}; unsigned long get_completed_synchronize_rcu(void); -void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +void get_completed_synchronize_rcu_full(struct rcu_gp_seq *gsp); // Maximum number of unsigned long values corresponding to // not-yet-completed RCU grace periods. diff --git a/include/linux/rcupdate_wait.h b/include/linux/rcupdate_wait.h index 4c92d4291cce..fa884704a3b7 100644 --- a/include/linux/rcupdate_wait.h +++ b/include/linux/rcupdate_wait.h @@ -18,7 +18,7 @@ struct rcu_synchronize { struct completion completion; /* This is for debugging. */ - struct rcu_gp_oldstate oldstate; + struct rcu_gp_seq oldstate; }; void wakeme_after_rcu(struct rcu_head *head); diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index f519cd680228..e56ded733b1b 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h @@ -14,11 +14,7 @@ #include /* for HZ */ -struct rcu_gp_oldstate { - unsigned long rgos_norm; -}; - -// Maximum number of rcu_gp_oldstate values corresponding to +// Maximum number of rcu_gp_seq values corresponding to // not-yet-completed RCU grace periods. #define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 2 @@ -26,31 +22,31 @@ struct rcu_gp_oldstate { * Are the two oldstate values the same? See the Tree RCU version for * docbook header. */ -static inline bool same_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp1, - struct rcu_gp_oldstate *rgosp2) +static inline bool same_state_synchronize_rcu_full(struct rcu_gp_seq *rgosp1, + struct rcu_gp_seq *rgosp2) { - return rgosp1->rgos_norm == rgosp2->rgos_norm; + return rgosp1->norm == rgosp2->norm; } unsigned long get_state_synchronize_rcu(void); -static inline void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +static inline void get_state_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - rgosp->rgos_norm = get_state_synchronize_rcu(); + gsp->norm = get_state_synchronize_rcu(); } unsigned long start_poll_synchronize_rcu(void); -static inline void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +static inline void start_poll_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - rgosp->rgos_norm = start_poll_synchronize_rcu(); + gsp->norm = start_poll_synchronize_rcu(); } bool poll_state_synchronize_rcu(unsigned long oldstate); -static inline bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +static inline bool poll_state_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - return poll_state_synchronize_rcu(rgosp->rgos_norm); + return poll_state_synchronize_rcu(gsp->norm); } static inline void cond_synchronize_rcu(unsigned long oldstate) @@ -58,9 +54,9 @@ static inline void cond_synchronize_rcu(unsigned long oldstate) might_sleep(); } -static inline void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +static inline void cond_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - cond_synchronize_rcu(rgosp->rgos_norm); + cond_synchronize_rcu(gsp->norm); } static inline unsigned long start_poll_synchronize_rcu_expedited(void) @@ -68,9 +64,9 @@ static inline unsigned long start_poll_synchronize_rcu_expedited(void) return start_poll_synchronize_rcu(); } -static inline void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) +static inline void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp) { - rgosp->rgos_norm = start_poll_synchronize_rcu_expedited(); + gsp->norm = start_poll_synchronize_rcu_expedited(); } static inline void cond_synchronize_rcu_expedited(unsigned long oldstate) @@ -78,9 +74,9 @@ static inline void cond_synchronize_rcu_expedited(unsigned long oldstate) cond_synchronize_rcu(oldstate); } -static inline void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) +static inline void cond_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp) { - cond_synchronize_rcu_expedited(rgosp->rgos_norm); + cond_synchronize_rcu_expedited(gsp->norm); } extern void rcu_barrier(void); diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 9d2d7bd251d4..16a04202888b 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -38,12 +38,7 @@ void synchronize_rcu_expedited(void); void rcu_barrier(void); void rcu_momentary_eqs(void); -struct rcu_gp_oldstate { - unsigned long rgos_norm; - unsigned long rgos_exp; -}; - -// Maximum number of rcu_gp_oldstate values corresponding to +// Maximum number of rcu_gp_seq values corresponding to // not-yet-completed RCU grace periods. #define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 4 @@ -60,29 +55,29 @@ struct rcu_gp_oldstate { * to a list header, allowing those structures to be slightly smaller. * * Note that equality is judged on a bitwise basis, so that an - * @rcu_gp_oldstate structure with an already-completed state in one field + * @rcu_gp_seq structure with an already-completed state in one field * will compare not-equal to a structure with an already-completed state - * in the other field. After all, the @rcu_gp_oldstate structure is opaque + * in the other field. After all, the @rcu_gp_seq structure is opaque * so how did such a situation come to pass in the first place? */ -static inline bool same_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp1, - struct rcu_gp_oldstate *rgosp2) +static inline bool same_state_synchronize_rcu_full(struct rcu_gp_seq *rgosp1, + struct rcu_gp_seq *rgosp2) { - return rgosp1->rgos_norm == rgosp2->rgos_norm && rgosp1->rgos_exp == rgosp2->rgos_exp; + return rgosp1->norm == rgosp2->norm && rgosp1->exp == rgosp2->exp; } unsigned long start_poll_synchronize_rcu_expedited(void); -void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp); +void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp); void cond_synchronize_rcu_expedited(unsigned long oldstate); -void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp); +void cond_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp); unsigned long get_state_synchronize_rcu(void); -void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +void get_state_synchronize_rcu_full(struct rcu_gp_seq *gsp); unsigned long start_poll_synchronize_rcu(void); -void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +void start_poll_synchronize_rcu_full(struct rcu_gp_seq *gsp); bool poll_state_synchronize_rcu(unsigned long oldstate); -bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +bool poll_state_synchronize_rcu_full(struct rcu_gp_seq *gsp); void cond_synchronize_rcu(unsigned long oldstate); -void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +void cond_synchronize_rcu_full(struct rcu_gp_seq *gsp); #ifdef CONFIG_PROVE_RCU void rcu_irq_exit_check_preempt(void); -- cgit v1.2.3 From fb5c815f12b5c17ed6c8c5879b3acb1407e5bac4 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Wed, 24 Jun 2026 06:23:46 -0700 Subject: rcu/segcblist: Track segment grace periods with struct rcu_gp_seq Change the type of the per-segment ->gp_seq[] array in struct rcu_segcblist from unsigned long to struct rcu_gp_seq. This prepares the callback tracking infrastructure to record both normal and expedited grace periods per segment. The rcu_segcblist_nextgp(), rcu_segcblist_advance(), and rcu_segcblist_accelerate() helpers now take a struct rcu_gp_seq * instead of an unsigned long, and all callers use the .norm field for comparisons and assignments. The SRCU and Tasks RCU wrappers construct a struct rcu_gp_seq with only .norm set and forward to the core helpers. No functional change: only the .norm field is used. Signed-off-by: Puranjay Mohan Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- include/linux/rcu_segcblist.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/rcu_segcblist.h b/include/linux/rcu_segcblist.h index 2fdc2208f1ca..137cc23b024c 100644 --- a/include/linux/rcu_segcblist.h +++ b/include/linux/rcu_segcblist.h @@ -190,7 +190,7 @@ struct rcu_cblist { struct rcu_segcblist { struct rcu_head *head; struct rcu_head **tails[RCU_CBLIST_NSEGS]; - unsigned long gp_seq[RCU_CBLIST_NSEGS]; + struct rcu_gp_seq gp_seq[RCU_CBLIST_NSEGS]; #ifdef CONFIG_RCU_NOCB_CPU atomic_long_t len; #else -- cgit v1.2.3 From 573d76f7db5ba050bc7c7c73d30e1364fcee269b Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sun, 28 Jun 2026 09:00:01 -0700 Subject: rcu: Mark __rcu_access_pointer() as context_unsafe() A simple comparison of a pointer returned by rcu_access_pointer() results in a context-analysis warning for lockless inspection of the RCU-protected (also known as __rcu-protected) pointer. This can be suppressed by placing context_unsafe() calls around calls rcu_access_pointer(), but this is messy and distracting. This commit therefore wraps the underlying __rcu_access_pointer() macro with a call to context_unsafe(), thereby informing the context-analysis code that rcu_access_pointer() may safely be invoked outside of an RCU read-side critical section. Reported-by: Christoph Hellwig Suggested-by: Marco Elver Signed-off-by: Paul E. McKenney Tested-by: Nilay Shroff Reviewed-by: Marco Elver --- include/linux/rcupdate.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 5e95acc33989..e40dc2e20c5b 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -490,12 +490,12 @@ context_unsafe( \ */ #define unrcu_pointer(p) __unrcu_pointer(p, __UNIQUE_ID(rcu)) -#define __rcu_access_pointer(p, local, space) \ +#define __rcu_access_pointer(p, local, space) context_unsafe( \ ({ \ typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \ rcu_check_sparse(p, space); \ ((typeof(*p) __force __kernel *)(local)); \ -}) +}) ) #define __rcu_dereference_check(p, local, c, space) \ ({ \ /* Dependency order vs. p above. */ \ -- cgit v1.2.3 From 647dd6e09481daa23d93e927c2b7f46a3acf8a03 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sun, 12 Jul 2026 10:50:08 -0700 Subject: rcu-tasks: Apply READ_ONCE() and WRITE_ONCE() to fix data race Now that rcutorture tests readers from interrupt handlers, KCSAN spotted an additional data race. This commit therefore fixes it by applying READ_ONCE() and WRITE_ONCE(). Signed-off-by: Paul E. McKenney --- include/linux/rcupdate_trace.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index cee89e51e45c..fd3ddeb6aa3b 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -95,10 +95,13 @@ static inline void rcu_read_unlock_tasks_trace(struct srcu_ctr __percpu *scp) */ static inline void rcu_read_lock_trace(void) { + int n; struct task_struct *t = current; rcu_try_lock_acquire(&rcu_tasks_trace_srcu_struct.dep_map); - if (t->trc_reader_nesting++) { + n = READ_ONCE(t->trc_reader_nesting); + WRITE_ONCE(t->trc_reader_nesting, n + 1); + if (n) { // In case we interrupted a Tasks Trace RCU reader. return; } @@ -119,12 +122,15 @@ static inline void rcu_read_lock_trace(void) */ static inline void rcu_read_unlock_trace(void) { + int n; struct srcu_ctr __percpu *scp; struct task_struct *t = current; scp = t->trc_reader_scp; barrier(); // scp before nesting to protect against interrupt handler. - if (!--t->trc_reader_nesting) { + n = READ_ONCE(t->trc_reader_nesting) - 1; + WRITE_ONCE(t->trc_reader_nesting, n); + if (!n) { if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB)) smp_mb(); // Placeholder for more selective ordering __srcu_read_unlock_fast(&rcu_tasks_trace_srcu_struct, scp); -- cgit v1.2.3 From 942b3e8f3f3f51011161cb728375481ebbc84e22 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Wed, 24 Jun 2026 06:23:49 -0700 Subject: rcu: Update comments for gp_seq and expedited GP tracking Update documentation comments throughout the RCU callback infrastructure to reflect the transition from a single grace-period sequence number to the full struct rcu_gp_seq that tracks both normal and expedited grace periods. The ->gp_seq[] array documentation in rcu_segcblist.h is updated to describe dual (normal and expedited) GP tracking. The rcu_segcblist_advance(), rcu_segcblist_accelerate(), and rcu_advance_cbs() comments are updated to refer to the struct rcu_gp_seq state (gsp) instead of the old bare grace-period sequence number (seq). Reviewed-by: Paul E. McKenney Signed-off-by: Puranjay Mohan Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- include/linux/rcu_segcblist.h | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcu_segcblist.h b/include/linux/rcu_segcblist.h index 137cc23b024c..08b63ecf719b 100644 --- a/include/linux/rcu_segcblist.h +++ b/include/linux/rcu_segcblist.h @@ -50,12 +50,14 @@ struct rcu_cblist { * Note that RCU_WAIT_TAIL cannot be empty unless RCU_NEXT_READY_TAIL is also * empty. * - * The ->gp_seq[] array contains the grace-period number at which the - * corresponding segment of callbacks will be ready to invoke. A given - * element of this array is meaningful only when the corresponding segment - * is non-empty, and it is never valid for RCU_DONE_TAIL (whose callbacks - * are already ready to invoke) or for RCU_NEXT_TAIL (whose callbacks have - * not yet been assigned a grace-period number). + * The ->gp_seq[] array contains the grace-period state at which the + * corresponding segment of callbacks will be ready to invoke. This tracks + * both normal and expedited grace periods, allowing callbacks to complete + * when either type of GP finishes. A given element of this array is + * meaningful only when the corresponding segment is non-empty, and it is + * never valid for RCU_DONE_TAIL (whose callbacks are already ready to + * invoke) or for RCU_NEXT_TAIL (whose callbacks have not yet been assigned + * a grace-period state). */ #define RCU_DONE_TAIL 0 /* Also RCU_WAIT head. */ #define RCU_WAIT_TAIL 1 /* Also RCU_NEXT_READY head. */ -- cgit v1.2.3 From f99dc9e288c5a2e191cca8a9af59113a8bc2768a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 20 Jul 2026 08:27:59 -0700 Subject: rcu-tasks: Fix IRQ read lock/unlock data race As noted by Marco Elver: rcu_read_lock_trace() .... t->trc_reader_scp = __srcu_read_lock_fast(&rcu_tasks_trace_srcu_struct); rcu_read_unlock_trace() < ... var decls only ... > scp = t->trc_reader_scp; This constitutes a data race between these two accesses to t->trc_reader_scp. If rcu_read_lock_trace() were to tear its store, this value would be corrupted. This commit therefore defers the rcu_read_lock_untrace() function's load from t->trc_reader_scp until after it has verified that this is the outermost rcu_read_unlock_trace(). With this change, the interrupt handler increments and decrements t->trc_reader_nesting and does not access t->trc_reader_scp, thus avoiding the data race. KCSAN located this issue. Signed-off-by: Paul E. McKenney --- include/linux/rcupdate_trace.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index fd3ddeb6aa3b..70decf877348 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -126,11 +126,13 @@ static inline void rcu_read_unlock_trace(void) struct srcu_ctr __percpu *scp; struct task_struct *t = current; - scp = t->trc_reader_scp; - barrier(); // scp before nesting to protect against interrupt handler. n = READ_ONCE(t->trc_reader_nesting) - 1; - WRITE_ONCE(t->trc_reader_nesting, n); - if (!n) { + if (n) { + WRITE_ONCE(t->trc_reader_nesting, n); + } else { + scp = t->trc_reader_scp; // Compiler cannot hoist load due to data raciness. + barrier(); // scp before nesting to protect against interrupt handler. + WRITE_ONCE(t->trc_reader_nesting, n); if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB)) smp_mb(); // Placeholder for more selective ordering __srcu_read_unlock_fast(&rcu_tasks_trace_srcu_struct, scp); -- cgit v1.2.3 From 0f5fe218d9d4e23d47a49873c9bfacd717b9c145 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 27 Jul 2026 15:11:04 -0700 Subject: rcu-tasks: Convert cond_resched_tasks_rcu_qs() to static inline In order to make "cc -E" output less annoying, this commit converts cond_resched_tasks_rcu_qs() to static inline. You know, the READ_ONCE() and WRITE_ONCE() macros used to be *so* simple. ;-) Reported-by: Andrew Morton Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 5e95acc33989..5e8770fd7b72 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -208,15 +208,15 @@ static inline void exit_tasks_rcu_finish(void) { } /** * cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU * - * This macro resembles cond_resched(), except that it is defined to + * This function resembles cond_resched(), except that it is defined to * report potential quiescent states to RCU-tasks even if the cond_resched() * machinery were to be shut off, as some advocate for PREEMPTION kernels. */ -#define cond_resched_tasks_rcu_qs() \ -do { \ - rcu_tasks_qs(current, false); \ - cond_resched(); \ -} while (0) +static inline void cond_resched_tasks_rcu_qs(void) +{ + rcu_tasks_qs(current, false); + cond_resched(); +} /** * rcu_softirq_qs_periodic - Report RCU and RCU-Tasks quiescent states -- cgit v1.2.3 From 86fa5387b473c160c26b781dff55014c4e1b3db0 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 8 Jul 2026 17:34:07 -0700 Subject: rcutorture: Make RCU Tasks Trace track Reader Batches This commit adds the ->get_sp_seq and ->gp_diff fields to the tasks_tracing_ops structure so that RCU Tasks Trace rcutorture runs will track Reader Batch. Signed-off-by: Paul E. McKenney --- include/linux/rcupdate_trace.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index cee89e51e45c..f2c4173bf14f 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -198,10 +198,13 @@ static inline void rcu_tasks_trace_expedite_current(void) srcu_expedite_current(&rcu_tasks_trace_srcu_struct); } +unsigned long rcu_tasks_trace_batches_completed(void); + // Placeholders to enable stepwise transition. void __init rcu_tasks_trace_suppress_unused(void); #else +static inline unsigned long rcu_tasks_trace_batches_completed(void) { return 0; } /* * The BPF JIT forms these addresses even when it doesn't call these * functions, so provide definitions that result in runtime errors. -- cgit v1.2.3