From a47dea1baa3b445365ad0757e6dee4ef8fde49e6 Mon Sep 17 00:00:00 2001 From: Onur Özkan Date: Sat, 13 Jun 2026 09:40:07 +0300 Subject: srcu: make init_srcu_struct() consistently wrap __init_srcu_struct() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restructure the SRCU initialization functions so it always follows one direction: init_srcu_struct() -> __init_srcu_struct() -> lockdep or generic This uses the same wrapper style as mutex. It avoids the old confusing style where init_srcu_struct() and __init_srcu_struct() called each other in different configs. It also helps Rust side to have simpler helper for SRCU initialization. Signed-off-by: Onur Özkan Reviewed-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Boqun Feng Signed-off-by: Paul E. McKenney --- include/linux/srcu.h | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) (limited to 'include') diff --git a/include/linux/srcu.h b/include/linux/srcu.h index a54ce9e808b9..c5ab7df6fe5c 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -25,20 +25,19 @@ context_lock_struct(srcu_struct, __reentrant_ctx_lock); #ifdef CONFIG_DEBUG_LOCK_ALLOC -int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); +int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key); +static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) +{ + return init_srcu_struct_lockdep(ssp, name, key); +} #ifndef CONFIG_TINY_SRCU int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); #endif // #ifndef CONFIG_TINY_SRCU -#define init_srcu_struct(ssp) \ -({ \ - static struct lock_class_key __srcu_key; \ - \ - __init_srcu_struct((ssp), #ssp, &__srcu_key); \ -}) - #define init_srcu_struct_fast(ssp) \ ({ \ static struct lock_class_key __srcu_key; \ @@ -56,7 +55,12 @@ int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, #define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name }, #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ -int init_srcu_struct(struct srcu_struct *ssp); +int init_srcu_struct_generic(struct srcu_struct *ssp); +static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) +{ + return init_srcu_struct_generic(ssp); +} #ifndef CONFIG_TINY_SRCU int init_srcu_struct_fast(struct srcu_struct *ssp); int init_srcu_struct_fast_updown(struct srcu_struct *ssp); @@ -65,6 +69,13 @@ int init_srcu_struct_fast_updown(struct srcu_struct *ssp); #define __SRCU_DEP_MAP_INIT(srcu_name) #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ +#define init_srcu_struct(ssp) \ +({ \ + static struct lock_class_key __srcu_key; \ + \ + __init_srcu_struct((ssp), #ssp, &__srcu_key); \ +}) + /* Values for SRCU Tree srcu_data ->srcu_reader_flavor, but also used by rcutorture. */ #define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock(). #define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe(). -- cgit From 91d5185187e97611e328d03ee5e00e54b5d6d2b0 Mon Sep 17 00:00:00 2001 From: Onur Özkan Date: Sat, 13 Jun 2026 09:40:09 +0300 Subject: srcu: expose srcu_readers_active() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is needed by rust/helpers/srcu.c which now adds rust_helper_srcu_readers_active() as a wrapper around the SRCU helper for Rust callers. To achive this: 1- Move the srcu_readers_active() implementation from "kernel/rcu/srcutree.c" to "include/linux/srcutree.h". 2- Implement a matching srcu_readers_active() in "include/linux/srcutiny.h" and use it on the existing open-coded WARN_ON() check in cleanup_srcu_struct(). Signed-off-by: Onur Özkan Reviewed-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Boqun Feng Signed-off-by: Paul E. McKenney --- include/linux/srcutiny.h | 13 +++++++++++++ include/linux/srcutree.h | 24 ++++++++++++++++++++++++ kernel/rcu/srcutiny.c | 2 +- kernel/rcu/srcutree.c | 25 ------------------------- rust/helpers/srcu.c | 5 +++++ 5 files changed, 43 insertions(+), 26 deletions(-) (limited to 'include') diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h index 905b629e8fa3..fbcf13bc12d1 100644 --- a/include/linux/srcutiny.h +++ b/include/linux/srcutiny.h @@ -154,4 +154,17 @@ static inline void srcu_torture_stats_print(struct srcu_struct *ssp, data_race(READ_ONCE(ssp->srcu_idx_max))); } +/** + * srcu_readers_active - returns true if there are readers. and false otherwise. + * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). + * + * Note that this is not an atomic primitive, and can therefore suffer + * severe errors when invoked on an active srcu_struct. That said, it + * can be useful as an error check at cleanup time. + */ +static inline bool srcu_readers_active(struct srcu_struct *ssp) +{ + return READ_ONCE(ssp->srcu_lock_nesting[0]) || READ_ONCE(ssp->srcu_lock_nesting[1]); +} + #endif diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h index fd1a9270cb9a..75e54e4f963f 100644 --- a/include/linux/srcutree.h +++ b/include/linux/srcutree.h @@ -374,4 +374,28 @@ static inline void srcu_check_read_flavor(struct srcu_struct *ssp, int read_flav __srcu_check_read_flavor(ssp, read_flavor); } +/** + * srcu_readers_active - returns true if there are readers. and false otherwise. + * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). + * + * Note that this is not an atomic primitive, and can therefore suffer + * severe errors when invoked on an active srcu_struct. That said, it + * can be useful as an error check at cleanup time. + */ +static inline bool srcu_readers_active(struct srcu_struct *ssp) +{ + int cpu; + unsigned long sum = 0; + + for_each_possible_cpu(cpu) { + struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); + + sum += atomic_long_read(&sdp->srcu_ctrs[0].srcu_locks); + sum += atomic_long_read(&sdp->srcu_ctrs[1].srcu_locks); + sum -= atomic_long_read(&sdp->srcu_ctrs[0].srcu_unlocks); + sum -= atomic_long_read(&sdp->srcu_ctrs[1].srcu_unlocks); + } + return sum; +} + #endif diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c index 47d48ed31848..558ba8d316db 100644 --- a/kernel/rcu/srcutiny.c +++ b/kernel/rcu/srcutiny.c @@ -85,7 +85,7 @@ EXPORT_SYMBOL_GPL(init_srcu_struct_generic); */ void cleanup_srcu_struct(struct srcu_struct *ssp) { - WARN_ON(ssp->srcu_lock_nesting[0] || ssp->srcu_lock_nesting[1]); + WARN_ON(srcu_readers_active(ssp)); irq_work_sync(&ssp->srcu_irq_work); flush_work(&ssp->srcu_work); WARN_ON(ssp->srcu_gp_running); diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 56c5db5c29a0..4a00e90e17fc 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -599,31 +599,6 @@ static bool srcu_readers_active_idx_check(struct srcu_struct *ssp, int idx) return srcu_readers_lock_idx(ssp, idx, did_gp, unlocks); } -/** - * srcu_readers_active - returns true if there are readers. and false - * otherwise - * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). - * - * Note that this is not an atomic primitive, and can therefore suffer - * severe errors when invoked on an active srcu_struct. That said, it - * can be useful as an error check at cleanup time. - */ -static bool srcu_readers_active(struct srcu_struct *ssp) -{ - int cpu; - unsigned long sum = 0; - - for_each_possible_cpu(cpu) { - struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); - - sum += atomic_long_read(&sdp->srcu_ctrs[0].srcu_locks); - sum += atomic_long_read(&sdp->srcu_ctrs[1].srcu_locks); - sum -= atomic_long_read(&sdp->srcu_ctrs[0].srcu_unlocks); - sum -= atomic_long_read(&sdp->srcu_ctrs[1].srcu_unlocks); - } - return sum; -} - /* * We use an adaptive strategy for synchronize_srcu() and especially for * synchronize_srcu_expedited(). We spin for a fixed time period diff --git a/rust/helpers/srcu.c b/rust/helpers/srcu.c index 225b3bf9334a..1a2f563640e0 100644 --- a/rust/helpers/srcu.c +++ b/rust/helpers/srcu.c @@ -9,6 +9,11 @@ __rust_helper int rust_helper_init_srcu_struct_with_key(struct srcu_struct *ssp, return __init_srcu_struct(ssp, name, key); } +__rust_helper bool rust_helper_srcu_readers_active(struct srcu_struct *ssp) +{ + return srcu_readers_active(ssp); +} + __rust_helper int rust_helper_srcu_read_lock(struct srcu_struct *ssp) { return srcu_read_lock(ssp); -- cgit From 573d76f7db5ba050bc7c7c73d30e1364fcee269b Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sun, 28 Jun 2026 09:00:01 -0700 Subject: rcu: Mark __rcu_access_pointer() as context_unsafe() A simple comparison of a pointer returned by rcu_access_pointer() results in a context-analysis warning for lockless inspection of the RCU-protected (also known as __rcu-protected) pointer. This can be suppressed by placing context_unsafe() calls around calls rcu_access_pointer(), but this is messy and distracting. This commit therefore wraps the underlying __rcu_access_pointer() macro with a call to context_unsafe(), thereby informing the context-analysis code that rcu_access_pointer() may safely be invoked outside of an RCU read-side critical section. Reported-by: Christoph Hellwig Suggested-by: Marco Elver Signed-off-by: Paul E. McKenney Tested-by: Nilay Shroff Reviewed-by: Marco Elver --- include/linux/rcupdate.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 5e95acc33989..e40dc2e20c5b 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -490,12 +490,12 @@ context_unsafe( \ */ #define unrcu_pointer(p) __unrcu_pointer(p, __UNIQUE_ID(rcu)) -#define __rcu_access_pointer(p, local, space) \ +#define __rcu_access_pointer(p, local, space) context_unsafe( \ ({ \ typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \ rcu_check_sparse(p, space); \ ((typeof(*p) __force __kernel *)(local)); \ -}) +}) ) #define __rcu_dereference_check(p, local, c, space) \ ({ \ /* Dependency order vs. p above. */ \ -- cgit From 647dd6e09481daa23d93e927c2b7f46a3acf8a03 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sun, 12 Jul 2026 10:50:08 -0700 Subject: rcu-tasks: Apply READ_ONCE() and WRITE_ONCE() to fix data race Now that rcutorture tests readers from interrupt handlers, KCSAN spotted an additional data race. This commit therefore fixes it by applying READ_ONCE() and WRITE_ONCE(). Signed-off-by: Paul E. McKenney --- include/linux/rcupdate_trace.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index cee89e51e45c..fd3ddeb6aa3b 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -95,10 +95,13 @@ static inline void rcu_read_unlock_tasks_trace(struct srcu_ctr __percpu *scp) */ static inline void rcu_read_lock_trace(void) { + int n; struct task_struct *t = current; rcu_try_lock_acquire(&rcu_tasks_trace_srcu_struct.dep_map); - if (t->trc_reader_nesting++) { + n = READ_ONCE(t->trc_reader_nesting); + WRITE_ONCE(t->trc_reader_nesting, n + 1); + if (n) { // In case we interrupted a Tasks Trace RCU reader. return; } @@ -119,12 +122,15 @@ static inline void rcu_read_lock_trace(void) */ static inline void rcu_read_unlock_trace(void) { + int n; struct srcu_ctr __percpu *scp; struct task_struct *t = current; scp = t->trc_reader_scp; barrier(); // scp before nesting to protect against interrupt handler. - if (!--t->trc_reader_nesting) { + n = READ_ONCE(t->trc_reader_nesting) - 1; + WRITE_ONCE(t->trc_reader_nesting, n); + if (!n) { if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB)) smp_mb(); // Placeholder for more selective ordering __srcu_read_unlock_fast(&rcu_tasks_trace_srcu_struct, scp); -- cgit From f99dc9e288c5a2e191cca8a9af59113a8bc2768a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 20 Jul 2026 08:27:59 -0700 Subject: rcu-tasks: Fix IRQ read lock/unlock data race As noted by Marco Elver: rcu_read_lock_trace() .... t->trc_reader_scp = __srcu_read_lock_fast(&rcu_tasks_trace_srcu_struct); rcu_read_unlock_trace() < ... var decls only ... > scp = t->trc_reader_scp; This constitutes a data race between these two accesses to t->trc_reader_scp. If rcu_read_lock_trace() were to tear its store, this value would be corrupted. This commit therefore defers the rcu_read_lock_untrace() function's load from t->trc_reader_scp until after it has verified that this is the outermost rcu_read_unlock_trace(). With this change, the interrupt handler increments and decrements t->trc_reader_nesting and does not access t->trc_reader_scp, thus avoiding the data race. KCSAN located this issue. Signed-off-by: Paul E. McKenney --- include/linux/rcupdate_trace.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'include') diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index fd3ddeb6aa3b..70decf877348 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -126,11 +126,13 @@ static inline void rcu_read_unlock_trace(void) struct srcu_ctr __percpu *scp; struct task_struct *t = current; - scp = t->trc_reader_scp; - barrier(); // scp before nesting to protect against interrupt handler. n = READ_ONCE(t->trc_reader_nesting) - 1; - WRITE_ONCE(t->trc_reader_nesting, n); - if (!n) { + if (n) { + WRITE_ONCE(t->trc_reader_nesting, n); + } else { + scp = t->trc_reader_scp; // Compiler cannot hoist load due to data raciness. + barrier(); // scp before nesting to protect against interrupt handler. + WRITE_ONCE(t->trc_reader_nesting, n); if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB)) smp_mb(); // Placeholder for more selective ordering __srcu_read_unlock_fast(&rcu_tasks_trace_srcu_struct, scp); -- cgit From 0f5fe218d9d4e23d47a49873c9bfacd717b9c145 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 27 Jul 2026 15:11:04 -0700 Subject: rcu-tasks: Convert cond_resched_tasks_rcu_qs() to static inline In order to make "cc -E" output less annoying, this commit converts cond_resched_tasks_rcu_qs() to static inline. You know, the READ_ONCE() and WRITE_ONCE() macros used to be *so* simple. ;-) Reported-by: Andrew Morton Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'include') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 5e95acc33989..5e8770fd7b72 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -208,15 +208,15 @@ static inline void exit_tasks_rcu_finish(void) { } /** * cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU * - * This macro resembles cond_resched(), except that it is defined to + * This function resembles cond_resched(), except that it is defined to * report potential quiescent states to RCU-tasks even if the cond_resched() * machinery were to be shut off, as some advocate for PREEMPTION kernels. */ -#define cond_resched_tasks_rcu_qs() \ -do { \ - rcu_tasks_qs(current, false); \ - cond_resched(); \ -} while (0) +static inline void cond_resched_tasks_rcu_qs(void) +{ + rcu_tasks_qs(current, false); + cond_resched(); +} /** * rcu_softirq_qs_periodic - Report RCU and RCU-Tasks quiescent states -- cgit From 86fa5387b473c160c26b781dff55014c4e1b3db0 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 8 Jul 2026 17:34:07 -0700 Subject: rcutorture: Make RCU Tasks Trace track Reader Batches This commit adds the ->get_sp_seq and ->gp_diff fields to the tasks_tracing_ops structure so that RCU Tasks Trace rcutorture runs will track Reader Batch. Signed-off-by: Paul E. McKenney --- include/linux/rcupdate_trace.h | 3 +++ kernel/rcu/rcutorture.c | 3 ++- kernel/rcu/tasks.h | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index cee89e51e45c..f2c4173bf14f 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -198,10 +198,13 @@ static inline void rcu_tasks_trace_expedite_current(void) srcu_expedite_current(&rcu_tasks_trace_srcu_struct); } +unsigned long rcu_tasks_trace_batches_completed(void); + // Placeholders to enable stepwise transition. void __init rcu_tasks_trace_suppress_unused(void); #else +static inline unsigned long rcu_tasks_trace_batches_completed(void) { return 0; } /* * The BPF JIT forms these addresses even when it doesn't call these * functions, so provide definitions that result in runtime errors. diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index c8118c38812a..8514fef3ec10 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -1252,7 +1252,8 @@ static struct rcu_torture_ops tasks_tracing_ops = { .read_delay = srcu_read_delay, /* just reuse srcu's version. */ .readunlock = tasks_tracing_torture_read_unlock, .readlock_held = rcu_read_lock_trace_held, - .get_gp_seq = rcu_no_completed, + .get_gp_seq = rcu_tasks_trace_batches_completed, + .gp_diff = rcu_seq_diff, .deferred_free = rcu_tasks_tracing_torture_deferred_free, .sync = synchronize_rcu_tasks_trace, .exp_sync = synchronize_rcu_tasks_trace, diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index f4da5fad70f5..496acac8796f 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1606,4 +1606,10 @@ static inline void rcu_tasks_bootup_oddness(void) {} DEFINE_SRCU_FAST(rcu_tasks_trace_srcu_struct); EXPORT_SYMBOL_GPL(rcu_tasks_trace_srcu_struct); +unsigned long rcu_tasks_trace_batches_completed(void) +{ + return srcu_batches_completed(&rcu_tasks_trace_srcu_struct); +} +EXPORT_SYMBOL_GPL(rcu_tasks_trace_batches_completed); + #endif /* #else #ifdef CONFIG_TASKS_TRACE_RCU */ -- cgit