From 051f5d223dfc1806e216f60e4b29c1bf35f5c2d3 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Sun, 5 Jul 2026 11:05:17 +0200 Subject: lockdep: Enable the printing of held locks of remote running tasks and print task CPU MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Background: ========== Currently lockdep does not print out the held locks of non-current tasks that are running on some other CPU, due to the fact that the held locks array is in flux and may be unreliable to print. Syzkaller on the other hand found it that the analysis of locking bugs is easier if we print this information too, because the more locking information the merrier. In particular races are bound to have multiple tasks running on different CPUs, and the exclusion of their held locks information is unnecessarily limiting. So while it's still true that printing out their held locks array is racy, it's not as bad as it seems. There's 16 internal callers to lockdep_print_held_locks(): - 14 callers call it with the current task, which should be safe out of box. - 1 caller, debug_show_all_locks(), calls it with RCU held, which should guarantee that 'p' cannot go away under us. - 1 caller, debug_show_held_locks(), exposes the internal API with the constraint that it should only be called by drivers or platform code if the task isn't actively running - we can assume that if it nevertheless does, it will be Their Problem™. As for held locks being changed from under debug_show_held_locks(), while the task cannot go away, so the held-locks array itself is safe (although potentially non-stable), AFAICS the worst-case race can be garbage printed out by print_lock(), not any actual crashes. In particular: unsigned int class_idx = hlock->class_idx; may be stale (belong to a lock that already got released on another CPU), but it should still be a valid class index bound by MAX_LOCKDEP_KEYS, and thus the lock_classes_in_use bitmap use should be safe. The other two accesses are ::acquire_ip and ::instance: printk(KERN_CONT "%px", hlock->instance); print_lock_name(hlock, lock); printk(KERN_CONT ", at: %pS\n", (void *)hlock->acquire_ip); But both are printed out as pointers, so no risk of dereference of a dangling pointer. We may print a garbage pointer. Also note that the check itself doesn't protect debug_show_held_locks() from printing garbage, as there's nothing that keeps a task from becoming runnable a nanosecond after we've run the task_is_running() check. In fact I'd argue that it's better to make this function *more* racy, for the simple robustness reason that we absolutely do not want it to crash even in the racy case. TL;DR: it should be fine to print the held locks of running tasks too, as long as we print out the information as well that a task is running, so that users are aware of any racy output. Implementation: ============== Implement that change. Also re-flow the function and streamline the printout into a single statement for all cases, which changes the 'no locks held by' / '%d lock[s] held by' phrasing that had a dependency on English spelling of plurals, to a uniform: locks held by bash/1234: %d Which spells correctly for 0, 1 and higher values, and should also be easier to parse both for humans and for scripts. Finally, print out the last CPU a task has ran on. This is very useful information for races and for locking bugs in particular. This basically extends the 'on CPU#%d' message we print for running tasks to all tasks we print. Reported-by: Tetsuo Handa Suggested-by: Tetsuo Handa Tested-by: Tetsuo Handa Signed-off-by: Ingo Molnar Cc: Boqun Feng Cc: Gary Guo Cc: Mark Brown Cc: Theodore Tso Cc: Miguel Ojeda Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Will Deacon Cc: Greg Kroah-Hartman Cc: Waiman Long Link: https://patch.msgid.link/akoeSIQGwqd9cZwd@gmail.com --- kernel/locking/lockdep.c | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index 2d4c5bab5af8..ff4bbf14665e 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -787,17 +787,33 @@ static void lockdep_print_held_locks(struct task_struct *p) { int i, depth = READ_ONCE(p->lockdep_depth); - if (!depth) - printk("no locks held by %s/%d.\n", p->comm, task_pid_nr(p)); - else - printk("%d lock%s held by %s/%d:\n", depth, - str_plural(depth), p->comm, task_pid_nr(p)); /* - * It's not reliable to print a task's held locks if it's not sleeping - * and it's not the current task. + * Note that it's always somewhat unreliable to print held locks + * of a task that is running on another CPU, but we cannot guarantee + * the stability of ->held_locks without actually stopping all active + * remote CPUs, which we absolutely do not want to do because it's + * very intrusive and thus slow. + * + * So we do the next best thing here: we print out the held lock + * array on a best-effort basis, without crashing even if the + * fields are being modified on another CPU. Note the careful + * construction of print_lock() so that it never crashes. + * + * We also print out the CPU the task is or was last running on, with + * the message saying 'on CPU...' if the task is running, and + * 'last CPU' if it's not. + * + * Also note that the task_is_running(p) information is fundamentally + * racy: even if the message says the task is 'on CPU', the task may + * have scheduled out already, or if it says 'last CPU', it may just + * have scheduled in on another CPU. But even with these limitations + * it's still useful debuggining information. */ - if (p != current && task_is_running(p)) - return; + printk("locks held by %s/%d: %d, %s CPU#%d%s\n", + p->comm, task_pid_nr(p), depth, + task_is_running(p) ? "last" : "on", task_cpu(p), + depth > 0 ? ":" : ""); + for (i = 0; i < depth; i++) { printk(" #%d: ", i); print_lock(p->held_locks + i); -- cgit v1.2.3 From 775e0f2284554e24855c4372faeb957a5e515337 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 28 Jul 2026 05:25:33 +0000 Subject: x86/runtime-const: Introduce runtime_const_mask_32() Futex hash computation requires a mask operation with read-only after init data that will be converted to a runtime constant in the subsequent commit. Introduce runtime_const_mask_32 to further optimize the mask operation in the futex hash computation hot path. [ prateek: Broke off the x86 chunk, commit message. ] Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: K Prateek Nayak Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260227161841.GH606826@noisy.programming.kicks-ass.net Link: https://patch.msgid.link/20260728052540.4728-2-kprateek.nayak@amd.com --- arch/x86/include/asm/runtime-const.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/arch/x86/include/asm/runtime-const.h b/arch/x86/include/asm/runtime-const.h index 4cd94fdcb45e..b13f7036c1c9 100644 --- a/arch/x86/include/asm/runtime-const.h +++ b/arch/x86/include/asm/runtime-const.h @@ -41,6 +41,15 @@ :"+r" (__ret)); \ __ret; }) +#define runtime_const_mask_32(val, sym) ({ \ + typeof(0u+(val)) __ret = (val); \ + asm_inline("and $0x12345678, %k0\n1:\n" \ + ".pushsection runtime_mask_" #sym ",\"a\"\n\t"\ + ".long 1b - 4 - .\n" \ + ".popsection" \ + : "+r" (__ret)); \ + __ret; }) + #define runtime_const_init(type, sym) do { \ extern s32 __start_runtime_##type##_##sym[]; \ extern s32 __stop_runtime_##type##_##sym[]; \ @@ -65,6 +74,11 @@ static inline void __runtime_fixup_shift(void *where, unsigned long val) *(unsigned char *)where = val; } +static inline void __runtime_fixup_mask(void *where, unsigned long val) +{ + *(unsigned int *)where = val; +} + static inline void runtime_const_fixup(void (*fn)(void *, unsigned long), unsigned long val, s32 *start, s32 *end) { -- cgit v1.2.3 From a6690350a4ae7558b39d15fcc776c4d1ab5bbf39 Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Tue, 28 Jul 2026 05:25:34 +0000 Subject: arm64/runtime-const: Use aarch64_insn_patch_text_nosync() for patching The current scheme to directly patch the kernel text for runtime constants runs into the following issue with futex adapted to using runtime constants on arm64: Unable to handle kernel write to read-only memory at virtual address ... The pc points to the *p assignment in the following call chain: futex_init() runtime_const_init(shift, __futex_shift) __runtime_fixup_shift() *p = cpu_to_le32(insn); which suggests that core_initcall() is too late to patch the kernel text directly unlike the "d_hash_shift" which is initialized during vfs_caches_init_early() before the protections are in place. Use aarch64_insn_patch_text_nosync() to patch the runtime constants instead of doing it directly to allow runtime_const_init() slightly later into the boot. Since aarch64_insn_patch_text_nosync() calls caches_clean_inval_pou() internally, __runtime_fixup_caches() ends up being redundant. runtime_const_init() are rare and the overheads of multiple calls to caches_clean_inval_pou() instead of batching them together should be negligible in practice. The cpu_to_le32() conversion of instruction isn't necessary since it is handled later in the aarch64_insn_patch_text_nosync() call-chain: aarch64_insn_patch_text_nosync(addr, insn) aarch64_insn_write(addr, insn) __aarch64_insn_write(addr, cpu_to_le32(insn)) Sashiko noted that aarch64_insn_patch_text_nosync() does not expect a lm_alias() address and Catalin suggested it is safe to drop the lm_alias() for runtime patching since the kernel text is readable. The address passed to fixup function is interpreted as a __le32 and dereferenced as is to read the opcode at the patch site. No functional changes are intended. Signed-off-by: K Prateek Nayak Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Catalin Marinas Reviewed-by: Charlie Jenkins Tested-by: Charlie Jenkins Link: https://patch.msgid.link/20260728052540.4728-3-kprateek.nayak@amd.com --- arch/arm64/include/asm/runtime-const.h | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/arch/arm64/include/asm/runtime-const.h b/arch/arm64/include/asm/runtime-const.h index c3dbd3ae68f6..838145bc289d 100644 --- a/arch/arm64/include/asm/runtime-const.h +++ b/arch/arm64/include/asm/runtime-const.h @@ -7,6 +7,7 @@ #endif #include +#include /* Sigh. You can still run arm64 in BE mode */ #include @@ -50,34 +51,26 @@ static inline void __runtime_fixup_16(__le32 *p, unsigned int val) u32 insn = le32_to_cpu(*p); insn &= 0xffe0001f; insn |= (val & 0xffff) << 5; - *p = cpu_to_le32(insn); -} - -static inline void __runtime_fixup_caches(void *where, unsigned int insns) -{ - unsigned long va = (unsigned long)where; - caches_clean_inval_pou(va, va + 4*insns); + aarch64_insn_patch_text_nosync(p, insn); } static inline void __runtime_fixup_ptr(void *where, unsigned long val) { - __le32 *p = lm_alias(where); + __le32 *p = where; __runtime_fixup_16(p, val); __runtime_fixup_16(p+1, val >> 16); __runtime_fixup_16(p+2, val >> 32); __runtime_fixup_16(p+3, val >> 48); - __runtime_fixup_caches(where, 4); } /* Immediate value is 6 bits starting at bit #16 */ static inline void __runtime_fixup_shift(void *where, unsigned long val) { - __le32 *p = lm_alias(where); + __le32 *p = where; u32 insn = le32_to_cpu(*p); insn &= 0xffc0ffff; insn |= (val & 63) << 16; - *p = cpu_to_le32(insn); - __runtime_fixup_caches(where, 1); + aarch64_insn_patch_text_nosync(p, insn); } static inline void runtime_const_fixup(void (*fn)(void *, unsigned long), -- cgit v1.2.3 From 7eccf137dc044c802f11d3d60bc7fc29066fdf92 Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Tue, 28 Jul 2026 05:25:35 +0000 Subject: arm64/runtime-const: Introduce runtime_const_mask_32() Futex hash computation requires a mask operation with read-only after init data that will be converted to a runtime constant in the subsequent commit. Introduce runtime_const_mask_32 to further optimize the mask operation in the futex hash computation hot path. Since all the current use-cases are of the form GENMASK(n, 0), with n > 0, a single: ubfx w0, w0, #0, #widthm1 // w0 = w0 [widthm1:0] instruction is used for amd64 to improve instruction dinsity and performance. "Arm A-profile A64 Instruction Set Architecture" manual, Sec. "A64 -- Base Instructions" [1] for UBFX instruction highlights the immediate "width" is encoded as width minus 1 in imms (Bits [15:10]) which is patched by __runtime_fixup_mask() once the mask is known. If a future use case arises that needs to tackle arbitrary mask, consider using: movz w1, #lo16, lsl #0 movk w1, #hi16, lsl #16 to patch the 32-bit mask in the asm block and return "__ret & (val)" from runtime_const_mask_32() which allows compiler to further optimize the logical and operation. __runtime_fixup_ptr() already patches a "movz, + movk lsl #16" sequence which can be reused when the need arises. A possible implementation for this alternate scheme can be found at [2]. Suggested-by: Samuel Holland Suggested-by: Charlie Jenkins Assisted-by: Claude:claude-sonnet-4-6 Signed-off-by: K Prateek Nayak Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Charlie Jenkins Tested-by: Charlie Jenkins Link: https://developer.arm.com/documentation/ddi0602/2026-03/Base-Instructions/ [1] Link: https://lore.kernel.org/lkml/20260430094730.31624-4-kprateek.nayak@amd.com/ [2] Link: https://patch.msgid.link/20260728052540.4728-4-kprateek.nayak@amd.com --- arch/arm64/include/asm/runtime-const.h | 46 ++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/arch/arm64/include/asm/runtime-const.h b/arch/arm64/include/asm/runtime-const.h index 838145bc289d..e221a868c48e 100644 --- a/arch/arm64/include/asm/runtime-const.h +++ b/arch/arm64/include/asm/runtime-const.h @@ -36,6 +36,17 @@ :"r" (0u+(val))); \ __ret; }) +#define runtime_const_mask_32(val, sym) ({ \ + unsigned long __ret; \ + asm_inline("1:\t" \ + "ubfx %w0, %w1, #0, #32\n\t" \ + ".pushsection runtime_mask_" #sym ",\"a\"\n\t" \ + ".long 1b - .\n\t" \ + ".popsection" \ + :"=r" (__ret) \ + :"r" (0u+(val))); \ + __ret; }) + #define runtime_const_init(type, sym) do { \ extern s32 __start_runtime_##type##_##sym[]; \ extern s32 __stop_runtime_##type##_##sym[]; \ @@ -73,6 +84,41 @@ static inline void __runtime_fixup_shift(void *where, unsigned long val) aarch64_insn_patch_text_nosync(p, insn); } +static inline void __runtime_fixup_mask(void *where, unsigned long val) +{ + unsigned int width = (val) ? __fls(val) + 1 : 0; + __le32 *p = where; + u32 insn; + + /* + * XXX: Current implementation only supports patching masks of + * form GENMASK(n, 0) (n >= 0) using a single UBFX instruction + * to improve performance, density, and covers all the current + * use-cases. + * + * When the need arises to support any generic mask, and this + * BUG_ON() is tripped, consider using a: + * + * movz %w0, #imm16 + * movk %w0, #imm16, lsl #16 + * + * sequence to load the 32bit const mask, and perform a logical + * and outside the asm block before returning the result. Fixup + * can simply reuse the existing __runtime_fixup_16() to patch + * the individual mov instructions. + */ + BUG_ON(!val || width > 32 || (GENMASK(width - 1, 0) != val)); + + /* + * The width of the mask is encoded as (width - 1) in imms + * which is 6 bits starting at bit #10. + */ + insn = le32_to_cpu(*p); + insn &= 0xffff03ff; + insn |= ((width - 1) & 0x1f) << 10; + aarch64_insn_patch_text_nosync(p, insn); +} + static inline void runtime_const_fixup(void (*fn)(void *, unsigned long), unsigned long val, s32 *start, s32 *end) { -- cgit v1.2.3 From ee10b1028129a73857593801614d5bf75a15b9c7 Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Tue, 28 Jul 2026 05:25:36 +0000 Subject: riscv/runtime-const: Replace open-coded placeholder with RUNTIME_MAGIC Define the placeholder used for lui + addi[w] patching sequence as RUNTIME_MAGIC and use that instead of open coding the constants in the inline assembly. No functional changes intended. Suggested-by: Guo Ren Signed-off-by: K Prateek Nayak Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Charlie Jenkins Reviewed-by: Guo Ren Tested-by: Charlie Jenkins Link: https://patch.msgid.link/20260728052540.4728-5-kprateek.nayak@amd.com --- arch/riscv/include/asm/runtime-const.h | 38 ++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/arch/riscv/include/asm/runtime-const.h b/arch/riscv/include/asm/runtime-const.h index 900db0a103d0..1ce02605d2e4 100644 --- a/arch/riscv/include/asm/runtime-const.h +++ b/arch/riscv/include/asm/runtime-const.h @@ -15,21 +15,23 @@ #include +#define RUNTIME_MAGIC __ASM_STR(0x89ABCDEF) + #ifdef CONFIG_32BIT -#define runtime_const_ptr(sym) \ -({ \ - typeof(sym) __ret; \ - asm_inline(".option push\n\t" \ - ".option norvc\n\t" \ - "1:\t" \ - "lui %[__ret],0x89abd\n\t" \ - "addi %[__ret],%[__ret],-0x211\n\t" \ - ".option pop\n\t" \ - ".pushsection runtime_ptr_" #sym ",\"a\"\n\t" \ - ".long 1b - .\n\t" \ - ".popsection" \ - : [__ret] "=r" (__ret)); \ - __ret; \ +#define runtime_const_ptr(sym) \ +({ \ + typeof(sym) __ret; \ + asm_inline(".option push\n\t" \ + ".option norvc\n\t" \ + "1:\t" \ + "lui %[__ret], %%hi(" RUNTIME_MAGIC ")\n\t" \ + "addi %[__ret],%[__ret], %%lo(" RUNTIME_MAGIC ")\n\t" \ + ".option pop\n\t" \ + ".pushsection runtime_ptr_" #sym ",\"a\"\n\t" \ + ".long 1b - .\n\t" \ + ".popsection" \ + : [__ret] "=r" (__ret)); \ + __ret; \ }) #else /* @@ -46,10 +48,10 @@ ".option push\n\t" \ ".option norvc\n\t" \ "1:\t" \ - "lui %[__ret],0x89abd\n\t" \ - "lui %[__tmp],0x1234\n\t" \ - "addiw %[__ret],%[__ret],-0x211\n\t" \ - "addiw %[__tmp],%[__tmp],0x567\n\t" \ + "lui %[__ret], %%hi(" RUNTIME_MAGIC ")\n\t" \ + "lui %[__tmp], %%hi(" RUNTIME_MAGIC ")\n\t" \ + "addiw %[__ret],%[__ret], %%lo(" RUNTIME_MAGIC ")\n\t" \ + "addiw %[__tmp],%[__tmp], %%lo(" RUNTIME_MAGIC ")\n\t" \ #define RISCV_RUNTIME_CONST_64_BASE \ "slli %[__tmp],%[__tmp],32\n\t" \ -- cgit v1.2.3 From e412c77541c7926ff61baec712616837a2c6887f Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Tue, 28 Jul 2026 05:25:37 +0000 Subject: riscv/runtime-const: Introduce runtime_const_mask_32() Futex hash computation requires a mask operation with read-only after init data that will be converted to a runtime constant in the subsequent commit. Introduce runtime_const_mask_32 to further optimize the mask operation in the futex hash computation hot path. Since all the current use-cases are of the form GENMASK(n, 0), with n > 0, following sequence: srli a0, a1, imm slli a0, a0, imm is used for RISC-V where imm = (31 - width) to improve instruction density and performance. "The RISC-V Instruction Set Manual, Volume I - Unprivileged Architecture" [1] Sec. 2.4.1 "Integer Register-Immediate Instructions" notes the immediate shift for SRLI and SLLI are 5 bits wide starting at bit #10. __runtime_fixup_shift() is reused to patch the immediate shifts for the two instructions. If a future use case arises that needs to tackle arbitrary mask, consider using: lui a0, 0x12346 # upper; +0x800 then >>12 for correct rounding addi a0, a0, 0x678 # lower 12 bits to patch the 32-bit mask in the asm block and return "__ret & (val)" from runtime_const_mask_32() which allows compiler to further optimize the logical and operation. __runtime_fixup_ptr() already patches a lui + addi sequence which can be reused when the need arises. A possible implementation for this alternate scheme can be found at [2]. Suggested-by: Samuel Holland Suggested-by: Charlie Jenkins Assisted-by: Claude:claude-sonnet-4-5 Signed-off-by: K Prateek Nayak Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Charlie Jenkins Tested-by: Charlie Jenkins Link: https://docs.riscv.org/reference/isa/_attachments/riscv-unprivileged.pdf [1] Link: https://lore.kernel.org/lkml/20260430094730.31624-6-kprateek.nayak@amd.com/ [2] Link: https://patch.msgid.link/20260728052540.4728-6-kprateek.nayak@amd.com --- arch/riscv/include/asm/asm.h | 1 + arch/riscv/include/asm/runtime-const.h | 44 ++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h index e9e8ba83e632..b8bf842d4c13 100644 --- a/arch/riscv/include/asm/asm.h +++ b/arch/riscv/include/asm/asm.h @@ -34,6 +34,7 @@ #define SZREG __REG_SEL(8, 4) #define LGREG __REG_SEL(3, 2) #define SRLI __REG_SEL(srliw, srli) +#define SLLI __REG_SEL(slliw, slli) #if __SIZEOF_POINTER__ == 8 #ifdef __ASSEMBLER__ diff --git a/arch/riscv/include/asm/runtime-const.h b/arch/riscv/include/asm/runtime-const.h index 1ce02605d2e4..a472ebbf0971 100644 --- a/arch/riscv/include/asm/runtime-const.h +++ b/arch/riscv/include/asm/runtime-const.h @@ -159,6 +159,23 @@ __ret; \ }) +#define runtime_const_mask_32(val, sym) \ +({ \ + u32 __ret; \ + asm_inline(".option push\n\t" \ + ".option norvc\n\t" \ + "1:\t" \ + SLLI " %[__ret],%[__val],12\n\t" \ + SRLI " %[__ret],%[__ret],12\n\t" \ + ".option pop\n\t" \ + ".pushsection runtime_mask_" #sym ",\"a\"\n\t" \ + ".long 1b - .\n\t" \ + ".popsection" \ + : [__ret] "=r" (__ret) \ + : [__val] "r" (val)); \ + __ret; \ +}) + #define runtime_const_init(type, sym) do { \ extern s32 __start_runtime_##type##_##sym[]; \ extern s32 __stop_runtime_##type##_##sym[]; \ @@ -262,6 +279,33 @@ static inline void __runtime_fixup_shift(void *where, unsigned long val) mutex_unlock(&text_mutex); } +static inline void __runtime_fixup_mask(void *where, unsigned long val) +{ + unsigned int width = (val) ? __fls(val) + 1 : 0; + + /* + * XXX: Current implementation only supports patching masks of + * form GENMASK(width, 0) (width >= 0) using a SRLI + SLLI + * sequence instead of LUI + ADDI + AND sequence to improve + * performance, density, and covers all the current use-cases. + * + * When the need arises to support any generic mask, and this + * BUG_ON() is tripped, consider using a: + * + * lui %[__ret], #imm16 + * addi %[__ret], #imm16 + * + * sequence to load the 32bit const mask, and perform a logical + * and outside the asm block before returning the result. Fixup + * can simply reuse the existing __runtime_fixup_32() to patch + * the LUI + ADDI sequence. + */ + BUG_ON(!val || width > 31 || (GENMASK(width - 1, 0) != val)); + + __runtime_fixup_shift(where, 32 - width); + __runtime_fixup_shift(where + 4, 32 - width); +} + static inline void runtime_const_fixup(void (*fn)(void *, unsigned long), unsigned long val, s32 *start, s32 *end) { -- cgit v1.2.3 From cf0de88094a7b4df18008b6efb1290f4efed731f Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Tue, 28 Jul 2026 05:25:38 +0000 Subject: s390/runtime-const: Introduce runtime_const_mask_32() Futex hash computation requires a mask operation with read-only after init data that will be converted to a runtime constant in the subsequent commit. Introduce runtime_const_mask_32 to further optimize the mask operation in the futex hash computation hot path. GCC generates a: nilf %r1, to tackle arbitrary 32-bit masks and the same is implemented here. Immediate patching pattern for __runtime_fixup_mask() has been adopted from __runtime_fixup_ptr(). Signed-off-by: K Prateek Nayak Signed-off-by: Peter Zijlstra (Intel) Acked-by: Heiko Carstens Link: https://patch.msgid.link/20260728052540.4728-7-kprateek.nayak@amd.com --- arch/s390/include/asm/runtime-const.h | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/arch/s390/include/asm/runtime-const.h b/arch/s390/include/asm/runtime-const.h index 17878b1d048c..7b71156031ec 100644 --- a/arch/s390/include/asm/runtime-const.h +++ b/arch/s390/include/asm/runtime-const.h @@ -33,6 +33,20 @@ __ret; \ }) +#define runtime_const_mask_32(val, sym) \ +({ \ + unsigned int __ret = (val); \ + \ + asm_inline( \ + "0: nilf %[__ret],12\n" \ + ".pushsection runtime_mask_" #sym ",\"a\"\n" \ + ".long 0b - .\n" \ + ".popsection" \ + : [__ret] "+d" (__ret) \ + : : "cc"); \ + __ret; \ +}) + #define runtime_const_init(type, sym) do { \ extern s32 __start_runtime_##type##_##sym[]; \ extern s32 __stop_runtime_##type##_##sym[]; \ @@ -43,12 +57,12 @@ __stop_runtime_##type##_##sym); \ } while (0) -/* 32-bit immediate for iihf and iilf in bits in I2 field */ static inline void __runtime_fixup_32(u32 *p, unsigned int val) { s390_kernel_write(p, &val, sizeof(val)); } +/* 32-bit immediate for iihf and iilf in bits in I2 field */ static inline void __runtime_fixup_ptr(void *where, unsigned long val) { __runtime_fixup_32(where + 2, val >> 32); @@ -65,6 +79,12 @@ static inline void __runtime_fixup_shift(void *where, unsigned long val) s390_kernel_write(where, &insn, sizeof(insn)); } +/* 32-bit immediate for nilf in bits in I2 field */ +static inline void __runtime_fixup_mask(void *where, unsigned long val) +{ + __runtime_fixup_32(where + 2, val); +} + static inline void runtime_const_fixup(void (*fn)(void *, unsigned long), unsigned long val, s32 *start, s32 *end) { -- cgit v1.2.3 From cb9362dddcd167662ad7c6347ce96fd47890a27f Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 28 Jul 2026 05:25:39 +0000 Subject: asm-generic/runtime-const: Add dummy runtime_const_mask_32() Add a dummy runtime_const_mask_32() for all the architectures that do not support runtime-const. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: K Prateek Nayak Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Charlie Jenkins Tested-by: Charlie Jenkins Link: https://patch.msgid.link/20260227161841.GH606826@noisy.programming.kicks-ass.net Link: https://patch.msgid.link/20260728052540.4728-8-kprateek.nayak@amd.com --- include/asm-generic/runtime-const.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/asm-generic/runtime-const.h b/include/asm-generic/runtime-const.h index 670499459514..03e6e3e02401 100644 --- a/include/asm-generic/runtime-const.h +++ b/include/asm-generic/runtime-const.h @@ -10,6 +10,7 @@ */ #define runtime_const_ptr(sym) (sym) #define runtime_const_shift_right_32(val, sym) ((u32)(val)>>(sym)) +#define runtime_const_mask_32(val, sym) ((u32)(val)&(sym)) #define runtime_const_init(type,sym) do { } while (0) #endif -- cgit v1.2.3 From b78b0b65825275f58336a43611a700de174be8c3 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 28 Jul 2026 05:25:40 +0000 Subject: futex: Use runtime constants for __futex_hash() hot path Runtime constify the read-only after init data __futex_shift(shift_32), __futex_mask(mask_32), and __futex_queues(ptr) used in __futex_hash() hot path to avoid referencing global variable. This also allows __futex_queues to be allocated dynamically to "nr_node_ids" slots instead of reserving config dependent MAX_NUMNODES (1 << CONFIG_NODES_SHIFT) worth of slots upfront. Runtime constants are initialized before their first access and runtime_const_init() provides necessary barrier to ensure subsequent accesses are not reordered against their initialization. No functional changes intended. perf bench futex on a 3rd Gen EPYC (2 x 64C/128T): +----------------+-----------+-----------+-----------+--------------+ | Benchmark | Kernel 1 | Kernel 2 | Unit | % Improvement| | | (avg/5) | (avg/5) | | (K2 vs K1) | +----------------+-----------+-----------+-----------+--------------+ | Wake-parallel | 0.01614 | 0.00456 | ms | +71.75% | | Requeue | 0.26394 | 0.24644 | ms | +6.63% | | Lock-pi | 34.0 | 57.2 | ops/sec | +68.24% | +----------------+-----------+-----------+-----------+--------------+ Performance testing on a 144-thread Intel(R) Xeon(R) CPU E7-8890 v3 (4 NUMA nodes): +-------------------------------------------------------------+ | perf bench futex hash -b 0 | +----------------------+------------+------------+------------+ | Configuration | As-is | Patched | Delta | +----------------------+------------+------------+------------+ | 1 thread, 1 futex | 6,449,632 | 6,532,004 | +1.28% | | 144 threads, 1024 fx | 2,111,486 | 2,139,685 | +1.34% | +----------------------+------------+------------+------------+i [ prateek: Dynamically allocate __futex_queues, mark the global data __ro_after_init since they are constified after futex_init(). ] Signed-off-by: Peter Zijlstra (Intel) Reported-by: Sebastian Andrzej Siewior # MAX_NUMNODES bloat Signed-off-by: K Prateek Nayak Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Charlie Jenkins Tested-by: Charlie Jenkins Link: https://patch.msgid.link/20260227161841.GH606826@noisy.programming.kicks-ass.net Link: https://patch.msgid.link/20260728052540.4728-9-kprateek.nayak@amd.com --- include/asm-generic/vmlinux.lds.h | 5 ++++- kernel/futex/core.c | 44 ++++++++++++++++++++++----------------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 5659f4b5a125..53207901d4c1 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -970,7 +970,10 @@ RUNTIME_CONST(ptr, __dentry_cache) \ RUNTIME_CONST(ptr, __names_cache) \ RUNTIME_CONST(ptr, __filp_cache) \ - RUNTIME_CONST(ptr, __bfilp_cache) + RUNTIME_CONST(ptr, __bfilp_cache) \ + RUNTIME_CONST(shift, __futex_shift) \ + RUNTIME_CONST(mask, __futex_mask) \ + RUNTIME_CONST(ptr, __futex_queues) /* Alignment must be consistent with (kunit_suite *) in include/kunit/test.h */ #define KUNIT_TABLE() \ diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 179b26e9c934..3bf6cb79f71a 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -48,23 +48,19 @@ #include +#include + #include "futex.h" #include "../locking/rtmutex_common.h" -/* - * The base of the bucket array and its size are always used together - * (after initialization only in futex_hash()), so ensure that they - * reside in the same cacheline. - */ -static struct { - unsigned long hashmask; - unsigned int hashshift; - struct futex_hash_bucket *queues[MAX_NUMNODES]; -} __futex_data __read_mostly __aligned(2*sizeof(long)); +static u32 __futex_mask __ro_after_init; +static u32 __futex_shift __ro_after_init; +static struct futex_hash_bucket **__futex_queues __ro_after_init; -#define futex_hashmask (__futex_data.hashmask) -#define futex_hashshift (__futex_data.hashshift) -#define futex_queues (__futex_data.queues) +static __always_inline struct futex_hash_bucket **futex_queues(void) +{ + return runtime_const_ptr(__futex_queues); +} struct futex_private_hash { int state; @@ -395,13 +391,13 @@ __futex_hash(union futex_key *key, struct futex_private_hash *fph, struct futex_ * NOTE: this isn't perfectly uniform, but it is fast and * handles sparse node masks. */ - node = (hash >> futex_hashshift) % nr_node_ids; + node = runtime_const_shift_right_32(hash, __futex_shift) % nr_node_ids; if (!node_possible(node)) { node = find_next_bit_wrap(node_possible_map.bits, nr_node_ids, node); } } - return &futex_queues[node][hash & futex_hashmask]; + return &futex_queues()[node][runtime_const_mask_32(hash, __futex_mask)]; } /** @@ -1922,7 +1918,7 @@ int futex_hash_allocate_default(void) * 16 <= threads * 4 <= global hash size */ buckets = roundup_pow_of_two(4 * threads); - buckets = clamp(buckets, 16, futex_hashmask + 1); + buckets = clamp(buckets, 16, __futex_mask + 1); if (current_buckets >= buckets) return 0; @@ -2020,10 +2016,21 @@ static int __init futex_init(void) hashsize = max(4, hashsize); hashsize = roundup_pow_of_two(hashsize); #endif - futex_hashshift = ilog2(hashsize); + __futex_mask = hashsize - 1; + __futex_shift = ilog2(hashsize); size = sizeof(struct futex_hash_bucket) * hashsize; order = get_order(size); + __futex_queues = kcalloc(nr_node_ids, sizeof(*__futex_queues), GFP_KERNEL); + + runtime_const_init(shift, __futex_shift); + runtime_const_init(mask, __futex_mask); + runtime_const_init(ptr, __futex_queues); + + barrier(); + + BUG_ON(!futex_queues()); + for_each_node(n) { struct futex_hash_bucket *table; @@ -2037,10 +2044,9 @@ static int __init futex_init(void) for (i = 0; i < hashsize; i++) futex_hash_bucket_init(&table[i]); - futex_queues[n] = table; + futex_queues()[n] = table; } - futex_hashmask = hashsize - 1; pr_info("futex hash table entries: %lu (%lu bytes on %d NUMA nodes, total %lu KiB, %s).\n", hashsize, size, num_possible_nodes(), size * num_possible_nodes() / 1024, order > MAX_PAGE_ORDER ? "vmalloc" : "linear"); -- cgit v1.2.3 From 5e601ab3615c86be7c4068ce992f94654693a032 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Wed, 1 Jul 2026 18:17:36 +0200 Subject: futex: Optimise the size check get_futex_key() The futex address must be naturally aligned and this is checked via "address % size" where `address' is the supplied address and `size' is the expected size of futex. It is guaranteed that `size' is power of two but the compiler does not see it and creates here a `div' operation (x86, arm, gcc-15). We can take advantage of the pow2 property and rewrite it as "address & (size-1)". As per testing, the command |perf bench futex hash -f 1 -b 16384 -t 1 -r 30 improved from | [thread 0] futex: 0x5619f931f740 [ 7001583 ops/sec ] to | [thread 0] futex: 0x55da173e5740 [ 7376137 ops/sec ] or by 5.3% Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260701161736.xYYizA0e@linutronix.de --- kernel/futex/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 3bf6cb79f71a..0864c6e8cde7 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -516,7 +516,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, * The futex address must be "naturally" aligned. */ key->both.offset = address % PAGE_SIZE; - if (unlikely((address % size) != 0)) + if (unlikely((address & (size-1)) != 0)) return -EINVAL; address -= key->both.offset; -- cgit v1.2.3 From 7577e00b9ab506202b9f1a33de3cc8cc6413a4db Mon Sep 17 00:00:00 2001 From: Naveen Kumar Chaudhary Date: Thu, 11 Jun 2026 23:08:17 +0530 Subject: locking/lockdep: Fix NULL pointer dereference in __lock_set_class() register_lock_class() can return NULL when the lock class pool is exhausted, graph_lock() fails, or key validation fails. However, __lock_set_class() uses the return value directly in pointer arithmetic without a NULL check: class = register_lock_class(lock, subclass, 0); hlock->class_idx = class - lock_classes; If class is NULL, this computes a wild offset that corrupts hlock->class_idx. The subsequent reacquire_held_locks() call will invoke hlock_class() with this corrupted index, leading to a NULL or out-of-bounds pointer dereference. Add the missing NULL check, consistent with how __lock_acquire() already handles this case at the same call site. Fixes: 64aa348edc61 ("lockdep: lock_set_subclass - reset a held lock's subclass") Signed-off-by: Naveen Kumar Chaudhary Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Waiman Long Reviewed-by: Dmitry Ilvokhin Link: https://patch.msgid.link/h2kfw43n4527x6mgi2lwpz2rieqnfzgictpv4wr5nyfjkc47co@2r5vz4uz44db --- kernel/locking/lockdep.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index ff4bbf14665e..25d77d4a1061 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -5453,6 +5453,8 @@ __lock_set_class(struct lockdep_map *lock, const char *name, lock->wait_type_outer, lock->lock_type); class = register_lock_class(lock, subclass, 0); + if (!class) + return 0; hlock->class_idx = class - lock_classes; curr->lockdep_depth = i; -- cgit v1.2.3 From 6cb423c96e842248ac4131ae0a25f0a50cef2776 Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Fri, 19 Jun 2026 14:49:36 +0200 Subject: static_call / jump_label: Replace __ASSEMBLY__ with __ASSEMBLER__ in headers While the GCC and Clang compilers already define __ASSEMBLER__ automatically when compiling assembly code, __ASSEMBLY__ is a macro that only gets defined by the Makefiles in the kernel. This can be very confusing when switching between userspace and kernelspace coding, or when dealing with uapi headers that rather should use __ASSEMBLER__ instead. So let's standardize now on the __ASSEMBLER__ macro that is provided by the compilers. This is a completely mechanical patch (done with a simple "sed -i" statement). Signed-off-by: Thomas Huth Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260619124936.208519-1-thuth@redhat.com --- arch/arm/include/asm/jump_label.h | 4 ++-- include/linux/jump_label.h | 10 +++++----- include/linux/static_call_types.h | 4 ++-- tools/include/linux/static_call_types.h | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/arch/arm/include/asm/jump_label.h b/arch/arm/include/asm/jump_label.h index a35aba7f548c..11024b7f1ef4 100644 --- a/arch/arm/include/asm/jump_label.h +++ b/arch/arm/include/asm/jump_label.h @@ -2,7 +2,7 @@ #ifndef _ASM_ARM_JUMP_LABEL_H #define _ASM_ARM_JUMP_LABEL_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include #include @@ -49,5 +49,5 @@ struct jump_entry { jump_label_t key; }; -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index b9c7b0ebf7b9..c95350a10987 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -71,7 +71,7 @@ * Additional babbling in: Documentation/staging/static-keys.rst */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include #include @@ -100,12 +100,12 @@ struct static_key { #endif /* CONFIG_JUMP_LABEL */ }; -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #ifdef CONFIG_JUMP_LABEL #include -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #ifdef CONFIG_HAVE_ARCH_JUMP_LABEL_RELATIVE struct jump_entry { @@ -180,7 +180,7 @@ static inline int jump_entry_size(struct jump_entry *entry) #endif #endif -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ enum jump_label_type { JUMP_LABEL_NOP = 0, @@ -524,6 +524,6 @@ extern bool ____wrong_branch_error(void); #define static_branch_enable_cpuslocked(x) static_key_enable_cpuslocked(&(x)->key) #define static_branch_disable_cpuslocked(x) static_key_disable_cpuslocked(&(x)->key) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _LINUX_JUMP_LABEL_H */ diff --git a/include/linux/static_call_types.h b/include/linux/static_call_types.h index cfb6ddeb292b..ac55bc966a56 100644 --- a/include/linux/static_call_types.h +++ b/include/linux/static_call_types.h @@ -25,7 +25,7 @@ #define STATIC_CALL_SITE_INIT 2UL /* init section */ #define STATIC_CALL_SITE_FLAGS 3UL -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * The static call site table needs to be created by external tooling (objtool @@ -102,6 +102,6 @@ struct static_call_key { #endif /* CONFIG_HAVE_STATIC_CALL */ -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _STATIC_CALL_TYPES_H */ diff --git a/tools/include/linux/static_call_types.h b/tools/include/linux/static_call_types.h index cfb6ddeb292b..ac55bc966a56 100644 --- a/tools/include/linux/static_call_types.h +++ b/tools/include/linux/static_call_types.h @@ -25,7 +25,7 @@ #define STATIC_CALL_SITE_INIT 2UL /* init section */ #define STATIC_CALL_SITE_FLAGS 3UL -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * The static call site table needs to be created by external tooling (objtool @@ -102,6 +102,6 @@ struct static_call_key { #endif /* CONFIG_HAVE_STATIC_CALL */ -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _STATIC_CALL_TYPES_H */ -- cgit v1.2.3 From c9c8578ad58e66a64ca887731e2b6ebf9d71174b Mon Sep 17 00:00:00 2001 From: Sun Shaojie Date: Tue, 23 Jun 2026 18:41:32 +0800 Subject: locking/percpu-rwsem: Annotate intentional data race in readers_active_check() KCSAN reports a data race between readers_active_check() and a concurrently executing reader: BUG: KCSAN: data-race in readers_active_check / percpu_down_write race at unknown origin, with read to 0xffff9f3eb5bf5f30 of 4 bytes by task 1271 on cpu 14: readers_active_check+0x... percpu_down_write+0x152/0x1f0 value changed: 0xfffffff9 -> 0xfffffff8 readers_active_check() calls per_cpu_sum(*sem->read_count), which iterates over all CPUs and reads each CPU's per-CPU read_count variable. Concurrently, a reader on a remote CPU is modifying its own CPU's read_count via this_cpu_inc() / this_cpu_dec() as it enters and exits the critical section. These are plain reads and writes to the same per-CPU storage, hence KCSAN flags a data race. This race is benign. readers_active_check() is called from the percpu_down_write() wait loop (rcuwait_wait_event) after sem->block is already set. At this point: - New readers must immediately back out (they see block set, decrement their counter, and wake the writer), so counters can only decrease. - If the sum catches a reader's increment before its decrement, readers_active_check() sees a non-zero sum and returns false. The writer merely iterates the wait loop again -- a harmless retry. - A false zero (observing sum == 0 while a reader is still active) cannot happen: per_cpu_sum() reads each CPU's counter, and each per-CPU int read is atomic on all architectures, so an active reader's counter is always seen as non-zero. Annotate the read with data_race() to suppress the KCSAN warning and document the intentional nature of this unlocked access. Signed-off-by: Sun Shaojie Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260623104132.505117-1-sunshaojie@kylinos.cn --- kernel/locking/percpu-rwsem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c index f7e152c40d6d..6c78961fe753 100644 --- a/kernel/locking/percpu-rwsem.c +++ b/kernel/locking/percpu-rwsem.c @@ -211,7 +211,7 @@ EXPORT_SYMBOL_GPL(percpu_is_read_locked); */ static bool readers_active_check(struct percpu_rw_semaphore *sem) { - if (per_cpu_sum(*sem->read_count) != 0) + if (data_race(per_cpu_sum(*sem->read_count)) != 0) return false; /* -- cgit v1.2.3 From 0f289334af68a1331d4a2d2a95799ffa4ff8a046 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sun, 2 Aug 2026 10:43:42 -0700 Subject: riscv/runtime-const: Disable linker relaxation for RUNTIME_MAGIC Commit ee10b1028129 ("riscv/runtime-const: Replace open-coded placeholder with RUNTIME_MAGIC") switched the lui + addi[w] placeholder from hand-encoded immediates to %hi()/%lo() of RUNTIME_MAGIC. GNU Assembler folds %hi()/%lo() of an absolute expression at assembly time. LLVM's integrated assembler since 21.1 defers it to the linker instead, which range checks it: on riscv64 R_RISCV_HI20 must fit a signed 20-bit field, and RUNTIME_MAGIC (0x89ABCDEF) is a positive 64-bit value rather than a sign-extended 32-bit one, so it does not: ld.lld: error: relocation R_RISCV_HI20 out of range: 563901 is not in [-524288, 524287] The sequence is patched at runtime and its instruction offsets are recorded via ".long 1b - .", so the linker must not touch it in the first place. Add ".option norelax", as is already done for ALTERNATIVE() and static keys; the integrated assembler then resolves %hi()/%lo() itself and emits no relocation, restoring the exact encoding the open-coded placeholder produced. Fixes: ee10b1028129 ("riscv/runtime-const: Replace open-coded placeholder with RUNTIME_MAGIC") Closes: https://github.com/ClangBuiltLinux/linux/issues/2179 Reported-by: Nathan Chancellor Signed-off-by: Fangrui Song Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260802174342.597092-1-i@maskray.me --- arch/riscv/include/asm/runtime-const.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/riscv/include/asm/runtime-const.h b/arch/riscv/include/asm/runtime-const.h index a472ebbf0971..ce2008bb5227 100644 --- a/arch/riscv/include/asm/runtime-const.h +++ b/arch/riscv/include/asm/runtime-const.h @@ -23,6 +23,7 @@ typeof(sym) __ret; \ asm_inline(".option push\n\t" \ ".option norvc\n\t" \ + ".option norelax\n\t" \ "1:\t" \ "lui %[__ret], %%hi(" RUNTIME_MAGIC ")\n\t" \ "addi %[__ret],%[__ret], %%lo(" RUNTIME_MAGIC ")\n\t" \ @@ -47,6 +48,7 @@ #define RISCV_RUNTIME_CONST_64_PREAMBLE \ ".option push\n\t" \ ".option norvc\n\t" \ + ".option norelax\n\t" \ "1:\t" \ "lui %[__ret], %%hi(" RUNTIME_MAGIC ")\n\t" \ "lui %[__tmp], %%hi(" RUNTIME_MAGIC ")\n\t" \ -- cgit v1.2.3 From 1c7efabfbaf796f11000a46094a69955a01ec6cc Mon Sep 17 00:00:00 2001 From: Felix Hoffmann Date: Fri, 31 Jul 2026 17:50:24 +0200 Subject: futex: Avoid private hash use-after-free on final put futex_private_hash_put() drops the reference to fph before evaluating fph->mm for wake_up_var(). futex_ref_put() enables preemption again before returning. If that put drops the final reference and the task is preempted, another task can pivot to the replacement hash and free the old hash after an RCU grace period. The first task then reads fph->mm from the freed allocation when it resumes. KASAN reports a slab-use-after-free in futex_private_hash_put(), with the read at offset 24 in a freed kmalloc-512 allocation. The allocation and free stacks point to futex_hash_allocate() and the RCU free path, respectively. Load the mm pointer while the fph reference is still held and pass the saved value to wake_up_var(). wake_up_var() uses the pointer as a waitqueue key and does not dereference the mm through it. Fixes: bd54df5ea7ca ("futex: Allow to resize the private local hash") Signed-off-by: Felix Hoffmann Signed-off-by: Peter Zijlstra (Intel) Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260731155024.1150011-1-f3lix.dev@gmx.de --- kernel/futex/core.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 0864c6e8cde7..d3fcad4ee716 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -139,8 +139,14 @@ static bool futex_private_hash_get(struct futex_private_hash *fph) void futex_private_hash_put(struct futex_private_hash *fph) { - if (fph && futex_ref_put(fph)) - wake_up_var(fph->mm); + struct mm_struct *mm; + + if (!fph) + return; + + mm = fph->mm; + if (futex_ref_put(fph)) + wake_up_var(mm); } static struct futex_hash_bucket * -- cgit v1.2.3 From 1fb92e0625596985b3941925afe7906fef41214b Mon Sep 17 00:00:00 2001 From: Philipp Stanner Date: Wed, 24 Jun 2026 17:07:02 +0200 Subject: rust: sync: Add abstraction for synchronize_rcu() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit synchronize_rcu() is a frequently used C function which is always safe to be called. Add a safe abstraction for synchronize_rcu(). Signed-off-by: Philipp Stanner Reviewed-by: Onur Özkan Reviewed-by: Danilo Krummrich Reviewed-by: Gary Guo [boqun: Fix rustdoc reported by kernel test robot ] Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260624150704.1504001-3-phasta@kernel.org --- rust/kernel/sync/rcu.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/rust/kernel/sync/rcu.rs b/rust/kernel/sync/rcu.rs index a32bef6e490b..d867240be736 100644 --- a/rust/kernel/sync/rcu.rs +++ b/rust/kernel/sync/rcu.rs @@ -50,3 +50,19 @@ impl Drop for Guard { pub fn read_lock() -> Guard { Guard::new() } + +/// Wait for one RCU grace period. +/// +/// Waits for all RCU read-side critical sections (such as those established by +/// a [`Guard`]) at the moment of the function call to finish. +/// +/// Does not prevent new read-side critical sections from starting, which may +/// begin and run while this call is blocking. +/// +/// Note that this is one of the RCU primitives which must not be called in +/// atomic context. +#[inline] +pub fn synchronize_rcu() { + // SAFETY: `synchronize_rcu()` is always safe to be called from process context. + unsafe { bindings::synchronize_rcu() }; +} -- cgit v1.2.3 From 042278f5fa9e681420ab486d6cbf464e26667461 Mon Sep 17 00:00:00 2001 From: Philipp Stanner Date: Wed, 24 Jun 2026 17:07:03 +0200 Subject: rust: revocable: Use safe synchronize_rcu() abstraction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We now have a safe wrapper for the foreign function synchronize_rcu(). Use it in revocable.rs. Signed-off-by: Philipp Stanner Reviewed-by: Onur Özkan Reviewed-by: Danilo Krummrich Reviewed-by: Gary Guo Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260624150704.1504001-4-phasta@kernel.org --- rust/kernel/revocable.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/rust/kernel/revocable.rs b/rust/kernel/revocable.rs index 0f4ae673256d..f539603349f1 100644 --- a/rust/kernel/revocable.rs +++ b/rust/kernel/revocable.rs @@ -7,7 +7,11 @@ use pin_init::Wrapper; -use crate::{bindings, prelude::*, sync::rcu, types::Opaque}; +use crate::{ + prelude::*, + sync::rcu, + types::Opaque, // +}; use core::{ marker::PhantomData, ops::Deref, @@ -161,8 +165,7 @@ impl Revocable { if revoke { if SYNC { - // SAFETY: Just an FFI call, there are no further requirements. - unsafe { bindings::synchronize_rcu() }; + rcu::synchronize_rcu(); } // SAFETY: We know `self.data` is valid because only one CPU can succeed the -- cgit v1.2.3 From 47c3367ff096e66f614ed3cdadece26ffc04b5e8 Mon Sep 17 00:00:00 2001 From: Philipp Stanner Date: Wed, 24 Jun 2026 17:07:04 +0200 Subject: rust: sync: Use safe synchronize_rcu() abstraction in poll MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We now have a safe wrapper for the foreign function synchronize_rcu(). Use it in poll.rs. Signed-off-by: Philipp Stanner Reviewed-by: Alice Ryhl Reviewed-by: Onur Özkan Reviewed-by: Danilo Krummrich Reviewed-by: Gary Guo Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260624150704.1504001-5-phasta@kernel.org --- rust/kernel/sync/poll.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/rust/kernel/sync/poll.rs b/rust/kernel/sync/poll.rs index 0ec985d560c8..5aa0ce9ba01b 100644 --- a/rust/kernel/sync/poll.rs +++ b/rust/kernel/sync/poll.rs @@ -8,7 +8,11 @@ use crate::{ bindings, fs::File, prelude::*, - sync::{CondVar, LockClassKey}, + sync::{ + rcu::synchronize_rcu, + CondVar, + LockClassKey, // + }, // }; use core::{marker::PhantomData, ops::Deref}; @@ -99,8 +103,6 @@ impl PinnedDrop for PollCondVar { unsafe { bindings::__wake_up_pollfree(self.inner.wait_queue_head.get()) }; // Wait for epoll items to be properly removed. - // - // SAFETY: Just an FFI call. - unsafe { bindings::synchronize_rcu() }; + synchronize_rcu(); } } -- cgit v1.2.3 From 79d3d667af401eaa452f046595209aae6933c485 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 9 Jun 2026 16:38:37 +0100 Subject: rust: sync: Add helpers for mb, dma_mb and friends They supplement the existing smp_mb, smp_rmb and smp_wmb. Reviewed-by: Eliot Courtney Signed-off-by: Gary Guo Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260609-rust-barrier-v2-1-30fcc48e1cd0@garyguo.net --- rust/helpers/barrier.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/rust/helpers/barrier.c b/rust/helpers/barrier.c index fed8853745c8..dbc7a3017c78 100644 --- a/rust/helpers/barrier.c +++ b/rust/helpers/barrier.c @@ -2,6 +2,36 @@ #include +__rust_helper void rust_helper_mb(void) +{ + mb(); +} + +__rust_helper void rust_helper_rmb(void) +{ + rmb(); +} + +__rust_helper void rust_helper_wmb(void) +{ + wmb(); +} + +__rust_helper void rust_helper_dma_mb(void) +{ + dma_mb(); +} + +__rust_helper void rust_helper_dma_rmb(void) +{ + dma_rmb(); +} + +__rust_helper void rust_helper_dma_wmb(void) +{ + dma_wmb(); +} + __rust_helper void rust_helper_smp_mb(void) { smp_mb(); -- cgit v1.2.3 From 72856afd33f2fa166c06f1536003e300e51ecf09 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 9 Jun 2026 16:38:38 +0100 Subject: rust: sync: Add generic memory barriers Implement a generic interface for memory barriers (full system/DMA/SMP). The interface uses a parameter to force user to specify their intent with barriers. Provide `Read`, `Write`, `Full` orderings which map to the existing `rmb()`, `wmb()` and `mb()`. Generic is used here instead of providing individual standalone functions to reduce code duplication; for example, the `CONFIG_SMP` check in `smp_mb` is uniformly implemented for all SMP barriers. This could extend to `virt_mb`'s if they're introduced in the future. It would also make it easier if new ordering types are introduced in the future (e.g. `Acquire`, `Release`). Signed-off-by: Gary Guo Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260609-rust-barrier-v2-2-30fcc48e1cd0@garyguo.net --- rust/kernel/sync/atomic/ordering.rs | 2 +- rust/kernel/sync/barrier.rs | 127 ++++++++++++++++++++++++++++-------- 2 files changed, 100 insertions(+), 29 deletions(-) diff --git a/rust/kernel/sync/atomic/ordering.rs b/rust/kernel/sync/atomic/ordering.rs index 3f103aa8db99..c4e732e7212f 100644 --- a/rust/kernel/sync/atomic/ordering.rs +++ b/rust/kernel/sync/atomic/ordering.rs @@ -15,7 +15,7 @@ //! - It provides ordering between the annotated operation and all the following memory accesses. //! - It provides ordering between all the preceding memory accesses and all the following memory //! accesses. -//! - All the orderings are the same strength as a full memory barrier (i.e. `smp_mb()`). +//! - All the orderings are the same strength as a full memory barrier (i.e. `smp_mb(Full)`). //! - [`Relaxed`] provides no ordering except the dependency orderings. Dependency orderings are //! described in "DEPENDENCY RELATIONS" in [`LKMM`]'s [`explanation`]. //! diff --git a/rust/kernel/sync/barrier.rs b/rust/kernel/sync/barrier.rs index 8f2d435fcd94..1180695d533a 100644 --- a/rust/kernel/sync/barrier.rs +++ b/rust/kernel/sync/barrier.rs @@ -7,6 +7,38 @@ //! //! [`LKMM`]: srctree/tools/memory-model/ +#![expect(private_bounds, reason = "sealed implementation")] + +/// Memory barrier orderings. +/// +/// The semantics of these orderings follows the [`LKMM`] definitions and rules. +/// +/// - [`Read`] provides ordering between preceding load operations and succeeding load operations. +/// - [`Write`] provides ordering between preceding store operations and succeeding store +/// operations. +/// - [`Full`] provides ordering between all the preceding memory accesses and succeeding memory +/// accesses. +/// +/// [`LKMM`]: srctree/tools/memory-model/ +pub mod ordering { + pub use crate::sync::atomic::ordering::Full; + + /// The annotation type for read-read barrier ordering. + pub struct Read; + + /// The annotation type for write-write barrier ordering. + pub struct Write; +} + +pub use ordering::{ + Full, + Read, + Write, // +}; + +struct Smp; +struct Dma; + /// A compiler barrier. /// /// A barrier that prevents compiler from reordering memory accesses across the barrier. @@ -19,43 +51,82 @@ pub(crate) fn barrier() { unsafe { core::arch::asm!("") }; } -/// A full memory barrier. +trait MemoryBarrier { + fn run(); +} + +macro_rules! define_barrier { + ($([$flavour:ident])? $ordering:ident, $binding:ident) => { + impl MemoryBarrier$(<$flavour>)? for $ordering { + #[inline] + fn run() { + // SAFETY: barrier methods are safe to call. + unsafe { bindings::$binding() }; + } + } + }; +} + +define_barrier!(Full, mb); +define_barrier!(Read, rmb); +define_barrier!(Write, wmb); +define_barrier!([Dma] Full, dma_mb); +define_barrier!([Dma] Read, dma_rmb); +define_barrier!([Dma] Write, dma_wmb); +define_barrier!([Smp] Full, smp_mb); +define_barrier!([Smp] Read, smp_rmb); +define_barrier!([Smp] Write, smp_wmb); + +/// Memory barrier. /// /// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. -#[inline(always)] -pub fn smp_mb() { - if cfg!(CONFIG_SMP) { - // SAFETY: `smp_mb()` is safe to call. - unsafe { bindings::smp_mb() }; - } else { - barrier(); - } +/// +/// The specific forms of reordering can be specified using the parameter. +/// - `mb(Read)` provides a read-read barrier. +/// - `mb(Write)` provides a write-write barrier. +/// - `mb(Full)` provides a full barrier. +/// +/// # Examples +/// +/// ``` +/// # use kernel::sync::barrier::*; +/// mb(Read); +/// mb(Write); +/// mb(Full); +/// ``` +#[inline] +#[doc(alias = "rmb")] +#[doc(alias = "wmb")] +pub fn mb(_: T) { + T::run() } -/// A write-write memory barrier. +/// Memory barrier between CPUs. /// -/// A barrier that prevents compiler and CPU from reordering memory write accesses across the -/// barrier. -#[inline(always)] -pub fn smp_wmb() { +/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. +/// Does not prevent re-ordering with respect to other bus-mastering devices. +/// +/// See [`mb`] for usage. +#[inline] +#[doc(alias = "smp_rmb")] +#[doc(alias = "smp_wmb")] +pub fn smp_mb>(_: T) { if cfg!(CONFIG_SMP) { - // SAFETY: `smp_wmb()` is safe to call. - unsafe { bindings::smp_wmb() }; + T::run() } else { - barrier(); + barrier() } } -/// A read-read memory barrier. +/// Memory barrier between local CPU and bus-mastering devices. /// -/// A barrier that prevents compiler and CPU from reordering memory read accesses across the -/// barrier. -#[inline(always)] -pub fn smp_rmb() { - if cfg!(CONFIG_SMP) { - // SAFETY: `smp_rmb()` is safe to call. - unsafe { bindings::smp_rmb() }; - } else { - barrier(); - } +/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. +/// Does not prevent re-ordering with respect to other CPUs. +/// +/// See [`mb`] for usage. +#[inline] +#[doc(alias = "dma_rmb")] +#[doc(alias = "dma_wmb")] +pub fn dma_mb>(_: T) { + T::run() } -- cgit v1.2.3 From 3f90c16d413b2f68407f1c5bb112a1b16ea35dc4 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Thu, 16 Jul 2026 15:55:35 +0100 Subject: rust: revocable: Use LKMM atomics instead of Rust atomics Kernel code should use LKMM atomics. The existing code is `AtomicBool` with the need to use `xchg`, so convert it to `AtomicFlag`. Signed-off-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: FUJITA Tomonori Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260716145536.3681630-1-gary@kernel.org --- rust/kernel/revocable.rs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/rust/kernel/revocable.rs b/rust/kernel/revocable.rs index f539603349f1..0e55e2a0fb37 100644 --- a/rust/kernel/revocable.rs +++ b/rust/kernel/revocable.rs @@ -9,14 +9,19 @@ use pin_init::Wrapper; use crate::{ prelude::*, - sync::rcu, + sync::{ + atomic::{ + AtomicFlag, + Relaxed, // + }, + rcu, // + }, types::Opaque, // }; use core::{ marker::PhantomData, ops::Deref, - ptr::drop_in_place, - sync::atomic::{AtomicBool, Ordering}, + ptr::drop_in_place, // }; /// An object that can become inaccessible at runtime. @@ -69,7 +74,7 @@ use core::{ /// ``` #[pin_data(PinnedDrop)] pub struct Revocable { - is_available: AtomicBool, + is_available: AtomicFlag, #[pin] data: Opaque, } @@ -88,7 +93,7 @@ impl Revocable { /// Creates a new revocable instance of the given data. pub fn new(data: impl PinInit) -> impl PinInit { try_pin_init!(Self { - is_available: AtomicBool::new(true), + is_available: AtomicFlag::new(true), data <- Opaque::pin_init(data), }? E) } @@ -102,7 +107,7 @@ impl Revocable { /// because another CPU may be waiting to complete the revocation of this object. pub fn try_access(&self) -> Option> { let guard = rcu::read_lock(); - if self.is_available.load(Ordering::Relaxed) { + if self.is_available.load(Relaxed) { // Since `self.is_available` is true, data is initialised and has to remain valid // because the RCU read side lock prevents it from being dropped. Some(RevocableGuard::new(self.data.get(), guard)) @@ -120,7 +125,7 @@ impl Revocable { /// allowed to sleep because another CPU may be waiting to complete the revocation of this /// object. pub fn try_access_with_guard<'a>(&'a self, _guard: &'a rcu::Guard) -> Option<&'a T> { - if self.is_available.load(Ordering::Relaxed) { + if self.is_available.load(Relaxed) { // SAFETY: Since `self.is_available` is true, data is initialised and has to remain // valid because the RCU read side lock prevents it from being dropped. Some(unsafe { &*self.data.get() }) @@ -161,7 +166,7 @@ impl Revocable { /// /// Callers must ensure that there are no more concurrent users of the revocable object. unsafe fn revoke_internal(&self) -> bool { - let revoke = self.is_available.swap(false, Ordering::Relaxed); + let revoke = self.is_available.xchg(false, Relaxed); if revoke { if SYNC { -- cgit v1.2.3 From fc6ad5eadcd1694d2f1ee2717c8024efd8152e20 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 4 Aug 2026 07:15:41 +0000 Subject: x86/paravirt: Use static_call() for the paravirt spinlock ops queued_spin_lock_slowpath() and queued_spin_unlock() are dispatched through pv_ops_lock via the paravirt-ops ALTERNATIVE machinery, which picks the target (native inline store / hypervisor call) once at boot and cannot change at runtime. Convert both to static_call(). The site becomes a direct call patched in place (one byte smaller), and on native the unlock still collapses to the inline "movb $0, (%rdi)" store, so the fast path is unchanged. Unlike the ALTERNATIVE mechanism, a static_call() target can also be updated at runtime via static_call_update(). This is a prerequisite for the contended_release tracepoint, which has to swap in a traced unlock while the system is running. [ ilvokhin: commit message; fix PARAVIRT_SPINLOCKS=n build; teach __static_call_validate() about the inline unlock insn; make the slowpath site module-safe: static_call_mod() + EXPORT_STATIC_CALL_TRAMP(); pass @lock to the callee-save unlock, fixing a boot hang under CALL_DEPTH_TRACKING. Boot tested native + KVM PV guest. ] Signed-off-by: Peter Zijlstra (Intel) Co-developed-by: Dmitry Ilvokhin Signed-off-by: Dmitry Ilvokhin Signed-off-by: Peter Zijlstra (Intel) Acked-by: Juergen Gross Link: https://lore.kernel.org/all/20260603120811.GW3493090@noisy.programming.kicks-ass.net/ Link: https://patch.msgid.link/9a32ae399eb804a02a31af04dcabe7e7ee4f3fdf.1785778551.git.d@ilvokhin.com --- arch/x86/hyperv/hv_spinlock.c | 4 ++-- arch/x86/include/asm/cpufeatures.h | 2 +- arch/x86/include/asm/paravirt-spinlock.h | 19 ++++++++++++------- arch/x86/kernel/kvm.c | 5 ++--- arch/x86/kernel/paravirt-spinlocks.c | 12 ++++++------ arch/x86/kernel/static_call.c | 27 +++++++++++++++++++++++++++ arch/x86/xen/spinlock.c | 5 ++--- tools/arch/x86/include/asm/cpufeatures.h | 2 +- 8 files changed, 53 insertions(+), 23 deletions(-) diff --git a/arch/x86/hyperv/hv_spinlock.c b/arch/x86/hyperv/hv_spinlock.c index 210b494e4de0..6b4bdea18218 100644 --- a/arch/x86/hyperv/hv_spinlock.c +++ b/arch/x86/hyperv/hv_spinlock.c @@ -78,8 +78,8 @@ void __init hv_init_spinlocks(void) pr_info("PV spinlocks enabled\n"); __pv_init_lock_hash(); - pv_ops_lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; - pv_ops_lock.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock); + static_call_update(queued_spin_lock_slowpath, __pv_queued_spin_lock_slowpath); + static_call_update(queued_spin_unlock, __raw_callee_save___pv_queued_spin_unlock); pv_ops_lock.wait = hv_qlock_wait; pv_ops_lock.kick = hv_qlock_kick; pv_ops_lock.vcpu_is_preempted = PV_CALLEE_SAVE(hv_vcpu_is_preempted); diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 1b4a48bff18f..55224157740b 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -225,7 +225,7 @@ #define X86_FEATURE_EPT_AD ( 8*32+17) /* "ept_ad" Intel Extended Page Table access-dirty bit */ #define X86_FEATURE_VMCALL ( 8*32+18) /* Hypervisor supports the VMCALL instruction */ #define X86_FEATURE_VMW_VMMCALL ( 8*32+19) /* VMware prefers VMMCALL hypercall instruction */ -#define X86_FEATURE_PVUNLOCK ( 8*32+20) /* PV unlock function */ +// free: was #define X86_FEATURE_PVUNLOCK ( 8*32+20) /* PV unlock function */ #define X86_FEATURE_VCPUPREEMPT ( 8*32+21) /* PV vcpu_is_preempted function */ #define X86_FEATURE_TDX_GUEST ( 8*32+22) /* "tdx_guest" Intel Trust Domain Extensions Guest */ diff --git a/arch/x86/include/asm/paravirt-spinlock.h b/arch/x86/include/asm/paravirt-spinlock.h index 7beffcb08ed6..ff735830de4a 100644 --- a/arch/x86/include/asm/paravirt-spinlock.h +++ b/arch/x86/include/asm/paravirt-spinlock.h @@ -3,6 +3,7 @@ #define _ASM_X86_PARAVIRT_SPINLOCK_H #include +#include #ifdef CONFIG_SMP #include @@ -11,9 +12,6 @@ struct qspinlock; struct pv_lock_ops { - void (*queued_spin_lock_slowpath)(struct qspinlock *lock, u32 val); - struct paravirt_callee_save queued_spin_unlock; - void (*wait)(u8 *ptr, u8 val); void (*kick)(int cpu); @@ -26,20 +24,27 @@ extern struct pv_lock_ops pv_ops_lock; extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); extern void __pv_init_lock_hash(void); extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); +extern void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock); extern void __raw_callee_save___pv_queued_spin_unlock(struct qspinlock *lock); extern bool nopvspin; +DECLARE_STATIC_CALL(queued_spin_lock_slowpath, native_queued_spin_lock_slowpath); +DECLARE_STATIC_CALL(queued_spin_unlock, __raw_callee_save___native_queued_spin_unlock); + static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) { - PVOP_VCALL2(pv_ops_lock, queued_spin_lock_slowpath, lock, val); + static_call_mod(queued_spin_lock_slowpath)(lock, val); } static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock) { - PVOP_ALT_VCALLEE1(pv_ops_lock, queued_spin_unlock, lock, - "movb $0, (%%" _ASM_ARG1 ")", - ALT_NOT(X86_FEATURE_PVUNLOCK)); + PVOP_CALL_ARGS; + __STATIC_CALL_MOD_ADDRESSABLE(queued_spin_unlock); + asm volatile ("call " STATIC_CALL_TRAMP_STR(queued_spin_unlock) + : PVOP_VCALLEE_CLOBBERS, ASM_CALL_CONSTRAINT + : PVOP_CALL_ARG1(lock) + : "memory", "cc"); } static __always_inline bool pv_vcpu_is_preempted(long cpu) diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index dcef84da304b..253c159c4abe 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -1136,9 +1136,8 @@ void __init kvm_spinlock_init(void) pr_info("PV spinlocks enabled\n"); __pv_init_lock_hash(); - pv_ops_lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; - pv_ops_lock.queued_spin_unlock = - PV_CALLEE_SAVE(__pv_queued_spin_unlock); + static_call_update(queued_spin_lock_slowpath, __pv_queued_spin_lock_slowpath); + static_call_update(queued_spin_unlock, __raw_callee_save___pv_queued_spin_unlock); pv_ops_lock.wait = kvm_wait; pv_ops_lock.kick = kvm_kick_cpu; diff --git a/arch/x86/kernel/paravirt-spinlocks.c b/arch/x86/kernel/paravirt-spinlocks.c index 95452444868f..ddc19dc28ba1 100644 --- a/arch/x86/kernel/paravirt-spinlocks.c +++ b/arch/x86/kernel/paravirt-spinlocks.c @@ -25,9 +25,14 @@ __visible void __native_queued_spin_unlock(struct qspinlock *lock) } PV_CALLEE_SAVE_REGS_THUNK(__native_queued_spin_unlock); +DEFINE_STATIC_CALL(queued_spin_lock_slowpath, native_queued_spin_lock_slowpath); +EXPORT_STATIC_CALL_TRAMP(queued_spin_lock_slowpath); +DEFINE_STATIC_CALL(queued_spin_unlock, __raw_callee_save___native_queued_spin_unlock); +EXPORT_STATIC_CALL_TRAMP(queued_spin_unlock); + bool pv_is_native_spin_unlock(void) { - return pv_ops_lock.queued_spin_unlock.func == + return static_call_query(queued_spin_unlock) == __raw_callee_save___native_queued_spin_unlock; } @@ -45,16 +50,11 @@ bool pv_is_native_vcpu_is_preempted(void) void __init paravirt_set_cap(void) { - if (!pv_is_native_spin_unlock()) - setup_force_cpu_cap(X86_FEATURE_PVUNLOCK); - if (!pv_is_native_vcpu_is_preempted()) setup_force_cpu_cap(X86_FEATURE_VCPUPREEMPT); } struct pv_lock_ops pv_ops_lock = { - .queued_spin_lock_slowpath = native_queued_spin_lock_slowpath, - .queued_spin_unlock = PV_CALLEE_SAVE(__native_queued_spin_unlock), .wait = paravirt_nop, .kick = paravirt_nop, .vcpu_is_preempted = PV_CALLEE_SAVE(__native_vcpu_is_preempted), diff --git a/arch/x86/kernel/static_call.c b/arch/x86/kernel/static_call.c index 61592e41a6b1..bab9406e6d6a 100644 --- a/arch/x86/kernel/static_call.c +++ b/arch/x86/kernel/static_call.c @@ -4,6 +4,12 @@ #include #include +/* Declared locally to avoid pulling asm/paravirt-spinlock.h header. */ +#ifdef CONFIG_PARAVIRT_SPINLOCKS +struct qspinlock; +void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock); +#endif + enum insn_type { CALL = 0, /* site call */ NOP = 1, /* site cond-call */ @@ -31,6 +37,17 @@ static const u8 retinsn[] = { RET_INSN_OPCODE, 0xcc, 0xcc, 0xcc, 0xcc }; */ static const u8 warninsn[] = { 0x67, 0x48, 0x0f, 0xb9, 0x3a }; +#ifdef CONFIG_PARAVIRT_SPINLOCKS +/* + * ds ds movb $0, (_ASM_ARG1) + */ +#ifdef CONFIG_64BIT +static const u8 unlockinsn[] = { 0x3e, 0x3e, 0xc6, 0x07, 0x00 }; +#else +static const u8 unlockinsn[] = { 0x3e, 0x3e, 0xc6, 0x00, 0x00 }; +#endif +#endif + static u8 __is_Jcc(u8 *insn) /* Jcc.d32 */ { u8 ret = 0; @@ -78,6 +95,12 @@ static void __ref __static_call_transform(void *insn, enum insn_type type, emulate = code; code = &warninsn; } +#ifdef CONFIG_PARAVIRT_SPINLOCKS + if (func == &__raw_callee_save___native_queued_spin_unlock) { + emulate = code; + code = &unlockinsn; + } +#endif break; case NOP: @@ -139,6 +162,10 @@ static void __static_call_validate(u8 *insn, bool tail, bool tramp) !memcmp(insn, xor5rax, 5) || !memcmp(insn, warninsn, 5)) return; +#ifdef CONFIG_PARAVIRT_SPINLOCKS + if (!memcmp(insn, unlockinsn, 5)) + return; +#endif } /* diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c index 83ac24ead289..f718e535ea7c 100644 --- a/arch/x86/xen/spinlock.c +++ b/arch/x86/xen/spinlock.c @@ -134,9 +134,8 @@ void __init xen_init_spinlocks(void) printk(KERN_DEBUG "xen: PV spinlocks enabled\n"); __pv_init_lock_hash(); - pv_ops_lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; - pv_ops_lock.queued_spin_unlock = - PV_CALLEE_SAVE(__pv_queued_spin_unlock); + static_call_update(queued_spin_lock_slowpath, __pv_queued_spin_lock_slowpath); + static_call_update(queued_spin_unlock, __raw_callee_save___pv_queued_spin_unlock); pv_ops_lock.wait = xen_qlock_wait; pv_ops_lock.kick = xen_qlock_kick; pv_ops_lock.vcpu_is_preempted = PV_CALLEE_SAVE(xen_vcpu_stolen); diff --git a/tools/arch/x86/include/asm/cpufeatures.h b/tools/arch/x86/include/asm/cpufeatures.h index 86d17b195e79..425452f8c375 100644 --- a/tools/arch/x86/include/asm/cpufeatures.h +++ b/tools/arch/x86/include/asm/cpufeatures.h @@ -225,7 +225,7 @@ #define X86_FEATURE_EPT_AD ( 8*32+17) /* "ept_ad" Intel Extended Page Table access-dirty bit */ #define X86_FEATURE_VMCALL ( 8*32+18) /* Hypervisor supports the VMCALL instruction */ #define X86_FEATURE_VMW_VMMCALL ( 8*32+19) /* VMware prefers VMMCALL hypercall instruction */ -#define X86_FEATURE_PVUNLOCK ( 8*32+20) /* PV unlock function */ +// free: was #define X86_FEATURE_PVUNLOCK ( 8*32+20) /* PV unlock function */ #define X86_FEATURE_VCPUPREEMPT ( 8*32+21) /* PV vcpu_is_preempted function */ #define X86_FEATURE_TDX_GUEST ( 8*32+22) /* "tdx_guest" Intel Trust Domain Extensions Guest */ -- cgit v1.2.3 From 216c6c67f7f1292474de1a3d1ae7d1cb95514264 Mon Sep 17 00:00:00 2001 From: Dmitry Ilvokhin Date: Tue, 4 Aug 2026 07:15:42 +0000 Subject: locking: Factor out queued_spin_release() The contended_release tracepoint needs to hook queued_spin_unlock(), but architectures with a custom unlock define queued_spin_unlock() directly, leaving no single generic place to add the tracing. Introduce queued_spin_release() as the arch-overridable release primitive and make queued_spin_unlock() a generic wrapper around it. An architecture that only customizes the release can then override queued_spin_release() and inherit the generic wrapper. Rename the MIPS override to queued_spin_release() accordingly. x86 paravirt overrides queued_spin_unlock() directly and is left unchanged. No functional change intended. Signed-off-by: Dmitry Ilvokhin Signed-off-by: Peter Zijlstra (Intel) Acked-by: Juergen Gross Link: https://patch.msgid.link/b8daabae6469ad72cc784a911f6cc43a6d45df3a.1785778551.git.d@ilvokhin.com --- arch/mips/include/asm/spinlock.h | 6 +++--- include/asm-generic/qspinlock.h | 17 ++++++++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/arch/mips/include/asm/spinlock.h b/arch/mips/include/asm/spinlock.h index 6ce2117e49f6..c349162f15eb 100644 --- a/arch/mips/include/asm/spinlock.h +++ b/arch/mips/include/asm/spinlock.h @@ -13,12 +13,12 @@ #include -#define queued_spin_unlock queued_spin_unlock +#define queued_spin_release queued_spin_release /** - * queued_spin_unlock - release a queued spinlock + * queued_spin_release - release a queued spinlock * @lock : Pointer to queued spinlock structure */ -static inline void queued_spin_unlock(struct qspinlock *lock) +static inline void queued_spin_release(struct qspinlock *lock) { /* This could be optimised with ARCH_HAS_MMIOWB */ mmiowb(); diff --git a/include/asm-generic/qspinlock.h b/include/asm-generic/qspinlock.h index bf47cca2c375..ae45289e8ec7 100644 --- a/include/asm-generic/qspinlock.h +++ b/include/asm-generic/qspinlock.h @@ -115,12 +115,12 @@ static __always_inline void queued_spin_lock(struct qspinlock *lock) } #endif -#ifndef queued_spin_unlock +#ifndef queued_spin_release /** - * queued_spin_unlock - release a queued spinlock + * queued_spin_release - release a queued spinlock * @lock : Pointer to queued spinlock structure */ -static __always_inline void queued_spin_unlock(struct qspinlock *lock) +static __always_inline void queued_spin_release(struct qspinlock *lock) { /* * unlock() needs release semantics: @@ -129,6 +129,17 @@ static __always_inline void queued_spin_unlock(struct qspinlock *lock) } #endif +#ifndef queued_spin_unlock +/** + * queued_spin_unlock - unlock a queued spinlock + * @lock : Pointer to queued spinlock structure + */ +static __always_inline void queued_spin_unlock(struct qspinlock *lock) +{ + queued_spin_release(lock); +} +#endif + #ifndef virt_spin_lock static __always_inline bool virt_spin_lock(struct qspinlock *lock) { -- cgit v1.2.3 From f7e2cb6d495aa1a88368650970a230b45870859b Mon Sep 17 00:00:00 2001 From: Dmitry Ilvokhin Date: Tue, 4 Aug 2026 07:15:43 +0000 Subject: locking/qspinlock: Add contended_release tracepoint Unlike mutex and rw_semaphore, qspinlock has no owner field, so "perf lock contention --lock-owner" cannot attribute a contended spinlock to its holder. The waiter-side contention_begin event records that a spinlock is contended, but not by whom. Firing contended_release in the holder's context at unlock is the only way to capture the holder of a contended spinlock. Combine the contention check, trace call and release in an out-of-line queued_spin_release_traced() so the compiler need not preserve the lock pointer in a callee-saved register across the call. The check in queued_spin_unlock() is paid on every unlock, even while the tracepoint is disabled: a static-branch NOP on x86_64, and a few more instructions to manage a stack frame elsewhere. Gate it behind CONFIG_QUEUED_SPINLOCKS_TRACE_CONTENDED_RELEASE (default n) so nobody pays for a tracepoint they do not use. Sleeping locks fire contended_release regardless. On x86 this generic path is used only with PARAVIRT_SPINLOCKS=n (e.g. defconfig). PARAVIRT_SPINLOCKS=y kernels keep the paravirt static_call unlock and are wired up separately. All below are with the QUEUED_SPINLOCKS_TRACE_CONTENDED_RELEASE option enabled. _raw_spin_unlock(), x86_64 defconfig, GCC 11, tracepoint compiled in but disabled. The unlock is the single 'movb'. The only instruction added to the executed path is the 2-byte static-branch NOP. The CALL to the traced helper and the JMP back are emitted out of line and are reached only once the static branch is patched on: endbr64 ; 4 bytes xchg %ax,%ax ; 2 static-branch NOP ; (added) movb $0x0,(%rdi) ; 3 unlock (single store) A: decl %gs:__preempt_count ; 7 je B ; 2 jmp __x86_return_thunk ; 5 call queued_spin_release_traced ; 5 out of line, reached ; only when the ; tracepoint is on jmp A ; 2 (added) B: call __SCT__preempt_schedule ; 5 jmp __x86_return_thunk ; 5 Baseline is the same stream without the NOP and the out-of-line CALL/JMP: 31 bytes vs 40 (+9 bytes). Binary size impact on x86_64, defconfig: +680 bytes (+0.00%), since all standard configs out-of-line unlock. Architectures with inlined unlock (s390 (always), csky and loongarch (both when !PREEMPTION)) will see a bigger increase in binary size. On the same path (x86_64, PARAVIRT_SPINLOCKS=n) with the tracepoint disabled, a _raw_spin_unlock()-heavy nginx workload [1] shows no measurable difference between baseline and patched kernels in throughput, latency, cycles, instructions, IPC, or L1 instruction-cache misses (kernel and total): all deltas stay within run-to-run noise. Unlike x86, on arm64 the frame setup code (STP, MOV and LDP) lands on the executed path in addition to static-branch NOP. Binary size impact on arm64, defconfig: +932 bytes (+0.00%). The _raw_spin_unlock()-heavy nginx workload reflects the larger hot path: L1 instruction-cache misses rise ~1.4% (kernel and total) and instruction count ~0.4%, consistent with the per-unlock frame. cpu_cycles, throughput and latency show no measurable change and are within run-to-run noise. Architectures with fully custom qspinlock implementations (e.g. PowerPC) are not covered by this change. [1]: https://lore.kernel.org/all/aiphFXe_TPNPxZ_n@shell.ilvokhin.com/ Signed-off-by: Dmitry Ilvokhin Signed-off-by: Peter Zijlstra (Intel) Acked-by: Juergen Gross Link: https://patch.msgid.link/0d998e22a0c595f670cfc6725bb683323aced5cb.1785778551.git.d@ilvokhin.com --- include/asm-generic/qspinlock.h | 21 +++++++++++++++++++++ kernel/Kconfig.locks | 20 ++++++++++++++++++++ kernel/locking/qspinlock.c | 22 ++++++++++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/include/asm-generic/qspinlock.h b/include/asm-generic/qspinlock.h index ae45289e8ec7..2ca94e41823b 100644 --- a/include/asm-generic/qspinlock.h +++ b/include/asm-generic/qspinlock.h @@ -41,6 +41,7 @@ #include #include +#include #ifndef queued_spin_is_locked /** @@ -130,12 +131,32 @@ static __always_inline void queued_spin_release(struct qspinlock *lock) #endif #ifndef queued_spin_unlock + +DECLARE_TRACEPOINT(contended_release); + +extern void queued_spin_release_traced(struct qspinlock *lock); + /** * queued_spin_unlock - unlock a queued spinlock * @lock : Pointer to queued spinlock structure + * + * Generic tracing wrapper around the arch-overridable + * queued_spin_release(). */ static __always_inline void queued_spin_unlock(struct qspinlock *lock) { + /* + * Trace and release are combined in queued_spin_release_traced() so + * the compiler does not need to preserve the lock pointer across the + * function call, avoiding callee-saved register save/restore on the + * hot path. queued_spin_release() is therefore called both here and in + * queued_spin_release_traced(). Keep the two in sync. + */ + if (IS_ENABLED(CONFIG_QUEUED_SPINLOCKS_TRACE_CONTENDED_RELEASE) && + tracepoint_enabled(contended_release)) { + queued_spin_release_traced(lock); + return; + } queued_spin_release(lock); } #endif diff --git a/kernel/Kconfig.locks b/kernel/Kconfig.locks index 4198f0273ecd..1c6423aafcd4 100644 --- a/kernel/Kconfig.locks +++ b/kernel/Kconfig.locks @@ -243,6 +243,26 @@ config QUEUED_SPINLOCKS def_bool y if ARCH_USE_QUEUED_SPINLOCKS depends on SMP +config QUEUED_SPINLOCKS_TRACE_CONTENDED_RELEASE + bool "Trace contended_release on queued spinlocks" + depends on QUEUED_SPINLOCKS && TRACEPOINTS + help + Fire the lock:contended_release tracepoint when a contended queued + spinlock is released, so it is possible to attribute a contended + spinlock to its holder. + + Architectures that can patch the unlock site do this at no cost and + do not need this option. + + Everywhere else the check is compiled into queued_spin_unlock() and + a small cost is paid on every unlock even when the tracepoint is + disabled: a static-branch NOP and possibly a few more instructions + to manage a stack frame. + + Sleeping locks fire lock:contended_release regardless of this option. + + If unsure, say N. + config BPF_ARCH_SPINLOCK bool diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c index af8d122bb649..33fe6d437c8f 100644 --- a/kernel/locking/qspinlock.c +++ b/kernel/locking/qspinlock.c @@ -104,6 +104,28 @@ static __always_inline u32 __pv_wait_head_or_lock(struct qspinlock *lock, #define queued_spin_lock_slowpath native_queued_spin_lock_slowpath #endif +#if !defined(queued_spin_unlock) && \ + IS_ENABLED(CONFIG_QUEUED_SPINLOCKS_TRACE_CONTENDED_RELEASE) +/* + * Out-of-line trace-and-release path for queued_spin_unlock(), used when + * the contended_release tracepoint is enabled. + * + * queued_spin_release() is duplicated here on purpose: doing the release + * in this function (rather than tracing here and releasing in the caller) + * lets queued_spin_unlock() return right after the call, so the + * tracepoint-disabled hot path never has to keep lock live across a call + * in a callee-saved register. Keep this release in sync with the one in + * queued_spin_unlock(). + */ +void __lockfunc queued_spin_release_traced(struct qspinlock *lock) +{ + if (queued_spin_is_contended(lock)) + trace_call__contended_release(lock); + queued_spin_release(lock); +} +EXPORT_SYMBOL(queued_spin_release_traced); +#endif + #endif /* _GEN_PV_LOCK_SLOWPATH */ /** -- cgit v1.2.3 From b359800c6970cb653d41ed2af18fd8e95dbb822f Mon Sep 17 00:00:00 2001 From: Dmitry Ilvokhin Date: Tue, 4 Aug 2026 07:15:44 +0000 Subject: tracing/lock: Use TRACE_EVENT_FN() for contended_release queued_spin_unlock() gates its contended_release trace call behind a static branch, so a NOP sits on the unlock path even while the tracepoint is disabled. Removing that requires replacing the unlock implementation only while contended_release is enabled, which needs a callback when the tracepoint is toggled. Convert contended_release to TRACE_EVENT_FN() and add weak no-op arch_contended_release_trace_reg()/arch_contended_release_trace_unreg() hooks. The default hooks are empty, so this is a no-op until an architecture overrides them. No functional change intended. Signed-off-by: Dmitry Ilvokhin Signed-off-by: Peter Zijlstra (Intel) Acked-by: Juergen Gross Link: https://patch.msgid.link/1c2fcccfb584c075c02890c484f22c76a1948bf1.1785778551.git.d@ilvokhin.com --- include/trace/events/lock.h | 10 ++++++++-- kernel/locking/mutex.c | 4 ++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/include/trace/events/lock.h b/include/trace/events/lock.h index 1ded869cd619..b1d5b18c4514 100644 --- a/include/trace/events/lock.h +++ b/include/trace/events/lock.h @@ -137,7 +137,11 @@ TRACE_EVENT(contention_end, TP_printk("%p (ret=%d)", __entry->lock_addr, __entry->ret) ); -TRACE_EVENT(contended_release, +/* kernel/locking/mutex.c */ +int arch_contended_release_trace_reg(void); +void arch_contended_release_trace_unreg(void); + +TRACE_EVENT_FN(contended_release, TP_PROTO(void *lock), @@ -151,7 +155,9 @@ TRACE_EVENT(contended_release, __entry->lock_addr = lock; ), - TP_printk("%p", __entry->lock_addr) + TP_printk("%p", __entry->lock_addr), + + arch_contended_release_trace_reg, arch_contended_release_trace_unreg ); #endif /* _TRACE_LOCK_H */ diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c index 8a85912d7ee6..942a939cee95 100644 --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -1272,6 +1272,10 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(contention_begin); EXPORT_TRACEPOINT_SYMBOL_GPL(contention_end); EXPORT_TRACEPOINT_SYMBOL_GPL(contended_release); +__weak int arch_contended_release_trace_reg(void) { return 0; } + +__weak void arch_contended_release_trace_unreg(void) { } + /** * atomic_dec_and_mutex_lock - return holding mutex if we dec to 0 * @cnt: the atomic which we are to dec -- cgit v1.2.3 From 087116fefbf343ce63b8911cf494e059e9679432 Mon Sep 17 00:00:00 2001 From: Dmitry Ilvokhin Date: Tue, 4 Aug 2026 07:15:45 +0000 Subject: x86/paravirt: Trace contended_release on unlock On PARAVIRT_SPINLOCKS=y kernels queued_spin_unlock() is dispatched through a static_call(). Those PARAVIRT_SPINLOCKS=y kernels are quite popular. Gating contended_release behind a static branch would leave a NOP on the unlock hot path even, when the tracepoint is disabled. Since the static_call() is already present, swap its target to a traced unlock, when the tracepoint is enabled instead. When contended_release tracepoint is disabled the target is the plain unlock (an inline store on native x86_64), so the unlock path is unchanged and the tracepoint is truly zero-cost. Provide two traced variants, native_queued_spin_unlock_traced() and pv_queued_spin_unlock_traced(), so each tail-calls its own base unlock directly rather than recursing through the now-traced static_call(). Teach pv_is_native_spin_unlock() that the traced native variant still counts as native. Only PARAVIRT_SPINLOCKS=y is affected. PARAVIRT_SPINLOCKS=n keeps the generic static-branch path. Suggested-by: Peter Zijlstra Signed-off-by: Dmitry Ilvokhin Signed-off-by: Peter Zijlstra (Intel) Acked-by: Juergen Gross Link: https://patch.msgid.link/17fa67f9fa4cf93f1150725e89f5f916e41a9b6f.1785778551.git.d@ilvokhin.com --- arch/x86/include/asm/paravirt-spinlock.h | 2 ++ arch/x86/kernel/paravirt-spinlocks.c | 53 ++++++++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/paravirt-spinlock.h b/arch/x86/include/asm/paravirt-spinlock.h index ff735830de4a..302bc2ba3a75 100644 --- a/arch/x86/include/asm/paravirt-spinlock.h +++ b/arch/x86/include/asm/paravirt-spinlock.h @@ -99,6 +99,8 @@ bool __raw_callee_save___native_vcpu_is_preempted(long cpu); void __init native_pv_lock_init(void); __visible void __native_queued_spin_unlock(struct qspinlock *lock); +__visible void native_queued_spin_unlock_traced(struct qspinlock *lock); +__visible void pv_queued_spin_unlock_traced(struct qspinlock *lock); bool pv_is_native_spin_unlock(void); __visible bool __native_vcpu_is_preempted(long cpu); bool pv_is_native_vcpu_is_preempted(void); diff --git a/arch/x86/kernel/paravirt-spinlocks.c b/arch/x86/kernel/paravirt-spinlocks.c index ddc19dc28ba1..ca12b3655307 100644 --- a/arch/x86/kernel/paravirt-spinlocks.c +++ b/arch/x86/kernel/paravirt-spinlocks.c @@ -7,6 +7,7 @@ #include #include #include +#include DEFINE_STATIC_KEY_FALSE(virt_spin_lock_key); @@ -30,10 +31,58 @@ EXPORT_STATIC_CALL_TRAMP(queued_spin_lock_slowpath); DEFINE_STATIC_CALL(queued_spin_unlock, __raw_callee_save___native_queued_spin_unlock); EXPORT_STATIC_CALL_TRAMP(queued_spin_unlock); +/* + * Traced unlock variants, swapped in via static_call while the + * contended_release tracepoint is enabled. Two of them, so each tail calls its + * own base directly. + */ +__visible void native_queued_spin_unlock_traced(struct qspinlock *lock) +{ + if (queued_spin_is_contended(lock)) + trace_call__contended_release(lock); + native_queued_spin_unlock(lock); +} +PV_CALLEE_SAVE_REGS_THUNK(native_queued_spin_unlock_traced); + +__visible void pv_queued_spin_unlock_traced(struct qspinlock *lock) +{ + if (queued_spin_is_contended(lock)) + trace_call__contended_release(lock); + __raw_callee_save___pv_queued_spin_unlock(lock); +} +PV_CALLEE_SAVE_REGS_THUNK(pv_queued_spin_unlock_traced); + bool pv_is_native_spin_unlock(void) { - return static_call_query(queued_spin_unlock) == - __raw_callee_save___native_queued_spin_unlock; + void *unlock = static_call_query(queued_spin_unlock); + + return unlock == __raw_callee_save___native_queued_spin_unlock || + unlock == __raw_callee_save_native_queued_spin_unlock_traced; +} + +int arch_contended_release_trace_reg(void) +{ + void *cur = static_call_query(queued_spin_unlock); + + if (cur == __raw_callee_save___native_queued_spin_unlock) + static_call_update(queued_spin_unlock, + __raw_callee_save_native_queued_spin_unlock_traced); + else if (cur == __raw_callee_save___pv_queued_spin_unlock) + static_call_update(queued_spin_unlock, + __raw_callee_save_pv_queued_spin_unlock_traced); + return 0; +} + +void arch_contended_release_trace_unreg(void) +{ + void *cur = static_call_query(queued_spin_unlock); + + if (cur == __raw_callee_save_native_queued_spin_unlock_traced) + static_call_update(queued_spin_unlock, + __raw_callee_save___native_queued_spin_unlock); + else if (cur == __raw_callee_save_pv_queued_spin_unlock_traced) + static_call_update(queued_spin_unlock, + __raw_callee_save___pv_queued_spin_unlock); } __visible bool __native_vcpu_is_preempted(long cpu) -- cgit v1.2.3 From fcb8ada1287227a1392930ee52d7b6c6cab0f0b2 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 7 Aug 2026 17:23:53 +0200 Subject: futex: Tell kmemleak we're not leaking __futex_queues Kmemleak doesn't know about runtime_const stuff and figures we're leaking __futex_queues. So add this little annotation to tell it all is well. Fixes: b78b0b658252 ("futex: Use runtime constants for __futex_hash() hot path") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-lkp/202608071053.6db6276e-lkp@intel.com Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260807152353.GP687043@noisy.programming.kicks-ass.net --- kernel/futex/core.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/futex/core.c b/kernel/futex/core.c index d3fcad4ee716..45538acecbf1 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -45,6 +45,7 @@ #include #include #include +#include #include @@ -2028,6 +2029,7 @@ static int __init futex_init(void) order = get_order(size); __futex_queues = kcalloc(nr_node_ids, sizeof(*__futex_queues), GFP_KERNEL); + kmemleak_not_leak(__futex_queues); runtime_const_init(shift, __futex_shift); runtime_const_init(mask, __futex_mask); -- cgit v1.2.3 From b54aa0edf0594a6e1138e626bfd8a352ed582ae9 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Tue, 4 Aug 2026 09:14:23 -0700 Subject: preempt: Track NMI nesting to separate per-CPU counter Move NMI nesting tracking from the preempt_count bits to a separate per-CPU counter (nmi_nesting). This is to free up the NMI bits in the preempt_count, allowing those bits to be repurposed for other uses. Reduce NMI_BITS from 4 to 1, using it only to detect if we're in an NMI. The per-CPU counter currently caps nesting at 15. [boqun: Address Steven Rostedt's comment on the BUG_ON() condition] [boqun: Use preempt_count_set() in __nmi_exit() to avoid underflow] Suggested-by: Boqun Feng Signed-off-by: Joel Fernandes Signed-off-by: Lyude Paul Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260121223933.1568682-3-lyude@redhat.com Link: https://patch.msgid.link/20260804161447.84806-2-boqun@kernel.org --- include/linux/hardirq.h | 17 +++++++++++++---- include/linux/preempt.h | 9 +++++++-- kernel/softirq.c | 2 ++ tools/testing/selftests/bpf/bpf_experimental.h | 2 +- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index d57cab4d4c06..8d4895531a45 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -10,6 +10,8 @@ #include #include +DECLARE_PER_CPU(unsigned int, nmi_nesting); + extern void synchronize_irq(unsigned int irq); extern bool synchronize_hardirq(unsigned int irq); @@ -102,14 +104,17 @@ void irq_exit_rcu(void); */ /* - * nmi_enter() can nest up to 15 times; see NMI_BITS. + * nmi_enter() can nest - nesting is tracked in a per-CPU counter. */ #define __nmi_enter() \ do { \ lockdep_off(); \ arch_nmi_enter(); \ - BUG_ON(in_nmi() == NMI_MASK); \ - __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \ + /* Maximum NMI nesting is 15. */ \ + BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \ + __this_cpu_inc(nmi_nesting); \ + __preempt_count_add(HARDIRQ_OFFSET); \ + preempt_count_set(preempt_count() | NMI_MASK); \ } while (0) #define nmi_enter() \ @@ -124,8 +129,12 @@ void irq_exit_rcu(void); #define __nmi_exit() \ do { \ + unsigned int nesting; \ BUG_ON(!in_nmi()); \ - __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \ + __preempt_count_sub(HARDIRQ_OFFSET); \ + nesting = __this_cpu_dec_return(nmi_nesting); \ + if (!nesting) \ + preempt_count_set(preempt_count() & ~NMI_MASK); \ arch_nmi_exit(); \ lockdep_on(); \ } while (0) diff --git a/include/linux/preempt.h b/include/linux/preempt.h index d964f965c8ff..586f96688325 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -17,6 +17,8 @@ * * - bits 0-7 are the preemption count (max preemption depth: 256) * - bits 8-15 are the softirq count (max # of softirqs: 256) + * - bits 16-19 are the hardirq count (max # of hardirqs: 16) + * - bit 20 is the NMI flag (no nesting count, tracked separately) * * The hardirq count could in theory be the same as the number of * interrupts in the system, but we run all interrupt handlers with @@ -24,16 +26,19 @@ * there are a few palaeontologic drivers which reenable interrupts in * the handler, so we need more than one bit here. * + * NMI nesting depth is tracked in a separate per-CPU variable + * (nmi_nesting) to save bits in preempt_count. + * * PREEMPT_MASK: 0x000000ff * SOFTIRQ_MASK: 0x0000ff00 * HARDIRQ_MASK: 0x000f0000 - * NMI_MASK: 0x00f00000 + * NMI_MASK: 0x00100000 * PREEMPT_NEED_RESCHED: 0x80000000 */ #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 #define HARDIRQ_BITS 4 -#define NMI_BITS 4 +#define NMI_BITS 1 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) diff --git a/kernel/softirq.c b/kernel/softirq.c index 4425d8dce44b..10af5ed859e7 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -88,6 +88,8 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled); EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context); #endif +DEFINE_PER_CPU(unsigned int, nmi_nesting); + /* * SOFTIRQ_OFFSET usage: * diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing/selftests/bpf/bpf_experimental.h index 67ff7882299e..e4e12001fce9 100644 --- a/tools/testing/selftests/bpf/bpf_experimental.h +++ b/tools/testing/selftests/bpf/bpf_experimental.h @@ -367,7 +367,7 @@ extern int bpf_cgroup_read_xattr(struct cgroup *cgroup, const char *name__str, #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 #define HARDIRQ_BITS 4 -#define NMI_BITS 4 +#define NMI_BITS 1 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) -- cgit v1.2.3 From 9634d860ac15dad51a62955566424c9d2996b15c Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Tue, 4 Aug 2026 09:14:24 -0700 Subject: preempt: Introduce HARDIRQ_DISABLE_BITS In order to support preempt_disable()-like interrupt disabling, that is, using part of preempt_count() to track interrupt disabling nesting level, change the preempt_count() layout to contain 8-bit HARDIRQ_DISABLE count. Signed-off-by: Boqun Feng Signed-off-by: Lyude Paul Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260121223933.1568682-2-lyude@redhat.com Link: https://patch.msgid.link/20260804161447.84806-3-boqun@kernel.org --- include/linux/preempt.h | 16 +++++++++++----- tools/testing/selftests/bpf/bpf_experimental.h | 5 ++++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/include/linux/preempt.h b/include/linux/preempt.h index 586f96688325..e2d3079d3f5f 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -17,8 +17,9 @@ * * - bits 0-7 are the preemption count (max preemption depth: 256) * - bits 8-15 are the softirq count (max # of softirqs: 256) - * - bits 16-19 are the hardirq count (max # of hardirqs: 16) - * - bit 20 is the NMI flag (no nesting count, tracked separately) + * - bits 16-23 are the hardirq disable count (max # of hardirq disable: 256) + * - bits 24-27 are the hardirq count (max # of hardirqs: 16) + * - bit 28 is the NMI flag (no nesting count, tracked separately) * * The hardirq count could in theory be the same as the number of * interrupts in the system, but we run all interrupt handlers with @@ -31,29 +32,34 @@ * * PREEMPT_MASK: 0x000000ff * SOFTIRQ_MASK: 0x0000ff00 - * HARDIRQ_MASK: 0x000f0000 - * NMI_MASK: 0x00100000 + * HARDIRQ_DISABLE_MASK: 0x00ff0000 + * HARDIRQ_MASK: 0x0f000000 + * NMI_MASK: 0x10000000 * PREEMPT_NEED_RESCHED: 0x80000000 */ #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 +#define HARDIRQ_DISABLE_BITS 8 #define HARDIRQ_BITS 4 #define NMI_BITS 1 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) -#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_DISABLE_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_SHIFT (HARDIRQ_DISABLE_SHIFT + HARDIRQ_DISABLE_BITS) #define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS) #define __IRQ_MASK(x) ((1UL << (x))-1) #define PREEMPT_MASK (__IRQ_MASK(PREEMPT_BITS) << PREEMPT_SHIFT) #define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT) +#define HARDIRQ_DISABLE_MASK (__IRQ_MASK(HARDIRQ_DISABLE_BITS) << HARDIRQ_DISABLE_SHIFT) #define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT) #define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT) #define PREEMPT_OFFSET (1UL << PREEMPT_SHIFT) #define SOFTIRQ_OFFSET (1UL << SOFTIRQ_SHIFT) +#define HARDIRQ_DISABLE_OFFSET (1UL << HARDIRQ_DISABLE_SHIFT) #define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT) #define NMI_OFFSET (1UL << NMI_SHIFT) diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing/selftests/bpf/bpf_experimental.h index e4e12001fce9..0159a3d365c8 100644 --- a/tools/testing/selftests/bpf/bpf_experimental.h +++ b/tools/testing/selftests/bpf/bpf_experimental.h @@ -366,17 +366,20 @@ extern int bpf_cgroup_read_xattr(struct cgroup *cgroup, const char *name__str, #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 +#define HARDIRQ_DISABLE_BITS 8 #define HARDIRQ_BITS 4 #define NMI_BITS 1 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) -#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_DISABLE_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_SHIFT (HARDIRQ_DISABLE_SHIFT + HARDIRQ_DISABLE_BITS) #define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS) #define __IRQ_MASK(x) ((1UL << (x))-1) #define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT) +#define HARDIRQ_DISABLE_MASK (__IRQ_MASK(HARDIRQ_DISABLE_BITS) << HARDIRQ_DISABLE_SHIFT) #define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT) #define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT) -- cgit v1.2.3 From 4ad87eee546ef97e3a5ad80414327f41f48a4e70 Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Tue, 4 Aug 2026 09:14:25 -0700 Subject: preempt: Introduce __preempt_count_{sub,add}_return() In order to use preempt_count() to track the interrupt disable nesting level, __preempt_count_{add,sub}_return() are introduced, as their names suggest, these primitives return the new value of the preempt_count() after changing it. The following example shows the usage of it in local_interrupt_disable(): // increase the HARDIRQ_DISABLE bit new_count = __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET); // if it's the first-time increment, then disable the interrupt // at hardware level. if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET) { local_irq_save(flags); raw_cpu_write(local_interrupt_disable_state, flags); } Having these primitives will avoid a read of preempt_count() after changing preempt_count() on certain architectures. Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Acked-by: Heiko Carstens # s390 Link: https://patch.msgid.link/20260804161447.84806-4-boqun@kernel.org --- arch/arm64/include/asm/preempt.h | 20 ++++++++++++++++++++ arch/s390/include/asm/preempt.h | 10 ++++++++++ arch/x86/include/asm/preempt.h | 10 ++++++++++ include/asm-generic/preempt.h | 14 ++++++++++++++ 4 files changed, 54 insertions(+) diff --git a/arch/arm64/include/asm/preempt.h b/arch/arm64/include/asm/preempt.h index 932ea4b62042..9ecc2766a9f2 100644 --- a/arch/arm64/include/asm/preempt.h +++ b/arch/arm64/include/asm/preempt.h @@ -55,6 +55,26 @@ static inline void __preempt_count_sub(int val) WRITE_ONCE(current_thread_info()->preempt.count, pc); } +static inline int __preempt_count_add_return(int val) +{ + u32 pc = READ_ONCE(current_thread_info()->preempt.count); + + pc += val; + WRITE_ONCE(current_thread_info()->preempt.count, pc); + + return pc; +} + +static inline int __preempt_count_sub_return(int val) +{ + u32 pc = READ_ONCE(current_thread_info()->preempt.count); + + pc -= val; + WRITE_ONCE(current_thread_info()->preempt.count, pc); + + return pc; +} + static inline bool __preempt_count_dec_and_test(void) { struct thread_info *ti = current_thread_info(); diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preempt.h index 6e5821bb047e..0a25d4648b4c 100644 --- a/arch/s390/include/asm/preempt.h +++ b/arch/s390/include/asm/preempt.h @@ -139,6 +139,16 @@ static __always_inline bool should_resched(int preempt_offset) return unlikely(READ_ONCE(get_lowcore()->preempt_count) == preempt_offset); } +static __always_inline int __preempt_count_add_return(int val) +{ + return val + __atomic_add(val, &get_lowcore()->preempt_count); +} + +static __always_inline int __preempt_count_sub_return(int val) +{ + return __preempt_count_add_return(-val); +} + #define init_task_preempt_count(p) do { } while (0) /* Deferred to CPU bringup time */ #define init_idle_preempt_count(p, cpu) do { } while (0) diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h index 578441db09f0..1220656f3370 100644 --- a/arch/x86/include/asm/preempt.h +++ b/arch/x86/include/asm/preempt.h @@ -85,6 +85,16 @@ static __always_inline void __preempt_count_sub(int val) raw_cpu_add_4(__preempt_count, -val); } +static __always_inline int __preempt_count_add_return(int val) +{ + return raw_cpu_add_return_4(__preempt_count, val); +} + +static __always_inline int __preempt_count_sub_return(int val) +{ + return raw_cpu_add_return_4(__preempt_count, -val); +} + /* * Because we keep PREEMPT_NEED_RESCHED set when we do _not_ need to reschedule * a decrement which hits zero means we have no preempt_count and should diff --git a/include/asm-generic/preempt.h b/include/asm-generic/preempt.h index 51f8f3881523..c8683c046615 100644 --- a/include/asm-generic/preempt.h +++ b/include/asm-generic/preempt.h @@ -59,6 +59,20 @@ static __always_inline void __preempt_count_sub(int val) *preempt_count_ptr() -= val; } +static __always_inline int __preempt_count_add_return(int val) +{ + *preempt_count_ptr() += val; + + return *preempt_count_ptr(); +} + +static __always_inline int __preempt_count_sub_return(int val) +{ + *preempt_count_ptr() -= val; + + return *preempt_count_ptr(); +} + static __always_inline bool __preempt_count_dec_and_test(void) { /* -- cgit v1.2.3 From e35e7700029b936356d736d6378cf596019e4cc8 Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Tue, 4 Aug 2026 09:14:26 -0700 Subject: openrisc: Include in smp.h While OpenRISC currently doesn't fail to build upstream, it appears that including in the right headers is enough to break that - primarily because OpenRISC's asm/smp.h header doesn't actually provide any definition for struct cpumask. Which means the only reason we aren't failing to build the kernel is because we've been lucky enough that every spot including asm/smp.h already has definitions for struct cpumask pulled in. This became evident when trying to work on a patch series for adding ref-counted interrupt enable/disable to the kernel, where introducing a new interrupt_rc.h header suddenly introduced a build error on OpenRISC: In file included from include/linux/interrupt_rc.h:17, from include/linux/spinlock.h:60, from include/linux/mmzone.h:8, from include/linux/gfp.h:7, from include/linux/mm.h:7, from arch/openrisc/include/asm/pgalloc.h:20, from arch/openrisc/include/asm/io.h:18, from include/linux/io.h:12, from drivers/irqchip/irq-ompic.c:61: arch/openrisc/include/asm/smp.h:21:59: warning: 'struct cpumask' declared inside parameter list will not be visible outside of this definition or declaration 21 | extern void arch_send_call_function_ipi_mask(const struct cpumask *mask); | ^~~~~~~ arch/openrisc/include/asm/smp.h:23:54: warning: 'struct cpumask' declared inside parameter list will not be visible outside of this definition or declaration 23 | extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int)); | ^~~~~~~ drivers/irqchip/irq-ompic.c: In function 'ompic_of_init': >> drivers/irqchip/irq-ompic.c:191:28: error: passing argument 1 of 'set_smp_cross_call' from incompatible pointer type [-Werror=incompatible-pointer-types] 191 | set_smp_cross_call(ompic_raise_softirq); | ^~~~~~~~~~~~~~~~~~~ | | | void (*)(const struct cpumask *, unsigned int) arch/openrisc/include/asm/smp.h:23:32: note: expected 'void (*)(const struct cpumask *, unsigned int)' but argument is of type 'void (*)(const struct cpumask *, unsigned int)' 23 | extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int)); To fix this, let's take an example from the smp.h headers of other architectures (x86, hexagon, arm64, probably more): just include linux/cpumask.h at the top. Signed-off-by: Lyude Paul Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Acked-by: Stafford Horne Link: https://patch.msgid.link/20260804161447.84806-5-boqun@kernel.org --- arch/openrisc/include/asm/smp.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/openrisc/include/asm/smp.h b/arch/openrisc/include/asm/smp.h index 007296f160ef..84653aaffa96 100644 --- a/arch/openrisc/include/asm/smp.h +++ b/arch/openrisc/include/asm/smp.h @@ -9,6 +9,8 @@ #ifndef __ASM_OPENRISC_SMP_H #define __ASM_OPENRISC_SMP_H +#include + #include #include -- cgit v1.2.3 From e901c1510e24726dcbd6340ee927b3ac8b992043 Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Tue, 4 Aug 2026 11:26:57 -0700 Subject: irq,spin_lock: Add counted interrupt disabling/enabling Currently the nested interrupt disabling and enabling is represented by _irqsave() and _irqrestore() APIs, which are relatively unsafe, for example: spin_lock_irqsave(l1, flag1); spin_lock_irqsave(l2, flag2); spin_unlock_irqrestore(l1, flags1); // accesses to interrupt-disable protected data will cause races This is even easier to trigger with guard facilities: unsigned long flag2; scoped_guard(spin_lock_irqsave, l1) { spin_lock_irqsave(l2, flag2); } // l2 locked but interrupts are enabled. spin_unlock_irqrestore(l2, flag2); (Hand-to-hand locking critical sections are not uncommon for a fine-grained lock design) And because of this unsafety, Rust cannot easily wrap the interrupt-disabling locks in a safe API, which complicates the design. To resolve this, introduce a new set of interrupt disabling APIs: * local_interrupt_disable(); * local_interrupt_enable(); They work like local_irq_save() and local_irq_restore() except that 1) the outermost local_interrupt_disable() call saves the interrupt state into a per-CPU variable, so that the outermost local_interrupt_enable() can restore the state, and 2) a per-CPU counter is added to record the nest level of these calls, so that interrupts are not accidentally enabled inside the outermost critical section. Also add the corresponding spin_lock primitives: spin_lock_irq_disable() and spin_unlock_irq_enable(), as a result, code as follows: spin_lock_irq_disable(l1); spin_lock_irq_disable(l2); spin_unlock_irq_enable(l1); // Interrupts are still disabled. spin_unlock_irq_enable(l2); doesn't have the issue that interrupts are accidentally enabled. This also makes the wrapper of interrupt-disabling locks on Rust easier to design. [boqun: Apply Peter's feedback and fix spell errors reported by Ingo] [boqun: Address the duplicate spin_acquire() spotted by sashiko] Co-developed-by: Lyude Paul Signed-off-by: Lyude Paul Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260804182657.87716-1-boqun@kernel.org --- include/linux/interrupt_rc.h | 82 ++++++++++++++++++++++++++++++++++++++++ include/linux/preempt.h | 4 ++ include/linux/spinlock.h | 23 +++++++++++ include/linux/spinlock_api_smp.h | 41 ++++++++++++++++++++ include/linux/spinlock_api_up.h | 15 ++++++++ include/linux/spinlock_rt.h | 18 +++++++++ kernel/locking/spinlock.c | 31 +++++++++++++++ kernel/softirq.c | 28 +++++++++++++- 8 files changed, 240 insertions(+), 2 deletions(-) create mode 100644 include/linux/interrupt_rc.h diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h new file mode 100644 index 000000000000..b9a7f05ecf42 --- /dev/null +++ b/include/linux/interrupt_rc.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_INTERRUPT_RC_H +#define __LINUX_INTERRUPT_RC_H + +/* + * include/linux/interrupt_rc.h - refcounted local processor interrupt + * management. + * + * Since the implementation of this API currently depends on + * local_irq_save()/local_irq_restore(), we split this into its own header to + * make it easier to include without hitting circular header dependencies. + */ + +#include +#include +#include +#include + +#ifndef MODULE +/* Per-CPU interrupt disabling state for local_interrupt_{disable,enable}(). */ +DECLARE_PER_CPU(unsigned long, local_interrupt_disable_state); + +static __always_inline void __local_interrupt_disable(void) +{ + unsigned long flags; + + local_irq_save(flags); + raw_cpu_write(local_interrupt_disable_state, flags); +} + +static __always_inline void __local_interrupt_enable(void) +{ + unsigned long flags = raw_cpu_read(local_interrupt_disable_state); + + local_irq_restore(flags); +} + +#ifndef INSTANTIATE_EXPORTED_INTERRUPT_DISABLE +static __always_inline void _local_interrupt_disable(void) +{ + __local_interrupt_disable(); +} + +static __always_inline void _local_interrupt_enable(void) +{ + __local_interrupt_enable(); +} +#else +extern void _local_interrupt_disable(void); +extern void _local_interrupt_enable(void); +#endif + +#else /* !MODULE */ +extern void _local_interrupt_disable(void); +extern void _local_interrupt_enable(void); +#endif /* !MODULE */ + +static inline void local_interrupt_disable(void) +{ + int new_count; + + WARN_ON_ONCE(in_nmi()); + + new_count = hardirq_disable_enter(); + + /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */ + + if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET) + _local_interrupt_disable(); +} + +static inline void local_interrupt_enable(void) +{ + int new_count; + + new_count = hardirq_disable_exit(); + + if ((new_count & HARDIRQ_DISABLE_MASK) == 0) + _local_interrupt_enable(); +} + +#endif /* !__LINUX_INTERRUPT_RC_H */ diff --git a/include/linux/preempt.h b/include/linux/preempt.h index e2d3079d3f5f..33fc4c814a9f 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -151,6 +151,10 @@ static __always_inline unsigned char interrupt_context_level(void) #define in_softirq() (softirq_count()) #define in_interrupt() (irq_count()) +#define hardirq_disable_count() ((preempt_count() & HARDIRQ_DISABLE_MASK) >> HARDIRQ_DISABLE_SHIFT) +#define hardirq_disable_enter() __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET) +#define hardirq_disable_exit() __preempt_count_sub_return(HARDIRQ_DISABLE_OFFSET) + /* * The preempt_count offset after preempt_disable(); */ diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index 241277cd34cf..3d405cc4c121 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -57,6 +57,7 @@ #include #include #include +#include #include #include #include @@ -273,9 +274,11 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock) #endif #define raw_spin_lock_irq(lock) _raw_spin_lock_irq(lock) +#define raw_spin_lock_irq_disable(lock) _raw_spin_lock_irq_disable(lock) #define raw_spin_lock_bh(lock) _raw_spin_lock_bh(lock) #define raw_spin_unlock(lock) _raw_spin_unlock(lock) #define raw_spin_unlock_irq(lock) _raw_spin_unlock_irq(lock) +#define raw_spin_unlock_irq_enable(lock) _raw_spin_unlock_irq_enable(lock) #define raw_spin_unlock_irqrestore(lock, flags) \ do { \ @@ -290,6 +293,8 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock) #define raw_spin_trylock_irqsave(lock, flags) _raw_spin_trylock_irqsave(lock, &(flags)) +#define raw_spin_trylock_irq_disable(lock) _raw_spin_trylock_irq_disable(lock) + #ifndef CONFIG_PREEMPT_RT /* Include rwlock functions for !RT */ #include @@ -372,6 +377,12 @@ static __always_inline void spin_lock_irq(spinlock_t *lock) raw_spin_lock_irq(&lock->rlock); } +static __always_inline void spin_lock_irq_disable(spinlock_t *lock) + __acquires(lock) __no_context_analysis +{ + raw_spin_lock_irq_disable(&lock->rlock); +} + #define spin_lock_irqsave(lock, flags) \ do { \ raw_spin_lock_irqsave(spinlock_check(lock), flags); \ @@ -402,6 +413,12 @@ static __always_inline void spin_unlock_irq(spinlock_t *lock) raw_spin_unlock_irq(&lock->rlock); } +static __always_inline void spin_unlock_irq_enable(spinlock_t *lock) + __releases(lock) __no_context_analysis +{ + raw_spin_unlock_irq_enable(&lock->rlock); +} + static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) __releases(lock) __no_context_analysis { @@ -427,6 +444,12 @@ static __always_inline bool _spin_trylock_irqsave(spinlock_t *lock, unsigned lon } #define spin_trylock_irqsave(lock, flags) _spin_trylock_irqsave(lock, &(flags)) +static __always_inline int spin_trylock_irq_disable(spinlock_t *lock) + __cond_acquires(true, lock) __no_context_analysis +{ + return raw_spin_trylock_irq_disable(&lock->rlock); +} + /** * spin_is_locked() - Check whether a spinlock is locked. * @lock: Pointer to the spinlock. diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_smp.h index bda5e7a390cd..90909d933dab 100644 --- a/include/linux/spinlock_api_smp.h +++ b/include/linux/spinlock_api_smp.h @@ -28,6 +28,8 @@ _raw_spin_lock_nest_lock(raw_spinlock_t *lock, struct lockdep_map *map) void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) __acquires(lock); void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock) __acquires(lock); +void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock) + __acquires(lock); unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock) __acquires(lock); @@ -39,6 +41,7 @@ int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock) __cond_acquires(true, void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) __releases(lock); void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) __releases(lock); void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) __releases(lock); +void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock) __releases(lock); void __lockfunc _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) __releases(lock); @@ -55,6 +58,11 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) #define _raw_spin_lock_irq(lock) __raw_spin_lock_irq(lock) #endif +/* Use the same config as spin_lock_irq() temporarily. */ +#ifdef CONFIG_INLINE_SPIN_LOCK_IRQ +#define _raw_spin_lock_irq_disable(lock) __raw_spin_lock_irq_disable(lock) +#endif + #ifdef CONFIG_INLINE_SPIN_LOCK_IRQSAVE #define _raw_spin_lock_irqsave(lock) __raw_spin_lock_irqsave(lock) #endif @@ -79,6 +87,11 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) #define _raw_spin_unlock_irq(lock) __raw_spin_unlock_irq(lock) #endif +/* Use the same config as spin_unlock_irq() temporarily. */ +#ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQ +#define _raw_spin_unlock_irq_enable(lock) __raw_spin_unlock_irq_enable(lock) +#endif + #ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE #define _raw_spin_unlock_irqrestore(lock, flags) __raw_spin_unlock_irqrestore(lock, flags) #endif @@ -105,6 +118,16 @@ static __always_inline bool _raw_spin_trylock_irq(raw_spinlock_t *lock) return false; } +static __always_inline bool _raw_spin_trylock_irq_disable(raw_spinlock_t *lock) + __cond_acquires(true, lock) +{ + local_interrupt_disable(); + if (_raw_spin_trylock(lock)) + return true; + local_interrupt_enable(); + return false; +} + static __always_inline bool _raw_spin_trylock_irqsave(raw_spinlock_t *lock, unsigned long *flags) __cond_acquires(true, lock) { @@ -143,6 +166,15 @@ static inline void __raw_spin_lock_irq(raw_spinlock_t *lock) LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); } +static inline void __raw_spin_lock_irq_disable(raw_spinlock_t *lock) + __acquires(lock) __no_context_analysis +{ + local_interrupt_disable(); + preempt_disable(); + spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); + LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); +} + static inline void __raw_spin_lock_bh(raw_spinlock_t *lock) __acquires(lock) __no_context_analysis { @@ -188,6 +220,15 @@ static inline void __raw_spin_unlock_irq(raw_spinlock_t *lock) preempt_enable(); } +static inline void __raw_spin_unlock_irq_enable(raw_spinlock_t *lock) + __releases(lock) +{ + spin_release(&lock->dep_map, _RET_IP_); + do_raw_spin_unlock(lock); + local_interrupt_enable(); + preempt_enable(); +} + static inline void __raw_spin_unlock_bh(raw_spinlock_t *lock) __releases(lock) { diff --git a/include/linux/spinlock_api_up.h b/include/linux/spinlock_api_up.h index a9d5c7c66e03..d03d3065ee04 100644 --- a/include/linux/spinlock_api_up.h +++ b/include/linux/spinlock_api_up.h @@ -42,6 +42,9 @@ #define __LOCK_IRQSAVE(lock, flags, ...) \ do { local_irq_save(flags); __LOCK(lock, ##__VA_ARGS__); } while (0) +#define __LOCK_IRQ_DISABLE(lock, ...) \ + do { local_interrupt_disable(); __LOCK(lock, ##__VA_ARGS__); } while (0) + #define ___UNLOCK_(lock) \ do { __release(lock); (void)(lock); } while (0) @@ -61,6 +64,9 @@ #define __UNLOCK_IRQRESTORE(lock, flags, ...) \ do { local_irq_restore(flags); __UNLOCK(lock, ##__VA_ARGS__); } while (0) +#define __UNLOCK_IRQ_ENABLE(lock, ...) \ + do { __UNLOCK(lock, ##__VA_ARGS__); local_interrupt_enable(); } while (0) + #define _raw_spin_lock(lock) __LOCK(lock) #define _raw_spin_lock_nested(lock, subclass) __LOCK(lock) #define _raw_read_lock(lock) __LOCK(lock, shared) @@ -70,6 +76,7 @@ #define _raw_read_lock_bh(lock) __LOCK_BH(lock, shared) #define _raw_write_lock_bh(lock) __LOCK_BH(lock) #define _raw_spin_lock_irq(lock) __LOCK_IRQ(lock) +#define _raw_spin_lock_irq_disable(lock) __LOCK_IRQ_DISABLE(lock) #define _raw_read_lock_irq(lock) __LOCK_IRQ(lock, shared) #define _raw_write_lock_irq(lock) __LOCK_IRQ(lock) #define _raw_spin_lock_irqsave(lock, flags) __LOCK_IRQSAVE(lock, flags) @@ -97,6 +104,13 @@ static __always_inline int _raw_spin_trylock_irq(raw_spinlock_t *lock) return 1; } +static __always_inline int _raw_spin_trylock_irq_disable(raw_spinlock_t *lock) + __cond_acquires(true, lock) +{ + __LOCK_IRQ_DISABLE(lock); + return 1; +} + static __always_inline int _raw_spin_trylock_irqsave(raw_spinlock_t *lock, unsigned long *flags) __cond_acquires(true, lock) { @@ -132,6 +146,7 @@ static __always_inline int _raw_write_trylock_irqsave(rwlock_t *lock, unsigned l #define _raw_write_unlock_bh(lock) __UNLOCK_BH(lock) #define _raw_read_unlock_bh(lock) __UNLOCK_BH(lock, shared) #define _raw_spin_unlock_irq(lock) __UNLOCK_IRQ(lock) +#define _raw_spin_unlock_irq_enable(lock) __UNLOCK_IRQ_ENABLE(lock) #define _raw_read_unlock_irq(lock) __UNLOCK_IRQ(lock, shared) #define _raw_write_unlock_irq(lock) __UNLOCK_IRQ(lock) #define _raw_spin_unlock_irqrestore(lock, flags) \ diff --git a/include/linux/spinlock_rt.h b/include/linux/spinlock_rt.h index 373618a4243c..560d06384e0c 100644 --- a/include/linux/spinlock_rt.h +++ b/include/linux/spinlock_rt.h @@ -96,6 +96,12 @@ static __always_inline void spin_lock_irq(spinlock_t *lock) rt_spin_lock(lock); } +static __always_inline void spin_lock_irq_disable(spinlock_t *lock) + __acquires(lock) +{ + rt_spin_lock(lock); +} + #define spin_lock_irqsave(lock, flags) \ do { \ typecheck(unsigned long, flags); \ @@ -122,6 +128,12 @@ static __always_inline void spin_unlock_irq(spinlock_t *lock) rt_spin_unlock(lock); } +static __always_inline void spin_unlock_irq_enable(spinlock_t *lock) + __releases(lock) +{ + rt_spin_unlock(lock); +} + static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) __releases(lock) @@ -131,6 +143,12 @@ static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, #define spin_trylock(lock) rt_spin_trylock(lock) +static __always_inline int spin_trylock_irq_disable(spinlock_t *lock) + __cond_acquires(true, lock) +{ + return rt_spin_trylock(lock); +} + #define spin_trylock_bh(lock) rt_spin_trylock_bh(lock) #define spin_trylock_irq(lock) rt_spin_trylock(lock) diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c index b42d293da38b..83a17eaf5717 100644 --- a/kernel/locking/spinlock.c +++ b/kernel/locking/spinlock.c @@ -129,6 +129,21 @@ static void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \ */ BUILD_LOCK_OPS(spin, raw_spinlock, __acquires); +/* No rwlock_t variants for now, so just build this function by hand */ +static void __lockfunc __raw_spin_lock_irq_disable(raw_spinlock_t *lock) +{ + for (;;) { + preempt_disable(); + local_interrupt_disable(); + if (likely(do_raw_spin_trylock(lock))) + break; + local_interrupt_enable(); + preempt_enable(); + + arch_spin_relax(&lock->raw_lock); + } +} + #ifndef CONFIG_PREEMPT_RT BUILD_LOCK_OPS(read, rwlock, __acquires_shared); BUILD_LOCK_OPS(write, rwlock, __acquires); @@ -176,6 +191,14 @@ noinline void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock) EXPORT_SYMBOL(_raw_spin_lock_irq); #endif +#ifndef CONFIG_INLINE_SPIN_LOCK_IRQ +noinline void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock) +{ + __raw_spin_lock_irq_disable(lock); +} +EXPORT_SYMBOL_GPL(_raw_spin_lock_irq_disable); +#endif + #ifndef CONFIG_INLINE_SPIN_LOCK_BH noinline void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) { @@ -208,6 +231,14 @@ noinline void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) EXPORT_SYMBOL(_raw_spin_unlock_irq); #endif +#ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ +noinline void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock) +{ + __raw_spin_unlock_irq_enable(lock); +} +EXPORT_SYMBOL_GPL(_raw_spin_unlock_irq_enable); +#endif + #ifndef CONFIG_INLINE_SPIN_UNLOCK_BH noinline void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) { diff --git a/kernel/softirq.c b/kernel/softirq.c index 10af5ed859e7..0c9b2269a8d6 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -9,6 +9,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#define INSTANTIATE_EXPORTED_INTERRUPT_DISABLE #include #include #include @@ -88,6 +89,20 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled); EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context); #endif +DEFINE_PER_CPU(unsigned long, local_interrupt_disable_state); + +void _local_interrupt_disable(void) +{ + __local_interrupt_disable(); +} +EXPORT_SYMBOL(_local_interrupt_disable); + +void _local_interrupt_enable(void) +{ + __local_interrupt_enable(); +} +EXPORT_SYMBOL(_local_interrupt_enable); + DEFINE_PER_CPU(unsigned int, nmi_nesting); /* @@ -728,10 +743,19 @@ static inline void __irq_exit_rcu(void) #endif account_hardirq_exit(current); preempt_count_sub(HARDIRQ_OFFSET); - if (!in_interrupt() && local_softirq_pending()) { + /* + * Interrupts may happen between hardirq_disable_enter() and + * local_irq_save() in local_interrupt_disable(), if irq_exit() invokes + * softirq here, we may have a softirq handler calling + * local_interrupt_disable() but it won't disable the IRQ because + * hardirq disabling count is already 1, hence we need to prevent + * invoking softirq when a local_interrupt_disable() is ongoing. + */ + if (!in_interrupt() && !hardirq_disable_count() && + local_softirq_pending()) { /* * If we left hrtimers unarmed, make sure to arm them now, - * before enabling interrupts to run SoftIRQ. + * before enabling interrupts to run softirq. */ hrtimer_rearm_deferred(); invoke_softirq(); -- cgit v1.2.3 From 07a88e2bcd5b5bd7881b2e12b6aad1897a7ee1de Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Tue, 4 Aug 2026 09:14:28 -0700 Subject: irq: Add KUnit test for refcounted interrupt enable/disable While making changes to the refcounted interrupt patch series, at some point on my local branch I broke something and ended up writing some kunit tests for testing refcounted interrupts as a result. So, let's include these tests now that we have refcounted interrupts. Signed-off-by: Lyude Paul Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260804161447.84806-7-boqun@kernel.org --- kernel/irq/Makefile | 1 + kernel/irq/refcount_interrupt_test.c | 109 +++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 kernel/irq/refcount_interrupt_test.c diff --git a/kernel/irq/Makefile b/kernel/irq/Makefile index 86a2e5ae08f9..44c4d6fc502a 100644 --- a/kernel/irq/Makefile +++ b/kernel/irq/Makefile @@ -16,3 +16,4 @@ obj-$(CONFIG_SMP) += affinity.o obj-$(CONFIG_GENERIC_IRQ_DEBUGFS) += debugfs.o obj-$(CONFIG_GENERIC_IRQ_MATRIX_ALLOCATOR) += matrix.o obj-$(CONFIG_IRQ_KUNIT_TEST) += irq_test.o +obj-$(CONFIG_KUNIT) += refcount_interrupt_test.o diff --git a/kernel/irq/refcount_interrupt_test.c b/kernel/irq/refcount_interrupt_test.c new file mode 100644 index 000000000000..ca904dba24b9 --- /dev/null +++ b/kernel/irq/refcount_interrupt_test.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit test for refcounted interrupt enable/disables. + */ + +#include +#include + +#define TEST_IRQ_ON() KUNIT_EXPECT_FALSE(test, irqs_disabled()) +#define TEST_IRQ_OFF() KUNIT_EXPECT_TRUE(test, irqs_disabled()) + +/* ===== Test cases ===== */ +static void test_single_irq_change(struct kunit *test) +{ + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); +} + +static void test_nested_irq_change(struct kunit *test) +{ + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_disable(); + TEST_IRQ_OFF(); + + local_interrupt_enable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); +} + +static void test_multiple_irq_change(struct kunit *test) +{ + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_disable(); + TEST_IRQ_OFF(); + + local_interrupt_enable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); + + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); +} + +static void test_irq_save(struct kunit *test) +{ + unsigned long flags; + + local_irq_save(flags); + TEST_IRQ_OFF(); + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_OFF(); + local_irq_restore(flags); + TEST_IRQ_ON(); + + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_irq_save(flags); + TEST_IRQ_OFF(); + local_irq_restore(flags); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); +} + +static struct kunit_case test_cases[] = { + KUNIT_CASE(test_single_irq_change), + KUNIT_CASE(test_nested_irq_change), + KUNIT_CASE(test_multiple_irq_change), + KUNIT_CASE(test_irq_save), + {}, +}; + +/* init and exit are the same. */ +static int test_init(struct kunit *test) +{ + TEST_IRQ_ON(); + + return 0; +} + +static void test_exit(struct kunit *test) +{ + TEST_IRQ_ON(); +} + +static struct kunit_suite refcount_interrupt_test_suite = { + .name = "refcount_interrupt", + .test_cases = test_cases, + .init = test_init, + .exit = test_exit, +}; + +kunit_test_suite(refcount_interrupt_test_suite); +MODULE_AUTHOR("Lyude Paul "); +MODULE_DESCRIPTION("Refcounted interrupt unit test suite"); +MODULE_LICENSE("GPL"); -- cgit v1.2.3 From 1b086687483371621e684e2a05b2bcd2cf07b634 Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Tue, 4 Aug 2026 09:14:29 -0700 Subject: locking: Switch to _irq_{disable,enable}() variants in cleanup guards The semantics of various IRQ disabling guards match what *_irq_{disable,enable}() provide, i.e. the interrupt disabling is properly nested, therefore it's OK to switch to use *_irq_{disable,enable}() primitives. [boqun: Adjust the user-side changes in do_sched_cfs_*_timer() provided by Peter and Lyude] Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260804161447.84806-8-boqun@kernel.org --- include/linux/spinlock.h | 26 ++++++++++++-------------- kernel/sched/fair.c | 12 ++++++------ 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index 3d405cc4c121..799a8f7d2741 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -572,12 +572,12 @@ DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_nested, __acquires(_T), __releases(*(raw #define class_raw_spinlock_nested_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_nested, _T) DEFINE_LOCK_GUARD_1(raw_spinlock_irq, raw_spinlock_t, - raw_spin_lock_irq(_T->lock), - raw_spin_unlock_irq(_T->lock)) + raw_spin_lock_irq_disable(_T->lock), + raw_spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irq, __acquires(_T), __releases(*(raw_spinlock_t **)_T)) #define class_raw_spinlock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irq, _T) -DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irq, _try, raw_spin_trylock_irq(_T->lock)) +DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irq, _try, raw_spin_trylock_irq_disable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irq_try, __acquires(_T), __releases(*(raw_spinlock_t **)_T)) #define class_raw_spinlock_irq_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irq_try, _T) @@ -592,14 +592,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_bh_try, __acquires(_T), __releases(*(raw #define class_raw_spinlock_bh_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_bh_try, _T) DEFINE_LOCK_GUARD_1(raw_spinlock_irqsave, raw_spinlock_t, - raw_spin_lock_irqsave(_T->lock, _T->flags), - raw_spin_unlock_irqrestore(_T->lock, _T->flags), - unsigned long flags) + raw_spin_lock_irq_disable(_T->lock), + raw_spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave, __acquires(_T), __releases(*(raw_spinlock_t **)_T)) #define class_raw_spinlock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave, _T) DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irqsave, _try, - raw_spin_trylock_irqsave(_T->lock, _T->flags)) + raw_spin_trylock_irq_disable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave_try, __acquires(_T), __releases(*(raw_spinlock_t **)_T)) #define class_raw_spinlock_irqsave_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave_try, _T) @@ -618,13 +617,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(spinlock_try, __acquires(_T), __releases(*(spinlock_t #define class_spinlock_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_try, _T) DEFINE_LOCK_GUARD_1(spinlock_irq, spinlock_t, - spin_lock_irq(_T->lock), - spin_unlock_irq(_T->lock)) + spin_lock_irq_disable(_T->lock), + spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irq, __acquires(_T), __releases(*(spinlock_t **)_T)) #define class_spinlock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irq, _T) DEFINE_LOCK_GUARD_1_COND(spinlock_irq, _try, - spin_trylock_irq(_T->lock)) + spin_trylock_irq_disable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irq_try, __acquires(_T), __releases(*(spinlock_t **)_T)) #define class_spinlock_irq_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irq_try, _T) @@ -640,14 +639,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(spinlock_bh_try, __acquires(_T), __releases(*(spinloc #define class_spinlock_bh_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_bh_try, _T) DEFINE_LOCK_GUARD_1(spinlock_irqsave, spinlock_t, - spin_lock_irqsave(_T->lock, _T->flags), - spin_unlock_irqrestore(_T->lock, _T->flags), - unsigned long flags) + spin_lock_irq_disable(_T->lock), + spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irqsave, __acquires(_T), __releases(*(spinlock_t **)_T)) #define class_spinlock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irqsave, _T) DEFINE_LOCK_GUARD_1_COND(spinlock_irqsave, _try, - spin_trylock_irqsave(_T->lock, _T->flags)) + spin_trylock_irq_disable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irqsave_try, __acquires(_T), __releases(*(spinlock_t **)_T)) #define class_spinlock_irqsave_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irqsave_try, _T) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index d78467ec6ee1..a46c4ff49f74 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -7120,7 +7120,7 @@ static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b) * period the timer is deactivated until scheduling resumes; cfs_b->idle is * used to track this state. */ -static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags) +static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun) __must_hold(&cfs_b->lock) { int throttled; @@ -7155,10 +7155,10 @@ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, u * This check is repeated as we release cfs_b->lock while we unthrottle. */ while (throttled && cfs_b->runtime > 0) { - raw_spin_unlock_irqrestore(&cfs_b->lock, flags); + raw_spin_unlock_irq_enable(&cfs_b->lock); /* we can't nest cfs_b->lock while distributing bandwidth */ throttled = distribute_cfs_runtime(cfs_b); - raw_spin_lock_irqsave(&cfs_b->lock, flags); + raw_spin_lock_irq_disable(&cfs_b->lock); } /* @@ -7266,7 +7266,7 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) { /* confirm we're still not at a refresh boundary */ - scoped_guard(raw_spinlock_irqsave, &cfs_b->lock) { + scoped_guard(raw_spinlock_irq, &cfs_b->lock) { u64 runtime = 0, slice = sched_cfs_bandwidth_slice(); cfs_b->slack_started = false; @@ -7351,14 +7351,14 @@ static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) int idle = 0; int count = 0; - CLASS(raw_spinlock_irqsave, cfsb_guard)(&cfs_b->lock); + guard(raw_spinlock_irq)(&cfs_b->lock); for (;;) { overrun = hrtimer_forward_now(timer, cfs_b->period); if (!overrun) break; - idle = do_sched_cfs_period_timer(cfs_b, overrun, cfsb_guard.flags); + idle = do_sched_cfs_period_timer(cfs_b, overrun); if (++count > 3) { u64 new, old = ktime_to_ns(cfs_b->period); -- cgit v1.2.3 From ac4231a77973fc20808ed84c4af343eca2342d4b Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Tue, 4 Aug 2026 09:14:30 -0700 Subject: sched: Remove the unused preempt_offset parameter of __cant_sleep() The preempt_offset is always 0 in all the callsites of __cant_sleep(), hence remove it. It also allows us to clear up the code a bit by no longer using a "preempt_count() > .." comparison. Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260804161447.84806-9-boqun@kernel.org --- include/linux/kernel.h | 4 ++-- kernel/sched/core.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/linux/kernel.h b/include/linux/kernel.h index e5570a16cbb1..24414c79e59a 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -72,7 +72,7 @@ extern int dynamic_might_resched(void); #ifdef CONFIG_DEBUG_ATOMIC_SLEEP extern void __might_resched(const char *file, int line, unsigned int offsets); extern void __might_sleep(const char *file, int line); -extern void __cant_sleep(const char *file, int line, int preempt_offset); +extern void __cant_sleep(const char *file, int line); extern void __cant_migrate(const char *file, int line); /** @@ -95,7 +95,7 @@ extern void __cant_migrate(const char *file, int line); * this macro will print a stack trace if it is executed with preemption enabled */ # define cant_sleep() \ - do { __cant_sleep(__FILE__, __LINE__, 0); } while (0) + do { __cant_sleep(__FILE__, __LINE__); } while (0) # define sched_annotate_sleep() (current->task_state_change = 0) /** diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 96226707c2f6..aa116daf21bd 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -9199,7 +9199,7 @@ void __might_resched(const char *file, int line, unsigned int offsets) } EXPORT_SYMBOL(__might_resched); -void __cant_sleep(const char *file, int line, int preempt_offset) +void __cant_sleep(const char *file, int line) { static unsigned long prev_jiffy; @@ -9209,7 +9209,7 @@ void __cant_sleep(const char *file, int line, int preempt_offset) if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) return; - if (preempt_count() > preempt_offset) + if (preempt_count()) return; if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) -- cgit v1.2.3 From 560fcaa92ef983315023eb4ebfc1ebae1132fb2a Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Tue, 4 Aug 2026 09:14:31 -0700 Subject: sched: Avoid signed comparison of preempt_count() in __cant_migrate() Currently preempt_count() is always a non-negative int on all archs (PREEMPT_NEED_RESCHED archs will mask out the MSB when returning preempt_count()), hence the checking in __cant_migrate() is in fact just checking whether preempt_count() is 0 or not. In a future change, we are going to use all the 32 bits of preempt_count(), which would make negative int values possible from preempt_count(). Therefore convert the "> 0" comparison into a zero check to prepare for the future change. No functional changes are intended. Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260804161447.84806-10-boqun@kernel.org --- kernel/sched/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index aa116daf21bd..9b3f1764fa9e 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -9241,7 +9241,7 @@ void __cant_migrate(const char *file, int line) if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) return; - if (preempt_count() > 0) + if (preempt_count()) return; if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) -- cgit v1.2.3 From 3b0e2a22d4086ed7c1294584c4417f7d19f1ac67 Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Tue, 4 Aug 2026 09:14:32 -0700 Subject: preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS With the changes that enable preempt count to track IRQ disabling nesting, we don't have enough bits in 32-bit preempt count implementation, as a result we move NMI nesting bits out of the 32-bit preempt count. However on the architectures that can support 64-bit preempt count implementation, we can keep the NMI nesting bits in the 32-bit preempt count and avoid maintaining NMI nesting bits outside of the same cache line. Therefore HAS_SEPARATE_PREEMPT_RESCHED_BITS is introduced to allow architectures to select this. Note that under this Kconfig, preempt count is maintained in a 64-bit word however preempt_count() still remains as an int because all the effective bits still fit in (previously we mask out NEED_RESCHED bit in preempt_count()). This should make no functional changes for existing preempt_count() users. Enable this for x86_64 along with the introduction of the Kconfig. [boqun: Undo the __preempt_count_{add,sub}() optimization in 32-bit preempt count since it may introduce {over,under}flow] Originally-by: Peter Zijlstra Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260804161447.84806-11-boqun@kernel.org --- arch/x86/Kconfig | 1 + arch/x86/include/asm/preempt.h | 55 +++++++++++++++++++++++++++++------------- arch/x86/kernel/cpu/common.c | 2 +- include/linux/hardirq.h | 47 ++++++++++++++++++++++++++---------- include/linux/preempt.h | 23 +++++++++++++++--- kernel/Kconfig.preempt | 4 +++ kernel/sched/core.c | 12 +++++++-- kernel/softirq.c | 6 +++++ lib/locking-selftest.c | 2 +- 9 files changed, 115 insertions(+), 37 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index bdad90f210e4..6a7067d20a6a 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -326,6 +326,7 @@ config X86 select USER_STACKTRACE_SUPPORT select HAVE_ARCH_KCSAN if X86_64 select PROC_PID_ARCH_STATUS if PROC_FS + select HAS_SEPARATE_PREEMPT_RESCHED_BITS if X86_64 && PREEMPT_COUNT select HAVE_ARCH_NODE_DEV_GROUP if X86_SGX select FUNCTION_ALIGNMENT_16B if X86_64 || X86_ALIGNMENT_16 select FUNCTION_ALIGNMENT_4B diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h index 1220656f3370..fafb6f8cdac3 100644 --- a/arch/x86/include/asm/preempt.h +++ b/arch/x86/include/asm/preempt.h @@ -7,10 +7,20 @@ #include -DECLARE_PER_CPU_CACHE_HOT(int, __preempt_count); +DECLARE_PER_CPU_CACHE_HOT(unsigned long, __preempt_count); -/* We use the MSB mostly because its available */ -#define PREEMPT_NEED_RESCHED 0x80000000 +/* + * We use the MSB for PREEMPT_NEED_RESCHED mostly because it is available. + */ +#define PREEMPT_NEED_RESCHED (~(((unsigned long)-1L) >> 1)) + +#ifdef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS +#define __pc_dec "decq" +#define __pc_op(op, ...) raw_cpu_##op##_8(__VA_ARGS__) +#else +#define __pc_dec "decl" +#define __pc_op(op, ...) raw_cpu_##op##_4(__VA_ARGS__) +#endif /* * We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such @@ -24,18 +34,26 @@ DECLARE_PER_CPU_CACHE_HOT(int, __preempt_count); */ static __always_inline int preempt_count(void) { - return raw_cpu_read_4(__preempt_count) & ~PREEMPT_NEED_RESCHED; + return __pc_op(read, __preempt_count) & ~PREEMPT_NEED_RESCHED; } -static __always_inline void preempt_count_set(int pc) +/* + * unsigned long preempt count parameter works for both 32bit and 64bit cases: + * + * - For 32bit, "int" (the return of preempt_count()) and "unsigned long" have + * the same size. + * - For 64bit, the effective bits of a preempt count sit in 32bit, and we + * preserve the NEED_RESCHED bit from the old count. + */ +static __always_inline void preempt_count_set(unsigned long pc) { - int old, new; + unsigned long old, new; - old = raw_cpu_read_4(__preempt_count); + old = __pc_op(read, __preempt_count); do { new = (old & PREEMPT_NEED_RESCHED) | (pc & ~PREEMPT_NEED_RESCHED); - } while (!raw_cpu_try_cmpxchg_4(__preempt_count, &old, new)); + } while (!__pc_op(try_cmpxchg, __preempt_count, &old, new)); } /* @@ -58,17 +76,17 @@ static __always_inline void preempt_count_set(int pc) static __always_inline void set_preempt_need_resched(void) { - raw_cpu_and_4(__preempt_count, ~PREEMPT_NEED_RESCHED); + __pc_op(and, __preempt_count, ~PREEMPT_NEED_RESCHED); } static __always_inline void clear_preempt_need_resched(void) { - raw_cpu_or_4(__preempt_count, PREEMPT_NEED_RESCHED); + __pc_op(or, __preempt_count, PREEMPT_NEED_RESCHED); } static __always_inline bool test_preempt_need_resched(void) { - return !(raw_cpu_read_4(__preempt_count) & PREEMPT_NEED_RESCHED); + return !(__pc_op(read, __preempt_count) & PREEMPT_NEED_RESCHED); } /* @@ -77,22 +95,22 @@ static __always_inline bool test_preempt_need_resched(void) static __always_inline void __preempt_count_add(int val) { - raw_cpu_add_4(__preempt_count, val); + __pc_op(add, __preempt_count, val); } static __always_inline void __preempt_count_sub(int val) { - raw_cpu_add_4(__preempt_count, -val); + __pc_op(add, __preempt_count, -val); } static __always_inline int __preempt_count_add_return(int val) { - return raw_cpu_add_return_4(__preempt_count, val); + return __pc_op(add_return, __preempt_count, val); } static __always_inline int __preempt_count_sub_return(int val) { - return raw_cpu_add_return_4(__preempt_count, -val); + return __pc_op(add_return, __preempt_count, -val); } /* @@ -102,7 +120,7 @@ static __always_inline int __preempt_count_sub_return(int val) */ static __always_inline bool __preempt_count_dec_and_test(void) { - return GEN_UNARY_RMWcc("decl", __my_cpu_var(__preempt_count), e, + return GEN_UNARY_RMWcc(__pc_dec, __my_cpu_var(__preempt_count), e, __percpu_arg([var])); } @@ -111,7 +129,7 @@ static __always_inline bool __preempt_count_dec_and_test(void) */ static __always_inline bool should_resched(int preempt_offset) { - return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset); + return unlikely(__pc_op(read, __preempt_count) == preempt_offset); } #ifdef CONFIG_PREEMPTION @@ -158,4 +176,7 @@ do { \ #endif /* PREEMPTION */ +#undef __pc_op +#undef __pc_dec + #endif /* __ASM_PREEMPT_H */ diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index a3df21d26460..73a6d9f6a78e 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -2236,7 +2236,7 @@ DEFINE_PER_CPU_CACHE_HOT(struct task_struct *, current_task) = &init_task; EXPORT_PER_CPU_SYMBOL(current_task); EXPORT_PER_CPU_SYMBOL(const_current_task); -DEFINE_PER_CPU_CACHE_HOT(int, __preempt_count) = INIT_PREEMPT_COUNT; +DEFINE_PER_CPU_CACHE_HOT(unsigned long, __preempt_count) = INIT_PREEMPT_COUNT; EXPORT_PER_CPU_SYMBOL(__preempt_count); DEFINE_PER_CPU_CACHE_HOT(unsigned long, cpu_current_top_of_stack) = TOP_OF_INIT_STACK; diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index 8d4895531a45..73b48dd9f135 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -10,8 +10,6 @@ #include #include -DECLARE_PER_CPU(unsigned int, nmi_nesting); - extern void synchronize_irq(unsigned int irq); extern bool synchronize_hardirq(unsigned int irq); @@ -94,6 +92,37 @@ void irq_exit_rcu(void); #define arch_nmi_exit() do { } while (0) #endif +#ifdef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS +static __always_inline void __preempt_count_nmi_enter(void) +{ + __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); +} + +static __always_inline void __preempt_count_nmi_exit(void) +{ + __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); +} +#else +DECLARE_PER_CPU(unsigned int, nmi_nesting); + +#define __preempt_count_nmi_enter() \ + do { \ + __preempt_count_add(HARDIRQ_OFFSET); \ + /* Maximum NMI nesting is 15. */ \ + BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \ + __this_cpu_inc(nmi_nesting); \ + preempt_count_set(preempt_count() | NMI_MASK); \ + } while (0) + +#define __preempt_count_nmi_exit() \ + do { \ + __preempt_count_sub(HARDIRQ_OFFSET); \ + if (!__this_cpu_dec_return(nmi_nesting)) \ + preempt_count_set(preempt_count() & ~NMI_MASK); \ + } while (0) + +#endif + /* * NMI vs Tracing * -------------- @@ -110,18 +139,14 @@ void irq_exit_rcu(void); do { \ lockdep_off(); \ arch_nmi_enter(); \ - /* Maximum NMI nesting is 15. */ \ - BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \ - __this_cpu_inc(nmi_nesting); \ - __preempt_count_add(HARDIRQ_OFFSET); \ - preempt_count_set(preempt_count() | NMI_MASK); \ + __preempt_count_nmi_enter(); \ } while (0) #define nmi_enter() \ do { \ __nmi_enter(); \ lockdep_hardirq_enter(); \ - ct_nmi_enter(); \ + ct_nmi_enter(); \ instrumentation_begin(); \ ftrace_nmi_enter(); \ instrumentation_end(); \ @@ -129,12 +154,8 @@ void irq_exit_rcu(void); #define __nmi_exit() \ do { \ - unsigned int nesting; \ BUG_ON(!in_nmi()); \ - __preempt_count_sub(HARDIRQ_OFFSET); \ - nesting = __this_cpu_dec_return(nmi_nesting); \ - if (!nesting) \ - preempt_count_set(preempt_count() & ~NMI_MASK); \ + __preempt_count_nmi_exit(); \ arch_nmi_exit(); \ lockdep_on(); \ } while (0) diff --git a/include/linux/preempt.h b/include/linux/preempt.h index 33fc4c814a9f..8299657f0f86 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -34,14 +34,31 @@ * SOFTIRQ_MASK: 0x0000ff00 * HARDIRQ_DISABLE_MASK: 0x00ff0000 * HARDIRQ_MASK: 0x0f000000 + * + * When HAS_SEPARATE_PREEMPT_RESCHED_BITS=y, PREEMPT_NEED_RESCHED is put in a + * separate word and that allows 64bit load-store architectures to 'set' + * PREEMPT_NEED_RESCHED without messing up the otherwise symmetric + * modifications used on preempt_count and still load the whole thing + * (single-copy) atomically, without having to resort to full atomic + * operations. + * + * Because of the above, NMI_MASK bits are different depending on + * HAS_SEPARATE_PREEMPT_RESCHED_BITS: + * + * - HAS_SEPARATE_PREEMPT_RESCHED_BITS=n: + * * NMI_MASK: 0x10000000 * PREEMPT_NEED_RESCHED: 0x80000000 + * + * - HAS_SEPARATE_PREEMPT_RESCHED_BITS=y: + * NMI_MASK: 0xf0000000 + * (PREEMPT_NEED_RESCHED is in a different word) */ #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 #define HARDIRQ_DISABLE_BITS 8 #define HARDIRQ_BITS 4 -#define NMI_BITS 1 +#define NMI_BITS (1 + 3*IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS)) #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) @@ -116,8 +133,8 @@ static __always_inline unsigned char interrupt_context_level(void) * preempt_count() is commonly implemented with READ_ONCE(). */ -#define nmi_count() (preempt_count() & NMI_MASK) -#define hardirq_count() (preempt_count() & HARDIRQ_MASK) +#define nmi_count() (preempt_count() & NMI_MASK) +#define hardirq_count() (preempt_count() & HARDIRQ_MASK) #ifdef CONFIG_PREEMPT_RT # define softirq_count() (current->softirq_disable_cnt & SOFTIRQ_MASK) # define irq_count() ((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) | softirq_count()) diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index 88c594c6d7fc..35f546a042b1 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -122,6 +122,10 @@ config PREEMPT_RT_NEEDS_BH_LOCK config PREEMPT_COUNT bool +config HAS_SEPARATE_PREEMPT_RESCHED_BITS + bool + depends on PREEMPT_COUNT && 64BIT + config PREEMPTION bool select PREEMPT_COUNT diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 9b3f1764fa9e..6d88343c3bad 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5973,8 +5973,13 @@ void preempt_count_add(int val) #ifdef CONFIG_DEBUG_PREEMPT /* * Underflow? + * + * Cannot detect underflow based on the current preempt_count() value + * if using HAS_SEPARATE_PREEMPT_RESCHED_BITS because preempt count takes all 32 + * bits. */ - if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) + if (!IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS) && + DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) return; #endif __preempt_count_add(val); @@ -6006,7 +6011,10 @@ void preempt_count_sub(int val) /* * Underflow? */ - if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) + unsigned int uval = val; + unsigned int pc = preempt_count(); + + if (DEBUG_LOCKS_WARN_ON(pc - uval > pc)) return; /* * Is the spinlock portion underflowing? diff --git a/kernel/softirq.c b/kernel/softirq.c index 0c9b2269a8d6..7980a4a232f9 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -103,7 +103,13 @@ void _local_interrupt_enable(void) } EXPORT_SYMBOL(_local_interrupt_enable); +#ifndef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS +/* + * Any 32bit architecture that still cares about performance should + * probably ensure this is near preempt_count. + */ DEFINE_PER_CPU(unsigned int, nmi_nesting); +#endif /* * SOFTIRQ_OFFSET usage: diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c index bfafe1204c7b..c3d976c801bb 100644 --- a/lib/locking-selftest.c +++ b/lib/locking-selftest.c @@ -1429,7 +1429,7 @@ static int unexpected_testcase_failures; static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) { - int saved_preempt_count = preempt_count(); + long saved_preempt_count = preempt_count(); #ifdef CONFIG_PREEMPT_RT int saved_mgd_count = current->migration_disabled; int saved_rcu_count = current->rcu_read_lock_nesting; -- cgit v1.2.3 From 264bbd32a31ba80e634bec5c1df8514c0e97fe7e Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Tue, 4 Aug 2026 09:14:33 -0700 Subject: arm64: sched/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS Arm64 already uses 64-bit preempt count and the need reschedule bit is maintained in a separate 32-bit word from the preempt count. Therefore preempt count has enough bits to represent 16 levels of NMI nesting, hence enable it for arm64. This saves a per-CPU variable and additional instructions in the NMI path. Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260804161447.84806-12-boqun@kernel.org --- arch/arm64/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b3afe0688919..349c3533cd1e 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -247,6 +247,7 @@ config ARM64 select PCI_SYSCALL if PCI select POWER_RESET select POWER_SUPPLY + select HAS_SEPARATE_PREEMPT_RESCHED_BITS select SPARSE_IRQ select SWIOTLB select SYSCTL_EXCEPTION_TRACE -- cgit v1.2.3 From 3484440a7b194ed80fa4ae2d25fe1a3fcade98f3 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 4 Aug 2026 09:14:34 -0700 Subject: s390/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS Convert s390's preempt_count to 64 bit, and change the preempt primitives accordingly. [boqun: Apply the corrected comment for asm block] Signed-off-by: Heiko Carstens Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260804161447.84806-13-boqun@kernel.org --- arch/s390/Kconfig | 1 + arch/s390/include/asm/lowcore.h | 13 +++++++++---- arch/s390/include/asm/preempt.h | 43 +++++++++++++++++++---------------------- 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 84404e6778d5..378fcd2b6181 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -273,6 +273,7 @@ config S390 select PCI_MSI if PCI select PCI_MSI_ARCH_FALLBACKS if PCI_MSI select PCI_QUIRKS if PCI + select HAS_SEPARATE_PREEMPT_RESCHED_BITS select SPARSE_IRQ select SWIOTLB select SYSCTL_EXCEPTION_TRACE diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 3b3ecc647993..5cef215d30e7 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -160,10 +160,15 @@ struct lowcore { /* SMP info area */ __u32 cpu_nr; /* 0x03a0 */ __u32 softirq_pending; /* 0x03a4 */ - __s32 preempt_count; /* 0x03a8 */ - __u32 spinlock_lockval; /* 0x03ac */ - __u32 spinlock_index; /* 0x03b0 */ - __u8 pad_0x03b4[0x03b8-0x03b4]; /* 0x03b4 */ + union { + struct { + __u32 need_resched; /* 0x03a8 */ + __u32 count; /* 0x03ac */ + } preempt; + __u64 preempt_count; /* 0x03a8 */ + }; + __u32 spinlock_lockval; /* 0x03b0 */ + __u32 spinlock_index; /* 0x03b4 */ __u64 percpu_offset; /* 0x03b8 */ __u8 percpu_register; /* 0x03c0 */ __u8 pad_0x03c1[0x0400-0x03c1]; /* 0x03c1 */ diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preempt.h index 0a25d4648b4c..5560d5fca2a3 100644 --- a/arch/s390/include/asm/preempt.h +++ b/arch/s390/include/asm/preempt.h @@ -8,11 +8,8 @@ #include #include -/* - * Use MSB so it is possible to read preempt_count with LLGT which - * reads the least significant 31 bits with a single instruction. - */ -#define PREEMPT_NEED_RESCHED 0x80000000 +/* Use MSB for PREEMPT_NEED_RESCHED mostly because it is available. */ +#define PREEMPT_NEED_RESCHED 0x8000000000000000UL /* * We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such @@ -26,25 +23,25 @@ */ static __always_inline int preempt_count(void) { - unsigned long lc_preempt, count; + unsigned long lc_preempt; + int count; - BUILD_BUG_ON(sizeof_field(struct lowcore, preempt_count) != sizeof(int)); - lc_preempt = offsetof(struct lowcore, preempt_count); - /* READ_ONCE(get_lowcore()->preempt_count) & ~PREEMPT_NEED_RESCHED */ + lc_preempt = offsetof(struct lowcore, preempt.count); + /* READ_ONCE(get_lowcore()->preempt.count) (without PREEMPT_NEED_RESCHED) */ asm_inline( - ALTERNATIVE("llgt %[count],%[offzero](%%r0)\n", - "llgt %[count],%[offalt](%%r0)\n", + ALTERNATIVE("ly %[count],%[offzero](%%r0)\n", + "ly %[count],%[offalt](%%r0)\n", ALT_FEATURE(MFEATURE_LOWCORE)) : [count] "=d" (count) : [offzero] "i" (lc_preempt), [offalt] "i" (lc_preempt + LOWCORE_ALT_ADDRESS), - "m" (((struct lowcore *)0)->preempt_count)); + "m" (((struct lowcore *)0)->preempt.count)); return count; } -static __always_inline void preempt_count_set(int pc) +static __always_inline void preempt_count_set(unsigned long pc) { - int old, new; + unsigned long old, new; old = READ_ONCE(get_lowcore()->preempt_count); do { @@ -63,12 +60,12 @@ static __always_inline void preempt_count_set(int pc) static __always_inline void set_preempt_need_resched(void) { - __atomic_and(~PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count); + __atomic64_and(~PREEMPT_NEED_RESCHED, (long *)&get_lowcore()->preempt_count); } static __always_inline void clear_preempt_need_resched(void) { - __atomic_or(PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count); + __atomic64_or(PREEMPT_NEED_RESCHED, (long *)&get_lowcore()->preempt_count); } static __always_inline bool test_preempt_need_resched(void) @@ -88,8 +85,8 @@ static __always_inline void __preempt_count_add(int val) lc_preempt = offsetof(struct lowcore, preempt_count); asm_inline( - ALTERNATIVE("asi %[offzero](%%r0),%[val]\n", - "asi %[offalt](%%r0),%[val]\n", + ALTERNATIVE("agsi %[offzero](%%r0),%[val]\n", + "agsi %[offalt](%%r0),%[val]\n", ALT_FEATURE(MFEATURE_LOWCORE)) : "+m" (((struct lowcore *)0)->preempt_count) : [offzero] "i" (lc_preempt), [val] "i" (val), @@ -98,7 +95,7 @@ static __always_inline void __preempt_count_add(int val) return; } } - __atomic_add(val, &get_lowcore()->preempt_count); + __atomic64_add(val, (long *)&get_lowcore()->preempt_count); } static __always_inline void __preempt_count_sub(int val) @@ -119,15 +116,15 @@ static __always_inline bool __preempt_count_dec_and_test(void) lc_preempt = offsetof(struct lowcore, preempt_count); asm_inline( - ALTERNATIVE("alsi %[offzero](%%r0),%[val]\n", - "alsi %[offalt](%%r0),%[val]\n", + ALTERNATIVE("algsi %[offzero](%%r0),%[val]\n", + "algsi %[offalt](%%r0),%[val]\n", ALT_FEATURE(MFEATURE_LOWCORE)) : "=@cc" (cc), "+m" (((struct lowcore *)0)->preempt_count) : [offzero] "i" (lc_preempt), [val] "i" (-1), [offalt] "i" (lc_preempt + LOWCORE_ALT_ADDRESS)); return (cc == 0) || (cc == 2); #else - return __atomic_add_const_and_test(-1, &get_lowcore()->preempt_count); + return __atomic64_add_const_and_test(-1, (long *)&get_lowcore()->preempt_count); #endif } @@ -141,7 +138,7 @@ static __always_inline bool should_resched(int preempt_offset) static __always_inline int __preempt_count_add_return(int val) { - return val + __atomic_add(val, &get_lowcore()->preempt_count); + return val + __atomic64_add(val, (long *)&get_lowcore()->preempt_count); } static __always_inline int __preempt_count_sub_return(int val) -- cgit v1.2.3 From c1f0451ec748f7a58a4e50bcb8f75bd89718be78 Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Fri, 7 Aug 2026 00:02:11 -0700 Subject: rust: Introduce interrupt module This introduces a module for dealing with interrupt-disabled contexts, including the ability to enable and disable interrupts along with the ability to annotate functions as expecting that IRQs are already disabled on the local CPU. Signed-off-by: Lyude Paul Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Benno Lossin Reviewed-by: Andreas Hindborg Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260807070218.27144-15-boqun@kernel.org --- rust/helpers/helpers.c | 1 + rust/helpers/interrupt.c | 13 +++++++ rust/helpers/sync.c | 5 +++ rust/kernel/interrupt.rs | 89 ++++++++++++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 1 + 5 files changed, 109 insertions(+) create mode 100644 rust/helpers/interrupt.c create mode 100644 rust/kernel/interrupt.rs diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 998e31052e66..0d85b5e68ec2 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -65,6 +65,7 @@ #include "irq.c" #include "fs.c" #include "gpu.c" +#include "interrupt.c" #include "io.c" #include "jump_label.c" #include "kunit.c" diff --git a/rust/helpers/interrupt.c b/rust/helpers/interrupt.c new file mode 100644 index 000000000000..69595498620f --- /dev/null +++ b/rust/helpers/interrupt.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_local_interrupt_disable(void) +{ + local_interrupt_disable(); +} + +__rust_helper void rust_helper_local_interrupt_enable(void) +{ + local_interrupt_enable(); +} diff --git a/rust/helpers/sync.c b/rust/helpers/sync.c index 82d6aff73b04..4f474fe847c4 100644 --- a/rust/helpers/sync.c +++ b/rust/helpers/sync.c @@ -11,3 +11,8 @@ __rust_helper void rust_helper_lockdep_unregister_key(struct lock_class_key *k) { lockdep_unregister_key(k); } + +__rust_helper void rust_helper_lockdep_assert_irqs_disabled(void) +{ + lockdep_assert_irqs_disabled(); +} diff --git a/rust/kernel/interrupt.rs b/rust/kernel/interrupt.rs new file mode 100644 index 000000000000..a880ec3b8538 --- /dev/null +++ b/rust/kernel/interrupt.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Interrupt controls +//! +//! This module allows Rust code to annotate areas of code where local processor interrupts should +//! be disabled, along with actually disabling local processor interrupts. +//! +//! # ⚠️ Warning! ⚠️ +//! +//! The usage of this module can be more complicated than meets the eye, especially surrounding +//! [preemptible kernels]. It's recommended to take care when using the functions and types defined +//! here and familiarize yourself with the various documentation we have before using them, along +//! with the various documents we link to here. +//! +//! # Reading material +//! +//! - [Software interrupts and realtime (LWN)](https://lwn.net/Articles/520076) +//! +//! [preemptible kernels]: https://www.kernel.org/doc/html/latest/locking/preempt-locking.html + +use crate::types::NotThreadSafe; + +/// A guard that represents local processor interrupt disablement on preemptible kernels. +/// +/// [`LocalInterruptDisabled`] is a guard type that represents that local processor interrupts have +/// been disabled on a preemptible kernel. +/// +/// Certain functions take an immutable reference of [`LocalInterruptDisabled`] in order to require +/// that they may only be run in local-interrupt-disabled contexts on preemptible kernels. +/// +/// This is a marker type; it has no size, and is simply used as a compile-time guarantee that local +/// processor interrupts are disabled on preemptible kernels. Note that no guarantees about the +/// state of interrupts are made by this type on non-preemptible kernels. +/// +/// # Invariants +/// +/// Local processor interrupts are disabled on preemptible kernels for as long as an object of this +/// type exists. +pub struct LocalInterruptDisabled(NotThreadSafe); + +/// Disable local processor interrupts on a preemptible kernel. +/// +/// This function disables local processor interrupts on a preemptible kernel, and returns a +/// [`LocalInterruptDisabled`] token as proof of this. On non-preemptible kernels, this function is +/// a no-op. +/// +/// **Usage of this function is discouraged** unless you are absolutely sure you know what you are +/// doing, as kernel interfaces for Rust that deal with interrupt state will typically handle local +/// processor interrupt state management on their own and managing this by hand is quite error +/// prone. +#[inline] +pub fn local_interrupt_disable() -> LocalInterruptDisabled { + // SAFETY: It's always safe to call `local_interrupt_disable()`. + unsafe { bindings::local_interrupt_disable() }; + + LocalInterruptDisabled(NotThreadSafe) +} + +impl Drop for LocalInterruptDisabled { + #[inline] + fn drop(&mut self) { + // SAFETY: Per type invariants, a `local_interrupt_disable()` must be called to create this + // object, hence calling the corresponding `local_interrupt_enable()` is safe. + unsafe { bindings::local_interrupt_enable() }; + } +} + +impl LocalInterruptDisabled { + /// Assume that local processor interrupts are disabled on preemptible kernels. + /// + /// This can be used for annotating code that is known to be run in contexts where local + /// processor interrupts are disabled on preemptible kernels. It makes no changes to the local + /// interrupt state on its own. + /// + /// # Safety + /// + /// For the whole life `'a`, local interrupts must be disabled on preemptible kernels. This + /// could be a context like, for example, an interrupt handler. + #[inline] + pub unsafe fn assume_disabled<'a>() -> &'a LocalInterruptDisabled { + const ASSUME_DISABLED: &LocalInterruptDisabled = &LocalInterruptDisabled(NotThreadSafe); + + // Confirm they're actually disabled if lockdep is available + // SAFETY: It's always safe to call `lockdep_assert_irqs_disabled()`. + unsafe { bindings::lockdep_assert_irqs_disabled() }; + + ASSUME_DISABLED + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 9512af7156df..2ee6c24d39c2 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -82,6 +82,7 @@ pub mod id_pool; pub mod impl_flags; pub mod init; pub mod interop; +pub mod interrupt; pub mod io; pub mod ioctl; pub mod iommu; -- cgit v1.2.3 From 78c998b04ef53492f3502051349e55615681e00f Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Fri, 7 Aug 2026 00:02:12 -0700 Subject: rust: helper: Add spin_{un,}lock_irq_{enable,disable}() helpers spin_lock_irq_disable() and spin_unlock_irq_enable() are inline functions, to use them in Rust helpers are introduced. This is for interrupt disabling lock abstraction in Rust. Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Andreas Hindborg Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260807070218.27144-16-boqun@kernel.org --- rust/helpers/spinlock.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/rust/helpers/spinlock.c b/rust/helpers/spinlock.c index 4d13062cf253..d53400c15022 100644 --- a/rust/helpers/spinlock.c +++ b/rust/helpers/spinlock.c @@ -36,3 +36,18 @@ __rust_helper void rust_helper_spin_assert_is_held(spinlock_t *lock) { lockdep_assert_held(lock); } + +__rust_helper void rust_helper_spin_lock_irq_disable(spinlock_t *lock) +{ + spin_lock_irq_disable(lock); +} + +__rust_helper void rust_helper_spin_unlock_irq_enable(spinlock_t *lock) +{ + spin_unlock_irq_enable(lock); +} + +__rust_helper int rust_helper_spin_trylock_irq_disable(spinlock_t *lock) +{ + return spin_trylock_irq_disable(lock); +} -- cgit v1.2.3 From df9165ab5254acb2035919c6e0511de799c1e8c1 Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Fri, 7 Aug 2026 00:02:13 -0700 Subject: rust: sync: Use super::* in spinlock.rs No functional changes. Signed-off-by: Lyude Paul Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260807070218.27144-17-boqun@kernel.org --- rust/kernel/sync/lock/spinlock.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index ef76fa07ca3a..d75af32218ba 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -3,6 +3,7 @@ //! A kernel spinlock. //! //! This module allows Rust code to use the kernel's `spinlock_t`. +use super::*; /// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class. /// @@ -82,7 +83,7 @@ pub use new_spinlock; /// ``` /// /// [`spinlock_t`]: srctree/include/linux/spinlock.h -pub type SpinLock = super::Lock; +pub type SpinLock = Lock; /// A kernel `spinlock_t` lock backend. pub struct SpinLockBackend; @@ -91,13 +92,11 @@ pub struct SpinLockBackend; /// /// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLock`]. It will unlock /// the [`SpinLock`] upon being dropped. -/// -/// [`Guard`]: super::Guard -pub type SpinLockGuard<'a, T> = super::Guard<'a, T, SpinLockBackend>; +pub type SpinLockGuard<'a, T> = Guard<'a, T, SpinLockBackend>; // SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the // default implementation that always calls the same locking method. -unsafe impl super::Backend for SpinLockBackend { +unsafe impl Backend for SpinLockBackend { type State = bindings::spinlock_t; type GuardState = (); -- cgit v1.2.3 From 5967f4df55521de596cbe34e0c0c96e962ef3f2e Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Fri, 7 Aug 2026 00:02:14 -0700 Subject: rust: sync: Add SpinLockIrq A variant of `SpinLock` that ensures interrupts are disabled in the critical section. `lock()` will ensure that either interrupts are already disabled or disable them. `unlock()` will reverse the respective operation. [Boqun: Port to use spin_lock_irq_disable() and spin_unlock_irq_enable()] Signed-off-by: Lyude Paul Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260807070218.27144-18-boqun@kernel.org --- rust/kernel/sync.rs | 9 +- rust/kernel/sync/lock/global.rs | 3 + rust/kernel/sync/lock/spinlock.rs | 230 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 241 insertions(+), 1 deletion(-) diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 993dbf2caa0e..df4f2604ff9b 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -27,7 +27,14 @@ pub use completion::Completion; pub use condvar::{new_condvar, CondVar, CondVarTimeoutResult}; pub use lock::global::{global_lock, GlobalGuard, GlobalLock, GlobalLockBackend, GlobalLockedBy}; pub use lock::mutex::{new_mutex, Mutex, MutexGuard}; -pub use lock::spinlock::{new_spinlock, SpinLock, SpinLockGuard}; +pub use lock::spinlock::{ + new_spinlock, + new_spinlock_irq, + SpinLock, + SpinLockGuard, + SpinLockIrq, + SpinLockIrqGuard, // +}; pub use locked_by::LockedBy; pub use refcount::Refcount; pub use set_once::SetOnce; diff --git a/rust/kernel/sync/lock/global.rs b/rust/kernel/sync/lock/global.rs index ec2dd84316fc..ebb10521d8bd 100644 --- a/rust/kernel/sync/lock/global.rs +++ b/rust/kernel/sync/lock/global.rs @@ -306,4 +306,7 @@ macro_rules! global_lock_inner { (backend SpinLock) => { $crate::sync::lock::spinlock::SpinLockBackend }; + (backend SpinLockIrq) => { + $crate::sync::lock::spinlock::SpinLockIrqBackend + }; } diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index d75af32218ba..872544948e5d 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -4,6 +4,7 @@ //! //! This module allows Rust code to use the kernel's `spinlock_t`. use super::*; +use crate::prelude::*; /// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class. /// @@ -143,3 +144,232 @@ unsafe impl Backend for SpinLockBackend { unsafe { bindings::spin_assert_is_held(ptr) } } } + +/// Creates a [`SpinLockIrq`] initialiser with the given name and a newly-created lock class. +/// +/// It uses the name if one is given, otherwise it generates one based on the file name and line +/// number. +#[macro_export] +macro_rules! new_spinlock_irq { + ($inner:expr $(, $name:literal)? $(,)?) => { + $crate::sync::SpinLockIrq::new( + $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_spinlock_irq; + +/// A variant of `SpinLock` that ensures interrupts are disabled in the critical section. +/// +/// For more info on spinlocks, see [`SpinLock`]. For more information on interrupts, +/// [see the interrupt module](kernel::interrupt). +/// +/// # Examples +/// +/// The following example shows how to declare, allocate initialise and access a struct (`Example`) +/// that contains an inner struct (`Inner`) that is protected by a spinlock that requires local +/// processor interrupts to be disabled. +/// +/// ``` +/// use kernel::sync::{new_spinlock_irq, SpinLockIrq}; +/// +/// struct Inner { +/// a: u32, +/// b: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// #[pin] +/// c: SpinLockIrq, +/// #[pin] +/// d: SpinLockIrq, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit { +/// pin_init!(Self { +/// c <- new_spinlock_irq!(Inner { a: 0, b: 10 }), +/// d <- new_spinlock_irq!(Inner { a: 20, b: 30 }), +/// }) +/// } +/// } +/// +/// // Allocate a boxed `Example` +/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// +/// // Accessing an `Example` from a context where interrupts may not be disabled already. +/// let c_guard = e.c.lock(); // interrupts are disabled now, +1 interrupt disable refcount +/// let d_guard = e.d.lock(); // no interrupt state change, +1 interrupt disable refcount +/// +/// assert_eq!(c_guard.a, 0); +/// assert_eq!(c_guard.b, 10); +/// assert_eq!(d_guard.a, 20); +/// assert_eq!(d_guard.b, 30); +/// +/// drop(c_guard); // Dropping c_guard will not re-enable interrupts just yet, since d_guard is +/// // still in scope. +/// drop(d_guard); // Last interrupt disable reference dropped here, so interrupts are re-enabled +/// // now +/// # Ok::<(), Error>(()) +/// ``` +/// +/// [`lock()`]: SpinLockIrq::lock +pub type SpinLockIrq = super::Lock; + +/// A kernel `spinlock_t` lock backend that can only be acquired in interrupt disabled contexts. +pub struct SpinLockIrqBackend; + +/// A [`Guard`] acquired from locking a [`SpinLockIrq`] using [`lock()`]. +/// +/// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLockIrq`] using +/// [`lock()`]. It will unlock the [`SpinLockIrq`] and decrement the local processor's interrupt +/// disablement refcount upon being dropped. +/// +/// [`lock()`]: SpinLockIrq::lock +pub type SpinLockIrqGuard<'a, T> = Guard<'a, T, SpinLockIrqBackend>; + +// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the +// default implementation that always calls the same locking method. +unsafe impl Backend for SpinLockIrqBackend { + type State = bindings::spinlock_t; + type GuardState = (); + + #[inline] + unsafe fn init( + ptr: *mut Self::State, + name: *const crate::ffi::c_char, + key: *mut bindings::lock_class_key, + ) { + // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and + // `key` are valid for read indefinitely. + unsafe { bindings::__spin_lock_init(ptr, name, key) } + } + + #[inline] + unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { + // SAFETY: The safety requirements of this function ensure that `ptr` points to valid + // memory, and that it has been initialised before. + unsafe { bindings::spin_lock_irq_disable(ptr) } + } + + #[inline] + unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) { + // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the + // caller is the owner of the spinlock. + unsafe { bindings::spin_unlock_irq_enable(ptr) } + } + + #[inline] + unsafe fn try_lock(ptr: *mut Self::State) -> Option { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + let result = unsafe { bindings::spin_trylock_irq_disable(ptr) }; + + if result != 0 { + Some(()) + } else { + None + } + } + + #[inline] + unsafe fn assert_is_held(ptr: *mut Self::State) { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + unsafe { bindings::spin_assert_is_held(ptr) } + } +} + +#[kunit_tests(rust_spinlock_irq_condvar)] +mod tests { + use super::*; + use crate::{ + sync::*, + workqueue::{ + self, + impl_has_work, + new_work, + Work, + WorkItem, // + }, + }; + + struct TestState { + value: u32, + waiter_ready: bool, + } + + #[pin_data] + struct Test { + #[pin] + state: SpinLockIrq, + + #[pin] + state_changed: CondVar, + + #[pin] + waiter_state_changed: CondVar, + + #[pin] + wait_work: Work, + } + + impl_has_work! { + impl HasWork for Test { self.wait_work } + } + + impl Test { + pub(crate) fn new() -> Result> { + Arc::try_pin_init( + try_pin_init!( + Self { + state <- new_spinlock_irq!(TestState { + value: 1, + waiter_ready: false + }), + state_changed <- new_condvar!(), + waiter_state_changed <- new_condvar!(), + wait_work <- new_work!("IrqCondvarTest::wait_work") + } + ), + GFP_KERNEL, + ) + } + } + + impl WorkItem for Test { + type Pointer = Arc; + + fn run(this: Arc) { + // Wait for the test to be ready to wait for us + let mut state = this.state.lock(); + + // Make sure the interrupts actually turned off + // SAFETY: It's always safe to call `lockdep_assert_irqs_disabled()` + unsafe { bindings::lockdep_assert_irqs_disabled() }; + + while !state.waiter_ready { + this.waiter_state_changed.wait(&mut state); + } + + // Deliver the exciting value update our test has been waiting for + state.value += 1; + this.state_changed.notify_sync(); + } + } + + #[test] + fn spinlock_irq_condvar() -> Result { + let testdata = Test::new()?; + + let _ = workqueue::system().enqueue(testdata.clone()); + + // Let the updater know when we're ready to wait + let mut state = testdata.state.lock(); + state.waiter_ready = true; + testdata.waiter_state_changed.notify_sync(); + + // Wait for the exciting value update + testdata.state_changed.wait(&mut state); + assert_eq!(state.value, 2); + Ok(()) + } +} -- cgit v1.2.3 From ec4fad7c2bdc80c62fa73f799b77ad299f73dcc3 Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Fri, 7 Aug 2026 00:02:15 -0700 Subject: rust: sync: Introduce SpinLockIrq::lock_with() and friends `SpinLockIrq` and `SpinLock` use the exact same underlying C structure, with the only real difference being that the former uses the irq_disable() and irq_enable() variants for locking/unlocking. These variants can introduce some minor overhead in contexts where we already know that local processor interrupts are disabled, and as such we want a way to be able to skip modifying processor interrupt state in said contexts in order to avoid some overhead - just like the current C API allows us to do. In order to do this, we add some special functions for SpinLockIrq: lock_with() and try_lock_with(), which allow acquiring the lock without changing the interrupt state - as long as the caller can provide a LocalInterruptDisabled reference to prove that local processor interrupts have been disabled. In some hacked-together benchmarks we ran, most of the time this did actually seem to lead to a noticeable difference in overhead: From an aarch64 VM running on a MacBook M4: lock() when irq is disabled, 100 times cost Delta { nanos: 500 } lock_with() when irq is disabled, 100 times cost Delta { nanos: 292 } lock() when irq is enabled, 100 times cost Delta { nanos: 834 } lock() when irq is disabled, 100 times cost Delta { nanos: 459 } lock_with() when irq is disabled, 100 times cost Delta { nanos: 291 } lock() when irq is enabled, 100 times cost Delta { nanos: 709 } From an x86_64 VM (qemu/kvm) running on a i7-13700H lock() when irq is disabled, 100 times cost Delta { nanos: 1002 } lock_with() when irq is disabled, 100 times cost Delta { nanos: 729 } lock() when irq is enabled, 100 times cost Delta { nanos: 1516 } lock() when irq is disabled, 100 times cost Delta { nanos: 754 } lock_with() when irq is disabled, 100 times cost Delta { nanos: 966 } lock() when irq is enabled, 100 times cost Delta { nanos: 1227 } (note that there were some runs on x86_64 where lock() on irq disabled vs. lock_with() on irq disabled had equivalent benchmarks, but it very much appeared to be a minority of test runs.) While it's not clear how this affects real-world workloads yet, let's add this for the time being so we can find out. This makes it so that a `SpinLockIrq` will work like a `SpinLock` if interrupts are disabled. So a function: (&'a SpinLockIrq, &'a LocalInterruptDisabled) -> Guard<'a, .., SpinLockBackend> makes sense. Note that due to `Guard` and `LocalInterruptDisabled` having the same lifetime, interrupts cannot be enabled while the Guard exists. Signed-off-by: Lyude Paul Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260807070218.27144-19-boqun@kernel.org --- rust/kernel/sync/lock/spinlock.rs | 92 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index 872544948e5d..aafc80125f59 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -4,7 +4,10 @@ //! //! This module allows Rust code to use the kernel's `spinlock_t`. use super::*; -use crate::prelude::*; +use crate::{ + interrupt::LocalInterruptDisabled, + prelude::*, // +}; /// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class. /// @@ -160,6 +163,16 @@ pub use new_spinlock_irq; /// A variant of `SpinLock` that ensures interrupts are disabled in the critical section. /// +/// This lock can be acquired in two ways: +/// +/// - Using [`lock()`] like any other type of lock, in which case the bindings will modify the +/// interrupt state to ensure that local processor interrupts remain disabled for at least as +/// long as the [`SpinLockIrqGuard`] exists. +/// - Using [`lock_with()`] in contexts where a [`LocalInterruptDisabled`] token is present and +/// local processor interrupts are already known to be disabled, in which case the local +/// interrupt state will not be touched. This method should be preferred if a +/// [`LocalInterruptDisabled`] token is present in the scope. +/// /// For more info on spinlocks, see [`SpinLock`]. For more information on interrupts, /// [see the interrupt module](kernel::interrupt). /// @@ -213,7 +226,47 @@ pub use new_spinlock_irq; /// # Ok::<(), Error>(()) /// ``` /// +/// The next example demonstrates locking a [`SpinLockIrq`] using [`lock_with()`] in a function +/// which can only be called when local processor interrupts are already disabled. +/// +/// ``` +/// use kernel::sync::{new_spinlock_irq, SpinLockIrq}; +/// use kernel::interrupt::*; +/// +/// struct Inner { +/// a: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// #[pin] +/// inner: SpinLockIrq, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit { +/// pin_init!(Self { +/// inner <- new_spinlock_irq!(Inner { a: 20 }), +/// }) +/// } +/// } +/// +/// // Accessing an `Example` from a function that can only be called in no-interrupt contexts. +/// fn noirq_work(e: &Example, interrupt_disabled: &LocalInterruptDisabled) { +/// // Because we know interrupts are disabled from interrupt_disable, we can skip toggling +/// // interrupt state using lock_with() and the provided token +/// assert_eq!(e.inner.lock_with(interrupt_disabled).a, 20); +/// } +/// +/// # let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// # let interrupt_guard = local_interrupt_disable(); +/// # noirq_work(&e, &interrupt_guard); +/// # +/// # Ok::<(), Error>(()) +/// ``` +/// /// [`lock()`]: SpinLockIrq::lock +/// [`lock_with()`]: SpinLockIrq::lock_with pub type SpinLockIrq = super::Lock; /// A kernel `spinlock_t` lock backend that can only be acquired in interrupt disabled contexts. @@ -278,6 +331,43 @@ unsafe impl Backend for SpinLockIrqBackend { } } +impl Lock { + /// Casts the lock as a `Lock`. + #[inline] + fn as_lock_in_interrupt<'a>(&'a self, _context: &'a LocalInterruptDisabled) -> &'a SpinLock { + // SAFETY: + // - `Lock` and `Lock` both have identical data + // layouts. + // - As long as local interrupts are disabled (which is proven to be true by _context), it + // is safe to treat a lock with SpinLockIrqBackend as a SpinLockBackend lock. + unsafe { core::mem::transmute(self) } + } + + /// Acquires the lock without modifying local interrupt state. + /// + /// This function should be used in place of the more expensive [`Lock::lock()`] function when + /// possible for [`SpinLockIrq`] locks. + #[inline] + pub fn lock_with<'a>(&'a self, context: &'a LocalInterruptDisabled) -> SpinLockGuard<'a, T> { + self.as_lock_in_interrupt(context).lock() + } + + /// Tries to acquire the lock without modifying local interrupt state. + /// + /// This function should be used in place of the more expensive [`Lock::try_lock()`] function + /// when possible for [`SpinLockIrq`] locks. + /// + /// Returns a guard that can be used to access the data protected by the lock if successful. + #[must_use = "if unused, the lock will be immediately unlocked"] + #[inline] + pub fn try_lock_with<'a>( + &'a self, + context: &'a LocalInterruptDisabled, + ) -> Option> { + self.as_lock_in_interrupt(context).try_lock() + } +} + #[kunit_tests(rust_spinlock_irq_condvar)] mod tests { use super::*; -- cgit v1.2.3