diff options
| author | Peter Zijlstra <peterz@infradead.org> | 2026-08-04 07:15:41 +0000 |
|---|---|---|
| committer | Peter Zijlstra <peterz@infradead.org> | 2026-08-07 17:58:09 +0200 |
| commit | fc6ad5eadcd1694d2f1ee2717c8024efd8152e20 (patch) | |
| tree | 7ff7e6782eb3be4ef9488799c8fef40854b29b87 /arch/x86/include/asm | |
| parent | 1c7efabfbaf796f11000a46094a69955a01ec6cc (diff) | |
x86/paravirt: Use static_call() for the paravirt spinlock ops
queued_spin_lock_slowpath() and queued_spin_unlock() are dispatched
through pv_ops_lock via the paravirt-ops ALTERNATIVE machinery, which
picks the target (native inline store / hypervisor call) once at boot
and cannot change at runtime.
Convert both to static_call(). The site becomes a direct call patched in
place (one byte smaller), and on native the unlock still collapses to
the inline "movb $0, (%rdi)" store, so the fast path is unchanged.
Unlike the ALTERNATIVE mechanism, a static_call() target can also be
updated at runtime via static_call_update(). This is a prerequisite for
the contended_release tracepoint, which has to swap in a traced unlock
while the system is running.
[ ilvokhin: commit message; fix PARAVIRT_SPINLOCKS=n build; teach
__static_call_validate() about the inline unlock insn; make the
slowpath site module-safe: static_call_mod() +
EXPORT_STATIC_CALL_TRAMP(); pass @lock to the callee-save unlock,
fixing a boot hang under CALL_DEPTH_TRACKING. Boot tested native + KVM
PV guest. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Co-developed-by: Dmitry Ilvokhin <d@ilvokhin.com>
Signed-off-by: Dmitry Ilvokhin <d@ilvokhin.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Juergen Gross <jgross@suse.com>
Link: https://lore.kernel.org/all/20260603120811.GW3493090@noisy.programming.kicks-ass.net/
Link: https://patch.msgid.link/9a32ae399eb804a02a31af04dcabe7e7ee4f3fdf.1785778551.git.d@ilvokhin.com
Diffstat (limited to 'arch/x86/include/asm')
| -rw-r--r-- | arch/x86/include/asm/cpufeatures.h | 2 | ||||
| -rw-r--r-- | arch/x86/include/asm/paravirt-spinlock.h | 19 |
2 files changed, 13 insertions, 8 deletions
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 1b4a48bff18f..55224157740b 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -225,7 +225,7 @@ #define X86_FEATURE_EPT_AD ( 8*32+17) /* "ept_ad" Intel Extended Page Table access-dirty bit */ #define X86_FEATURE_VMCALL ( 8*32+18) /* Hypervisor supports the VMCALL instruction */ #define X86_FEATURE_VMW_VMMCALL ( 8*32+19) /* VMware prefers VMMCALL hypercall instruction */ -#define X86_FEATURE_PVUNLOCK ( 8*32+20) /* PV unlock function */ +// free: was #define X86_FEATURE_PVUNLOCK ( 8*32+20) /* PV unlock function */ #define X86_FEATURE_VCPUPREEMPT ( 8*32+21) /* PV vcpu_is_preempted function */ #define X86_FEATURE_TDX_GUEST ( 8*32+22) /* "tdx_guest" Intel Trust Domain Extensions Guest */ diff --git a/arch/x86/include/asm/paravirt-spinlock.h b/arch/x86/include/asm/paravirt-spinlock.h index 7beffcb08ed6..ff735830de4a 100644 --- a/arch/x86/include/asm/paravirt-spinlock.h +++ b/arch/x86/include/asm/paravirt-spinlock.h @@ -3,6 +3,7 @@ #define _ASM_X86_PARAVIRT_SPINLOCK_H #include <asm/paravirt_types.h> +#include <linux/static_call_types.h> #ifdef CONFIG_SMP #include <asm/spinlock_types.h> @@ -11,9 +12,6 @@ struct qspinlock; struct pv_lock_ops { - void (*queued_spin_lock_slowpath)(struct qspinlock *lock, u32 val); - struct paravirt_callee_save queued_spin_unlock; - void (*wait)(u8 *ptr, u8 val); void (*kick)(int cpu); @@ -26,20 +24,27 @@ extern struct pv_lock_ops pv_ops_lock; extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); extern void __pv_init_lock_hash(void); extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); +extern void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock); extern void __raw_callee_save___pv_queued_spin_unlock(struct qspinlock *lock); extern bool nopvspin; +DECLARE_STATIC_CALL(queued_spin_lock_slowpath, native_queued_spin_lock_slowpath); +DECLARE_STATIC_CALL(queued_spin_unlock, __raw_callee_save___native_queued_spin_unlock); + static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) { - PVOP_VCALL2(pv_ops_lock, queued_spin_lock_slowpath, lock, val); + static_call_mod(queued_spin_lock_slowpath)(lock, val); } static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock) { - PVOP_ALT_VCALLEE1(pv_ops_lock, queued_spin_unlock, lock, - "movb $0, (%%" _ASM_ARG1 ")", - ALT_NOT(X86_FEATURE_PVUNLOCK)); + PVOP_CALL_ARGS; + __STATIC_CALL_MOD_ADDRESSABLE(queued_spin_unlock); + asm volatile ("call " STATIC_CALL_TRAMP_STR(queued_spin_unlock) + : PVOP_VCALLEE_CLOBBERS, ASM_CALL_CONSTRAINT + : PVOP_CALL_ARG1(lock) + : "memory", "cc"); } static __always_inline bool pv_vcpu_is_preempted(long cpu) |