From d85f7dc8cf3c91a131946dae4c3aa8d9c97322d7 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 16:56:55 -0700 Subject: KVM: VMX: Formally define KVM's "0xdead" exit reason used for failed VM-Enter Add EXIT_REASON_UNDEFINED to track KVM's magic "0xdead" value that's used to stuff the exit reason on a failed VM-Enter so that the magic can be reused by TDX, and to make it more obvious that it's an arbitrary, KVM- defined value (though the whole "dead" thing makes it pretty darn obvious). Opportunistically add a compile-time assert to ensure the KVM-defined exit reason never collides with a "real" exit reason. No functional change intended. Cc: Xiaoyao Li Cc: Yan Zhao Cc: Binbin Wu Cc: Rick Edgecombe Cc: stable@vger.kernel.org Reviewed-by: Binbin Wu Reviewed-by: Xiaoyao Li Link: https://patch.msgid.link/20260921235656.1107929-1-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/vmx.h | 8 ++++++++ arch/x86/kvm/vmx/vmx.c | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index 3f1b3096ff04..e49b836c7313 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -20,6 +20,14 @@ #include #include +/* + * UNDEFINED is a KVM-defined value used to indicate that the basic exit reason + * is undefined/invalid, e.g. on VM-Fail or if TDX never attempted VM-Enter. + * The value itself is arbitrary, it just needs to not collide with a real exit + * reason. + */ +#define EXIT_REASON_UNDEFINED 0xdead + struct vmcs_hdr { u32 revision_id:31; u32 shadow_vmcs:1; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 612ab07d4100..c897dbf0e954 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6388,6 +6388,8 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { static const int kvm_vmx_max_exit_handlers = ARRAY_SIZE(kvm_vmx_exit_handlers); +static_assert(EXIT_REASON_UNDEFINED >= ARRAY_SIZE(kvm_vmx_exit_handlers)); + void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code) { @@ -7474,7 +7476,7 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, vmx_enable_fb_clear(vmx); if (unlikely(vmx->fail)) { - vmx->vt.exit_reason.full = 0xdead; + vmx->vt.exit_reason.full = EXIT_REASON_UNDEFINED; goto out; } -- cgit v1.2.3 From f305fd31f52a8d3ce4d2bf0c88c45bb7a373809c Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 18:03:46 -0700 Subject: KVM: VMX: Disallow using to_vmx() in common VT code Poison to_vmx() in main.c and posted_intr.c so that attempting to interpret the vCPU as a VMX vCPU will fail at compile time, as opposed to failing at runtime, or worse corrupting state without outright failing. Note, to_tdx() is buried in tdx.c, i.e. isn't broadly reachable, and so doesn't need the same treatment as to_vmx(). Reviewed-by: Xiaoyao Li Reviewed-by: Kai Huang Tested-by: Kai Huang Reviewed-by: Binbin Wu Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260904010353.3175819-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/main.c | 2 ++ arch/x86/kvm/vmx/posted_intr.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 4c52ab8d0786..ea0d825e556d 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -10,6 +10,8 @@ #include "tdx.h" #include "tdx_arch.h" +#pragma GCC poison to_vmx + #ifdef CONFIG_KVM_INTEL_TDX static_assert(offsetof(struct vcpu_vmx, vt) == offsetof(struct vcpu_tdx, vt)); diff --git a/arch/x86/kvm/vmx/posted_intr.c b/arch/x86/kvm/vmx/posted_intr.c index 4a6d9a17da23..24221ba553be 100644 --- a/arch/x86/kvm/vmx/posted_intr.c +++ b/arch/x86/kvm/vmx/posted_intr.c @@ -14,6 +14,8 @@ #include "vmx.h" #include "tdx.h" +#pragma GCC poison to_vmx + /* * Maintain a per-CPU list of vCPUs that need to be awakened by wakeup_handler() * when a WAKEUP_VECTOR interrupted is posted. vCPUs are added to the list when -- cgit v1.2.3 From d93395058c1951bb657c8895dd1fc0f22263f1cc Mon Sep 17 00:00:00 2001 From: Xiaoyao Li Date: Wed, 19 Aug 2026 17:48:55 +0800 Subject: KVM: TDX: Enable Notify VM exit Enable Notify VM exit functionality for TDX guests. Notify VM exit is an existing feature supported by KVM. Userspace can enable Notify VM exit through KVM_CAP_X86_NOTIFY_VMEXIT when it's reported as supported. However, KVM reports the support of this CAP just based on the hardware capability but doesn't differentiate between VMX and TDX. This leads to the issue that userspace can enable this cap for TDX guests without getting an error, but the feature is not actually enabled because KVM doesn't call the TDX module API to program the relevant TD VMCS fields. Enable Notify VM exit for TDX guests by: - Invoking TDX module API calls to set NOTIFY_VM_EXITING and Notify Window in TD VMCS. It's done in tdx_vcpu_init() where other TD VMCS bits are set. Since TDX vCPU cannot be reset, it only needs to be configured once when initializing the TDX vCPU. - Adding corresponding exit handler for TDX Notify VM Exit. Notify VM exit can happen when executing the IRET instruction. If the IRET unblocks the NMI blocking state, bit 12 of the exit qualification is set. In this case, the VMM needs to restore the "blocked by NMI" state when it decides to re-enter the guest. For TDX, KVM cannot manage the GUEST_INTERRUPTIBILITY_INFO and it's TDX module's responsibility to handle it. Extract the common part without NMI blocking handling into a helper in common.h so that it can be shared between VMX and TDX. Note, KVM uses "pre-production" terminology for the feature formally called Notify VM-Exit. All public versions of the SDM refer to the feature as Instruction Timeout. This will be remedied in the near future, for now, use KVM's terminology for consistency. Note, #2, there is no enumeration bit for Notify VM exit by TDX module because all TDX modules support it, and allow to set the corresponding TD VMCS fields as long as the hardware supports the feature. Fixes: 161d34609f9b ("KVM: TDX: Make TDX VM type supported") Cc: stable@vger.kernel.org Signed-off-by: Xiaoyao Li Reviewed-by: Rick Edgecombe Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260819094903.3060020-2-xiaoyao.li@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/common.h | 19 +++++++++++++++++++ arch/x86/kvm/vmx/tdx.c | 10 ++++++++++ arch/x86/kvm/vmx/vmx.c | 13 +------------ 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index 08005676702c..7fce1bbabc78 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h @@ -4,6 +4,7 @@ #include #include +#include #include "mmu.h" @@ -183,6 +184,24 @@ static inline void __vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_VECTOR); } +static inline int __vt_handle_notify(struct kvm_vcpu *vcpu, + unsigned long exit_qual) +{ + bool context_invalid = exit_qual & NOTIFY_VM_CONTEXT_INVALID; + + ++vcpu->stat.notify_window_exits; + + if (vcpu->kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_USER || + context_invalid) { + vcpu->run->exit_reason = KVM_EXIT_NOTIFY; + vcpu->run->notify.flags = context_invalid ? + KVM_NOTIFY_CONTEXT_INVALID : 0; + return 0; + } + + return 1; +} + noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu); #endif /* __KVM_X86_VMX_COMMON_H */ diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index b272c20586a7..d557840687d2 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -2126,6 +2126,9 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) * - If it's not an MSMI, no need to do anything here. */ return 1; + case EXIT_REASON_NOTIFY: + /* NMI blocking state is handled by TDX module */ + return __vt_handle_notify(vcpu, vmx_get_exit_qual(vcpu)); default: break; } @@ -3154,6 +3157,13 @@ static int tdx_vcpu_init(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *cmd) td_vmcs_write64(tdx, POSTED_INTR_DESC_ADDR, __pa(&tdx->vt.pi_desc)); td_vmcs_setbit32(tdx, PIN_BASED_VM_EXEC_CONTROL, PIN_BASED_POSTED_INTR); + if (kvm_notify_vmexit_enabled(vcpu->kvm)) { + td_vmcs_setbit32(tdx, SECONDARY_VM_EXEC_CONTROL, + SECONDARY_EXEC_NOTIFY_VM_EXITING); + td_vmcs_write32(tdx, NOTIFY_WINDOW, + vcpu->kvm->arch.notify_window); + } + tdx->state = VCPU_TD_STATE_INITIALIZED; return 0; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index c897dbf0e954..965141e0d4c8 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6281,9 +6281,6 @@ static int handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) static int handle_notify(struct kvm_vcpu *vcpu) { unsigned long exit_qual = vmx_get_exit_qual(vcpu); - bool context_invalid = exit_qual & NOTIFY_VM_CONTEXT_INVALID; - - ++vcpu->stat.notify_window_exits; /* * Notify VM exit happened while executing iret from NMI, @@ -6293,15 +6290,7 @@ static int handle_notify(struct kvm_vcpu *vcpu) vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI); - if (vcpu->kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_USER || - context_invalid) { - vcpu->run->exit_reason = KVM_EXIT_NOTIFY; - vcpu->run->notify.flags = context_invalid ? - KVM_NOTIFY_CONTEXT_INVALID : 0; - return 0; - } - - return 1; + return __vt_handle_notify(vcpu, exit_qual); } static int vmx_get_msr_imm_reg(struct kvm_vcpu *vcpu) -- cgit v1.2.3 From 0966c881ea13be5ec89820c185de4d55568e80ed Mon Sep 17 00:00:00 2001 From: Xiaoyao Li Date: Wed, 19 Aug 2026 17:48:56 +0800 Subject: KVM: TDX: Check if there is valid exit infos based on vp_enter_ret Check if there is valid exit info based on vp_enter_ret instead of relying on the clobbered Exit Reason, in tdx_get_exit_info(). Current KVM uses "Exit Reason is not equal to the synthesized invalid Exit Reason, -1u," as the condition to identify there is a real TD Exit and valid exit infos. However, there is one issue with this approach: KVM updates the Exit Reason to the synthesized invalid Exit Reason for real EPT MISCONFIG as well. This is a false positive for real EPT MISCONFIG, which has valid exit infos. Though the issue can be addressed by changing the handling for real EPT MISCONFIG to not update the Exit Reason to the synthesized one, relying on the clobbered Exit Reason itself is brittle. Instead, check vp_enter_ret directly to identify if it is a valid Exit Reason. Fixes: da407fe45908 ("KVM: TDX: Handle EPT violation/misconfig exit") Cc: stable@vger.kernel.org Suggested-by: Sean Christopherson Signed-off-by: Xiaoyao Li Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260819094903.3060020-3-xiaoyao.li@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index d557840687d2..1dead84e6077 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -921,21 +921,27 @@ static __always_inline u32 tdcall_to_vmx_exit_reason(struct kvm_vcpu *vcpu) return EXIT_REASON_TDCALL; } -static __always_inline u32 tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu) +static __always_inline bool tdx_is_exit_reason_valid(u64 vp_enter_ret) { - struct vcpu_tdx *tdx = to_tdx(vcpu); - u32 exit_reason; - - switch (tdx->vp_enter_ret & TDX_SEAMCALL_STATUS_MASK) { + switch (vp_enter_ret & TDX_SEAMCALL_STATUS_MASK) { case TDX_SUCCESS: case TDX_NON_RECOVERABLE_VCPU: case TDX_NON_RECOVERABLE_TD: case TDX_NON_RECOVERABLE_TD_NON_ACCESSIBLE: case TDX_NON_RECOVERABLE_TD_WRONG_APIC_MODE: - break; + return true; default: - return -1u; + return false; } +} + +static __always_inline u32 tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu) +{ + struct vcpu_tdx *tdx = to_tdx(vcpu); + u32 exit_reason; + + if (!tdx_is_exit_reason_valid(tdx->vp_enter_ret)) + return -1u; exit_reason = tdx->vp_enter_ret; @@ -2144,7 +2150,7 @@ void tdx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, struct vcpu_tdx *tdx = to_tdx(vcpu); *reason = tdx->vt.exit_reason.full; - if (*reason != -1u) { + if (tdx_is_exit_reason_valid(tdx->vp_enter_ret)) { *info1 = vmx_get_exit_qual(vcpu); *info2 = tdx->ext_exit_qualification; *intr_info = vmx_get_intr_info(vcpu); -- cgit v1.2.3 From b43c520b198ab70868307a489ef2117ed6d6c4cc Mon Sep 17 00:00:00 2001 From: Xiaoyao Li Date: Wed, 19 Aug 2026 17:48:57 +0800 Subject: KVM: TDX: Set bits 31:16 to 0 and use UNDEFINED when synthesizing exit reason Set bits 31:16 to 0 instead of all-1s when synthesizing an exit reason, e.g. if the TDX Module never attempted VM-Enter, or when stuffing the exit reason due to an unexpected "real" EPT Misconfig. Opportunistically use VMX's KVM-defined EXIT_REASON_UNDEFINED (0xdead) instead of -1u. KVM is going to support Bus Lock VM exit for TDX, after which bit 26 of the Exit Reason becomes meaningful and indicates that a bus lock happened. The existing synthesized Exit Reason, -1u, will cause a false positive in that case. Change the synthesized Exit Reason from -1u to U16_MAX, so that bits 31:16 are set to 0. This also avoids the potential issues when other bits in 31:16 become valid in the future. As a bonus, the check for synthesized Exit Reason in tdx_failed_vmentry() becomes unnecessary. Just drop it. Cc: stable@vger.kernel.org Signed-off-by: Xiaoyao Li Reviewed-by: Rick Edgecombe Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260819094903.3060020-4-xiaoyao.li@intel.com [sean: use EXIT_REASON_UNDEFINED instead of another magic value] Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 1dead84e6077..2664123caf26 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -940,8 +940,13 @@ static __always_inline u32 tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu) struct vcpu_tdx *tdx = to_tdx(vcpu); u32 exit_reason; + /* + * Return the synthesized invalid Exit Reason, as the TDX module + * never attempted to run the vCPU, i.e. the Exit Reason is undefined, + * but this is NOT a failed VM-Enter. + */ if (!tdx_is_exit_reason_valid(tdx->vp_enter_ret)) - return -1u; + return EXIT_REASON_UNDEFINED; exit_reason = tdx->vp_enter_ret; @@ -956,7 +961,7 @@ static __always_inline u32 tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu) * Defer KVM_BUG_ON() until tdx_handle_exit() because this is in * non-instrumentable code with interrupts disabled. */ - return -1u; + return EXIT_REASON_UNDEFINED; default: break; } @@ -987,8 +992,7 @@ static noinstr void tdx_vcpu_enter_exit(struct kvm_vcpu *vcpu) static bool tdx_failed_vmentry(struct kvm_vcpu *vcpu) { - return vmx_get_exit_reason(vcpu).failed_vmentry && - vmx_get_exit_reason(vcpu).full != -1u; + return vmx_get_exit_reason(vcpu).failed_vmentry; } static fastpath_t tdx_exit_handlers_fastpath(struct kvm_vcpu *vcpu) -- cgit v1.2.3 From bc335ce04408750585b4b32f248c543c9493d545 Mon Sep 17 00:00:00 2001 From: Xiaoyao Li Date: Wed, 19 Aug 2026 17:48:58 +0800 Subject: KVM: TDX: Don't assume exit_reason[31:16] is all-0 in tdx_to_vmx_exit_reason() When handling the real Exit Reason, don't assume the upper 16 bits as all-0 in tdx_to_vmx_exit_reason(), in preparation for enabling Bus Lock VM exit. When Bus Lock VM exit is enabled, the bit 26 of Exit Reason becomes valid and it can be 1 with various exit reasons. Change the logic in tdx_to_vmx_exit_reason() to check the 'basic' Exit Reason for correctness. Also preserve bit[31:16] when changing the (basic) Exit Reason, to not lose the information in bit[31:16]. Change the return type of tdx_to_vmx_exit_reason() to "union vmx_exit_reason" for the convenience of manipulating the basic field. Fixes: c42856af8f70 ("KVM: TDX: Add a place holder for handler of TDX hypercalls (TDG.VP.VMCALL)") Cc: stable@vger.kernel.org Signed-off-by: Xiaoyao Li Link: https://patch.msgid.link/20260819094903.3060020-5-xiaoyao.li@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 2664123caf26..f4d74c02fa1e 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -935,10 +935,10 @@ static __always_inline bool tdx_is_exit_reason_valid(u64 vp_enter_ret) } } -static __always_inline u32 tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu) +static __always_inline union vmx_exit_reason tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu) { struct vcpu_tdx *tdx = to_tdx(vcpu); - u32 exit_reason; + union vmx_exit_reason exit_reason; /* * Return the synthesized invalid Exit Reason, as the TDX module @@ -946,22 +946,26 @@ static __always_inline u32 tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu) * but this is NOT a failed VM-Enter. */ if (!tdx_is_exit_reason_valid(tdx->vp_enter_ret)) - return EXIT_REASON_UNDEFINED; + return (union vmx_exit_reason) { + .basic = EXIT_REASON_UNDEFINED, + }; - exit_reason = tdx->vp_enter_ret; + exit_reason.full = (u32)tdx->vp_enter_ret; - switch (exit_reason) { + switch (exit_reason.basic) { case EXIT_REASON_TDCALL: if (tdvmcall_exit_type(vcpu)) - return EXIT_REASON_VMCALL; - - return tdcall_to_vmx_exit_reason(vcpu); + exit_reason.basic = EXIT_REASON_VMCALL; + else + exit_reason.basic = tdcall_to_vmx_exit_reason(vcpu); + break; case EXIT_REASON_EPT_MISCONFIG: /* * Defer KVM_BUG_ON() until tdx_handle_exit() because this is in * non-instrumentable code with interrupts disabled. */ - return EXIT_REASON_UNDEFINED; + exit_reason.basic = EXIT_REASON_UNDEFINED; + break; default: break; } @@ -978,7 +982,7 @@ static noinstr void tdx_vcpu_enter_exit(struct kvm_vcpu *vcpu) tdx->vp_enter_ret = tdh_vp_enter(&tdx->vp, &tdx->vp_enter_args); - vt->exit_reason.full = tdx_to_vmx_exit_reason(vcpu); + vt->exit_reason = tdx_to_vmx_exit_reason(vcpu); vt->exit_qualification = tdx->vp_enter_args.rcx; tdx->ext_exit_qualification = tdx->vp_enter_args.rdx; -- cgit v1.2.3 From e5f1d62c59aea4f270773de1bf6e8867f5b1124c Mon Sep 17 00:00:00 2001 From: Xiaoyao Li Date: Wed, 19 Aug 2026 17:48:59 +0800 Subject: KVM: TDX: Stuff exit_reason with UNDEFINED on wait_for_sept_zap return Forcefully update the exit_reason to KVM's "standard" EXIT_REASON_UNDEFINED if tdx_vcpu_run() bails without trying to run the vCPU because KVM needs to wait for an in-progress S-EPT zap to complete. This avoids handling the previous exit multiple times due to a stale exit_reason. Fixes: 4b2abc49712b ("KVM: TDX: Kick off vCPUs when SEAMCALL is busy during TD page removal") Cc: stable@vger.kernel.org Signed-off-by: Xiaoyao Li Link: https://patch.msgid.link/20260819094903.3060020-6-xiaoyao.li@intel.com [sean: use EXIT_REASON_UNDEFINED, drop comment about bits 31:16] Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index f4d74c02fa1e..6762724a80f5 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -1085,8 +1085,16 @@ fastpath_t tdx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) * allowing vCPU entry to avoid contention with tdh_vp_enter() and * TDCALLs. */ - if (unlikely(READ_ONCE(to_kvm_tdx(vcpu->kvm)->wait_for_sept_zap))) + if (unlikely(READ_ONCE(to_kvm_tdx(vcpu->kvm)->wait_for_sept_zap))) { + /* + * The vCPU never entered the guest, but this looks like a + * handled exit to the caller. Synthesize an invalid exit + * reason so the previous exit's stale value isn't consumed + * a second time. + */ + vt->exit_reason.full = EXIT_REASON_UNDEFINED; return EXIT_FASTPATH_EXIT_HANDLED; + } trace_kvm_entry(vcpu, run_flags & KVM_RUN_FORCE_IMMEDIATE_EXIT); -- cgit v1.2.3 From cef48693da1e964e9f192bfc16cd0ed287c48337 Mon Sep 17 00:00:00 2001 From: Xiaoyao Li Date: Wed, 19 Aug 2026 17:49:00 +0800 Subject: KVM: VMX: Preserve negative return value in vmx_handle_exit() with bus lock detected Preserve the negative return value from __vmx_handle_exit() when a bus lock is detected, instead of always overwriting it with 0. The purpose of bus_lock_detected handling is to force a userspace exit to inform userspace that a bus lock happened. The negative return value can achieve this purpose, and changing the negative value to 0 fails to return an error to userspace. So, preserve the negative return value. Fixes: fe6b6bc802b4 ("KVM: VMX: Enable bus lock VM exit") Cc: stable@vger.kernel.org Reported-by: sashiko-bot@kernel.org Closes: https://lore.kernel.org/all/20260805034602.5B2BB1F000E9@smtp.kernel.org/ Signed-off-by: Xiaoyao Li Link: https://patch.msgid.link/20260819094903.3060020-7-xiaoyao.li@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/vmx.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 965141e0d4c8..6df390ae8708 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6864,11 +6864,12 @@ int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) * a bus lock in guest. */ if (vmx_get_exit_reason(vcpu).bus_lock_detected) { - if (ret > 0) + if (ret > 0) { vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; + ret = 0; + } vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; - return 0; } return ret; } -- cgit v1.2.3 From 326fa2c9b4d3e6cab58c139eb55aaabd7179192e Mon Sep 17 00:00:00 2001 From: Xiaoyao Li Date: Wed, 19 Aug 2026 17:49:01 +0800 Subject: KVM: VMX: Make handle_bus_lock_vmexit() a shared helper Move handle_bus_lock_vmexit() from vmx.c to main.c to make it a shared helper so that TDX can use it for Bus Lock VM exit handling as well. Rename it to add vt_ prefix to reflect that it is a helper for both VMX and TDX. Cc: stable@vger.kernel.org Signed-off-by: Xiaoyao Li Reviewed-by: Nikolay Borisov Link: https://patch.msgid.link/20260819094903.3060020-8-xiaoyao.li@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/common.h | 1 + arch/x86/kvm/vmx/main.c | 11 +++++++++++ arch/x86/kvm/vmx/vmx.c | 13 +------------ 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index 7fce1bbabc78..bbeec197a507 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h @@ -203,5 +203,6 @@ static inline int __vt_handle_notify(struct kvm_vcpu *vcpu, } noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu); +int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu); #endif /* __KVM_X86_VMX_COMMON_H */ diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index ea0d825e556d..835de375e8c8 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -878,6 +878,17 @@ static int vt_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, #define vt_op_tdx_only(name) NULL #endif /* CONFIG_KVM_INTEL_TDX */ +int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) +{ + /* + * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK + * VM-Exits. Unconditionally set the flag here and leave the handling + * to .handle_exit() callback. + */ + to_vt(vcpu)->exit_reason.bus_lock_detected = true; + return 1; +} + #define VMX_REQUIRED_APICV_INHIBITS \ (BIT(APICV_INHIBIT_REASON_DISABLED) | \ BIT(APICV_INHIBIT_REASON_ABSENT) | \ diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 6df390ae8708..3ba1e762e7e7 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6267,17 +6267,6 @@ static int handle_encls(struct kvm_vcpu *vcpu) } #endif /* CONFIG_X86_SGX_KVM */ -static int handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) -{ - /* - * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK - * VM-Exits. Unconditionally set the flag here and leave the handling to - * vmx_handle_exit(). - */ - to_vt(vcpu)->exit_reason.bus_lock_detected = true; - return 1; -} - static int handle_notify(struct kvm_vcpu *vcpu) { unsigned long exit_qual = vmx_get_exit_qual(vcpu); @@ -6366,7 +6355,7 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { [EXIT_REASON_VMFUNC] = handle_vmx_instruction, [EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer, [EXIT_REASON_ENCLS] = handle_encls, - [EXIT_REASON_BUS_LOCK] = handle_bus_lock_vmexit, + [EXIT_REASON_BUS_LOCK] = vt_handle_bus_lock_vmexit, [EXIT_REASON_NOTIFY] = handle_notify, [EXIT_REASON_SEAMCALL] = handle_tdx_instruction, [EXIT_REASON_TDCALL] = handle_tdx_instruction, -- cgit v1.2.3 From 63d196e2e20429de5ba501f1b6059d9b93140007 Mon Sep 17 00:00:00 2001 From: Xiaoyao Li Date: Wed, 19 Aug 2026 17:49:02 +0800 Subject: KVM: TDX: Enable Bus Lock VM exit Enable Bus Lock VM exit functionality for TDX guests. Bus Lock VM exit is an existing feature supported by KVM. Userspace can enable Bus Lock VM exit through KVM_BUS_LOCK_DETECTION_EXIT when it's reported as supported. However, KVM reports the support of this CAP just based on the hardware capability but doesn't differentiate between VMX and TDX. This leads to the issue that userspace can enable this cap for TDX guests without getting an error, but the feature is not actually enabled because KVM doesn't call the TDX module API to program the relevant TD VMCS fields. Enable Bus Lock VM exit for TDX guests by programming the BUS_LOCK_DETECTION control in the TD VMCS and by adding the exit handler. Note, there is no enumeration bit for this feature by TDX module because all TDX modules support it and allow to set the TD VMCS as long as the hardware supports the feature. Fixes: 161d34609f9b ("KVM: TDX: Make TDX VM type supported") Cc: stable@vger.kernel.org Originally-by: Chenyi Qiang Signed-off-by: Xiaoyao Li Reviewed-by: Nikolay Borisov Link: https://patch.msgid.link/20260819094903.3060020-9-xiaoyao.li@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 6762724a80f5..3d093efe8820 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -2050,7 +2050,7 @@ int tdx_complete_emulated_msr(struct kvm_vcpu *vcpu, int err) } -int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) +static int __tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) { struct vcpu_tdx *tdx = to_tdx(vcpu); u64 vp_enter_ret = tdx->vp_enter_ret; @@ -2151,6 +2151,8 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) case EXIT_REASON_NOTIFY: /* NMI blocking state is handled by TDX module */ return __vt_handle_notify(vcpu, vmx_get_exit_qual(vcpu)); + case EXIT_REASON_BUS_LOCK: + return vt_handle_bus_lock_vmexit(vcpu); default: break; } @@ -2160,6 +2162,22 @@ unhandled_exit: return 0; } +int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) +{ + int ret = __tdx_handle_exit(vcpu, fastpath); + + /* Exit to user space when bus lock was detected */ + if (vmx_get_exit_reason(vcpu).bus_lock_detected) { + if (ret > 0) { + vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; + ret = 0; + } + + vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; + } + return ret; +} + void tdx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code) { @@ -3186,6 +3204,10 @@ static int tdx_vcpu_init(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *cmd) vcpu->kvm->arch.notify_window); } + if (vcpu->kvm->arch.bus_lock_detection_enabled) + td_vmcs_setbit32(tdx, SECONDARY_VM_EXEC_CONTROL, + SECONDARY_EXEC_BUS_LOCK_DETECTION); + tdx->state = VCPU_TD_STATE_INITIALIZED; return 0; -- cgit v1.2.3 From 213bbf18f07aed935e21df322fa61a84a19f9169 Mon Sep 17 00:00:00 2001 From: Xiaoyao Li Date: Wed, 19 Aug 2026 17:49:03 +0800 Subject: KVM: VMX: Consolidate the exit handler for VMX and TDX The exit handlers for VMX and TDX have the similar pattern. Consolidate them into a single vt_handle_exit() helper. Changes in v4: - new patch. Signed-off-by: Xiaoyao Li Reviewed-by: Nikolay Borisov Link: https://patch.msgid.link/20260819094903.3060020-10-xiaoyao.li@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/main.c | 38 ++++++++++++++++++++++++++++---------- arch/x86/kvm/vmx/tdx.c | 18 +----------------- arch/x86/kvm/vmx/vmx.c | 21 +-------------------- 3 files changed, 30 insertions(+), 47 deletions(-) diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 835de375e8c8..fc7787b2bbb5 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -156,15 +156,6 @@ static fastpath_t vt_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) return vmx_vcpu_run(vcpu, run_flags); } -static int vt_handle_exit(struct kvm_vcpu *vcpu, - enum exit_fastpath_completion fastpath) -{ - if (is_td_vcpu(vcpu)) - return tdx_handle_exit(vcpu, fastpath); - - return vmx_handle_exit(vcpu, fastpath); -} - static bool vt_unhandleable_emulation_required(struct kvm_vcpu *vcpu) { if (is_td_vcpu(vcpu)) { @@ -889,6 +880,33 @@ int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) return 1; } +static int vt_handle_exit(struct kvm_vcpu *vcpu, + enum exit_fastpath_completion fastpath) +{ + int ret; + +#ifdef CONFIG_KVM_INTEL_TDX + if (is_td_vcpu(vcpu)) + ret = tdx_handle_exit(vcpu, fastpath); + else +#endif + ret = vmx_handle_exit(vcpu, fastpath); + + /* + * Exit to user space when bus lock detected to inform that there is + * a bus lock in guest. + */ + if (vmx_get_exit_reason(vcpu).bus_lock_detected) { + if (ret > 0) { + vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; + ret = 0; + } + + vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; + } + return ret; +} + #define VMX_REQUIRED_APICV_INHIBITS \ (BIT(APICV_INHIBIT_REASON_DISABLED) | \ BIT(APICV_INHIBIT_REASON_ABSENT) | \ @@ -962,7 +980,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = { .vcpu_needs_initialization = vt_op_tdx_only(vcpu_needs_initialization), .vcpu_run = vt_op(vcpu_run), - .handle_exit = vt_op(handle_exit), + .handle_exit = vt_handle_exit, .skip_emulated_instruction = vmx_skip_emulated_instruction, .update_emulated_instruction = vmx_update_emulated_instruction, .unhandleable_emulation_required = vt_op(unhandleable_emulation_required), diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 3d093efe8820..8c0fdb032bb4 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -2050,7 +2050,7 @@ int tdx_complete_emulated_msr(struct kvm_vcpu *vcpu, int err) } -static int __tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) +int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) { struct vcpu_tdx *tdx = to_tdx(vcpu); u64 vp_enter_ret = tdx->vp_enter_ret; @@ -2162,22 +2162,6 @@ unhandled_exit: return 0; } -int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) -{ - int ret = __tdx_handle_exit(vcpu, fastpath); - - /* Exit to user space when bus lock was detected */ - if (vmx_get_exit_reason(vcpu).bus_lock_detected) { - if (ret > 0) { - vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; - ret = 0; - } - - vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; - } - return ret; -} - void tdx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code) { diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 3ba1e762e7e7..a9afffae951a 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6685,7 +6685,7 @@ void dump_vmcs(struct kvm_vcpu *vcpu) * The guest has exited. See if we can fix it or if we need userspace * assistance. */ -static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) +int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) { struct vcpu_vmx *vmx = to_vmx(vcpu); union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu); @@ -6844,25 +6844,6 @@ unexpected_vmexit: return 0; } -int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) -{ - int ret = __vmx_handle_exit(vcpu, exit_fastpath); - - /* - * Exit to user space when bus lock detected to inform that there is - * a bus lock in guest. - */ - if (vmx_get_exit_reason(vcpu).bus_lock_detected) { - if (ret > 0) { - vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; - ret = 0; - } - - vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; - } - return ret; -} - void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) { int tpr_threshold; -- cgit v1.2.3 From f502a81663dd8520f83af75849803b168479be05 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 18:03:47 -0700 Subject: KVM: VMX: Move the shared "IRQs off" exit handler(s) to common code Move vmx_handle_exit_irqoff() and its helpers to common.h / main.c to capture that it's a common handler and to allow guarding against incorrectly using to_vmx(). Opportunistically use a "vt" prefix instead of "vmx" to communicate that it's a shared handler. No functional change intended. Cc: Rick Edgecombe Cc: Xiaoyao Li Cc: Binbin Wu Cc: Kai Huang Cc: Yan Zhao Reviewed-by: Xiaoyao Li Reviewed-by: Kai Huang Tested-by: Kai Huang Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260904010353.3175819-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/common.h | 6 ++++ arch/x86/kvm/vmx/main.c | 71 +++++++++++++++++++++++++++++++++++++++++++- arch/x86/kvm/vmx/vmx.c | 74 ---------------------------------------------- arch/x86/kvm/vmx/x86_ops.h | 1 - 4 files changed, 76 insertions(+), 76 deletions(-) diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index bbeec197a507..88c525231e3a 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h @@ -75,6 +75,12 @@ static __always_inline bool is_td_vcpu(struct kvm_vcpu *vcpu) { return false; } #endif +static inline bool is_xfd_nm_fault(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.guest_fpu.fpstate->xfd && + !kvm_is_cr0_bit_set(vcpu, X86_CR0_TS); +} + static inline bool vt_is_tdx_private_gpa(struct kvm *kvm, gpa_t gpa) { /* For TDX the direct mask is the shared mask. */ diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index fc7787b2bbb5..87e9893d5ada 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include "x86_ops.h" @@ -880,6 +881,74 @@ int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) return 1; } +static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu) +{ + /* + * Save xfd_err to guest_fpu before interrupt is enabled, so the + * MSR value is not clobbered by the host activity before the guest + * has chance to consume it. + * + * Update the guest's XFD_ERR if and only if XFD is enabled, as the #NM + * interception may have been caused by L1 interception. Per the SDM, + * XFD_ERR is not modified for non-XFD #NM, i.e. if CR0.TS=1. + * + * Note, XFD_ERR is updated _before_ the #NM interception check, i.e. + * unlike CR2 and DR6, the value is not a payload that is attached to + * the #NM exception. + */ + if (is_xfd_nm_fault(vcpu)) + rdmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); +} + +static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info) +{ + /* if exit due to PF check for async PF */ + if (is_page_fault(intr_info)) + vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); + /* if exit due to NM, handle before interrupts are enabled */ + else if (is_nm_fault(intr_info)) + handle_nm_fault_irqoff(vcpu); + /* Handle machine checks before interrupts are enabled */ + else if (is_machine_check(intr_info)) + kvm_machine_check(); +} + +static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu, + u32 intr_info) +{ + unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK; + + if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm, + "unexpected VM-Exit interrupt info: 0x%x", intr_info)) + return; + + kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ); + x86_entry_from_kvm(EVENT_TYPE_EXTINT, vector); + kvm_after_interrupt(vcpu); + + vcpu->arch.at_instruction_boundary = true; +} + +static void vt_handle_exit_irqoff(struct kvm_vcpu *vcpu) +{ + if (to_vt(vcpu)->emulation_required) + return; + + switch (vmx_get_exit_reason(vcpu).basic) { + case EXIT_REASON_EXTERNAL_INTERRUPT: + handle_external_interrupt_irqoff(vcpu, vmx_get_intr_info(vcpu)); + break; + case EXIT_REASON_EXCEPTION_NMI: + handle_exception_irqoff(vcpu, vmx_get_intr_info(vcpu)); + break; + case EXIT_REASON_MCE_DURING_VMENTRY: + kvm_machine_check(); + break; + default: + break; + } +} + static int vt_handle_exit(struct kvm_vcpu *vcpu, enum exit_fastpath_completion fastpath) { @@ -1031,7 +1100,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = { .load_mmu_pgd = vt_op(load_mmu_pgd), .check_intercept = vmx_check_intercept, - .handle_exit_irqoff = vmx_handle_exit_irqoff, + .handle_exit_irqoff = vt_handle_exit_irqoff, .update_cpu_dirty_logging = vt_op(update_cpu_dirty_logging), diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index a9afffae951a..4c444a3cae33 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -5381,12 +5381,6 @@ bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu) (kvm_get_rflags(vcpu) & X86_EFLAGS_AC); } -static bool is_xfd_nm_fault(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.guest_fpu.fpstate->xfd && - !kvm_is_cr0_bit_set(vcpu, X86_CR0_TS); -} - static int vmx_handle_page_fault(struct kvm_vcpu *vcpu, u32 error_code) { unsigned long cr2 = vmx_get_exit_qual(vcpu); @@ -7107,74 +7101,6 @@ void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]); } -static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu) -{ - /* - * Save xfd_err to guest_fpu before interrupt is enabled, so the - * MSR value is not clobbered by the host activity before the guest - * has chance to consume it. - * - * Update the guest's XFD_ERR if and only if XFD is enabled, as the #NM - * interception may have been caused by L1 interception. Per the SDM, - * XFD_ERR is not modified for non-XFD #NM, i.e. if CR0.TS=1. - * - * Note, XFD_ERR is updated _before_ the #NM interception check, i.e. - * unlike CR2 and DR6, the value is not a payload that is attached to - * the #NM exception. - */ - if (is_xfd_nm_fault(vcpu)) - rdmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); -} - -static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info) -{ - /* if exit due to PF check for async PF */ - if (is_page_fault(intr_info)) - vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); - /* if exit due to NM, handle before interrupts are enabled */ - else if (is_nm_fault(intr_info)) - handle_nm_fault_irqoff(vcpu); - /* Handle machine checks before interrupts are enabled */ - else if (is_machine_check(intr_info)) - kvm_machine_check(); -} - -static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu, - u32 intr_info) -{ - unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK; - - if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm, - "unexpected VM-Exit interrupt info: 0x%x", intr_info)) - return; - - kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ); - x86_entry_from_kvm(EVENT_TYPE_EXTINT, vector); - kvm_after_interrupt(vcpu); - - vcpu->arch.at_instruction_boundary = true; -} - -void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu) -{ - if (to_vt(vcpu)->emulation_required) - return; - - switch (vmx_get_exit_reason(vcpu).basic) { - case EXIT_REASON_EXTERNAL_INTERRUPT: - handle_external_interrupt_irqoff(vcpu, vmx_get_intr_info(vcpu)); - break; - case EXIT_REASON_EXCEPTION_NMI: - handle_exception_irqoff(vcpu, vmx_get_intr_info(vcpu)); - break; - case EXIT_REASON_MCE_DURING_VMENTRY: - kvm_machine_check(); - break; - default: - break; - } -} - /* * The kvm parameter can be NULL (module initialization, or invocation before * VM creation). Be sure to check the kvm parameter before using it. diff --git a/arch/x86/kvm/vmx/x86_ops.h b/arch/x86/kvm/vmx/x86_ops.h index 054fd14bb2e1..7ac02c68457f 100644 --- a/arch/x86/kvm/vmx/x86_ops.h +++ b/arch/x86/kvm/vmx/x86_ops.h @@ -27,7 +27,6 @@ void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event); void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu); void vmx_vcpu_put(struct kvm_vcpu *vcpu); int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath); -void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu); int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu); void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu); bool vmx_unhandleable_emulation_required(struct kvm_vcpu *vcpu); -- cgit v1.2.3 From a06210034254b6a5800ff0a8c1b69739adf16014 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 09:59:15 -0700 Subject: KVM: VMX: Move the shared NMI handler/trampoline to common code Move vmx_handle_nmi() to main.c to capture that it's a common handler and to allow guarding against incorrectly using to_vmx(). Opportunistically use a "vt" prefix instead of "vmx" to communicate that it's a shared handler, and move the declaration higher up in common.h so that it doesn't get buried in a pile of inline helpers (more are on the way). No functional change intended. Reviewed-by: Xiaoyao Li Reviewed-by: Kai Huang Tested-by: Kai Huang Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260904010353.3175819-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/common.h | 3 ++- arch/x86/kvm/vmx/main.c | 11 +++++++++++ arch/x86/kvm/vmx/tdx.c | 2 +- arch/x86/kvm/vmx/vmx.c | 13 +------------ 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index 88c525231e3a..dbc1bca47ed6 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h @@ -56,6 +56,8 @@ struct vcpu_vt { #endif }; +noinstr void vt_handle_nmi(struct kvm_vcpu *vcpu); + #ifdef CONFIG_KVM_INTEL_TDX static __always_inline bool is_td(struct kvm *kvm) @@ -208,7 +210,6 @@ static inline int __vt_handle_notify(struct kvm_vcpu *vcpu, return 1; } -noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu); int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu); #endif /* __KVM_X86_VMX_COMMON_H */ diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 87e9893d5ada..0f2f9532e048 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -881,6 +881,17 @@ int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) return 1; } +noinstr void vt_handle_nmi(struct kvm_vcpu *vcpu) +{ + if ((u16)vmx_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI || + !is_nmi(vmx_get_intr_info(vcpu))) + return; + + kvm_before_interrupt(vcpu, KVM_HANDLING_NMI); + x86_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR); + kvm_after_interrupt(vcpu); +} + static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu) { /* diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 8c0fdb032bb4..ed1bd4e37026 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -989,7 +989,7 @@ static noinstr void tdx_vcpu_enter_exit(struct kvm_vcpu *vcpu) tdx->exit_gpa = tdx->vp_enter_args.r8; vt->exit_intr_info = tdx->vp_enter_args.r9; - vmx_handle_nmi(vcpu); + vt_handle_nmi(vcpu); guest_state_exit_irqoff(); } diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 4c444a3cae33..7b867e273e59 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -7327,17 +7327,6 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu, } } -noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu) -{ - if ((u16)vmx_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI || - !is_nmi(vmx_get_intr_info(vcpu))) - return; - - kvm_before_interrupt(vcpu, KVM_HANDLING_NMI); - x86_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR); - kvm_after_interrupt(vcpu); -} - static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, unsigned int flags) { @@ -7370,7 +7359,7 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, if (likely(!vmx_get_exit_reason(vcpu).failed_vmentry)) vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); - vmx_handle_nmi(vcpu); + vt_handle_nmi(vcpu); out: guest_state_exit_irqoff(); -- cgit v1.2.3 From 8f0140d8c777f0fbe89cc59ac7d3ca1f47da9f49 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 18:03:49 -0700 Subject: KVM: VMX: Rename posted interrupt prefixes from "vmx" to "vt" Rename all common posted interrupt APIs that use a "vmx" prefix to instead use a "vt" prefix to capture that they are used for both VMX and TDX vCPUs. No functional change intended. Reviewed-by: Xiaoyao Li Reviewed-by: Kai Huang Tested-by: Kai Huang Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260904010353.3175819-5-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/common.h | 4 ++-- arch/x86/kvm/vmx/main.c | 4 ++-- arch/x86/kvm/vmx/posted_intr.c | 22 +++++++++++----------- arch/x86/kvm/vmx/posted_intr.h | 8 ++++---- arch/x86/kvm/vmx/tdx.c | 6 +++--- arch/x86/kvm/vmx/vmx.c | 6 +++--- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index dbc1bca47ed6..f19fe019ee8a 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h @@ -173,8 +173,8 @@ static inline void kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu, * Post an interrupt to a vCPU's PIR and trigger the vCPU to process the * interrupt if necessary. */ -static inline void __vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, - struct pi_desc *pi_desc, int vector) +static inline void __vt_deliver_posted_interrupt(struct kvm_vcpu *vcpu, + struct pi_desc *pi_desc, int vector) { if (pi_test_and_set_pir(vector, pi_desc)) return; diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 0f2f9532e048..4e342fbd4d67 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -1115,8 +1115,8 @@ struct kvm_x86_ops vt_x86_ops __initdata = { .update_cpu_dirty_logging = vt_op(update_cpu_dirty_logging), - .pi_update_irte = vmx_pi_update_irte, - .pi_start_bypass = vmx_pi_start_bypass, + .pi_update_irte = vt_pi_update_irte, + .pi_start_bypass = vt_pi_start_bypass, #ifdef CONFIG_X86_64 .set_hv_timer = vt_op(set_hv_timer), diff --git a/arch/x86/kvm/vmx/posted_intr.c b/arch/x86/kvm/vmx/posted_intr.c index 24221ba553be..2644add54b76 100644 --- a/arch/x86/kvm/vmx/posted_intr.c +++ b/arch/x86/kvm/vmx/posted_intr.c @@ -56,7 +56,7 @@ static int pi_try_set_control(struct pi_desc *pi_desc, u64 *pold, u64 new) return 0; } -void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu) +void vt_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu) { struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); struct vcpu_vt *vt = to_vt(vcpu); @@ -146,11 +146,11 @@ after_clear_sn: pi_set_on(pi_desc); } -static bool vmx_can_use_vtd_pi(struct kvm *kvm) +static bool vt_can_use_vtd_pi(struct kvm *kvm) { /* * Note, reading the number of possible bypass IRQs can race with a - * bypass IRQ being attached to the VM. vmx_pi_start_bypass() ensures + * bypass IRQ being attached to the VM. vt_pi_start_bypass() ensures * blockng vCPUs will see an elevated count or get KVM_REQ_UNBLOCK. */ return irqchip_in_kernel(kvm) && kvm_arch_has_irq_bypass() && @@ -208,7 +208,7 @@ static void pi_enable_wakeup_handler(struct kvm_vcpu *vcpu) __apic_send_IPI_self(POSTED_INTR_WAKEUP_VECTOR); } -static bool vmx_needs_pi_wakeup(struct kvm_vcpu *vcpu) +static bool vt_needs_pi_wakeup(struct kvm_vcpu *vcpu) { /* * The default posted interrupt vector does nothing when @@ -219,14 +219,14 @@ static bool vmx_needs_pi_wakeup(struct kvm_vcpu *vcpu) * back to the pi_wakeup_handler() function. */ return (vmx_can_use_ipiv(vcpu) && !is_td_vcpu(vcpu)) || - vmx_can_use_vtd_pi(vcpu->kvm); + vt_can_use_vtd_pi(vcpu->kvm); } -void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu) +void vt_vcpu_pi_put(struct kvm_vcpu *vcpu) { struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); - if (!vmx_needs_pi_wakeup(vcpu)) + if (!vt_needs_pi_wakeup(vcpu)) return; /* @@ -294,17 +294,17 @@ bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu) /* * Kick all vCPUs when the first possible bypass IRQ is attached to a VM, as * blocking vCPUs may scheduled out without reconfiguring PID.NV to the wakeup - * vector, i.e. if the bypass IRQ came along after vmx_vcpu_pi_put(). + * vector, i.e. if the bypass IRQ came along after vt_vcpu_pi_put(). */ -void vmx_pi_start_bypass(struct kvm *kvm) +void vt_pi_start_bypass(struct kvm *kvm) { - if (WARN_ON_ONCE(!vmx_can_use_vtd_pi(kvm))) + if (WARN_ON_ONCE(!vt_can_use_vtd_pi(kvm))) return; kvm_make_all_cpus_request(kvm, KVM_REQ_UNBLOCK); } -int vmx_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, +int vt_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, unsigned int host_irq, uint32_t guest_irq, struct kvm_vcpu *vcpu, u32 vector) { diff --git a/arch/x86/kvm/vmx/posted_intr.h b/arch/x86/kvm/vmx/posted_intr.h index a4af39948cf0..c5860ea828ae 100644 --- a/arch/x86/kvm/vmx/posted_intr.h +++ b/arch/x86/kvm/vmx/posted_intr.h @@ -8,16 +8,16 @@ #include -void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu); -void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu); +void vt_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu); +void vt_vcpu_pi_put(struct kvm_vcpu *vcpu); void pi_wakeup_handler(void); void __init pi_init_cpu(int cpu); void pi_apicv_pre_state_restore(struct kvm_vcpu *vcpu); bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu); -int vmx_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, +int vt_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, unsigned int host_irq, uint32_t guest_irq, struct kvm_vcpu *vcpu, u32 vector); -void vmx_pi_start_bypass(struct kvm *kvm); +void vt_pi_start_bypass(struct kvm *kvm); static inline int pi_find_highest_vector(struct pi_desc *pi_desc) { diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index ed1bd4e37026..3a685c4ce332 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -716,7 +716,7 @@ void tdx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { struct vcpu_tdx *tdx = to_tdx(vcpu); - vmx_vcpu_pi_load(vcpu, cpu); + vt_vcpu_pi_load(vcpu, cpu); if (vcpu->cpu == cpu || !is_hkid_assigned(to_kvm_tdx(vcpu->kvm))) return; @@ -826,7 +826,7 @@ static void tdx_prepare_switch_to_host(struct kvm_vcpu *vcpu) void tdx_vcpu_put(struct kvm_vcpu *vcpu) { - vmx_vcpu_pi_put(vcpu); + vt_vcpu_pi_put(vcpu); tdx_prepare_switch_to_host(vcpu); } @@ -1917,7 +1917,7 @@ void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, struct vcpu_tdx *tdx = to_tdx(vcpu); /* TDX supports only posted interrupt. No lapic emulation. */ - __vmx_deliver_posted_interrupt(vcpu, &tdx->vt.pi_desc, vector); + __vt_deliver_posted_interrupt(vcpu, &tdx->vt.pi_desc, vector); trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector); } diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 7b867e273e59..e99b5184518a 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1577,12 +1577,12 @@ void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) vmx_vcpu_load_vmcs(vcpu, cpu); - vmx_vcpu_pi_load(vcpu, cpu); + vt_vcpu_pi_load(vcpu, cpu); } void vmx_vcpu_put(struct kvm_vcpu *vcpu) { - vmx_vcpu_pi_put(vcpu); + vt_vcpu_pi_put(vcpu); vmx_prepare_switch_to_host(to_vmx(vcpu)); } @@ -4426,7 +4426,7 @@ static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector) if (!vcpu->arch.apic->apicv_active) return -1; - __vmx_deliver_posted_interrupt(vcpu, &vt->pi_desc, vector); + __vt_deliver_posted_interrupt(vcpu, &vt->pi_desc, vector); return 0; } -- cgit v1.2.3 From 3d2ad6cfb6e1ef5fc6f3e86aa8b0f2e4bcab1b02 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 18:03:50 -0700 Subject: KVM: VMX: Rename EPT violation handler prefix from "vmx" to "vt" Rename __vmx_handle_ept_violation() to __vt_handle_ept_violation to capture that it is shared between VMX and TDX. No functional change intended. Reviewed-by: Xiaoyao Li Reviewed-by: Kai Huang Tested-by: Kai Huang Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260904010353.3175819-6-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/common.h | 4 ++-- arch/x86/kvm/vmx/tdx.c | 2 +- arch/x86/kvm/vmx/vmx.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index f19fe019ee8a..4af85ec70989 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h @@ -89,8 +89,8 @@ static inline bool vt_is_tdx_private_gpa(struct kvm *kvm, gpa_t gpa) return !kvm_is_addr_direct(kvm, gpa); } -static inline int __vmx_handle_ept_violation(struct kvm_vcpu *vcpu, gpa_t gpa, - unsigned long exit_qualification) +static inline int __vt_handle_ept_violation(struct kvm_vcpu *vcpu, gpa_t gpa, + unsigned long exit_qualification) { u64 error_code; diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 3a685c4ce332..76bcf2fabb83 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -2009,7 +2009,7 @@ static int tdx_handle_ept_violation(struct kvm_vcpu *vcpu) while (1) { struct kvm_memory_slot *slot; - ret = __vmx_handle_ept_violation(vcpu, gpa, exit_qual); + ret = __vt_handle_ept_violation(vcpu, gpa, exit_qual); if (ret != RET_PF_RETRY || !local_retry) break; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index e99b5184518a..d0a617797661 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -5993,7 +5993,7 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu) if (unlikely(allow_smaller_maxphyaddr && !kvm_vcpu_is_legal_gpa(vcpu, gpa))) return kvm_emulate_instruction(vcpu, 0); - return __vmx_handle_ept_violation(vcpu, gpa, exit_qualification); + return __vt_handle_ept_violation(vcpu, gpa, exit_qualification); } static int handle_ept_misconfig(struct kvm_vcpu *vcpu) -- cgit v1.2.3 From 978540f908addea83672e66b3af3461a8ddfc744 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 18:03:51 -0700 Subject: KVM: VMX: Use dummy pseudo-overlay struct for to_vt() and vt_to_vcpu() Define a combined vcpu_vmx_tdx structure to represent the common layout of vcpu_vmx and vcpu_tdx, instead of having vcpu_tdx piggyback vcpu_vmx. I.e. require VMX and TDX to match the common definition, not for TDX to match VMX. This will allow moving to_vt() and vt_to_vcpu() to common.h where they belong. No functional change intended. Reviewed-by: Xiaoyao Li Reviewed-by: Kai Huang Tested-by: Kai Huang Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260904010353.3175819-7-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/main.c | 4 +++- arch/x86/kvm/vmx/vmx.h | 9 +++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 4e342fbd4d67..9b304f420d4c 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -13,8 +13,10 @@ #pragma GCC poison to_vmx +static_assert(offsetof(struct vcpu_vmx_tdx, vt) == offsetof(struct vcpu_vmx, vt)); + #ifdef CONFIG_KVM_INTEL_TDX -static_assert(offsetof(struct vcpu_vmx, vt) == offsetof(struct vcpu_tdx, vt)); +static_assert(offsetof(struct vcpu_vmx_tdx, vt) == offsetof(struct vcpu_tdx, vt)); static void vt_disable_virtualization_cpu(void) { diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index dc8517f15bc4..0415d328e1a5 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -304,14 +304,19 @@ struct kvm_vmx { u64 *pid_table; }; +struct vcpu_vmx_tdx { + struct kvm_vcpu vcpu; + struct vcpu_vt vt; +}; + static __always_inline struct vcpu_vt *to_vt(struct kvm_vcpu *vcpu) { - return &(container_of(vcpu, struct vcpu_vmx, vcpu)->vt); + return &(container_of(vcpu, struct vcpu_vmx_tdx, vcpu)->vt); } static __always_inline struct kvm_vcpu *vt_to_vcpu(struct vcpu_vt *vt) { - return &(container_of(vt, struct vcpu_vmx, vt)->vcpu); + return &(container_of(vt, struct vcpu_vmx_tdx, vt)->vcpu); } static __always_inline union vmx_exit_reason vmx_get_exit_reason(struct kvm_vcpu *vcpu) -- cgit v1.2.3 From 82e11cefb7464d402ee3323c0dbff0b44abf7509 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 18:03:52 -0700 Subject: KVM: VMX: Move common VT getters/converters to common.h Move the "to" vCPU converters and common exit information getters from vmx.h to common.h to capture that they are indeed common VT code. Defer renaming their prefixes to isolate the code movement from the massive churn that will result from said renaming. No functional change intended. Reviewed-by: Xiaoyao Li Reviewed-by: Kai Huang Tested-by: Kai Huang Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260904010353.3175819-8-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/common.h | 44 ++++++++++++++++++++++++++++++++++++++++++++ arch/x86/kvm/vmx/vmx.h | 42 ------------------------------------------ 2 files changed, 44 insertions(+), 42 deletions(-) diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index 4af85ec70989..4e00832f132a 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h @@ -2,11 +2,13 @@ #ifndef __KVM_X86_VMX_COMMON_H #define __KVM_X86_VMX_COMMON_H +#include #include #include #include #include "mmu.h" +#include "vmx_ops.h" union vmx_exit_reason { struct { @@ -77,6 +79,48 @@ static __always_inline bool is_td_vcpu(struct kvm_vcpu *vcpu) { return false; } #endif +struct vcpu_vmx_tdx { + struct kvm_vcpu vcpu; + struct vcpu_vt vt; +}; + +static __always_inline struct vcpu_vt *to_vt(struct kvm_vcpu *vcpu) +{ + return &(container_of(vcpu, struct vcpu_vmx_tdx, vcpu)->vt); +} + +static __always_inline struct kvm_vcpu *vt_to_vcpu(struct vcpu_vt *vt) +{ + return &(container_of(vt, struct vcpu_vmx_tdx, vt)->vcpu); +} + +static __always_inline union vmx_exit_reason vmx_get_exit_reason(struct kvm_vcpu *vcpu) +{ + return to_vt(vcpu)->exit_reason; +} + +static __always_inline unsigned long vmx_get_exit_qual(struct kvm_vcpu *vcpu) +{ + struct vcpu_vt *vt = to_vt(vcpu); + + if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_1) && + !WARN_ON_ONCE(is_td_vcpu(vcpu))) + vt->exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + + return vt->exit_qualification; +} + +static __always_inline u32 vmx_get_intr_info(struct kvm_vcpu *vcpu) +{ + struct vcpu_vt *vt = to_vt(vcpu); + + if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_2) && + !WARN_ON_ONCE(is_td_vcpu(vcpu))) + vt->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); + + return vt->exit_intr_info; +} + static inline bool is_xfd_nm_fault(struct kvm_vcpu *vcpu) { return vcpu->arch.guest_fpu.fpstate->xfd && diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 0415d328e1a5..3247b55b3484 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -304,48 +304,6 @@ struct kvm_vmx { u64 *pid_table; }; -struct vcpu_vmx_tdx { - struct kvm_vcpu vcpu; - struct vcpu_vt vt; -}; - -static __always_inline struct vcpu_vt *to_vt(struct kvm_vcpu *vcpu) -{ - return &(container_of(vcpu, struct vcpu_vmx_tdx, vcpu)->vt); -} - -static __always_inline struct kvm_vcpu *vt_to_vcpu(struct vcpu_vt *vt) -{ - return &(container_of(vt, struct vcpu_vmx_tdx, vt)->vcpu); -} - -static __always_inline union vmx_exit_reason vmx_get_exit_reason(struct kvm_vcpu *vcpu) -{ - return to_vt(vcpu)->exit_reason; -} - -static __always_inline unsigned long vmx_get_exit_qual(struct kvm_vcpu *vcpu) -{ - struct vcpu_vt *vt = to_vt(vcpu); - - if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_1) && - !WARN_ON_ONCE(is_td_vcpu(vcpu))) - vt->exit_qualification = vmcs_readl(EXIT_QUALIFICATION); - - return vt->exit_qualification; -} - -static __always_inline u32 vmx_get_intr_info(struct kvm_vcpu *vcpu) -{ - struct vcpu_vt *vt = to_vt(vcpu); - - if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_2) && - !WARN_ON_ONCE(is_td_vcpu(vcpu))) - vt->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); - - return vt->exit_intr_info; -} - void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu); int allocate_vpid(void); void free_vpid(int vpid); -- cgit v1.2.3 From 72dea4f43c78207a6828fc1b2deb7e3727342807 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 18:03:53 -0700 Subject: KVM: VMX: Rename common exit info getters prefixes from "vmx" to "vt" Rename all common prefixes for the APIs that get exit information common to VMX and TDX from "vmx" to "vt" to capture that they are used for both VMX and TDX vCPUs. No functional change intended. Reviewed-by: Xiaoyao Li Reviewed-by: Kai Huang Tested-by: Kai Huang Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260904010353.3175819-9-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/common.h | 6 ++--- arch/x86/kvm/vmx/main.c | 12 +++++----- arch/x86/kvm/vmx/nested.c | 36 ++++++++++++++--------------- arch/x86/kvm/vmx/tdx.c | 22 +++++++++--------- arch/x86/kvm/vmx/vmx.c | 58 +++++++++++++++++++++++------------------------ 5 files changed, 67 insertions(+), 67 deletions(-) diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index 4e00832f132a..e22c159c3b84 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h @@ -94,12 +94,12 @@ static __always_inline struct kvm_vcpu *vt_to_vcpu(struct vcpu_vt *vt) return &(container_of(vt, struct vcpu_vmx_tdx, vt)->vcpu); } -static __always_inline union vmx_exit_reason vmx_get_exit_reason(struct kvm_vcpu *vcpu) +static __always_inline union vmx_exit_reason vt_get_exit_reason(struct kvm_vcpu *vcpu) { return to_vt(vcpu)->exit_reason; } -static __always_inline unsigned long vmx_get_exit_qual(struct kvm_vcpu *vcpu) +static __always_inline unsigned long vt_get_exit_qual(struct kvm_vcpu *vcpu) { struct vcpu_vt *vt = to_vt(vcpu); @@ -110,7 +110,7 @@ static __always_inline unsigned long vmx_get_exit_qual(struct kvm_vcpu *vcpu) return vt->exit_qualification; } -static __always_inline u32 vmx_get_intr_info(struct kvm_vcpu *vcpu) +static __always_inline u32 vt_get_intr_info(struct kvm_vcpu *vcpu) { struct vcpu_vt *vt = to_vt(vcpu); diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 9b304f420d4c..c11732cfadf0 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -885,8 +885,8 @@ int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) noinstr void vt_handle_nmi(struct kvm_vcpu *vcpu) { - if ((u16)vmx_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI || - !is_nmi(vmx_get_intr_info(vcpu))) + if ((u16)vt_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI || + !is_nmi(vt_get_intr_info(vcpu))) return; kvm_before_interrupt(vcpu, KVM_HANDLING_NMI); @@ -947,12 +947,12 @@ static void vt_handle_exit_irqoff(struct kvm_vcpu *vcpu) if (to_vt(vcpu)->emulation_required) return; - switch (vmx_get_exit_reason(vcpu).basic) { + switch (vt_get_exit_reason(vcpu).basic) { case EXIT_REASON_EXTERNAL_INTERRUPT: - handle_external_interrupt_irqoff(vcpu, vmx_get_intr_info(vcpu)); + handle_external_interrupt_irqoff(vcpu, vt_get_intr_info(vcpu)); break; case EXIT_REASON_EXCEPTION_NMI: - handle_exception_irqoff(vcpu, vmx_get_intr_info(vcpu)); + handle_exception_irqoff(vcpu, vt_get_intr_info(vcpu)); break; case EXIT_REASON_MCE_DURING_VMENTRY: kvm_machine_check(); @@ -978,7 +978,7 @@ static int vt_handle_exit(struct kvm_vcpu *vcpu, * Exit to user space when bus lock detected to inform that there is * a bus lock in guest. */ - if (vmx_get_exit_reason(vcpu).bus_lock_detected) { + if (vt_get_exit_reason(vcpu).bus_lock_detected) { if (ret > 0) { vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; ret = 0; diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 151873407abd..47599e7312cf 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -446,7 +446,7 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, * "NMI unblocking due to IRET", i.e. the bit can be propagated * as-is from the original EXIT_QUALIFICATION. */ - exit_qualification = vmx_get_exit_qual(vcpu) & INTR_INFO_UNBLOCK_NMI; + exit_qualification = vt_get_exit_qual(vcpu) & INTR_INFO_UNBLOCK_NMI; } else { if (fault->error_code & PFERR_RSVD_MASK) { vm_exit_reason = EXIT_REASON_EPT_MISCONFIG; @@ -472,7 +472,7 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, * and walks the faulting GPA. */ if (from_hardware) - exit_qualification |= vmx_get_exit_qual(vcpu) & mask; + exit_qualification |= vt_get_exit_qual(vcpu) & mask; else exit_qualification |= fault->exit_qualification & mask; @@ -4747,7 +4747,7 @@ static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, { /* update exit information fields: */ vmcs12->vm_exit_reason = vm_exit_reason; - if (vmx_get_exit_reason(vcpu).enclave_mode) + if (vt_get_exit_reason(vcpu).enclave_mode) vmcs12->vm_exit_reason |= VMX_EXIT_REASONS_SGX_ENCLAVE_MODE; vmcs12->exit_qualification = exit_qualification; @@ -5373,7 +5373,7 @@ static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer, struct x86_exception e; int r; - if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), + if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), vmcs_read32(VMX_INSTRUCTION_INFO), false, sizeof(*vmpointer), &gva)) { *ret = 1; @@ -5665,7 +5665,7 @@ static int handle_vmread(struct kvm_vcpu *vcpu) { struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) : get_vmcs12(vcpu); - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); struct vcpu_vmx *vmx = to_vmx(vcpu); struct x86_exception e; @@ -5771,7 +5771,7 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu) { struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) : get_vmcs12(vcpu); - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); struct vcpu_vmx *vmx = to_vmx(vcpu); struct x86_exception e; @@ -5960,7 +5960,7 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) /* Emulate the VMPTRST instruction */ static int handle_vmptrst(struct kvm_vcpu *vcpu) { - unsigned long exit_qual = vmx_get_exit_qual(vcpu); + unsigned long exit_qual = vt_get_exit_qual(vcpu); u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr; struct x86_exception e; @@ -6021,7 +6021,7 @@ static int handle_invept(struct kvm_vcpu *vcpu) /* According to the Intel VMX instruction reference, the memory * operand is read even if it isn't needed (e.g., for type==global) */ - if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), + if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), vmx_instruction_info, false, sizeof(operand), &gva)) return 1; r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); @@ -6104,7 +6104,7 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) /* according to the intel vmx instruction reference, the memory * operand is read even if it isn't needed (e.g., for type==global) */ - if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), + if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), vmx_instruction_info, false, sizeof(operand), &gva)) return 1; r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); @@ -6236,8 +6236,8 @@ fail: * EXIT_REASON_VMFUNC as the exit reason. */ nested_vmx_vmexit(vcpu, vmx->vt.exit_reason.full, - vmx_get_intr_info(vcpu), - vmx_get_exit_qual(vcpu)); + vt_get_intr_info(vcpu), + vt_get_exit_qual(vcpu)); return 1; } @@ -6288,7 +6288,7 @@ static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu, if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING); - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); port = exit_qualification >> 16; size = (exit_qualification & 7) + 1; @@ -6314,7 +6314,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, if (exit_reason.basic == EXIT_REASON_MSR_READ_IMM || exit_reason.basic == EXIT_REASON_MSR_WRITE_IMM) - msr_index = vmx_get_exit_qual(vcpu); + msr_index = vt_get_exit_qual(vcpu); else msr_index = kvm_ecx_read(vcpu); @@ -6350,7 +6350,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); int cr = exit_qualification & 15; int reg; unsigned long val; @@ -6484,7 +6484,7 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, switch ((u16)exit_reason.basic) { case EXIT_REASON_EXCEPTION_NMI: - intr_info = vmx_get_intr_info(vcpu); + intr_info = vt_get_intr_info(vcpu); if (is_nmi(intr_info)) return true; else if (is_page_fault(intr_info)) @@ -6567,7 +6567,7 @@ static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu, switch ((u16)exit_reason.basic) { case EXIT_REASON_EXCEPTION_NMI: - intr_info = vmx_get_intr_info(vcpu); + intr_info = vt_get_intr_info(vcpu); if (is_nmi(intr_info)) return true; else if (is_page_fault(intr_info)) @@ -6737,14 +6737,14 @@ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu) * need to be synthesized by querying the in-kernel LAPIC, but external * interrupts are never reflected to L1 so it's a non-issue. */ - exit_intr_info = vmx_get_intr_info(vcpu); + exit_intr_info = vt_get_intr_info(vcpu); if (is_exception_with_error_code(exit_intr_info)) { struct vmcs12 *vmcs12 = get_vmcs12(vcpu); vmcs12->vm_exit_intr_error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); } - exit_qual = vmx_get_exit_qual(vcpu); + exit_qual = vt_get_exit_qual(vcpu); reflect_vmexit: nested_vmx_vmexit(vcpu, exit_reason.full, exit_intr_info, exit_qual); diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 76bcf2fabb83..53fd4e63ce40 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -741,7 +741,7 @@ bool tdx_interrupt_allowed(struct kvm_vcpu *vcpu) * interrupt is always allowed unless TDX guest calls TDVMCALL with HLT, * which passes the interrupt blocked flag. */ - return vmx_get_exit_reason(vcpu).basic != EXIT_REASON_HLT || + return vt_get_exit_reason(vcpu).basic != EXIT_REASON_HLT || !to_tdx(vcpu)->vp_enter_args.r12; } @@ -759,7 +759,7 @@ static bool tdx_protected_apic_has_interrupt(struct kvm_vcpu *vcpu) * otherwise the interrupt would have been serviced at the instruction * boundary. */ - if (vmx_get_exit_reason(vcpu).basic != EXIT_REASON_HLT || + if (vt_get_exit_reason(vcpu).basic != EXIT_REASON_HLT || to_tdx(vcpu)->vp_enter_args.r12) return false; @@ -996,7 +996,7 @@ static noinstr void tdx_vcpu_enter_exit(struct kvm_vcpu *vcpu) static bool tdx_failed_vmentry(struct kvm_vcpu *vcpu) { - return vmx_get_exit_reason(vcpu).failed_vmentry; + return vt_get_exit_reason(vcpu).failed_vmentry; } static fastpath_t tdx_exit_handlers_fastpath(struct kvm_vcpu *vcpu) @@ -1153,7 +1153,7 @@ void tdx_inject_nmi(struct kvm_vcpu *vcpu) static int tdx_handle_exception_nmi(struct kvm_vcpu *vcpu) { - u32 intr_info = vmx_get_intr_info(vcpu); + u32 intr_info = vt_get_intr_info(vcpu); /* * Machine checks are handled by handle_exception_irqoff(), or by @@ -1925,7 +1925,7 @@ void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, static inline bool tdx_is_sept_violation_unexpected_pending(struct kvm_vcpu *vcpu) { u64 eeq_type = to_tdx(vcpu)->ext_exit_qualification & TDX_EXT_EXIT_QUAL_TYPE_MASK; - u64 eq = vmx_get_exit_qual(vcpu); + u64 eq = vt_get_exit_qual(vcpu); if (eeq_type != TDX_EXT_EXIT_QUAL_TYPE_PENDING_EPT_VIOLATION) return false; @@ -1961,7 +1961,7 @@ static int tdx_handle_ept_violation(struct kvm_vcpu *vcpu) /* Only private GPA triggers zero-step mitigation */ local_retry = true; } else { - exit_qual = vmx_get_exit_qual(vcpu); + exit_qual = vt_get_exit_qual(vcpu); /* * EPT violation due to instruction fetch should never be * triggered from shared memory in TDX guest. If such EPT @@ -2043,7 +2043,7 @@ int tdx_complete_emulated_msr(struct kvm_vcpu *vcpu, int err) return 1; } - if (vmx_get_exit_reason(vcpu).basic == EXIT_REASON_MSR_READ) + if (vt_get_exit_reason(vcpu).basic == EXIT_REASON_MSR_READ) tdvmcall_set_return_val(vcpu, kvm_read_edx_eax(vcpu)); return 1; @@ -2054,7 +2054,7 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) { struct vcpu_tdx *tdx = to_tdx(vcpu); u64 vp_enter_ret = tdx->vp_enter_ret; - union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu); + union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu); if (fastpath != EXIT_FASTPATH_NONE) return 1; @@ -2150,7 +2150,7 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) return 1; case EXIT_REASON_NOTIFY: /* NMI blocking state is handled by TDX module */ - return __vt_handle_notify(vcpu, vmx_get_exit_qual(vcpu)); + return __vt_handle_notify(vcpu, vt_get_exit_qual(vcpu)); case EXIT_REASON_BUS_LOCK: return vt_handle_bus_lock_vmexit(vcpu); default: @@ -2169,9 +2169,9 @@ void tdx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, *reason = tdx->vt.exit_reason.full; if (tdx_is_exit_reason_valid(tdx->vp_enter_ret)) { - *info1 = vmx_get_exit_qual(vcpu); + *info1 = vt_get_exit_qual(vcpu); *info2 = tdx->ext_exit_qualification; - *intr_info = vmx_get_intr_info(vcpu); + *intr_info = vt_get_intr_info(vcpu); } else { *info1 = 0; *info2 = 0; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index d0a617797661..21d2d96fd969 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1791,7 +1791,7 @@ int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, * so that guest userspace can't DoS the guest simply by triggering * emulation (enclaves are CPL3 only). */ - if (vmx_get_exit_reason(vcpu).enclave_mode) { + if (vt_get_exit_reason(vcpu).enclave_mode) { kvm_queue_exception(vcpu, UD_VECTOR); return X86EMUL_PROPAGATE_FAULT; } @@ -1806,7 +1806,7 @@ int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, static int skip_emulated_instruction(struct kvm_vcpu *vcpu) { - union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu); + union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu); unsigned long rip, orig_rip; u32 instr_len; @@ -5383,7 +5383,7 @@ bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu) static int vmx_handle_page_fault(struct kvm_vcpu *vcpu, u32 error_code) { - unsigned long cr2 = vmx_get_exit_qual(vcpu); + unsigned long cr2 = vt_get_exit_qual(vcpu); if (vcpu->arch.apf.host_apf_flags) goto handle_pf; @@ -5431,7 +5431,7 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) u32 vect_info; vect_info = vmx->idt_vectoring_info; - intr_info = vmx_get_intr_info(vcpu); + intr_info = vt_get_intr_info(vcpu); /* * Machine checks are handled by handle_exception_irqoff(), or by @@ -5512,7 +5512,7 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) switch (ex_no) { case DB_VECTOR: - dr6 = vmx_get_exit_qual(vcpu); + dr6 = vt_get_exit_qual(vcpu); if (!(vcpu->guest_debug & (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) { /* @@ -5588,7 +5588,7 @@ static int handle_io(struct kvm_vcpu *vcpu) int size, in, string; unsigned port; - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); string = (exit_qualification & 16) != 0; ++vcpu->stat.io_exits; @@ -5677,7 +5677,7 @@ static int handle_cr(struct kvm_vcpu *vcpu) int err; int ret; - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); cr = exit_qualification & 15; reg = (exit_qualification >> 8) & 15; switch ((exit_qualification >> 4) & 3) { @@ -5755,7 +5755,7 @@ static int handle_dr(struct kvm_vcpu *vcpu) int dr, dr7, reg; int err = 1; - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); dr = exit_qualification & DEBUG_REG_ACCESS_NUM; /* First, if DR does not exist, trigger UD */ @@ -5851,7 +5851,7 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu) static int handle_invlpg(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); kvm_mmu_invlpg(vcpu, exit_qualification); return kvm_skip_emulated_instruction(vcpu); @@ -5860,7 +5860,7 @@ static int handle_invlpg(struct kvm_vcpu *vcpu) static int handle_apic_access(struct kvm_vcpu *vcpu) { if (likely(fasteoi)) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); int access_type, offset; access_type = exit_qualification & APIC_ACCESS_TYPE; @@ -5881,7 +5881,7 @@ static int handle_apic_access(struct kvm_vcpu *vcpu) static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); int vector = exit_qualification & 0xff; /* EOI-induced VM exit is trap-like and thus no need to adjust IP */ @@ -5891,7 +5891,7 @@ static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu) static int handle_apic_write(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); /* * APIC-write VM-Exit is trap-like, KVM doesn't need to advance RIP and @@ -5919,7 +5919,7 @@ static int handle_task_switch(struct kvm_vcpu *vcpu) idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK); type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK); - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); reason = (u32)exit_qualification >> 30; if (reason == TASK_SWITCH_GATE && idt_v) { @@ -5965,7 +5965,7 @@ static int handle_task_switch(struct kvm_vcpu *vcpu) static int handle_ept_violation(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); gpa_t gpa; /* @@ -6156,7 +6156,7 @@ static int handle_invpcid(struct kvm_vcpu *vcpu) /* According to the Intel instruction reference, the memory operand * is read even if it isn't needed (e.g., for type==all) */ - if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), + if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), vmx_instruction_info, false, sizeof(operand), &gva)) return 1; @@ -6170,7 +6170,7 @@ static int handle_pml_full(struct kvm_vcpu *vcpu) trace_kvm_pml_full(vcpu->vcpu_id); - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); /* * PML buffer FULL happened while executing iret from NMI, @@ -6263,7 +6263,7 @@ static int handle_encls(struct kvm_vcpu *vcpu) static int handle_notify(struct kvm_vcpu *vcpu) { - unsigned long exit_qual = vmx_get_exit_qual(vcpu); + unsigned long exit_qual = vt_get_exit_qual(vcpu); /* * Notify VM exit happened while executing iret from NMI, @@ -6283,13 +6283,13 @@ static int vmx_get_msr_imm_reg(struct kvm_vcpu *vcpu) static int handle_rdmsr_imm(struct kvm_vcpu *vcpu) { - return kvm_emulate_rdmsr_imm(vcpu, vmx_get_exit_qual(vcpu), + return kvm_emulate_rdmsr_imm(vcpu, vt_get_exit_qual(vcpu), vmx_get_msr_imm_reg(vcpu)); } static int handle_wrmsr_imm(struct kvm_vcpu *vcpu) { - return kvm_emulate_wrmsr_imm(vcpu, vmx_get_exit_qual(vcpu), + return kvm_emulate_wrmsr_imm(vcpu, vt_get_exit_qual(vcpu), vmx_get_msr_imm_reg(vcpu)); } @@ -6368,10 +6368,10 @@ void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, struct vcpu_vmx *vmx = to_vmx(vcpu); *reason = vmx->vt.exit_reason.full; - *info1 = vmx_get_exit_qual(vcpu); + *info1 = vt_get_exit_qual(vcpu); if (!(vmx->vt.exit_reason.failed_vmentry)) { *info2 = vmx->idt_vectoring_info; - *intr_info = vmx_get_intr_info(vcpu); + *intr_info = vt_get_intr_info(vcpu); if (is_exception_with_error_code(*intr_info)) *error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); else @@ -6682,7 +6682,7 @@ void dump_vmcs(struct kvm_vcpu *vcpu) int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) { struct vcpu_vmx *vmx = to_vmx(vcpu); - union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu); + union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu); u32 vectoring_info = vmx->idt_vectoring_info; u16 exit_handler_index; @@ -7140,7 +7140,7 @@ static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx) if (vmx->loaded_vmcs->nmi_known_unmasked) return; - exit_intr_info = vmx_get_intr_info(&vmx->vcpu); + exit_intr_info = vt_get_intr_info(&vmx->vcpu); unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0; vector = exit_intr_info & INTR_INFO_VECTOR_MASK; /* @@ -7307,14 +7307,14 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu, * the fastpath even, all other exits must use the slow path. */ if (is_guest_mode(vcpu) && - vmx_get_exit_reason(vcpu).basic != EXIT_REASON_PREEMPTION_TIMER) + vt_get_exit_reason(vcpu).basic != EXIT_REASON_PREEMPTION_TIMER) return EXIT_FASTPATH_NONE; - switch (vmx_get_exit_reason(vcpu).basic) { + switch (vt_get_exit_reason(vcpu).basic) { case EXIT_REASON_MSR_WRITE: return handle_fastpath_wrmsr(vcpu); case EXIT_REASON_MSR_WRITE_IMM: - return handle_fastpath_wrmsr_imm(vcpu, vmx_get_exit_qual(vcpu), + return handle_fastpath_wrmsr_imm(vcpu, vt_get_exit_qual(vcpu), vmx_get_msr_imm_reg(vcpu)); case EXIT_REASON_PREEMPTION_TIMER: return handle_fastpath_preemption_timer(vcpu, force_immediate_exit); @@ -7356,7 +7356,7 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, } vmx->vt.exit_reason.full = vmcs_read32(VM_EXIT_REASON); - if (likely(!vmx_get_exit_reason(vcpu).failed_vmentry)) + if (likely(!vt_get_exit_reason(vcpu).failed_vmentry)) vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); vt_handle_nmi(vcpu); @@ -7496,7 +7496,7 @@ fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) * checking. */ if (vcpu->arch.nested_run_pending && - !vmx_get_exit_reason(vcpu).failed_vmentry) + !vt_get_exit_reason(vcpu).failed_vmentry) ++vcpu->stat.nested_run; vcpu->arch.nested_run_pending = 0; @@ -7507,7 +7507,7 @@ fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) trace_kvm_exit(vcpu, KVM_ISA_VMX); - if (unlikely(vmx_get_exit_reason(vcpu).failed_vmentry)) + if (unlikely(vt_get_exit_reason(vcpu).failed_vmentry)) return EXIT_FASTPATH_NONE; vmx->loaded_vmcs->launched = 1; -- cgit v1.2.3 From 6088c9a220127c1902902f2791d127dbbd9bca34 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 17:01:52 -0700 Subject: KVM: VMX: Move bus lock handler from main.c => common.h Move the bus lock VM-Exit handler, which is basically a one-liner, from main.c to vmx.h now that to_vt() is usable in common.h. No functional change intended. Reviewed-by: Binbin Wu Reviewed-by: Xiaoyao Li Link: https://patch.msgid.link/20260922000152.1113202-1-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/common.h | 11 ++++++++++- arch/x86/kvm/vmx/main.c | 11 ----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index e22c159c3b84..b2905335a2d2 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h @@ -254,6 +254,15 @@ static inline int __vt_handle_notify(struct kvm_vcpu *vcpu, return 1; } -int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu); +static inline int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) +{ + /* + * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK + * VM-Exits. Unconditionally set the flag here and leave the handling + * to .handle_exit() callback. + */ + to_vt(vcpu)->exit_reason.bus_lock_detected = true; + return 1; +} #endif /* __KVM_X86_VMX_COMMON_H */ diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index c11732cfadf0..2985742e20f1 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -872,17 +872,6 @@ static int vt_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, #define vt_op_tdx_only(name) NULL #endif /* CONFIG_KVM_INTEL_TDX */ -int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) -{ - /* - * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK - * VM-Exits. Unconditionally set the flag here and leave the handling - * to .handle_exit() callback. - */ - to_vt(vcpu)->exit_reason.bus_lock_detected = true; - return 1; -} - noinstr void vt_handle_nmi(struct kvm_vcpu *vcpu) { if ((u16)vt_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI || -- cgit v1.2.3 From 1a66112cbc0214743c0d6d52427609379ac858f7 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Wed, 26 Aug 2026 13:03:21 +0300 Subject: KVM: TDX: Remove always true ifdef CONFIG_KVM_INTEL_TDX depends on KVM_TDX_HOST, which in turn depends on CONFIG_X86_64. So it's guaranteed that any code under ifdef CONFIG_KVM_INTEL_TDX is also running under CONFIG_X86_64 being true, remove pointless ifdef guard. Signed-off-by: Nikolay Borisov Link: https://patch.msgid.link/20260826100321.1309685-1-nik.borisov@suse.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/main.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 2985742e20f1..250666cee0c2 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -803,7 +803,6 @@ static void vt_write_tsc_multiplier(struct kvm_vcpu *vcpu) vmx_write_tsc_multiplier(vcpu); } -#ifdef CONFIG_X86_64 static int vt_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, bool *expired) { @@ -822,7 +821,6 @@ static void vt_cancel_hv_timer(struct kvm_vcpu *vcpu) vmx_cancel_hv_timer(vcpu); } -#endif static void vt_setup_mce(struct kvm_vcpu *vcpu) { -- cgit v1.2.3 From aec196ba2affa5b396ed1bc16340067d66972f24 Mon Sep 17 00:00:00 2001 From: Haotian Jiang Date: Thu, 6 Aug 2026 10:06:01 -0700 Subject: KVM: TDX: Reject INIT_MEM_REGION if number of bytes would overflow a u64 The nr_pages field in struct kvm_tdx_init_mem_region is a u64 that comes directly from userspace via copy_from_user(). The current validation uses a manual overflow check: region.gpa + (region.nr_pages << PAGE_SHIFT) <= region.gpa When nr_pages >= 2^52, the shift (nr_pages << PAGE_SHIFT) wraps around to a small value, bypassing the wrap check. While downstream protections (gfn_to_memslot() returning NULL for GFNs outside any memslot, and kvm_slot_has_gmem() checking for NULL) prevent any actual out-of-bounds access, the overflow itself is a real bug that should be caught at the validation layer. Replace the manual overflow check with check_shl_overflow() to correctly detect the wrap-around. Note, the manual wrap-around check on the gpa+size technically has a benign off-by-one bug, and can also use check_add_overflow(). Those flaws will be addressed shortly. Opportunistically separate the initial sanity checks from the more involved checks to try and make the code easier to read. Fixes: c846b451d3c5 ("KVM: TDX: Add an ioctl to create initial guest memory") Reported-by: Sashiko Bot Closes: https://lore.kernel.org/all/20260630214952.98C851F000E9@smtp.kernel.org Cc: Yan Zhao Cc: Binbin Wu Cc: Ackerley Tng Signed-off-by: Haotian Jiang [sean: use gpa_t, isolate check_shl_overflow() change, tweak changelog] Reviewed-by: Binbin Wu Reviewed-by: Yan Zhao Tested-by: Yan Zhao Link: https://patch.msgid.link/20260806170602.4112602-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 53fd4e63ce40..d991996259a8 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -3257,6 +3257,7 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); struct kvm_tdx_init_mem_region region; struct tdx_gmem_post_populate_arg arg; + gpa_t nr_bytes; long gmem_ret; int ret; @@ -3274,10 +3275,13 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c return -EFAULT; if (!PAGE_ALIGNED(region.source_addr) || !region.source_addr || - !PAGE_ALIGNED(region.gpa) || !region.nr_pages || - region.gpa + (region.nr_pages << PAGE_SHIFT) <= region.gpa || + !PAGE_ALIGNED(region.gpa) || !region.nr_pages) + return -EINVAL; + + if (check_shl_overflow(region.nr_pages, PAGE_SHIFT, &nr_bytes) || + region.gpa + nr_bytes <= region.gpa || !vt_is_tdx_private_gpa(kvm, region.gpa) || - !vt_is_tdx_private_gpa(kvm, region.gpa + (region.nr_pages << PAGE_SHIFT) - 1)) + !vt_is_tdx_private_gpa(kvm, region.gpa + nr_bytes - 1)) return -EINVAL; ret = 0; -- cgit v1.2.3 From ea93efec4dda2ca8c33dc88cf4d389a980a358df Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 6 Aug 2026 10:06:02 -0700 Subject: KVM: TDX: Fix a benign off-by-one bug on the end GPA for INIT_MEM_REGION When verifying that the incoming GPA rage for INIT_MEM_REGION doesn't wrap, check the inclusive last GPA, not the exclusive last GPA. Super duper technically, it's ok if the very last GPA is -1ull. In practice, the flaw is benign as KVM x86 disallows memslots with GPAs that exceed MAXPHYADDR, i.e. INIT_MEM_REGION would fail with -EINVAL anyways due to the memslot check in kvm_gmem_populate(). Opportunistically use check_add_overflow() instead of manually checking for wrap, mostly so that the inclusive math doesn't need to be copy+pasted in the "is private" check. Fixes: c846b451d3c5 ("KVM: TDX: Add an ioctl to create initial guest memory") Reviewed-by: Yan Zhao Tested-by: Yan Zhao Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260806170602.4112602-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index d991996259a8..2a3acf493cb5 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -3257,7 +3257,7 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); struct kvm_tdx_init_mem_region region; struct tdx_gmem_post_populate_arg arg; - gpa_t nr_bytes; + gpa_t nr_bytes, end_gpa; long gmem_ret; int ret; @@ -3279,9 +3279,9 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c return -EINVAL; if (check_shl_overflow(region.nr_pages, PAGE_SHIFT, &nr_bytes) || - region.gpa + nr_bytes <= region.gpa || + check_add_overflow(region.gpa, nr_bytes - 1, &end_gpa) || !vt_is_tdx_private_gpa(kvm, region.gpa) || - !vt_is_tdx_private_gpa(kvm, region.gpa + nr_bytes - 1)) + !vt_is_tdx_private_gpa(kvm, end_gpa)) return -EINVAL; ret = 0; -- cgit v1.2.3 From 883719f57f5d3898637b130d6db4742471cad618 Mon Sep 17 00:00:00 2001 From: Keqiang Duan Date: Thu, 20 Aug 2026 20:35:12 +0800 Subject: KVM: x86: Clear hardware HLT state when userspace makes a vCPU not-halted Force a vCPU out of its hardware-tracked halted state when userspace puts the vCPU into any state other than HALTED via KVM_SET_MP_STATE, i.e. clear VMCS.GUEST_ACTIVITY_STATE if it says the vCPU is halted. Add an optional kvm_x86_ops hook to do the clearing, as SVM has no equivalent VMCB field. When HLT-exiting is disabled for a VM (KVM_CAP_X86_DISABLE_EXITS with KVM_X86_DISABLE_EXITS_HLT, e.g. QEMU's "-overcommit cpu-pm=on"), a guest HLT halts the physical CPU instead of exiting to KVM, and hardware saves GUEST_ACTIVITY_STATE=HLT into the VMCS on the next VM-Exit. That field is sticky: it survives VM-Exit/VM-Enter and is only cleared by vmx_clear_hlt() on event injection, or by vmx_vcpu_reset() on INIT / vCPU creation. Nothing clears it on a userspace-driven state change. KVM_SET_REGS only writes the software register cache and KVM_SET_MP_STATE only writes vcpu->arch.mp_state; kvm_vcpu_running() likewise consults software state only. A VMM that emulates a machine reset therefore ends up with a vCPU that KVM happily VM-Enters while hardware refuses to fetch instructions. Reproduce with a Linux guest by triggering a panic/kdump on a non-boot vCPU: nmi_shootdown_cpus() parks the other vCPUs -- including vCPU0 -- in crash_nmi_callback(), which does local_irq_disable() followed by a bare HLT. The capture kernel then resets the machine via port 0xCF9. QEMU rewrites RIP to 0xfff0 and sets mp_state to RUNNABLE, but vCPU0's GUEST_ACTIVITY_STATE is still HLT, so the BSP never executes the reset vector, never sends SIPIs, and the entire VM hangs at "reboot: machine restart" forever. Only destroying and recreating the VM recovers it. Key off "not HALTED" rather than "is RUNNABLE" as HALTED is the only target state for which retaining the hardware halted state is meaningful; every other transition either can't be reached on VMX, e.g. AP_RESET_HOLD, or goes through vmx_vcpu_reset(), which stuffs GUEST_ACTIVITY_ACTIVE anyways. This also captures the intent more directly: KVM only ever drops the hardware halted state, it never installs it. Hook KVM_SET_MP_STATE rather than the VM-Enter path so that the clearing is driven by an explicit userspace declaration, and so that no work is added to vmx_vcpu_run(). Gate the call on kvm_hlt_in_guest() even though vmx_clear_hlt() checks it as well, as the check is cheap and makes it obvious in common code when the hardware state can exist at all. Do nothing for TDX vCPUs as TDX doesn't support disabling HLT-exiting and KVM cannot access a TD's VMCS, i.e. the VMCS used to run the vCPU can't in HALTED state, and even if it were, KVM can't change it. Fixes: caa057a2cad6 ("KVM: X86: Provide a capability to disable HLT intercepts") Cc: stable@vger.kernel.org Tested-by: Qinguang Chen Signed-off-by: Zhiping Du Signed-off-by: Keqiang Duan Reviewed-by: Chao Gao Link: https://patch.msgid.link/20260820123512.87236-1-duankeqiangcym@gmail.com [sean: drop claim that spurious wakeups are ok, massage changelog] Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-ops.h | 1 + arch/x86/include/asm/kvm_host.h | 7 +++++++ arch/x86/kvm/vmx/main.c | 13 +++++++++++++ arch/x86/kvm/vmx/vmx.c | 2 +- arch/x86/kvm/vmx/x86_ops.h | 1 + arch/x86/kvm/x86.c | 13 +++++++++++++ 6 files changed, 36 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index e213c9ae3e30..dd9026c1071c 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -82,6 +82,7 @@ KVM_X86_OP(interrupt_allowed) KVM_X86_OP(nmi_allowed) KVM_X86_OP(get_nmi_mask) KVM_X86_OP(set_nmi_mask) +KVM_X86_OP_OPTIONAL(clear_hlt) KVM_X86_OP(enable_nmi_window) KVM_X86_OP(enable_irq_window) KVM_X86_OP_OPTIONAL(update_cr8_intercept) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 3af7395c2430..4cc39c3592ee 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1608,6 +1608,13 @@ struct kvm_x86_ops { int (*nmi_allowed)(struct kvm_vcpu *vcpu, bool for_injection); bool (*get_nmi_mask)(struct kvm_vcpu *vcpu); void (*set_nmi_mask)(struct kvm_vcpu *vcpu, bool masked); + /* + * Force the vCPU out of any hardware-tracked halted/inactive state so + * that it will fetch and execute instructions on the next VM-Enter. + * Only needed by VMX, where VMCS.GUEST_ACTIVITY_STATE persists across + * VM-Exit/VM-Enter; SVM has no equivalent VMCB field. + */ + void (*clear_hlt)(struct kvm_vcpu *vcpu); /* Whether or not a virtual NMI is pending in hardware. */ bool (*is_vnmi_pending)(struct kvm_vcpu *vcpu); /* diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 250666cee0c2..8bf1702f5165 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -597,6 +597,18 @@ static void vt_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) vmx_set_nmi_mask(vcpu, masked); } +static void vt_clear_hlt(struct kvm_vcpu *vcpu) +{ + /* + * TDX doesn't support disabling HLT-exiting, and KVM can't access a + * TD's VMCS, so there is never any hardware halted state to clear. + */ + if (is_td_vcpu(vcpu)) + return; + + vmx_clear_hlt(vcpu); +} + static void vt_enable_nmi_window(struct kvm_vcpu *vcpu) { /* Refer to the comments in tdx_inject_nmi(). */ @@ -1064,6 +1076,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = { .nmi_allowed = vt_op(nmi_allowed), .get_nmi_mask = vt_op(get_nmi_mask), .set_nmi_mask = vt_op(set_nmi_mask), + .clear_hlt = vt_op(clear_hlt), .enable_nmi_window = vt_op(enable_nmi_window), .enable_irq_window = vt_op(enable_irq_window), .update_cr8_intercept = vt_op(update_cr8_intercept), diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 21d2d96fd969..cf32aee0aab7 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1909,7 +1909,7 @@ int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu) return skip_emulated_instruction(vcpu); } -static void vmx_clear_hlt(struct kvm_vcpu *vcpu) +void vmx_clear_hlt(struct kvm_vcpu *vcpu) { /* * Ensure that we clear the HLT state in the VMCS. We don't need to diff --git a/arch/x86/kvm/vmx/x86_ops.h b/arch/x86/kvm/vmx/x86_ops.h index 7ac02c68457f..9ead79c90ba8 100644 --- a/arch/x86/kvm/vmx/x86_ops.h +++ b/arch/x86/kvm/vmx/x86_ops.h @@ -94,6 +94,7 @@ int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection); int vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection); bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu); void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked); +void vmx_clear_hlt(struct kvm_vcpu *vcpu); void vmx_enable_nmi_window(struct kvm_vcpu *vcpu); void vmx_enable_irq_window(struct kvm_vcpu *vcpu); void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 79468ddfe473..6f590f9be849 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -9058,6 +9058,19 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, } kvm_set_mp_state(vcpu, mp_state->mp_state); + + /* + * Force the vCPU out of any hardware-tracked halted state, e.g. VMX's + * GUEST_ACTIVITY_STATE=HLT, when userspace puts the vCPU into a state + * other than HALTED. The hardware state is sticky across VM-Exit and + * VM-Enter and is not touched by any other ioctl, so a vCPU that halted + * with HLT-exiting disabled stays wedged even after userspace rewrites + * its registers, e.g. when a VMM emulates a machine reset. + */ + if (kvm_hlt_in_guest(vcpu->kvm) && + mp_state->mp_state != KVM_MP_STATE_HALTED) + kvm_x86_call(clear_hlt)(vcpu); + kvm_make_request(KVM_REQ_EVENT, vcpu); ret = 0; -- cgit v1.2.3