diff options
| author | Paolo Bonzini <pbonzini@redhat.com> | 2026-06-12 10:11:09 +0200 |
|---|---|---|
| committer | Paolo Bonzini <pbonzini@redhat.com> | 2026-06-12 10:11:09 +0200 |
| commit | f6d6be78b24426eba424e24d06718f59a860e38a (patch) | |
| tree | 04342c95a0eebc4ccd9026ce5df0bba5bb8c1b1f | |
| parent | 06ba195f136a34f0a6e41f252002368d57875db9 (diff) | |
| parent | 65aa483f32ec674f0506658329d43403cafe5eb0 (diff) | |
Merge tag 'kvm-x86-misc-7.2' of https://github.com/kvm-x86/linux into HEAD
KVM misc x86 changes for 7.2
- Handle EXIT_FASTPATH_EXIT_USERSPACE in vendor code to ensure vendor code
gets a chance to handle things like reaping the PML buffer.
- Ensure KVM's copy of CR0 and CR3 are up-to-date on SVM prior to invoking
fastpath handlers.
- Update KVM's view of PV async enabling if and only if the MSR write fully
succeeds.
- Fix a variety of issues where the emulator doesn't honor guest-debug state,
and clean up related code along the way.
- Synthesize EPT Violation and #NPF "error code" bits when injecting faults
into L1 that didn't originate in hardware (in which case the VMCS/VMCB
doesn't hold relevant information).
- Add support for virtualizing (well, emulating) AMD's flavor of CPL>0 CPUID
faulting.
- Clean up the GPR APIs so that KVM's use of "raw" is consistent, and fix a
variety of minor bugs along the way.
- Fix an OOB memory access due to not checking the VP ID when handling a
Hyper-V PV TLB flush for L2.
- Fix a bug in the mediated PMU's handling of fixed counters that allowed the
guest to bypass the PMU event filter.
- Allow userspace to return EAGAIN when handling SNP and TDX hypercalls, so
the KVM can forward a "retry" status code to the guest, and reserve all
unused error codes for future usage.
- Misc fixes and cleanups.
42 files changed, 1112 insertions, 543 deletions
diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 52bbbb553ce1..7d55aac04c15 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -8818,6 +8818,9 @@ block sizes is exposed in KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES as a This capability, if enabled, will cause KVM to exit to userspace with KVM_EXIT_HYPERCALL exit reason to process some hypercalls. +Userspace may fail the hypercall by setting hypercall.ret to EINVAL +or may request the hypercall to be retried the next time the guest run +by setting hypercall.ret to EAGAIN. Calling KVM_CHECK_EXTENSION for this capability will return a bitmask of hypercalls that can be configured to exit to userspace. @@ -9363,6 +9366,8 @@ means the VM type with value @n is supported. Possible values of @n are:: #define KVM_X86_SW_PROTECTED_VM 1 #define KVM_X86_SEV_VM 2 #define KVM_X86_SEV_ES_VM 3 + #define KVM_X86_SNP_VM 4 + #define KVM_X86_TDX_VM 5 Note, KVM_X86_SW_PROTECTED_VM is currently only for development and testing. Do not use KVM_X86_SW_PROTECTED_VM for "real" VMs, and especially not in diff --git a/MAINTAINERS b/MAINTAINERS index 355e4ab2f9e5..f63c9bd97e32 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14162,6 +14162,7 @@ L: kvm@vger.kernel.org S: Supported P: Documentation/process/maintainer-kvm-x86.rst T: git git://git.kernel.org/pub/scm/virt/kvm/kvm.git +F: Documentation/process/maintainer-kvm-x86.rst F: arch/x86/include/asm/kvm* F: arch/x86/include/asm/svm.h F: arch/x86/include/asm/vmx*.h diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 5fdd8cb3680f..dcdf7f099266 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -284,6 +284,8 @@ enum x86_intercept_stage; #define PFERR_GUEST_RMP_MASK BIT_ULL(31) #define PFERR_GUEST_FINAL_MASK BIT_ULL(32) #define PFERR_GUEST_PAGE_MASK BIT_ULL(33) +#define PFERR_GUEST_FAULT_STAGE_MASK \ + (PFERR_GUEST_FINAL_MASK | PFERR_GUEST_PAGE_MASK) #define PFERR_GUEST_ENC_MASK BIT_ULL(34) #define PFERR_GUEST_SIZEM_MASK BIT_ULL(35) #define PFERR_GUEST_VMPL_MASK BIT_ULL(36) @@ -484,7 +486,8 @@ struct kvm_mmu { u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index); int (*page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault); void (*inject_page_fault)(struct kvm_vcpu *vcpu, - struct x86_exception *fault); + struct x86_exception *fault, + bool from_hardware); gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, gpa_t gva_or_gpa, u64 access, struct x86_exception *exception); @@ -1057,8 +1060,6 @@ struct kvm_vcpu_arch { u16 vec; u32 id; u32 host_apf_flags; - bool send_always; - bool delivery_as_pf_vmexit; bool pageready_pending; } apf; @@ -1441,6 +1442,7 @@ struct kvm_arch { bool has_private_mem; bool has_protected_state; bool has_protected_eoi; + bool has_protected_pmu; bool pre_fault_allowed; struct hlist_head *mmu_page_hash; struct list_head active_mmu_pages; @@ -2155,8 +2157,6 @@ int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3); extern bool tdp_enabled; -u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu); - /* * EMULTYPE_NO_DECODE - Set when re-emulating an instruction (after completing * userspace I/O) to indicate that the emulation context @@ -2276,7 +2276,6 @@ int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu); void kvm_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg); void kvm_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg); -int kvm_load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, int seg); void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector); int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index, @@ -2307,10 +2306,18 @@ void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr, unsigned long payload); void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned int nr, bool has_error_code, u32 error_code); -void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault); -void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, - struct x86_exception *fault); -bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl); +void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault, + bool from_hardware); +void __kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, + struct x86_exception *fault, + bool from_hardware); + +static inline void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, + struct x86_exception *fault) +{ + __kvm_inject_emulated_page_fault(vcpu, fault, false); +} + bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr); static inline int __kvm_irq_line_state(unsigned long *irq_state, diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 86554de9a3f5..18c4be75e927 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -899,6 +899,7 @@ #define MSR_K7_HWCR_IRPERF_EN_BIT 30 #define MSR_K7_HWCR_IRPERF_EN BIT_ULL(MSR_K7_HWCR_IRPERF_EN_BIT) #define MSR_K7_HWCR_CPUID_USER_DIS_BIT 35 +#define MSR_K7_HWCR_CPUID_USER_DIS BIT_ULL(MSR_K7_HWCR_CPUID_USER_DIS_BIT) #define MSR_K7_FID_VID_CTL 0xc0010041 #define MSR_K7_FID_VID_STATUS 0xc0010042 #define MSR_K7_HWCR_CPB_DIS_BIT 25 diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index e69156b54cff..591d2294acd7 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -11,7 +11,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/kvm_host.h> -#include "linux/lockdep.h" +#include <linux/lockdep.h> #include <linux/export.h> #include <linux/vmalloc.h> #include <linux/uaccess.h> @@ -1248,7 +1248,7 @@ void kvm_initialize_cpu_caps(void) F(AUTOIBRS), EMULATED_F(NO_SMM_CTL_MSR), /* PrefetchCtlMsr */ - /* GpOnUserCpuid */ + EMULATED_F(GP_ON_USER_CPUID), /* EPSF */ F(PREFETCHI), F(AVX512_BMM), @@ -2161,17 +2161,18 @@ int kvm_emulate_cpuid(struct kvm_vcpu *vcpu) { u32 eax, ebx, ecx, edx; - if (!is_smm(vcpu) && cpuid_fault_enabled(vcpu) && - !kvm_require_cpl(vcpu, 0)) + if (!kvm_is_cpuid_allowed(vcpu)) { + kvm_queue_exception_e(vcpu, GP_VECTOR, 0); return 1; + } - eax = kvm_rax_read(vcpu); - ecx = kvm_rcx_read(vcpu); + eax = kvm_eax_read(vcpu); + ecx = kvm_ecx_read(vcpu); kvm_cpuid(vcpu, &eax, &ebx, &ecx, &edx, false); - kvm_rax_write(vcpu, eax); - kvm_rbx_write(vcpu, ebx); - kvm_rcx_write(vcpu, ecx); - kvm_rdx_write(vcpu, edx); + kvm_eax_write(vcpu, eax); + kvm_ebx_write(vcpu, ebx); + kvm_ecx_write(vcpu, ecx); + kvm_edx_write(vcpu, edx); return kvm_skip_emulated_instruction(vcpu); } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_cpuid); diff --git a/arch/x86/kvm/cpuid.h b/arch/x86/kvm/cpuid.h index 039b8e6f40ba..8d863f45585d 100644 --- a/arch/x86/kvm/cpuid.h +++ b/arch/x86/kvm/cpuid.h @@ -7,6 +7,8 @@ #include <asm/processor.h> #include <uapi/asm/kvm_para.h> +#include "smm.h" + extern u32 kvm_cpu_caps[NR_KVM_CPU_CAPS] __read_mostly; extern bool kvm_is_configuring_cpu_caps __read_mostly; @@ -124,7 +126,7 @@ static __always_inline bool guest_cpuid_has(struct kvm_vcpu *vcpu, entry = kvm_find_cpuid_entry_index(vcpu, cpuid.function, cpuid.index); if (!entry) - return NULL; + return false; reg = __cpuid_entry_get_reg(entry, cpuid.reg); if (!reg) @@ -181,15 +183,17 @@ static inline int guest_cpuid_stepping(struct kvm_vcpu *vcpu) return x86_stepping(best->eax); } -static inline bool supports_cpuid_fault(struct kvm_vcpu *vcpu) +static inline bool cpuid_fault_enabled(struct kvm_vcpu *vcpu) { - return vcpu->arch.msr_platform_info & MSR_PLATFORM_INFO_CPUID_FAULT; + return (vcpu->arch.msr_misc_features_enables & + MSR_MISC_FEATURES_ENABLES_CPUID_FAULT) || + (vcpu->arch.msr_hwcr & MSR_K7_HWCR_CPUID_USER_DIS); } -static inline bool cpuid_fault_enabled(struct kvm_vcpu *vcpu) +static inline bool kvm_is_cpuid_allowed(struct kvm_vcpu *vcpu) { - return vcpu->arch.msr_misc_features_enables & - MSR_MISC_FEATURES_ENABLES_CPUID_FAULT; + return !cpuid_fault_enabled(vcpu) || is_smm(vcpu) || + !kvm_x86_call(get_cpl)(vcpu); } static __always_inline void kvm_cpu_cap_clear(unsigned int x86_feature) diff --git a/arch/x86/kvm/debugfs.c b/arch/x86/kvm/debugfs.c index 999227fc7c66..0074a56e45b4 100644 --- a/arch/x86/kvm/debugfs.c +++ b/arch/x86/kvm/debugfs.c @@ -121,7 +121,7 @@ static int kvm_mmu_rmaps_stat_show(struct seq_file *m, void *v) lpage_size = kvm_mmu_slot_lpages(slot, k + 1); cur = log[k]; for (l = 0; l < lpage_size; l++) { - index = ffs(pte_list_count(&rmap[l])); + index = fls(pte_list_count(&rmap[l])); if (WARN_ON_ONCE(index >= RMAP_LOG_SIZE)) index = RMAP_LOG_SIZE - 1; cur[index]++; diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index 8013dccb3110..b566ab5c7515 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -20,7 +20,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/kvm_host.h> -#include "kvm_cache_regs.h" +#include "regs.h" #include "kvm_emulate.h" #include <linux/stringify.h> #include <asm/debugreg.h> @@ -540,8 +540,9 @@ static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec, return X86EMUL_PROPAGATE_FAULT; } -static int emulate_db(struct x86_emulate_ctxt *ctxt) +static int emulate_db(struct x86_emulate_ctxt *ctxt, unsigned long dr6) { + ctxt->exception.dr6 = dr6; return emulate_exception(ctxt, DB_VECTOR, 0, false); } @@ -3593,12 +3594,8 @@ static int em_sti(struct x86_emulate_ctxt *ctxt) static int em_cpuid(struct x86_emulate_ctxt *ctxt) { u32 eax, ebx, ecx, edx; - u64 msr = 0; - ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr); - if (!ctxt->ops->is_smm(ctxt) && - (msr & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT) && - ctxt->ops->cpl(ctxt)) + if (!ctxt->ops->is_cpuid_allowed(ctxt)) return emulate_gp(ctxt, 0); eax = reg_read(ctxt, VCPU_REGS_RAX); @@ -3847,15 +3844,8 @@ static int check_dr_read(struct x86_emulate_ctxt *ctxt) if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5)) return emulate_ud(ctxt); - if (ctxt->ops->get_dr(ctxt, 7) & DR7_GD) { - ulong dr6; - - dr6 = ctxt->ops->get_dr(ctxt, 6); - dr6 &= ~DR_TRAP_BITS; - dr6 |= DR6_BD | DR6_ACTIVE_LOW; - ctxt->ops->set_dr(ctxt, 6, dr6); - return emulate_db(ctxt); - } + if (ctxt->ops->get_effective_dr7(ctxt) & DR7_GD) + return emulate_db(ctxt, DR6_BD); return X86EMUL_CONTINUE; } diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 015c6947b462..fd4eb1e561f7 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -1839,6 +1839,11 @@ static bool hv_is_vp_in_sparse_set(u32 vp_id, u64 valid_bank_mask, u64 sparse_ba int valid_bit_nr = vp_id / HV_VCPUS_PER_SPARSE_BANK; unsigned long sbank; + BUILD_BUG_ON(BITS_PER_TYPE(valid_bank_mask) != HV_MAX_SPARSE_VCPU_BANKS); + + if (valid_bit_nr >= HV_MAX_SPARSE_VCPU_BANKS) + return false; + if (!test_bit(valid_bit_nr, (unsigned long *)&valid_bank_mask)) return false; @@ -2377,10 +2382,10 @@ static void kvm_hv_hypercall_set_result(struct kvm_vcpu *vcpu, u64 result) longmode = is_64_bit_hypercall(vcpu); if (longmode) - kvm_rax_write(vcpu, result); + kvm_rax_write_raw(vcpu, result); else { - kvm_rdx_write(vcpu, result >> 32); - kvm_rax_write(vcpu, result & 0xffffffff); + kvm_edx_write(vcpu, result >> 32); + kvm_eax_write(vcpu, result); } } @@ -2544,18 +2549,15 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu) #ifdef CONFIG_X86_64 if (is_64_bit_hypercall(vcpu)) { - hc.param = kvm_rcx_read(vcpu); - hc.ingpa = kvm_rdx_read(vcpu); - hc.outgpa = kvm_r8_read(vcpu); + hc.param = kvm_rcx_read_raw(vcpu); + hc.ingpa = kvm_rdx_read_raw(vcpu); + hc.outgpa = kvm_r8_read_raw(vcpu); } else #endif { - hc.param = ((u64)kvm_rdx_read(vcpu) << 32) | - (kvm_rax_read(vcpu) & 0xffffffff); - hc.ingpa = ((u64)kvm_rbx_read(vcpu) << 32) | - (kvm_rcx_read(vcpu) & 0xffffffff); - hc.outgpa = ((u64)kvm_rdi_read(vcpu) << 32) | - (kvm_rsi_read(vcpu) & 0xffffffff); + hc.param = ((u64)kvm_edx_read(vcpu) << 32) | kvm_eax_read(vcpu); + hc.ingpa = ((u64)kvm_ebx_read(vcpu) << 32) | kvm_ecx_read(vcpu); + hc.outgpa = ((u64)kvm_edi_read(vcpu) << 32) | kvm_esi_read(vcpu); } hc.code = hc.param & 0xffff; diff --git a/arch/x86/kvm/hyperv.h b/arch/x86/kvm/hyperv.h index 6301f79fcbae..65e89ed65349 100644 --- a/arch/x86/kvm/hyperv.h +++ b/arch/x86/kvm/hyperv.h @@ -232,8 +232,8 @@ static inline bool kvm_hv_is_tlb_flush_hcall(struct kvm_vcpu *vcpu) if (!hv_vcpu) return false; - code = is_64_bit_hypercall(vcpu) ? kvm_rcx_read(vcpu) : - kvm_rax_read(vcpu); + code = is_64_bit_hypercall(vcpu) ? kvm_rcx_read_raw(vcpu) : + kvm_eax_read(vcpu); return (code == HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE || code == HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST || diff --git a/arch/x86/kvm/ioapic.c b/arch/x86/kvm/ioapic.c index eed96ff6e722..f3f4a483ca15 100644 --- a/arch/x86/kvm/ioapic.c +++ b/arch/x86/kvm/ioapic.c @@ -441,7 +441,7 @@ static void ioapic_write_indirect(struct kvm_ioapic *ioapic, u32 val) irq.dest_id = old_dest_id; irq.dest_mode = kvm_lapic_irq_dest_mode( - !!e->fields.dest_mode); + !!old_dest_mode); kvm_bitmap_or_dest_vcpus(ioapic->kvm, &irq, vcpu_bitmap); } diff --git a/arch/x86/kvm/irq.c b/arch/x86/kvm/irq.c index 9519fec09ee6..8c62c6d4d5c1 100644 --- a/arch/x86/kvm/irq.c +++ b/arch/x86/kvm/irq.c @@ -585,12 +585,16 @@ int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) r = 0; switch (chip->chip_id) { case KVM_IRQCHIP_PIC_MASTER: + spin_lock(&pic->lock); memcpy(&chip->chip.pic, &pic->pics[0], sizeof(struct kvm_pic_state)); + spin_unlock(&pic->lock); break; case KVM_IRQCHIP_PIC_SLAVE: + spin_lock(&pic->lock); memcpy(&chip->chip.pic, &pic->pics[1], sizeof(struct kvm_pic_state)); + spin_unlock(&pic->lock); break; case KVM_IRQCHIP_IOAPIC: kvm_get_ioapic(kvm, &chip->chip.ioapic); diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h index 0abff36d0994..3e375af15c03 100644 --- a/arch/x86/kvm/kvm_emulate.h +++ b/arch/x86/kvm/kvm_emulate.h @@ -22,9 +22,13 @@ enum x86_intercept_stage; struct x86_exception { u8 vector; bool error_code_valid; - u16 error_code; + u64 error_code; bool nested_page_fault; - u64 address; /* cr2 or nested page fault gpa */ + union { + u64 address; /* cr2 or nested page fault gpa */ + unsigned long dr6; + u64 payload; + }; u8 async_page_fault; unsigned long exit_qualification; }; @@ -211,6 +215,7 @@ struct x86_emulate_ops { ulong (*get_cr)(struct x86_emulate_ctxt *ctxt, int cr); int (*set_cr)(struct x86_emulate_ctxt *ctxt, int cr, ulong val); int (*cpl)(struct x86_emulate_ctxt *ctxt); + ulong (*get_effective_dr7)(struct x86_emulate_ctxt *ctxt); ulong (*get_dr)(struct x86_emulate_ctxt *ctxt, int dr); int (*set_dr)(struct x86_emulate_ctxt *ctxt, int dr, ulong value); int (*set_msr_with_filter)(struct x86_emulate_ctxt *ctxt, u32 msr_index, u64 data); @@ -225,6 +230,7 @@ struct x86_emulate_ops { struct x86_instruction_info *info, enum x86_intercept_stage stage); + bool (*is_cpuid_allowed)(struct x86_emulate_ctxt *ctxt); bool (*get_cpuid)(struct x86_emulate_ctxt *ctxt, u32 *eax, u32 *ebx, u32 *ecx, u32 *edx, bool exact_only); bool (*guest_has_movbe)(struct x86_emulate_ctxt *ctxt); @@ -520,13 +526,6 @@ enum x86_intercept { nr_x86_intercepts }; -/* Host execution mode. */ -#if defined(CONFIG_X86_32) -#define X86EMUL_MODE_HOST X86EMUL_MODE_PROT32 -#elif defined(CONFIG_X86_64) -#define X86EMUL_MODE_HOST X86EMUL_MODE_PROT64 -#endif - int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len, int emulation_type); bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt); #define EMULATION_FAILED -1 diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 4078e624ca66..27cca31308bd 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -37,7 +37,7 @@ #include <asm/delay.h> #include <linux/atomic.h> #include <linux/jump_label.h> -#include "kvm_cache_regs.h" +#include "regs.h" #include "irq.h" #include "ioapic.h" #include "trace.h" @@ -2744,6 +2744,32 @@ u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) return (tpr & 0xf0) >> 4; } +void kvm_lapic_update_cr8_intercept(struct kvm_vcpu *vcpu) +{ + int max_irr, tpr; + + if (!kvm_x86_ops.update_cr8_intercept) + return; + + if (!lapic_in_kernel(vcpu)) + return; + + if (vcpu->arch.apic->apicv_active) + return; + + if (!vcpu->arch.apic->vapic_addr) + max_irr = kvm_lapic_find_highest_irr(vcpu); + else + max_irr = -1; + + if (max_irr != -1) + max_irr >>= 4; + + tpr = kvm_lapic_get_cr8(vcpu); + + kvm_x86_call(update_cr8_intercept)(vcpu, tpr, max_irr); +} + static void __kvm_apic_set_base(struct kvm_vcpu *vcpu, u64 value) { u64 old_value = vcpu->arch.apic_base; diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index 274885af4ebc..533581d06151 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -100,6 +100,7 @@ int kvm_apic_accept_events(struct kvm_vcpu *vcpu); void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event); u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu); void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8); +void kvm_lapic_update_cr8_intercept(struct kvm_vcpu *vcpu); void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu); void kvm_apic_set_version(struct kvm_vcpu *vcpu); void kvm_apic_after_set_mcg_cap(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h index ddf4e467c071..e1bb663ebbd5 100644 --- a/arch/x86/kvm/mmu.h +++ b/arch/x86/kvm/mmu.h @@ -3,7 +3,7 @@ #define __KVM_X86_MMU_H #include <linux/kvm_host.h> -#include "kvm_cache_regs.h" +#include "regs.h" #include "x86.h" #include "cpuid.h" diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index c87c26bf4149..b8f2edf2cfeb 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -22,7 +22,7 @@ #include "mmu_internal.h" #include "tdp_mmu.h" #include "x86.h" -#include "kvm_cache_regs.h" +#include "regs.h" #include "smm.h" #include "kvm_emulate.h" #include "page_track.h" diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 07100bbfc270..df3ae0c7ec2c 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -328,6 +328,12 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, const int write_fault = access & PFERR_WRITE_MASK; const int user_fault = access & PFERR_USER_MASK; const int fetch_fault = access & PFERR_FETCH_MASK; + /* + * Note! Track the error_code that's common to legacy shadow paging + * and NPT shadow paging as a u16 to guard against unintentionally + * setting any of bits 63:16. Architecturally, the #PF error code is + * 32 bits, and Intel CPUs don't support settings bits 31:16. + */ u16 errcode = 0; gpa_t real_gpa; gfn_t gfn; @@ -391,16 +397,6 @@ retry_walk: nested_access | PFERR_GUEST_PAGE_MASK, &walker->fault, 0); - /* - * FIXME: This can happen if emulation (for of an INS/OUTS - * instruction) triggers a nested page fault. The exit - * qualification / exit info field will incorrectly have - * "guest page access" as the nested page fault's cause, - * instead of "guest page structure access". To fix this, - * the x86_exception struct should be augmented with enough - * information to fix the exit_qualification or exit_info_1 - * fields. - */ if (unlikely(real_gpa == INVALID_GPA)) return 0; @@ -506,7 +502,8 @@ error: * [2:0] - Derive from the access bits. The exit_qualification might be * out of date if it is serving an EPT misconfiguration. * [5:3] - Calculated by the page walk of the guest EPT page tables - * [7:11] - Derived from [7:11] of real exit_qualification + * [7:8] - Derived from "fault stage" access bits + * [9:11] - Derived from [9:11] of real exit_qualification * * The other bits are set to 0. */ @@ -521,12 +518,22 @@ error: walker->fault.exit_qualification |= EPT_VIOLATION_ACC_READ; /* + * KVM doesn't emulate features that access GPAs directly, e.g. + * Intel Processor Trace. Assume the GVA is always valid; when + * propagating faults from hardware, KVM will discard this info + * and use the EXIT_QUALIFICATION bits from the VMCS. + */ + walker->fault.exit_qualification |= EPT_VIOLATION_GVA_IS_VALID; + + /* * Accesses to guest paging structures are either "reads" or * "read+write" accesses, so consider them the latter if write_fault * is true. */ if (access & PFERR_GUEST_PAGE_MASK) walker->fault.exit_qualification |= EPT_VIOLATION_ACC_READ; + else + walker->fault.exit_qualification |= EPT_VIOLATION_GVA_TRANSLATED; /* * Note, pte_access holds the raw RWX bits from the EPTE, not @@ -542,6 +549,11 @@ error: walker->fault.nested_page_fault = mmu != vcpu->arch.walk_mmu; walker->fault.async_page_fault = false; +#if PTTYPE != PTTYPE_EPT + if (walker->fault.nested_page_fault) + walker->fault.error_code |= access & PFERR_GUEST_FAULT_STAGE_MASK; +#endif + trace_kvm_mmu_walker_error(walker->fault.error_code); return 0; } @@ -807,7 +819,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault */ if (!r) { if (!fault->prefetch) - kvm_inject_emulated_page_fault(vcpu, &walker.fault); + __kvm_inject_emulated_page_fault(vcpu, &walker.fault, true); return RET_PF_RETRY; } diff --git a/arch/x86/kvm/kvm_cache_regs.h b/arch/x86/kvm/regs.h index 2ae492ad6412..a57ba26279ed 100644 --- a/arch/x86/kvm/kvm_cache_regs.h +++ b/arch/x86/kvm/regs.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef ASM_KVM_CACHE_REGS_H -#define ASM_KVM_CACHE_REGS_H +#ifndef ARCH_X86_KVM_REGS_H +#define ARCH_X86_KVM_REGS_H #include <linux/kvm_host.h> @@ -16,32 +16,91 @@ static_assert(!(KVM_POSSIBLE_CR0_GUEST_BITS & X86_CR0_PDPTR_BITS)); -#define BUILD_KVM_GPR_ACCESSORS(lname, uname) \ -static __always_inline unsigned long kvm_##lname##_read(struct kvm_vcpu *vcpu)\ -{ \ - return vcpu->arch.regs[VCPU_REGS_##uname]; \ -} \ -static __always_inline void kvm_##lname##_write(struct kvm_vcpu *vcpu, \ - unsigned long val) \ -{ \ - vcpu->arch.regs[VCPU_REGS_##uname] = val; \ -} -BUILD_KVM_GPR_ACCESSORS(rax, RAX) -BUILD_KVM_GPR_ACCESSORS(rbx, RBX) -BUILD_KVM_GPR_ACCESSORS(rcx, RCX) -BUILD_KVM_GPR_ACCESSORS(rdx, RDX) -BUILD_KVM_GPR_ACCESSORS(rbp, RBP) -BUILD_KVM_GPR_ACCESSORS(rsi, RSI) -BUILD_KVM_GPR_ACCESSORS(rdi, RDI) +static inline bool is_long_mode(struct kvm_vcpu *vcpu) +{ +#ifdef CONFIG_X86_64 + return !!(vcpu->arch.efer & EFER_LMA); +#else + return false; +#endif +} + +static inline bool is_64_bit_mode(struct kvm_vcpu *vcpu) +{ + int cs_db, cs_l; + + WARN_ON_ONCE(vcpu->arch.guest_state_protected); + + if (!is_long_mode(vcpu)) + return false; + kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l); + return cs_l; +} + +static inline bool is_64_bit_hypercall(struct kvm_vcpu *vcpu) +{ #ifdef CONFIG_X86_64 -BUILD_KVM_GPR_ACCESSORS(r8, R8) -BUILD_KVM_GPR_ACCESSORS(r9, R9) -BUILD_KVM_GPR_ACCESSORS(r10, R10) -BUILD_KVM_GPR_ACCESSORS(r11, R11) -BUILD_KVM_GPR_ACCESSORS(r12, R12) -BUILD_KVM_GPR_ACCESSORS(r13, R13) -BUILD_KVM_GPR_ACCESSORS(r14, R14) -BUILD_KVM_GPR_ACCESSORS(r15, R15) + /* + * If running with protected guest state, the CS register is not + * accessible. The hypercall register values will have had to been + * provided in 64-bit mode, so assume the guest is in 64-bit. + */ + return vcpu->arch.guest_state_protected || is_64_bit_mode(vcpu); +#else + return false; +#endif +} + +static __always_inline unsigned long kvm_reg_mode_mask(struct kvm_vcpu *vcpu) +{ +#ifdef CONFIG_X86_64 + return is_64_bit_mode(vcpu) ? GENMASK(63, 0) : GENMASK(31, 0); +#else + return GENMASK(31, 0); +#endif +} + +#define __BUILD_KVM_GPR_ACCESSORS(lname, uname) \ +static __always_inline unsigned long kvm_##lname##_read(struct kvm_vcpu *vcpu) \ +{ \ + return vcpu->arch.regs[VCPU_REGS_##uname] & kvm_reg_mode_mask(vcpu); \ +} \ +static __always_inline unsigned long kvm_##lname##_read_raw(struct kvm_vcpu *vcpu) \ +{ \ + return vcpu->arch.regs[VCPU_REGS_##uname]; \ +} \ +static __always_inline void kvm_##lname##_write_raw(struct kvm_vcpu *vcpu, \ + unsigned long val) \ +{ \ + vcpu->arch.regs[VCPU_REGS_##uname] = val; \ +} +#define BUILD_KVM_GPR_ACCESSORS(lname, uname) \ +static __always_inline u32 kvm_e##lname##_read(struct kvm_vcpu *vcpu) \ +{ \ + return vcpu->arch.regs[VCPU_REGS_##uname]; \ +} \ +static __always_inline void kvm_e##lname##_write(struct kvm_vcpu *vcpu, u32 val) \ +{ \ + vcpu->arch.regs[VCPU_REGS_##uname] = val; \ +} \ +__BUILD_KVM_GPR_ACCESSORS(r##lname, uname) + +BUILD_KVM_GPR_ACCESSORS(ax, RAX) +BUILD_KVM_GPR_ACCESSORS(bx, RBX) +BUILD_KVM_GPR_ACCESSORS(cx, RCX) +BUILD_KVM_GPR_ACCESSORS(dx, RDX) +BUILD_KVM_GPR_ACCESSORS(bp, RBP) +BUILD_KVM_GPR_ACCESSORS(si, RSI) +BUILD_KVM_GPR_ACCESSORS(di, RDI) +#ifdef CONFIG_X86_64 +__BUILD_KVM_GPR_ACCESSORS(r8, R8) +__BUILD_KVM_GPR_ACCESSORS(r9, R9) +__BUILD_KVM_GPR_ACCESSORS(r10, R10) +__BUILD_KVM_GPR_ACCESSORS(r11, R11) +__BUILD_KVM_GPR_ACCESSORS(r12, R12) +__BUILD_KVM_GPR_ACCESSORS(r13, R13) +__BUILD_KVM_GPR_ACCESSORS(r14, R14) +__BUILD_KVM_GPR_ACCESSORS(r15, R15) #endif /* @@ -143,6 +202,11 @@ static inline unsigned long kvm_register_read_raw(struct kvm_vcpu *vcpu, int reg return vcpu->arch.regs[reg]; } +static inline unsigned long kvm_register_read(struct kvm_vcpu *vcpu, int reg) +{ + return kvm_register_read_raw(vcpu, reg) & kvm_reg_mode_mask(vcpu); +} + static inline void kvm_register_write_raw(struct kvm_vcpu *vcpu, int reg, unsigned long val) { @@ -153,6 +217,12 @@ static inline void kvm_register_write_raw(struct kvm_vcpu *vcpu, int reg, kvm_register_mark_dirty(vcpu, reg); } +static inline void kvm_register_write(struct kvm_vcpu *vcpu, + int reg, unsigned long val) +{ + return kvm_register_write_raw(vcpu, reg, val & kvm_reg_mode_mask(vcpu)); +} + static inline unsigned long kvm_rip_read(struct kvm_vcpu *vcpu) { if (!kvm_register_is_available(vcpu, VCPU_REG_RIP)) @@ -177,6 +247,11 @@ static inline void kvm_rsp_write(struct kvm_vcpu *vcpu, unsigned long val) kvm_register_write_raw(vcpu, VCPU_REGS_RSP, val); } +static inline u64 kvm_read_edx_eax(struct kvm_vcpu *vcpu) +{ + return kvm_eax_read(vcpu) | (u64)(kvm_edx_read(vcpu)) << 32; +} + static inline u64 kvm_pdptr_read(struct kvm_vcpu *vcpu, int index) { might_sleep(); /* on svm */ @@ -243,10 +318,75 @@ static inline ulong kvm_read_cr4(struct kvm_vcpu *vcpu) return kvm_read_cr4_bits(vcpu, ~0UL); } -static inline u64 kvm_read_edx_eax(struct kvm_vcpu *vcpu) +static inline bool __kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) +{ + return !(cr4 & vcpu->arch.cr4_guest_rsvd_bits); +} + +#define __cr4_reserved_bits(__cpu_has, __c) \ +({ \ + u64 __reserved_bits = CR4_RESERVED_BITS; \ + \ + if (!__cpu_has(__c, X86_FEATURE_XSAVE)) \ + __reserved_bits |= X86_CR4_OSXSAVE; \ + if (!__cpu_has(__c, X86_FEATURE_SMEP)) \ + __reserved_bits |= X86_CR4_SMEP; \ + if (!__cpu_has(__c, X86_FEATURE_SMAP)) \ + __reserved_bits |= X86_CR4_SMAP; \ + if (!__cpu_has(__c, X86_FEATURE_FSGSBASE)) \ + __reserved_bits |= X86_CR4_FSGSBASE; \ + if (!__cpu_has(__c, X86_FEATURE_PKU)) \ + __reserved_bits |= X86_CR4_PKE; \ + if (!__cpu_has(__c, X86_FEATURE_LA57)) \ + __reserved_bits |= X86_CR4_LA57; \ + if (!__cpu_has(__c, X86_FEATURE_UMIP)) \ + __reserved_bits |= X86_CR4_UMIP; \ + if (!__cpu_has(__c, X86_FEATURE_VMX)) \ + __reserved_bits |= X86_CR4_VMXE; \ + if (!__cpu_has(__c, X86_FEATURE_PCID)) \ + __reserved_bits |= X86_CR4_PCIDE; \ + if (!__cpu_has(__c, X86_FEATURE_LAM)) \ + __reserved_bits |= X86_CR4_LAM_SUP; \ + if (!__cpu_has(__c, X86_FEATURE_SHSTK) && \ + !__cpu_has(__c, X86_FEATURE_IBT)) \ + __reserved_bits |= X86_CR4_CET; \ + __reserved_bits; \ +}) + +static inline bool is_protmode(struct kvm_vcpu *vcpu) +{ + return kvm_is_cr0_bit_set(vcpu, X86_CR0_PE); +} + +static inline bool is_pae(struct kvm_vcpu *vcpu) +{ + return kvm_is_cr4_bit_set(vcpu, X86_CR4_PAE); +} + +static inline bool is_pse(struct kvm_vcpu *vcpu) +{ + return kvm_is_cr4_bit_set(vcpu, X86_CR4_PSE); +} + +static inline bool is_paging(struct kvm_vcpu *vcpu) +{ + return likely(kvm_is_cr0_bit_set(vcpu, X86_CR0_PG)); +} + +static inline bool is_pae_paging(struct kvm_vcpu *vcpu) +{ + return !is_long_mode(vcpu) && is_pae(vcpu) && is_paging(vcpu); +} + +static inline bool kvm_dr7_valid(u64 data) +{ + /* Bits [63:32] are reserved */ + return !(data >> 32); +} +static inline bool kvm_dr6_valid(u64 data) { - return (kvm_rax_read(vcpu) & -1u) - | ((u64)(kvm_rdx_read(vcpu) & -1u) << 32); + /* Bits [63:32] are reserved */ + return !(data >> 32); } static inline void enter_guest_mode(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/smm.c b/arch/x86/kvm/smm.c index f623c5986119..a446487bdd5c 100644 --- a/arch/x86/kvm/smm.c +++ b/arch/x86/kvm/smm.c @@ -3,7 +3,7 @@ #include <linux/kvm_host.h> #include "x86.h" -#include "kvm_cache_regs.h" +#include "regs.h" #include "kvm_emulate.h" #include "smm.h" #include "cpuid.h" diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 1bf3e4804ad0..a09038d3539b 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -34,23 +34,37 @@ #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, - struct x86_exception *fault) + struct x86_exception *fault, + bool from_hardware) { struct vcpu_svm *svm = to_svm(vcpu); struct vmcb *vmcb = svm->vmcb; + u64 fault_stage; - if (vmcb->control.exit_code != SVM_EXIT_NPF) { - /* - * TODO: track the cause of the nested page fault, and - * correctly fill in the high bits of exit_info_1. - */ - vmcb->control.exit_code = SVM_EXIT_NPF; - vmcb->control.exit_info_1 = (1ULL << 32); - vmcb->control.exit_info_2 = fault->address; - } + /* + * For hardware NPF exits, the GUEST_FAULT_STAGE bits are only + * available in the hardware exit_info_1, since the guest_mmu + * walker doesn't know whether the faulting GPA was a page table + * page or final page from L2's perspective. + */ + if (from_hardware) + fault_stage = vmcb->control.exit_info_1 & + PFERR_GUEST_FAULT_STAGE_MASK; + else + fault_stage = fault->error_code & PFERR_GUEST_FAULT_STAGE_MASK; + + /* + * All nested page faults should be annotated as occurring on the + * final translation *or* the page walk. Arbitrarily choose "final" + * if KVM is buggy and enumerated both or neither. + */ + if (WARN_ON_ONCE(hweight64(fault_stage) != 1)) + fault_stage = PFERR_GUEST_FINAL_MASK; - vmcb->control.exit_info_1 &= ~0xffffffffULL; - vmcb->control.exit_info_1 |= fault->error_code; + vmcb->control.exit_code = SVM_EXIT_NPF; + vmcb->control.exit_info_1 = fault_stage | + (fault->error_code & ~PFERR_GUEST_FAULT_STAGE_MASK); + vmcb->control.exit_info_2 = fault->address; nested_svm_vmexit(svm); } @@ -771,7 +785,7 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) svm->vcpu.arch.cr2 = save->cr2; - kvm_rax_write(vcpu, save->rax); + kvm_rax_write_raw(vcpu, save->rax); kvm_rsp_write(vcpu, save->rsp); kvm_rip_write(vcpu, save->rip); @@ -1112,7 +1126,7 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu) if (WARN_ON_ONCE(!svm->nested.initialized)) return -EINVAL; - vmcb12_gpa = kvm_register_read(vcpu, VCPU_REGS_RAX); + vmcb12_gpa = kvm_rax_read(vcpu); if (!page_address_valid(vcpu, vmcb12_gpa)) { kvm_inject_gp(vcpu, 0); return 1; @@ -1237,7 +1251,7 @@ static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu) vmcb12->save.rflags = kvm_get_rflags(vcpu); vmcb12->save.rip = kvm_rip_read(vcpu); vmcb12->save.rsp = kvm_rsp_read(vcpu); - vmcb12->save.rax = kvm_rax_read(vcpu); + vmcb12->save.rax = kvm_rax_read_raw(vcpu); vmcb12->save.dr7 = vmcb02->save.dr7; vmcb12->save.dr6 = svm->vcpu.arch.dr6; vmcb12->save.cpl = vmcb02->save.cpl; @@ -1384,7 +1398,7 @@ void nested_svm_vmexit(struct vcpu_svm *svm) svm_set_efer(vcpu, vmcb01->save.efer); svm_set_cr0(vcpu, vmcb01->save.cr0 | X86_CR0_PE); svm_set_cr4(vcpu, vmcb01->save.cr4); - kvm_rax_write(vcpu, vmcb01->save.rax); + kvm_rax_write_raw(vcpu, vmcb01->save.rax); kvm_rsp_write(vcpu, vmcb01->save.rsp); kvm_rip_write(vcpu, vmcb01->save.rip); diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 0c3962647116..0fde900ba8dd 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -3711,9 +3711,13 @@ static int snp_rmptable_psmash(kvm_pfn_t pfn) static int snp_complete_psc_msr(struct kvm_vcpu *vcpu) { + u64 hypercall_ret = READ_ONCE(vcpu->run->hypercall.ret); struct vcpu_svm *svm = to_svm(vcpu); - if (vcpu->run->hypercall.ret) + if (!kvm_is_valid_map_gpa_range_ret(hypercall_ret)) + return -EINVAL; + + if (hypercall_ret) set_ghcb_msr(svm, GHCB_MSR_PSC_RESP_ERROR); else set_ghcb_msr(svm, GHCB_MSR_PSC_RESP); @@ -3802,9 +3806,13 @@ static void __snp_complete_one_psc(struct vcpu_svm *svm) static int snp_complete_one_psc(struct kvm_vcpu *vcpu) { + u64 hypercall_ret = READ_ONCE(vcpu->run->hypercall.ret); struct vcpu_svm *svm = to_svm(vcpu); - if (vcpu->run->hypercall.ret) { + if (!kvm_is_valid_map_gpa_range_ret(hypercall_ret)) + return -EINVAL; + + if (hypercall_ret) { snp_complete_psc(svm, VMGEXIT_PSC_ERROR_GENERIC); return 1; /* resume guest */ } diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 81e5a889a794..74dcc7b5b07a 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -4,7 +4,7 @@ #include "irq.h" #include "mmu.h" -#include "kvm_cache_regs.h" +#include "regs.h" #include "x86.h" #include "smm.h" #include "cpuid.h" @@ -2217,7 +2217,7 @@ static int intr_interception(struct kvm_vcpu *vcpu) static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload) { - u64 vmcb12_gpa = kvm_register_read(vcpu, VCPU_REGS_RAX); + u64 vmcb12_gpa = kvm_rax_read(vcpu); struct vcpu_svm *svm = to_svm(vcpu); struct vmcb *vmcb12; struct kvm_host_map map; @@ -2325,7 +2325,7 @@ static int gp_interception(struct kvm_vcpu *vcpu) if (nested_svm_check_permissions(vcpu)) return 1; - if (!page_address_valid(vcpu, kvm_register_read(vcpu, VCPU_REGS_RAX))) + if (!page_address_valid(vcpu, kvm_rax_read(vcpu))) goto reinject; /* @@ -2408,16 +2408,13 @@ static int clgi_interception(struct kvm_vcpu *vcpu) static int invlpga_interception(struct kvm_vcpu *vcpu) { + /* FIXME: Handle an address size prefix. */ gva_t gva = kvm_rax_read(vcpu); - u32 asid = kvm_rcx_read(vcpu); + u32 asid = kvm_ecx_read(vcpu); if (nested_svm_check_permissions(vcpu)) return 1; - /* FIXME: Handle an address size prefix. */ - if (!is_long_mode(vcpu)) - gva = (u32)gva; - trace_kvm_invlpga(to_svm(vcpu)->vmcb->save.rip, asid, gva); /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */ @@ -3674,13 +3671,8 @@ static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) struct vcpu_svm *svm = to_svm(vcpu); struct kvm_run *kvm_run = vcpu->run; - /* SEV-ES guests must use the CR write traps to track CR registers. */ - if (!is_sev_es_guest(vcpu)) { - if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE)) - vcpu->arch.cr0 = svm->vmcb->save.cr0; - if (npt_enabled) - vcpu->arch.cr3 = svm->vmcb->save.cr3; - } + if (unlikely(exit_fastpath == EXIT_FASTPATH_EXIT_USERSPACE)) + return 0; if (is_guest_mode(vcpu)) { int vmexit; @@ -4535,11 +4527,17 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL)) x86_spec_ctrl_restore_host(svm->virt_spec_ctrl); + /* SEV-ES guests must use the CR write traps to track CR registers. */ if (!is_sev_es_guest(vcpu)) { vcpu->arch.cr2 = svm->vmcb->save.cr2; vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax; vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; vcpu->arch.rip = svm->vmcb->save.rip; + + if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE)) + vcpu->arch.cr0 = svm->vmcb->save.cr0; + if (npt_enabled) + vcpu->arch.cr3 = svm->vmcb->save.cr3; } kvm_reset_dirty_registers(vcpu); diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index f6c1b348ec8d..d013a5ceb62e 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -23,7 +23,7 @@ #include <asm/sev-common.h> #include "cpuid.h" -#include "kvm_cache_regs.h" +#include "regs.h" /* * Helpers to convert to/from physical addresses for pages whose address is diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 4690a4d23709..b2c851cc7d5c 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -411,7 +411,8 @@ static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp, } static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, - struct x86_exception *fault) + struct x86_exception *fault, + bool from_hardware) { struct vmcs12 *vmcs12 = get_vmcs12(vcpu); struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -444,13 +445,29 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, exit_qualification = 0; } else { u64 mask = EPT_VIOLATION_GVA_IS_VALID | - EPT_VIOLATION_GVA_TRANSLATED; + EPT_VIOLATION_GVA_TRANSLATED; + if (vmx->nested.msrs.ept_caps & VMX_EPT_ADVANCED_VMEXIT_INFO_BIT) mask |= EPT_VIOLATION_GVA_USER | - EPT_VIOLATION_GVA_WRITABLE | - EPT_VIOLATION_GVA_NX; - exit_qualification = fault->exit_qualification; - exit_qualification |= vmx_get_exit_qual(vcpu) & mask; + EPT_VIOLATION_GVA_WRITABLE | + EPT_VIOLATION_GVA_NX; + + exit_qualification = fault->exit_qualification & ~mask; + + /* + * Use the EXIT_QUALIFICATION from the VMCS if and only + * if the hardware VM-Exit from L2 was an EPT Violation. + * If the fault is synthesized, then EXIT_QUALIFICATION + * is stale and/or holds entirely different data. And + * conversely, KVM _must_ rely on EXIT_QUALIFICATION if + * the fault came from hardware, because KVM only sees + * and walks the faulting GPA. + */ + if (from_hardware) + exit_qualification |= vmx_get_exit_qual(vcpu) & mask; + else + exit_qualification |= fault->exit_qualification & mask; + vm_exit_reason = EXIT_REASON_EPT_VIOLATION; } @@ -6148,7 +6165,7 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { - u32 index = kvm_rcx_read(vcpu); + u32 index = kvm_ecx_read(vcpu); u64 new_eptp; if (WARN_ON_ONCE(!nested_cpu_has_ept(vmcs12))) @@ -6182,7 +6199,7 @@ static int handle_vmfunc(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); struct vmcs12 *vmcs12; - u32 function = kvm_rax_read(vcpu); + u32 function = kvm_eax_read(vcpu); /* * VMFUNC should never execute cleanly while L1 is active; KVM supports @@ -6304,7 +6321,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, exit_reason.basic == EXIT_REASON_MSR_WRITE_IMM) msr_index = vmx_get_exit_qual(vcpu); else - msr_index = kvm_rcx_read(vcpu); + msr_index = kvm_ecx_read(vcpu); /* * The MSR_BITMAP page is divided into four 1024-byte bitmaps, @@ -6414,7 +6431,7 @@ static bool nested_vmx_exit_handled_encls(struct kvm_vcpu *vcpu, !nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENCLS_EXITING)) return false; - encls_leaf = kvm_rax_read(vcpu); + encls_leaf = kvm_eax_read(vcpu); if (encls_leaf > 62) encls_leaf = 63; return vmcs12->encls_exiting_bitmap & BIT_ULL(encls_leaf); @@ -6535,6 +6552,8 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, nested_evmcs_l2_tlb_flush_enabled(vcpu) && kvm_hv_is_tlb_flush_hcall(vcpu); #endif + case EXIT_REASON_CPUID: + return !kvm_is_cpuid_allowed(vcpu); default: break; } diff --git a/arch/x86/kvm/vmx/nested.h b/arch/x86/kvm/vmx/nested.h index 213a448104af..6d6cd5904ddf 100644 --- a/arch/x86/kvm/vmx/nested.h +++ b/arch/x86/kvm/vmx/nested.h @@ -2,7 +2,7 @@ #ifndef __KVM_X86_VMX_NESTED_H #define __KVM_X86_VMX_NESTED_H -#include "kvm_cache_regs.h" +#include "regs.h" #include "hyperv.h" #include "vmcs12.h" #include "vmx.h" diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 27eb76e6b6a0..3281cc3e95c6 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -56,8 +56,16 @@ static struct x86_pmu_lbr *vcpu_to_lbr_records(struct kvm_vcpu *vcpu) static void reprogram_fixed_counters(struct kvm_pmu *pmu, u64 data) { + /* + * Compare against the value the mediated PMU shoves into hardware, not + * the guest's desired value. For the emulated PMU (proxied via perf), + * they are one and the same (fixed_ctr_ctrl_hw isn't used other than + * here). For the mediated PMU, KVM needs to reprogram the actual MSR, + * and so needs to react to potential changes in the value shoved into + * hardware, e.g. to ensure the event filter is enforced. + */ + u64 old_fixed_ctr_ctrl = pmu->fixed_ctr_ctrl_hw; struct kvm_pmc *pmc; - u64 old_fixed_ctr_ctrl = pmu->fixed_ctr_ctrl; int i; pmu->fixed_ctr_ctrl = data; diff --git a/arch/x86/kvm/vmx/sgx.c b/arch/x86/kvm/vmx/sgx.c index df1d0cf76947..876dc2814108 100644 --- a/arch/x86/kvm/vmx/sgx.c +++ b/arch/x86/kvm/vmx/sgx.c @@ -6,7 +6,7 @@ #include <asm/sgx.h> #include "x86.h" -#include "kvm_cache_regs.h" +#include "regs.h" #include "nested.h" #include "sgx.h" #include "vmx.h" @@ -352,7 +352,7 @@ static int handle_encls_einit(struct kvm_vcpu *vcpu) rflags &= ~X86_EFLAGS_ZF; vmx_set_rflags(vcpu, rflags); - kvm_rax_write(vcpu, ret); + kvm_eax_write(vcpu, ret); return kvm_skip_emulated_instruction(vcpu); } @@ -380,7 +380,7 @@ static inline bool sgx_enabled_in_guest_bios(struct kvm_vcpu *vcpu) int handle_encls(struct kvm_vcpu *vcpu) { - u32 leaf = (u32)kvm_rax_read(vcpu); + u32 leaf = kvm_eax_read(vcpu); if (!enable_sgx || !guest_cpu_cap_has(vcpu, X86_FEATURE_SGX) || !guest_cpu_cap_has(vcpu, X86_FEATURE_SGX1)) { diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index b8c3d3d8bbfe..6089aa8f9053 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -639,6 +639,12 @@ int tdx_vm_init(struct kvm *kvm) kvm->arch.disabled_quirks |= KVM_X86_QUIRK_IGNORE_GUEST_PAT; /* + * PMU support is provided by the TDX-Module (if enabled for the VM). + * From KVM's perspective, the VM doesn't have a virtual PMU. + */ + kvm->arch.has_protected_pmu = true; + + /* * Because guest TD is protected, VMM can't parse the instruction in TD. * Instead, guest uses MMIO hypercall. For unmodified device driver, * #VE needs to be injected for MMIO and #VE handler in TD converts MMIO @@ -1163,11 +1169,11 @@ static int complete_hypercall_exit(struct kvm_vcpu *vcpu) static int tdx_emulate_vmcall(struct kvm_vcpu *vcpu) { - kvm_rax_write(vcpu, to_tdx(vcpu)->vp_enter_args.r10); - kvm_rbx_write(vcpu, to_tdx(vcpu)->vp_enter_args.r11); - kvm_rcx_write(vcpu, to_tdx(vcpu)->vp_enter_args.r12); - kvm_rdx_write(vcpu, to_tdx(vcpu)->vp_enter_args.r13); - kvm_rsi_write(vcpu, to_tdx(vcpu)->vp_enter_args.r14); + kvm_rax_write_raw(vcpu, to_tdx(vcpu)->vp_enter_args.r10); + kvm_rbx_write_raw(vcpu, to_tdx(vcpu)->vp_enter_args.r11); + kvm_rcx_write_raw(vcpu, to_tdx(vcpu)->vp_enter_args.r12); + kvm_rdx_write_raw(vcpu, to_tdx(vcpu)->vp_enter_args.r13); + kvm_rsi_write_raw(vcpu, to_tdx(vcpu)->vp_enter_args.r14); return __kvm_emulate_hypercall(vcpu, 0, complete_hypercall_exit); } @@ -1182,12 +1188,22 @@ static void __tdx_map_gpa(struct vcpu_tdx *tdx); static int tdx_complete_vmcall_map_gpa(struct kvm_vcpu *vcpu) { + u64 hypercall_ret = READ_ONCE(vcpu->run->hypercall.ret); struct vcpu_tdx *tdx = to_tdx(vcpu); + long rc; - if (vcpu->run->hypercall.ret) { - tdvmcall_set_return_code(vcpu, TDVMCALL_STATUS_INVALID_OPERAND); - tdx->vp_enter_args.r11 = tdx->map_gpa_next; - return 1; + switch (hypercall_ret) { + case 0: + break; + case EAGAIN: + rc = TDVMCALL_STATUS_RETRY; + goto propagate_error; + case EINVAL: + rc = TDVMCALL_STATUS_INVALID_OPERAND; + goto propagate_error; + default: + WARN_ON_ONCE(kvm_is_valid_map_gpa_range_ret(hypercall_ret)); + return -EINVAL; } tdx->map_gpa_next += TDX_MAP_GPA_MAX_LEN; @@ -1200,13 +1216,17 @@ static int tdx_complete_vmcall_map_gpa(struct kvm_vcpu *vcpu) * TDVMCALL_MAP_GPA, see comments in tdx_protected_apic_has_interrupt(). */ if (kvm_vcpu_has_events(vcpu)) { - tdvmcall_set_return_code(vcpu, TDVMCALL_STATUS_RETRY); - tdx->vp_enter_args.r11 = tdx->map_gpa_next; - return 1; + rc = TDVMCALL_STATUS_RETRY; + goto propagate_error; } __tdx_map_gpa(tdx); return 0; + +propagate_error: + tdvmcall_set_return_code(vcpu, rc); + tdx->vp_enter_args.r11 = tdx->map_gpa_next; + return 1; } static void __tdx_map_gpa(struct vcpu_tdx *tdx) @@ -2031,12 +2051,12 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) case EXIT_REASON_IO_INSTRUCTION: return tdx_emulate_io(vcpu); case EXIT_REASON_MSR_READ: - kvm_rcx_write(vcpu, tdx->vp_enter_args.r12); + kvm_ecx_write(vcpu, tdx->vp_enter_args.r12); return kvm_emulate_rdmsr(vcpu); case EXIT_REASON_MSR_WRITE: - kvm_rcx_write(vcpu, tdx->vp_enter_args.r12); - kvm_rax_write(vcpu, tdx->vp_enter_args.r13 & -1u); - kvm_rdx_write(vcpu, tdx->vp_enter_args.r13 >> 32); + kvm_ecx_write(vcpu, tdx->vp_enter_args.r12); + kvm_eax_write(vcpu, tdx->vp_enter_args.r13); + kvm_edx_write(vcpu, tdx->vp_enter_args.r13 >> 32); return kvm_emulate_wrmsr(vcpu); case EXIT_REASON_EPT_MISCONFIG: return tdx_emulate_mmio(vcpu); diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 9324d6083941..c06a91e34aff 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -59,7 +59,7 @@ #include "hyperv.h" #include "kvm_onhyperv.h" #include "irq.h" -#include "kvm_cache_regs.h" +#include "regs.h" #include "lapic.h" #include "mmu.h" #include "nested.h" @@ -1912,6 +1912,24 @@ void vmx_inject_exception(struct kvm_vcpu *vcpu) u32 intr_info = ex->vector | INTR_INFO_VALID_MASK; struct vcpu_vmx *vmx = to_vmx(vcpu); + /* + * When injecting a #DB, single-stepping is enabled in RFLAGS, and STI + * or MOV-SS blocking is active, set vmcs.PENDING_DBG_EXCEPTIONS.BS to + * prevent a false positive from VM-Entry consistency check. VM-Entry + * asserts that a single-step #DB _must_ be pending in this scenario, + * as the previous instruction cannot have toggled RFLAGS.TF 0=>1 + * (because STI and POP/MOV don't modify RFLAGS), therefore the one + * instruction delay when activating single-step breakpoints must have + * already expired. However, the CPU isn't smart enough to peek at + * vmcs.VM_ENTRY_INTR_INFO_FIELD and so doesn't realize that yes, there + * is indeed a #DB pending/imminent. + */ + if (ex->vector == DB_VECTOR && + (vmx_get_rflags(vcpu) & X86_EFLAGS_TF) && + vmx_get_interrupt_shadow(vcpu)) + vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, + vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS) | DR6_BS); + kvm_deliver_exception_payload(vcpu, ex); if (ex->has_error_code) { @@ -5495,26 +5513,9 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) * avoid single-step #DB and MTF updates, as ICEBP is * higher priority. Note, skipping ICEBP still clears * STI and MOVSS blocking. - * - * For all other #DBs, set vmcs.PENDING_DBG_EXCEPTIONS.BS - * if single-step is enabled in RFLAGS and STI or MOVSS - * blocking is active, as the CPU doesn't set the bit - * on VM-Exit due to #DB interception. VM-Entry has a - * consistency check that a single-step #DB is pending - * in this scenario as the previous instruction cannot - * have toggled RFLAGS.TF 0=>1 (because STI and POP/MOV - * don't modify RFLAGS), therefore the one instruction - * delay when activating single-step breakpoints must - * have already expired. Note, the CPU sets/clears BS - * as appropriate for all other VM-Exits types. */ if (is_icebp(intr_info)) WARN_ON(!skip_emulated_instruction(vcpu)); - else if ((vmx_get_rflags(vcpu) & X86_EFLAGS_TF) && - (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & - (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS))) - vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, - vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS) | DR6_BS); kvm_queue_exception_p(vcpu, DB_VECTOR, dr6); return 1; @@ -6715,6 +6716,9 @@ static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) if (enable_pml && !is_guest_mode(vcpu)) vmx_flush_pml_buffer(vcpu); + if (unlikely(exit_fastpath == EXIT_FASTPATH_EXIT_USERSPACE)) + return 0; + /* * KVM should never reach this point with a pending nested VM-Enter. * More specifically, short-circuiting VM-Entry to emulate L2 due to diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index daedf663c0a9..de9de0d2016c 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -10,7 +10,7 @@ #include <asm/posted_intr.h> #include "capabilities.h" -#include "../kvm_cache_regs.h" +#include "../regs.h" #include "pmu_intel.h" #include "vmcs.h" #include "vmx_ops.h" diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index a5e09bf431ce..bc41717501e4 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -23,7 +23,7 @@ #include "mmu.h" #include "i8254.h" #include "tss.h" -#include "kvm_cache_regs.h" +#include "regs.h" #include "kvm_emulate.h" #include "mmu/page_track.h" #include "x86.h" @@ -128,12 +128,10 @@ static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE); KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST | \ KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST) -static void update_cr8_intercept(struct kvm_vcpu *vcpu); static void process_nmi(struct kvm_vcpu *vcpu); static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags); static void store_regs(struct kvm_vcpu *vcpu); static int sync_regs(struct kvm_vcpu *vcpu); -static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu); static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2); static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2); @@ -152,6 +150,7 @@ struct kvm_x86_ops kvm_x86_ops __read_mostly; #include <asm/kvm-x86-ops.h> EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits); EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg); +EXPORT_STATIC_CALL_GPL(kvm_x86_get_cpl); static bool __read_mostly ignore_msrs = 0; module_param(ignore_msrs, bool, 0644); @@ -576,13 +575,6 @@ static struct kmem_cache *kvm_alloc_emulator_cache(void) static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt); -static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu) -{ - int i; - for (i = 0; i < ASYNC_PF_PER_VCPU; i++) - vcpu->arch.apf.gfns[i] = ~0; -} - static void kvm_destroy_user_return_msrs(void) { int cpu; @@ -970,7 +962,8 @@ static int complete_emulated_insn_gp(struct kvm_vcpu *vcpu, int err) EMULTYPE_COMPLETE_USER_EXIT); } -void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault) +void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault, + bool from_hardware) { ++vcpu->stat.pf_guest; @@ -987,8 +980,9 @@ void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault) fault->address); } -void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, - struct x86_exception *fault) +void __kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, + struct x86_exception *fault, + bool from_hardware) { struct kvm_mmu *fault_mmu; WARN_ON_ONCE(fault->vector != PF_VECTOR); @@ -1005,9 +999,9 @@ void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, kvm_mmu_invalidate_addr(vcpu, fault_mmu, fault->address, KVM_MMU_ROOT_CURRENT); - fault_mmu->inject_page_fault(vcpu, fault); + fault_mmu->inject_page_fault(vcpu, fault, from_hardware); } -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_inject_emulated_page_fault); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_inject_emulated_page_fault); void kvm_inject_nmi(struct kvm_vcpu *vcpu) { @@ -1021,18 +1015,6 @@ void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_queue_exception_e); -/* - * Checks if cpl <= required_cpl; if true, return true. Otherwise queue - * a #GP and return false. - */ -bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl) -{ - if (kvm_x86_call(get_cpl)(vcpu) <= required_cpl) - return true; - kvm_queue_exception_e(vcpu, GP_VECTOR, 0); - return false; -} - bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr) { if ((dr != 4 && dr != 5) || !kvm_is_cr4_bit_set(vcpu, X86_CR4_DE)) @@ -1043,13 +1025,6 @@ bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_require_dr); -static bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu) -{ - u64 mask = KVM_ASYNC_PF_ENABLED | KVM_ASYNC_PF_DELIVERY_AS_INT; - - return (vcpu->arch.apf.msr_en_val & mask) == mask; -} - static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu) { return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2); @@ -1314,7 +1289,7 @@ int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu) { /* Note, #UD due to CR4.OSXSAVE=0 has priority over the intercept. */ if (kvm_x86_call(get_cpl)(vcpu) != 0 || - __kvm_set_xcr(vcpu, kvm_rcx_read(vcpu), kvm_read_edx_eax(vcpu))) { + __kvm_set_xcr(vcpu, kvm_ecx_read(vcpu), kvm_read_edx_eax(vcpu))) { kvm_inject_gp(vcpu, 0); return 1; } @@ -1601,9 +1576,17 @@ unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_dr); +static unsigned long kvm_get_effective_dr7(struct kvm_vcpu *vcpu) +{ + if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) + return vcpu->arch.guest_debug_dr7; + + return vcpu->arch.dr7; +} + int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu) { - u32 pmc = kvm_rcx_read(vcpu); + u32 pmc = kvm_ecx_read(vcpu); u64 data; if (kvm_pmu_rdpmc(vcpu, pmc, &data)) { @@ -1611,8 +1594,8 @@ int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu) return 1; } - kvm_rax_write(vcpu, (u32)data); - kvm_rdx_write(vcpu, data >> 32); + kvm_eax_write(vcpu, data); + kvm_edx_write(vcpu, data >> 32); return kvm_skip_emulated_instruction(vcpu); } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_rdpmc); @@ -2059,8 +2042,8 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_msr_write); static void complete_userspace_rdmsr(struct kvm_vcpu *vcpu) { if (!vcpu->run->msr.error) { - kvm_rax_write(vcpu, (u32)vcpu->run->msr.data); - kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32); + kvm_eax_write(vcpu, vcpu->run->msr.data); + kvm_edx_write(vcpu, vcpu->run->msr.data >> 32); } } @@ -2141,8 +2124,8 @@ static int __kvm_emulate_rdmsr(struct kvm_vcpu *vcpu, u32 msr, int reg, trace_kvm_msr_read(msr, data); if (reg < 0) { - kvm_rax_write(vcpu, data & -1u); - kvm_rdx_write(vcpu, (data >> 32) & -1u); + kvm_eax_write(vcpu, data); + kvm_edx_write(vcpu, data >> 32); } else { kvm_register_write(vcpu, reg, data); } @@ -2159,7 +2142,7 @@ static int __kvm_emulate_rdmsr(struct kvm_vcpu *vcpu, u32 msr, int reg, int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu) { - return __kvm_emulate_rdmsr(vcpu, kvm_rcx_read(vcpu), -1, + return __kvm_emulate_rdmsr(vcpu, kvm_ecx_read(vcpu), -1, complete_fast_rdmsr); } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_rdmsr); @@ -2195,7 +2178,7 @@ static int __kvm_emulate_wrmsr(struct kvm_vcpu *vcpu, u32 msr, u64 data) int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu) { - return __kvm_emulate_wrmsr(vcpu, kvm_rcx_read(vcpu), + return __kvm_emulate_wrmsr(vcpu, kvm_ecx_read(vcpu), kvm_read_edx_eax(vcpu)); } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_wrmsr); @@ -2305,7 +2288,7 @@ static fastpath_t __handle_fastpath_wrmsr(struct kvm_vcpu *vcpu, u32 msr, u64 da fastpath_t handle_fastpath_wrmsr(struct kvm_vcpu *vcpu) { - return __handle_fastpath_wrmsr(vcpu, kvm_rcx_read(vcpu), + return __handle_fastpath_wrmsr(vcpu, kvm_ecx_read(vcpu), kvm_read_edx_eax(vcpu)); } EXPORT_SYMBOL_FOR_KVM_INTERNAL(handle_fastpath_wrmsr); @@ -3648,23 +3631,19 @@ static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data) if (!lapic_in_kernel(vcpu)) return data ? 1 : 0; + if (__kvm_pv_async_pf_enabled(data) && + kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa, + sizeof(u64))) + return 1; + vcpu->arch.apf.msr_en_val = data; - if (!kvm_pv_async_pf_enabled(vcpu)) { + if (__kvm_pv_async_pf_enabled(data)) { + kvm_async_pf_wakeup_all(vcpu); + } else { kvm_clear_async_pf_completion_queue(vcpu); kvm_async_pf_hash_reset(vcpu); - return 0; } - - if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa, - sizeof(u64))) - return 1; - - vcpu->arch.apf.send_always = (data & KVM_ASYNC_PF_SEND_ALWAYS); - vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT; - - kvm_async_pf_wakeup_all(vcpu); - return 0; } @@ -4003,22 +3982,28 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) break; case MSR_EFER: return set_efer(vcpu, msr_info); - case MSR_K7_HWCR: - data &= ~(u64)0x40; /* ignore flush filter disable */ - data &= ~(u64)0x100; /* ignore ignne emulation enable */ - data &= ~(u64)0x8; /* ignore TLB cache disable */ - + case MSR_K7_HWCR: { /* * Allow McStatusWrEn and TscFreqSel. (Linux guests from v3.2 * through at least v6.6 whine if TscFreqSel is clear, * depending on F/M/S. */ - if (data & ~(BIT_ULL(18) | BIT_ULL(24))) { + u64 valid = BIT_ULL(18) | BIT_ULL(24); + + data &= ~(u64)0x40; /* ignore flush filter disable */ + data &= ~(u64)0x100; /* ignore ignne emulation enable */ + data &= ~(u64)0x8; /* ignore TLB cache disable */ + + if (guest_cpu_cap_has(vcpu, X86_FEATURE_GP_ON_USER_CPUID)) + valid |= MSR_K7_HWCR_CPUID_USER_DIS; + + if (data & ~valid) { kvm_pr_unimpl_wrmsr(vcpu, msr, data); return 1; } vcpu->arch.msr_hwcr = data; break; + } case MSR_FAM10H_MMIO_CONF_BASE: if (data != 0) { kvm_pr_unimpl_wrmsr(vcpu, msr, data); @@ -4265,7 +4250,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) case MSR_MISC_FEATURES_ENABLES: if (data & ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT || (data & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT && - !supports_cpuid_fault(vcpu))) + !(vcpu->arch.msr_platform_info & MSR_PLATFORM_INFO_CPUID_FAULT))) return 1; vcpu->arch.msr_misc_features_enables = data; break; @@ -5341,7 +5326,7 @@ static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, r = kvm_apic_set_state(vcpu, s); if (r) return r; - update_cr8_intercept(vcpu); + kvm_lapic_update_cr8_intercept(vcpu); return 0; } @@ -6915,6 +6900,10 @@ disable_exits_unlock: if (!enable_pmu || (cap->args[0] & ~KVM_CAP_PMU_VALID_MASK)) break; + if (kvm->arch.has_protected_pmu && + cap->args[0] != KVM_PMU_CAP_DISABLE) + break; + mutex_lock(&kvm->lock); if (!kvm->created_vcpus && !kvm->arch.created_mediated_pmu) { kvm->arch.enable_pmu = !(cap->args[0] & KVM_PMU_CAP_DISABLE); @@ -8553,6 +8542,11 @@ static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt) kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt)); } +static unsigned long emulator_get_effective_dr7(struct x86_emulate_ctxt *ctxt) +{ + return kvm_get_effective_dr7(emul_to_vcpu(ctxt)); +} + static unsigned long emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr) { return kvm_get_dr(emul_to_vcpu(ctxt), dr); @@ -8810,6 +8804,11 @@ static int emulator_intercept(struct x86_emulate_ctxt *ctxt, &ctxt->exception); } +static bool emulator_is_cpuid_allowed(struct x86_emulate_ctxt *ctxt) +{ + return kvm_is_cpuid_allowed(emul_to_vcpu(ctxt)); +} + static bool emulator_get_cpuid(struct x86_emulate_ctxt *ctxt, u32 *eax, u32 *ebx, u32 *ecx, u32 *edx, bool exact_only) @@ -8935,6 +8934,7 @@ static const struct x86_emulate_ops emulate_ops = { .get_cr = emulator_get_cr, .set_cr = emulator_set_cr, .cpl = emulator_get_cpl, + .get_effective_dr7 = emulator_get_effective_dr7, .get_dr = emulator_get_dr, .set_dr = emulator_set_dr, .set_msr_with_filter = emulator_set_msr_with_filter, @@ -8946,6 +8946,7 @@ static const struct x86_emulate_ops emulate_ops = { .wbinvd = emulator_wbinvd, .fix_hypercall = emulator_fix_hypercall, .intercept = emulator_intercept, + .is_cpuid_allowed = emulator_is_cpuid_allowed, .get_cpuid = emulator_get_cpuid, .guest_has_movbe = emulator_guest_has_movbe, .guest_has_fxsr = emulator_guest_has_fxsr, @@ -8981,17 +8982,36 @@ static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask) } } -static void inject_emulated_exception(struct kvm_vcpu *vcpu) +static int kvm_inject_emulated_db(struct kvm_vcpu *vcpu, unsigned long dr6) { - struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; + struct kvm_run *kvm_run = vcpu->run; + + if (vcpu->guest_debug & (KVM_GUESTDBG_USE_HW_BP | KVM_GUESTDBG_SINGLESTEP)) { + kvm_run->debug.arch.dr6 = dr6 | DR6_ACTIVE_LOW; + kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu); + kvm_run->debug.arch.exception = DB_VECTOR; + kvm_run->exit_reason = KVM_EXIT_DEBUG; + return 0; + } + + kvm_queue_exception_p(vcpu, DB_VECTOR, dr6); + return 1; +} + +static int inject_emulated_exception(struct kvm_vcpu *vcpu) +{ + struct x86_exception *ex = &vcpu->arch.emulate_ctxt->exception; + + if (ex->vector == DB_VECTOR) + return kvm_inject_emulated_db(vcpu, ex->dr6); - if (ctxt->exception.vector == PF_VECTOR) - kvm_inject_emulated_page_fault(vcpu, &ctxt->exception); - else if (ctxt->exception.error_code_valid) - kvm_queue_exception_e(vcpu, ctxt->exception.vector, - ctxt->exception.error_code); + if (ex->vector == PF_VECTOR) + kvm_inject_emulated_page_fault(vcpu, ex); + else if (ex->error_code_valid) + kvm_queue_exception_e(vcpu, ex->vector, ex->error_code); else - kvm_queue_exception(vcpu, ctxt->exception.vector); + kvm_queue_exception(vcpu, ex->vector); + return 1; } static struct x86_emulate_ctxt *alloc_emulate_ctxt(struct kvm_vcpu *vcpu) @@ -9031,6 +9051,7 @@ static void init_emulate_ctxt(struct kvm_vcpu *vcpu) ctxt->interruptibility = 0; ctxt->have_exception = false; ctxt->exception.vector = -1; + ctxt->exception.payload = 0; ctxt->perm_ok = false; init_decode_cache(ctxt); @@ -9248,21 +9269,6 @@ static int kvm_vcpu_check_hw_bp(unsigned long addr, u32 type, u32 dr7, return dr6; } -static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu) -{ - struct kvm_run *kvm_run = vcpu->run; - - if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) { - kvm_run->debug.arch.dr6 = DR6_BS | DR6_ACTIVE_LOW; - kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu); - kvm_run->debug.arch.exception = DB_VECTOR; - kvm_run->exit_reason = KVM_EXIT_DEBUG; - return 0; - } - kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS); - return 1; -} - int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu) { unsigned long rflags = kvm_x86_call(get_rflags)(vcpu); @@ -9283,13 +9289,16 @@ int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu) * that sets the TF flag". */ if (unlikely(rflags & X86_EFLAGS_TF)) - r = kvm_vcpu_do_singlestep(vcpu); + r = kvm_inject_emulated_db(vcpu, DR6_BS); return r; } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_skip_emulated_instruction); static bool kvm_is_code_breakpoint_inhibited(struct kvm_vcpu *vcpu) { + if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) + return false; + if (kvm_get_rflags(vcpu) & X86_EFLAGS_RF) return true; @@ -9306,6 +9315,8 @@ static bool kvm_is_code_breakpoint_inhibited(struct kvm_vcpu *vcpu) static bool kvm_vcpu_check_code_breakpoint(struct kvm_vcpu *vcpu, int emulation_type, int *r) { + unsigned long dr7 = kvm_get_effective_dr7(vcpu); + WARN_ON_ONCE(emulation_type & EMULTYPE_NO_DECODE); /* @@ -9326,34 +9337,14 @@ static bool kvm_vcpu_check_code_breakpoint(struct kvm_vcpu *vcpu, EMULTYPE_TRAP_UD | EMULTYPE_VMWARE_GP | EMULTYPE_PF)) return false; - if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) && - (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) { - struct kvm_run *kvm_run = vcpu->run; - unsigned long eip = kvm_get_linear_rip(vcpu); - u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0, - vcpu->arch.guest_debug_dr7, - vcpu->arch.eff_db); - - if (dr6 != 0) { - kvm_run->debug.arch.dr6 = dr6 | DR6_ACTIVE_LOW; - kvm_run->debug.arch.pc = eip; - kvm_run->debug.arch.exception = DB_VECTOR; - kvm_run->exit_reason = KVM_EXIT_DEBUG; - *r = 0; - return true; - } - } - - if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) && + if (unlikely(dr7 & DR7_BP_EN_MASK) && !kvm_is_code_breakpoint_inhibited(vcpu)) { unsigned long eip = kvm_get_linear_rip(vcpu); - u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0, - vcpu->arch.dr7, - vcpu->arch.db); + u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0, dr7, + vcpu->arch.eff_db); - if (dr6 != 0) { - kvm_queue_exception_p(vcpu, DB_VECTOR, dr6); - *r = 1; + if (dr6) { + *r = kvm_inject_emulated_db(vcpu, dr6); return true; } } @@ -9499,8 +9490,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, */ WARN_ON_ONCE(ctxt->exception.vector == UD_VECTOR || exception_type(ctxt->exception.vector) == EXCPT_TRAP); - inject_emulated_exception(vcpu); - return 1; + return inject_emulated_exception(vcpu); } return handle_emulation_failure(vcpu, emulation_type); } @@ -9595,8 +9585,7 @@ restart: if (ctxt->have_exception) { WARN_ON_ONCE(vcpu->mmio_needed && !vcpu->mmio_is_write); vcpu->mmio_needed = false; - r = 1; - inject_emulated_exception(vcpu); + r = inject_emulated_exception(vcpu); } else if (vcpu->arch.pio.count) { if (!vcpu->arch.pio.in) { /* FIXME: return into emulator if single-stepping. */ @@ -9639,7 +9628,7 @@ writeback: kvm_pmu_branch_retired(vcpu); kvm_rip_write(vcpu, ctxt->eip); if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP))) - r = kvm_vcpu_do_singlestep(vcpu); + r = kvm_inject_emulated_db(vcpu, DR6_BS); kvm_x86_call(update_emulated_instruction)(vcpu); __kvm_set_rflags(vcpu, ctxt->eflags); } @@ -9690,7 +9679,7 @@ static int complete_fast_pio_out(struct kvm_vcpu *vcpu) static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port) { - unsigned long val = kvm_rax_read(vcpu); + unsigned long val = kvm_rax_read_raw(vcpu); int ret = emulator_pio_out(vcpu, size, port, &val, 1); if (ret) @@ -9726,10 +9715,10 @@ static int complete_fast_pio_in(struct kvm_vcpu *vcpu) } /* For size less than 4 we merge, else we zero extend */ - val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0; + val = (vcpu->arch.pio.size < 4) ? kvm_rax_read_raw(vcpu) : 0; complete_emulator_pio_in(vcpu, &val); - kvm_rax_write(vcpu, val); + kvm_rax_write_raw(vcpu, val); return kvm_skip_emulated_instruction(vcpu); } @@ -9741,11 +9730,11 @@ static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size, int ret; /* For size less than 4 we merge, else we zero extend */ - val = (size < 4) ? kvm_rax_read(vcpu) : 0; + val = (size < 4) ? kvm_rax_read_raw(vcpu) : 0; ret = emulator_pio_in(vcpu, size, port, &val, 1); if (ret) { - kvm_rax_write(vcpu, val); + kvm_rax_write_raw(vcpu, val); return ret; } @@ -10412,33 +10401,34 @@ static int complete_hypercall_exit(struct kvm_vcpu *vcpu) if (!is_64_bit_hypercall(vcpu)) ret = (u32)ret; - kvm_rax_write(vcpu, ret); + kvm_rax_write_raw(vcpu, ret); return kvm_skip_emulated_instruction(vcpu); } int ____kvm_emulate_hypercall(struct kvm_vcpu *vcpu, int cpl, int (*complete_hypercall)(struct kvm_vcpu *)) { - unsigned long ret; - unsigned long nr = kvm_rax_read(vcpu); - unsigned long a0 = kvm_rbx_read(vcpu); - unsigned long a1 = kvm_rcx_read(vcpu); - unsigned long a2 = kvm_rdx_read(vcpu); - unsigned long a3 = kvm_rsi_read(vcpu); int op_64_bit = is_64_bit_hypercall(vcpu); + unsigned long ret, nr, a0, a1, a2, a3; ++vcpu->stat.hypercalls; - trace_kvm_hypercall(nr, a0, a1, a2, a3); - - if (!op_64_bit) { - nr &= 0xFFFFFFFF; - a0 &= 0xFFFFFFFF; - a1 &= 0xFFFFFFFF; - a2 &= 0xFFFFFFFF; - a3 &= 0xFFFFFFFF; + if (op_64_bit) { + nr = kvm_rax_read_raw(vcpu); + a0 = kvm_rbx_read_raw(vcpu); + a1 = kvm_rcx_read_raw(vcpu); + a2 = kvm_rdx_read_raw(vcpu); + a3 = kvm_rsi_read_raw(vcpu); + } else { + nr = kvm_eax_read(vcpu); + a0 = kvm_ebx_read(vcpu); + a1 = kvm_ecx_read(vcpu); + a2 = kvm_edx_read(vcpu); + a3 = kvm_esi_read(vcpu); } + trace_kvm_hypercall(nr, a0, a1, a2, a3); + if (cpl) { ret = -KVM_EPERM; goto out; @@ -10581,33 +10571,6 @@ static void post_kvm_run_save(struct kvm_vcpu *vcpu) kvm_run->flags |= KVM_RUN_X86_GUEST_MODE; } -static void update_cr8_intercept(struct kvm_vcpu *vcpu) -{ - int max_irr, tpr; - - if (!kvm_x86_ops.update_cr8_intercept) - return; - - if (!lapic_in_kernel(vcpu)) - return; - - if (vcpu->arch.apic->apicv_active) - return; - - if (!vcpu->arch.apic->vapic_addr) - max_irr = kvm_lapic_find_highest_irr(vcpu); - else - max_irr = -1; - - if (max_irr != -1) - max_irr >>= 4; - - tpr = kvm_lapic_get_cr8(vcpu); - - kvm_x86_call(update_cr8_intercept)(vcpu, tpr, max_irr); -} - - int kvm_check_nested_events(struct kvm_vcpu *vcpu) { if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { @@ -11348,7 +11311,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) kvm_x86_call(enable_irq_window)(vcpu); if (kvm_lapic_enabled(vcpu)) { - update_cr8_intercept(vcpu); + kvm_lapic_update_cr8_intercept(vcpu); kvm_lapic_sync_to_vapic(vcpu); } } @@ -11593,9 +11556,6 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (vcpu->arch.apic_attention) kvm_lapic_sync_from_vapic(vcpu); - if (unlikely(exit_fastpath == EXIT_FASTPATH_EXIT_USERSPACE)) - return 0; - r = kvm_x86_call(handle_exit)(vcpu, exit_fastpath); return r; @@ -12135,23 +12095,23 @@ static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) emulator_writeback_register_cache(vcpu->arch.emulate_ctxt); vcpu->arch.emulate_regs_need_sync_to_vcpu = false; } - regs->rax = kvm_rax_read(vcpu); - regs->rbx = kvm_rbx_read(vcpu); - regs->rcx = kvm_rcx_read(vcpu); - regs->rdx = kvm_rdx_read(vcpu); - regs->rsi = kvm_rsi_read(vcpu); - regs->rdi = kvm_rdi_read(vcpu); + regs->rax = kvm_rax_read_raw(vcpu); + regs->rbx = kvm_rbx_read_raw(vcpu); + regs->rcx = kvm_rcx_read_raw(vcpu); + regs->rdx = kvm_rdx_read_raw(vcpu); + regs->rsi = kvm_rsi_read_raw(vcpu); + regs->rdi = kvm_rdi_read_raw(vcpu); regs->rsp = kvm_rsp_read(vcpu); - regs->rbp = kvm_rbp_read(vcpu); + regs->rbp = kvm_rbp_read_raw(vcpu); #ifdef CONFIG_X86_64 - regs->r8 = kvm_r8_read(vcpu); - regs->r9 = kvm_r9_read(vcpu); - regs->r10 = kvm_r10_read(vcpu); - regs->r11 = kvm_r11_read(vcpu); - regs->r12 = kvm_r12_read(vcpu); - regs->r13 = kvm_r13_read(vcpu); - regs->r14 = kvm_r14_read(vcpu); - regs->r15 = kvm_r15_read(vcpu); + regs->r8 = kvm_r8_read_raw(vcpu); + regs->r9 = kvm_r9_read_raw(vcpu); + regs->r10 = kvm_r10_read_raw(vcpu); + regs->r11 = kvm_r11_read_raw(vcpu); + regs->r12 = kvm_r12_read_raw(vcpu); + regs->r13 = kvm_r13_read_raw(vcpu); + regs->r14 = kvm_r14_read_raw(vcpu); + regs->r15 = kvm_r15_read_raw(vcpu); #endif regs->rip = kvm_rip_read(vcpu); @@ -12175,23 +12135,23 @@ static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) vcpu->arch.emulate_regs_need_sync_from_vcpu = true; vcpu->arch.emulate_regs_need_sync_to_vcpu = false; - kvm_rax_write(vcpu, regs->rax); - kvm_rbx_write(vcpu, regs->rbx); - kvm_rcx_write(vcpu, regs->rcx); - kvm_rdx_write(vcpu, regs->rdx); - kvm_rsi_write(vcpu, regs->rsi); - kvm_rdi_write(vcpu, regs->rdi); + kvm_rax_write_raw(vcpu, regs->rax); + kvm_rbx_write_raw(vcpu, regs->rbx); + kvm_rcx_write_raw(vcpu, regs->rcx); + kvm_rdx_write_raw(vcpu, regs->rdx); + kvm_rsi_write_raw(vcpu, regs->rsi); + kvm_rdi_write_raw(vcpu, regs->rdi); kvm_rsp_write(vcpu, regs->rsp); - kvm_rbp_write(vcpu, regs->rbp); + kvm_rbp_write_raw(vcpu, regs->rbp); #ifdef CONFIG_X86_64 - kvm_r8_write(vcpu, regs->r8); - kvm_r9_write(vcpu, regs->r9); - kvm_r10_write(vcpu, regs->r10); - kvm_r11_write(vcpu, regs->r11); - kvm_r12_write(vcpu, regs->r12); - kvm_r13_write(vcpu, regs->r13); - kvm_r14_write(vcpu, regs->r14); - kvm_r15_write(vcpu, regs->r15); + kvm_r8_write_raw(vcpu, regs->r8); + kvm_r9_write_raw(vcpu, regs->r9); + kvm_r10_write_raw(vcpu, regs->r10); + kvm_r11_write_raw(vcpu, regs->r11); + kvm_r12_write_raw(vcpu, regs->r12); + kvm_r13_write_raw(vcpu, regs->r13); + kvm_r14_write_raw(vcpu, regs->r14); + kvm_r15_write_raw(vcpu, regs->r15); #endif kvm_rip_write(vcpu, regs->rip); @@ -12497,7 +12457,7 @@ static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs, kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR); kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); - update_cr8_intercept(vcpu); + kvm_lapic_update_cr8_intercept(vcpu); /* Older userspace won't unhalt the vcpu on reset. */ if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 && @@ -13094,7 +13054,7 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) * on RESET. But, go through the motions in case that's ever remedied. */ cpuid_0x1 = kvm_find_cpuid_entry(vcpu, 1); - kvm_rdx_write(vcpu, cpuid_0x1 ? cpuid_0x1->eax : 0x600); + kvm_edx_write(vcpu, cpuid_0x1 ? cpuid_0x1->eax : 0x600); kvm_x86_call(vcpu_reset)(vcpu, init_event); @@ -13368,7 +13328,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) kvm->arch.default_tsc_khz = max_tsc_khz ? : tsc_khz; kvm->arch.apic_bus_cycle_ns = APIC_BUS_CYCLE_NS_DEFAULT; kvm->arch.guest_can_read_msr_platform_info = true; - kvm->arch.enable_pmu = enable_pmu; + kvm->arch.enable_pmu = enable_pmu && !kvm->arch.has_protected_pmu; #if IS_ENABLED(CONFIG_HYPERV) spin_lock_init(&kvm->arch.hv_root_tdp_lock); @@ -14013,7 +13973,7 @@ static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu) if (!kvm_pv_async_pf_enabled(vcpu)) return false; - if (!vcpu->arch.apf.send_always && + if (!(vcpu->arch.apf.msr_en_val & KVM_ASYNC_PF_SEND_ALWAYS) && (vcpu->arch.guest_state_protected || !kvm_x86_call(get_cpl)(vcpu))) return false; @@ -14022,7 +13982,7 @@ static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu) * L1 needs to opt into the special #PF vmexits that are * used to deliver async page faults. */ - return vcpu->arch.apf.delivery_as_pf_vmexit; + return vcpu->arch.apf.msr_en_val & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT; } else { /* * Play it safe in case the guest temporarily disables paging. @@ -14066,7 +14026,7 @@ bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, fault.nested_page_fault = false; fault.address = work->arch.token; fault.async_page_fault = true; - kvm_inject_page_fault(vcpu, &fault); + kvm_inject_page_fault(vcpu, &fault, false); return true; } else { /* @@ -14237,7 +14197,7 @@ void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_c fault.address = gva; fault.async_page_fault = false; } - vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault); + vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault, true); } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_fixup_and_inject_pf_error); diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 38a905fa86de..dabbecc21378 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -6,7 +6,7 @@ #include <asm/fpu/xstate.h> #include <asm/mce.h> #include <asm/pvclock.h> -#include "kvm_cache_regs.h" +#include "regs.h" #include "kvm_emulate.h" #include "cpuid.h" @@ -243,42 +243,6 @@ static inline bool kvm_exception_is_soft(unsigned int nr) return (nr == BP_VECTOR) || (nr == OF_VECTOR); } -static inline bool is_protmode(struct kvm_vcpu *vcpu) -{ - return kvm_is_cr0_bit_set(vcpu, X86_CR0_PE); -} - -static inline bool is_long_mode(struct kvm_vcpu *vcpu) -{ -#ifdef CONFIG_X86_64 - return !!(vcpu->arch.efer & EFER_LMA); -#else - return false; -#endif -} - -static inline bool is_64_bit_mode(struct kvm_vcpu *vcpu) -{ - int cs_db, cs_l; - - WARN_ON_ONCE(vcpu->arch.guest_state_protected); - - if (!is_long_mode(vcpu)) - return false; - kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l); - return cs_l; -} - -static inline bool is_64_bit_hypercall(struct kvm_vcpu *vcpu) -{ - /* - * If running with protected guest state, the CS register is not - * accessible. The hypercall register values will have had to been - * provided in 64-bit mode, so assume the guest is in 64-bit. - */ - return vcpu->arch.guest_state_protected || is_64_bit_mode(vcpu); -} - static inline bool x86_exception_has_error_code(unsigned int vector) { static u32 exception_has_error_code = BIT(DF_VECTOR) | BIT(TS_VECTOR) | @@ -293,26 +257,6 @@ static inline bool mmu_is_nested(struct kvm_vcpu *vcpu) return vcpu->arch.walk_mmu == &vcpu->arch.nested_mmu; } -static inline bool is_pae(struct kvm_vcpu *vcpu) -{ - return kvm_is_cr4_bit_set(vcpu, X86_CR4_PAE); -} - -static inline bool is_pse(struct kvm_vcpu *vcpu) -{ - return kvm_is_cr4_bit_set(vcpu, X86_CR4_PSE); -} - -static inline bool is_paging(struct kvm_vcpu *vcpu) -{ - return likely(kvm_is_cr0_bit_set(vcpu, X86_CR0_PG)); -} - -static inline bool is_pae_paging(struct kvm_vcpu *vcpu) -{ - return !is_long_mode(vcpu) && is_pae(vcpu) && is_paging(vcpu); -} - static inline u8 vcpu_virt_addr_bits(struct kvm_vcpu *vcpu) { return kvm_is_cr4_bit_set(vcpu, X86_CR4_LA57) ? 57 : 48; @@ -421,21 +365,6 @@ static inline bool vcpu_match_mmio_gpa(struct kvm_vcpu *vcpu, gpa_t gpa) return false; } -static inline unsigned long kvm_register_read(struct kvm_vcpu *vcpu, int reg) -{ - unsigned long val = kvm_register_read_raw(vcpu, reg); - - return is_64_bit_mode(vcpu) ? val : (u32)val; -} - -static inline void kvm_register_write(struct kvm_vcpu *vcpu, - int reg, unsigned long val) -{ - if (!is_64_bit_mode(vcpu)) - val = (u32)val; - return kvm_register_write_raw(vcpu, reg, val); -} - static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk) { return !(kvm->arch.disabled_quirks & quirk); @@ -630,15 +559,23 @@ static inline bool kvm_pat_valid(u64 data) return (data | ((data & 0x0202020202020202ull) << 1)) == data; } -static inline bool kvm_dr7_valid(u64 data) +static inline bool __kvm_pv_async_pf_enabled(u64 data) +{ + u64 mask = KVM_ASYNC_PF_ENABLED | KVM_ASYNC_PF_DELIVERY_AS_INT; + + return (data & mask) == mask; +} + +static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu) { - /* Bits [63:32] are reserved */ - return !(data >> 32); + return __kvm_pv_async_pf_enabled(vcpu->arch.apf.msr_en_val); } -static inline bool kvm_dr6_valid(u64 data) + +static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu) { - /* Bits [63:32] are reserved */ - return !(data >> 32); + int i; + for (i = 0; i < ASYNC_PF_PER_VCPU; i++) + vcpu->arch.apf.gfns[i] = ~0; } /* @@ -687,41 +624,6 @@ enum kvm_msr_access { #define KVM_MSR_RET_UNSUPPORTED 2 #define KVM_MSR_RET_FILTERED 3 -static inline bool __kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) -{ - return !(cr4 & vcpu->arch.cr4_guest_rsvd_bits); -} - -#define __cr4_reserved_bits(__cpu_has, __c) \ -({ \ - u64 __reserved_bits = CR4_RESERVED_BITS; \ - \ - if (!__cpu_has(__c, X86_FEATURE_XSAVE)) \ - __reserved_bits |= X86_CR4_OSXSAVE; \ - if (!__cpu_has(__c, X86_FEATURE_SMEP)) \ - __reserved_bits |= X86_CR4_SMEP; \ - if (!__cpu_has(__c, X86_FEATURE_SMAP)) \ - __reserved_bits |= X86_CR4_SMAP; \ - if (!__cpu_has(__c, X86_FEATURE_FSGSBASE)) \ - __reserved_bits |= X86_CR4_FSGSBASE; \ - if (!__cpu_has(__c, X86_FEATURE_PKU)) \ - __reserved_bits |= X86_CR4_PKE; \ - if (!__cpu_has(__c, X86_FEATURE_LA57)) \ - __reserved_bits |= X86_CR4_LA57; \ - if (!__cpu_has(__c, X86_FEATURE_UMIP)) \ - __reserved_bits |= X86_CR4_UMIP; \ - if (!__cpu_has(__c, X86_FEATURE_VMX)) \ - __reserved_bits |= X86_CR4_VMXE; \ - if (!__cpu_has(__c, X86_FEATURE_PCID)) \ - __reserved_bits |= X86_CR4_PCIDE; \ - if (!__cpu_has(__c, X86_FEATURE_LAM)) \ - __reserved_bits |= X86_CR4_LAM_SUP; \ - if (!__cpu_has(__c, X86_FEATURE_SHSTK) && \ - !__cpu_has(__c, X86_FEATURE_IBT)) \ - __reserved_bits |= X86_CR4_CET; \ - __reserved_bits; \ -}) - int kvm_sev_es_mmio(struct kvm_vcpu *vcpu, bool is_write, gpa_t gpa, unsigned int bytes, void *data); int kvm_sev_es_string_io(struct kvm_vcpu *vcpu, unsigned int size, @@ -754,6 +656,12 @@ static inline void kvm_prepare_emulated_mmio_exit(struct kvm_vcpu *vcpu, frag->data, vcpu->mmio_is_write); } +static inline bool kvm_is_valid_map_gpa_range_ret(u64 hypercall_ret) +{ + return !hypercall_ret || hypercall_ret == EINVAL || + hypercall_ret == EAGAIN; +} + static inline bool user_exit_on_hypercall(struct kvm *kvm, unsigned long hc_nr) { return kvm->arch.hypercall_exit_enabled & BIT(hc_nr); diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 91fd3673c09a..694b31c1fcc9 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -1408,7 +1408,7 @@ int kvm_xen_hvm_config(struct kvm *kvm, struct kvm_xen_hvm_config *xhc) static int kvm_xen_hypercall_set_result(struct kvm_vcpu *vcpu, u64 result) { - kvm_rax_write(vcpu, result); + kvm_rax_write_raw(vcpu, result); return kvm_skip_emulated_instruction(vcpu); } @@ -1678,32 +1678,35 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) bool handled = false; u8 cpl; - input = (u64)kvm_register_read(vcpu, VCPU_REGS_RAX); - /* Hyper-V hypercalls get bit 31 set in EAX */ - if ((input & 0x80000000) && + if ((kvm_rax_read_raw(vcpu) & 0x80000000) && kvm_hv_hypercall_enabled(vcpu)) return kvm_hv_hypercall(vcpu); longmode = is_64_bit_hypercall(vcpu); if (!longmode) { - params[0] = (u32)kvm_rbx_read(vcpu); - params[1] = (u32)kvm_rcx_read(vcpu); - params[2] = (u32)kvm_rdx_read(vcpu); - params[3] = (u32)kvm_rsi_read(vcpu); - params[4] = (u32)kvm_rdi_read(vcpu); - params[5] = (u32)kvm_rbp_read(vcpu); + input = kvm_eax_read(vcpu); + params[0] = kvm_ebx_read(vcpu); + params[1] = kvm_ecx_read(vcpu); + params[2] = kvm_edx_read(vcpu); + params[3] = kvm_esi_read(vcpu); + params[4] = kvm_edi_read(vcpu); + params[5] = kvm_ebp_read(vcpu); } -#ifdef CONFIG_X86_64 else { - params[0] = (u64)kvm_rdi_read(vcpu); - params[1] = (u64)kvm_rsi_read(vcpu); - params[2] = (u64)kvm_rdx_read(vcpu); - params[3] = (u64)kvm_r10_read(vcpu); - params[4] = (u64)kvm_r8_read(vcpu); - params[5] = (u64)kvm_r9_read(vcpu); - } +#ifdef CONFIG_X86_64 + input = (u64)kvm_rax_read_raw(vcpu); + params[0] = (u64)kvm_rdi_read_raw(vcpu); + params[1] = (u64)kvm_rsi_read_raw(vcpu); + params[2] = (u64)kvm_rdx_read_raw(vcpu); + params[3] = (u64)kvm_r10_read_raw(vcpu); + params[4] = (u64)kvm_r8_read_raw(vcpu); + params[5] = (u64)kvm_r9_read_raw(vcpu); +#else + KVM_BUG_ON(1, vcpu->kvm); + return -EIO; #endif + } cpl = kvm_x86_call(get_cpl)(vcpu); trace_kvm_xen_hypercall(cpl, input, params[0], params[1], params[2], params[3], params[4], params[5]); diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 9118a5a51b89..0a0a02a1dffe 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -97,6 +97,7 @@ TEST_GEN_PROGS_x86 += x86/nested_emulation_test TEST_GEN_PROGS_x86 += x86/nested_exceptions_test TEST_GEN_PROGS_x86 += x86/nested_invalid_cr3_test TEST_GEN_PROGS_x86 += x86/nested_set_state_test +TEST_GEN_PROGS_x86 += x86/nested_tdp_fault_test TEST_GEN_PROGS_x86 += x86/nested_tsc_adjust_test TEST_GEN_PROGS_x86 += x86/nested_tsc_scaling_test TEST_GEN_PROGS_x86 += x86/nested_vmsave_vmload_test diff --git a/tools/testing/selftests/kvm/include/x86/processor.h b/tools/testing/selftests/kvm/include/x86/processor.h index 77f576ee7789..513e4a1075fa 100644 --- a/tools/testing/selftests/kvm/include/x86/processor.h +++ b/tools/testing/selftests/kvm/include/x86/processor.h @@ -38,7 +38,24 @@ extern u64 guest_tsc_khz; const char *ex_str(int vector); -#define X86_EFLAGS_FIXED (1u << 1) +#define X86_EFLAGS_CF BIT(0) /* Carry Flag */ +#define X86_EFLAGS_FIXED BIT(1) /* Bit 1 - always on */ +#define X86_EFLAGS_PF BIT(2) /* Parity Flag */ +#define X86_EFLAGS_AF BIT(4) /* Auxiliary carry Flag */ +#define X86_EFLAGS_ZF BIT(6) /* Zero Flag */ +#define X86_EFLAGS_SF BIT(7) /* Sign Flag */ +#define X86_EFLAGS_TF BIT(8) /* Trap Flag */ +#define X86_EFLAGS_IF BIT(9) /* Interrupt Flag */ +#define X86_EFLAGS_DF BIT(10) /* Direction Flag */ +#define X86_EFLAGS_OF BIT(11) /* Overflow Flag */ +#define X86_EFLAGS_IOPL BIT(12) /* I/O Privilege Level (2 bits) */ +#define X86_EFLAGS_NT BIT(14) /* Nested Task */ +#define X86_EFLAGS_RF BIT(16) /* Resume Flag */ +#define X86_EFLAGS_VM BIT(17) /* Virtual Mode */ +#define X86_EFLAGS_AC BIT(18) /* Alignment Check/Access Control */ +#define X86_EFLAGS_VIF BIT(19) /* Virtual Interrupt Flag */ +#define X86_EFLAGS_VIP BIT(20) /* Virtual Interrupt Pending */ +#define X86_EFLAGS_ID BIT(21) /* CPUID detection */ #define X86_CR4_VME (1ul << 0) #define X86_CR4_PVI (1ul << 1) @@ -209,6 +226,7 @@ struct kvm_x86_cpu_feature { #define X86_FEATURE_SEV KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 1) #define X86_FEATURE_SEV_ES KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 3) #define X86_FEATURE_SEV_SNP KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 4) +#define X86_FEATURE_GP_ON_USER_CPUID KVM_X86_CPU_FEATURE(0x80000021, 0, EAX, 17) #define X86_FEATURE_PERFMON_V2 KVM_X86_CPU_FEATURE(0x80000022, 0, EAX, 0) #define X86_FEATURE_LBR_PMC_FREEZE KVM_X86_CPU_FEATURE(0x80000022, 0, EAX, 2) @@ -1556,6 +1574,15 @@ u64 *tdp_get_pte(struct kvm_vm *vm, u64 l2_gpa); #define PFERR_GUEST_PAGE_MASK BIT_ULL(PFERR_GUEST_PAGE_BIT) #define PFERR_IMPLICIT_ACCESS BIT_ULL(PFERR_IMPLICIT_ACCESS_BIT) +#define EPT_VIOLATION_ACC_READ BIT(0) +#define EPT_VIOLATION_ACC_WRITE BIT(1) +#define EPT_VIOLATION_ACC_INSTR BIT(2) +#define EPT_VIOLATION_PROT_READ BIT(3) +#define EPT_VIOLATION_PROT_WRITE BIT(4) +#define EPT_VIOLATION_PROT_EXEC BIT(5) +#define EPT_VIOLATION_GVA_IS_VALID BIT(7) +#define EPT_VIOLATION_GVA_TRANSLATED BIT(8) + bool sys_clocksource_is_based_on_tsc(void); #endif /* SELFTEST_KVM_PROCESSOR_H */ diff --git a/tools/testing/selftests/kvm/lib/x86/processor.c b/tools/testing/selftests/kvm/lib/x86/processor.c index b51467d70f6e..4ca48de7a926 100644 --- a/tools/testing/selftests/kvm/lib/x86/processor.c +++ b/tools/testing/selftests/kvm/lib/x86/processor.c @@ -848,7 +848,7 @@ struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, u32 vcpu_id) /* Setup guest general purpose registers */ vcpu_regs_get(vcpu, ®s); - regs.rflags = regs.rflags | 0x2; + regs.rflags = regs.rflags | X86_EFLAGS_FIXED; regs.rsp = stack_gva; vcpu_regs_set(vcpu, ®s); diff --git a/tools/testing/selftests/kvm/lib/x86/vmx.c b/tools/testing/selftests/kvm/lib/x86/vmx.c index 67642759e4a0..7c10ba6e6fb4 100644 --- a/tools/testing/selftests/kvm/lib/x86/vmx.c +++ b/tools/testing/selftests/kvm/lib/x86/vmx.c @@ -360,7 +360,7 @@ static inline void init_vmcs_guest_state(void *rip, void *rsp) vmwrite(GUEST_DR7, 0x400); vmwrite(GUEST_RSP, (u64)rsp); vmwrite(GUEST_RIP, (u64)rip); - vmwrite(GUEST_RFLAGS, 2); + vmwrite(GUEST_RFLAGS, X86_EFLAGS_FIXED); vmwrite(GUEST_PENDING_DBG_EXCEPTIONS, 0); vmwrite(GUEST_SYSENTER_ESP, vmreadz(HOST_IA32_SYSENTER_ESP)); vmwrite(GUEST_SYSENTER_EIP, vmreadz(HOST_IA32_SYSENTER_EIP)); diff --git a/tools/testing/selftests/kvm/x86/debug_regs.c b/tools/testing/selftests/kvm/x86/debug_regs.c index 0dfaf03cd0a0..2a2ef3e179ff 100644 --- a/tools/testing/selftests/kvm/x86/debug_regs.c +++ b/tools/testing/selftests/kvm/x86/debug_regs.c @@ -15,10 +15,51 @@ #define IRQ_VECTOR 0xAA +#define CAST_TO_RIP(v) ((unsigned long long)&(v)) + /* For testing data access debug BP */ u32 guest_value; extern unsigned char sw_bp, hw_bp, write_data, ss_start, bd_start; +extern unsigned char fep_bd_start, fep_sti_start, fep_sti_end; + +static int irqs_received; + +static void guest_db_handler(struct ex_regs *regs) +{ + static int count; + unsigned long target_rips[2] = { + CAST_TO_RIP(fep_sti_start), + CAST_TO_RIP(fep_sti_end), + }; + + __GUEST_ASSERT(regs->rip == target_rips[count], + "STI[%u]: unexpected rip 0x%lx (should be 0x%lx)", + count, regs->rip, target_rips[count]); + regs->rflags &= ~X86_EFLAGS_TF; + count++; +} + +static void guest_irq_handler(struct ex_regs *regs) +{ + /* + * The pending IRQ should finally be take when KVM_GUESTDBG_BLOCKIRQ is + * cleared and IRQs are enabled. Note, the IRQ is expected to arrive + * on the instruction immediately after STI, even though its in an STI + * shadow. Because the next instruction has a coincident #DB, and #DBs + * are not subject to STI-blocking, the #DB will push RFLAGS.IF=1 on + * the stack, and the eventual IRET will unmask IRQs and obliterate the + * STI shadow in the process. + */ + unsigned long target_rip = CAST_TO_RIP(fep_sti_start); + + __GUEST_ASSERT(regs->rip == target_rip, + "IRQ: unexpected rip 0x%lx (should be 0x%lx)", + regs->rip, target_rip); + + irqs_received++; + x2apic_write_reg(APIC_EOI, 0); +} static void guest_code(void) { @@ -64,11 +105,33 @@ static void guest_code(void) /* DR6.BD test */ asm volatile("bd_start: mov %%dr0, %%rax" : : : "rax"); + + /* + * Note, the IRET from the #DB that occurs in the below STI-shadow will + * unmask IRQs, i.e. the pending interrupt will be delivered after #DB + * handling, on the CLI! + */ + if (is_forced_emulation_enabled) { + asm volatile(KVM_FEP "fep_bd_start: mov %%dr0, %%rax" : : : "rax"); + + /* pending debug exceptions for emulation */ + asm volatile("pushf\n\t" + "orq $" __stringify(X86_EFLAGS_TF) ", (%rsp)\n\t" + "popf\n\t" + "sti\n\t" + "fep_sti_start:" + "cli\n\t" + "pushf\n\t" + "orq $" __stringify(X86_EFLAGS_TF) ", (%rsp)\n\t" + "popf\n\t" + KVM_FEP "sti\n\t" + "fep_sti_end:" + "cli\n\t"); + GUEST_ASSERT(irqs_received == 1); + } GUEST_DONE(); } -#define CAST_TO_RIP(v) ((unsigned long long)&(v)) - static void vcpu_skip_insn(struct kvm_vcpu *vcpu, int insn_len) { struct kvm_regs regs; @@ -185,7 +248,7 @@ int main(void) target_dr6); } - /* Finally test global disable */ + /* test global disable */ memset(&debug, 0, sizeof(debug)); debug.control = KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP; debug.arch.debugreg[7] = 0x400 | DR7_GD; @@ -202,10 +265,29 @@ int main(void) run->debug.arch.pc, target_rip, run->debug.arch.dr6, target_dr6); + /* test global disable in emulation */ + if (is_forced_emulation_enabled) { + /* Skip the 3-bytes "mov dr0" */ + vcpu_skip_insn(vcpu, 3); + vcpu_run(vcpu); + TEST_ASSERT(run->exit_reason == KVM_EXIT_DEBUG && + run->debug.arch.exception == DB_VECTOR && + run->debug.arch.pc == CAST_TO_RIP(fep_bd_start) && + run->debug.arch.dr6 == target_dr6, + "DR7.GD: exit %d exception %d rip 0x%llx " + "(should be 0x%llx) dr6 0x%llx (should be 0x%llx)", + run->exit_reason, run->debug.arch.exception, + run->debug.arch.pc, CAST_TO_RIP(fep_bd_start), + run->debug.arch.dr6, target_dr6); + } + /* Disable all debug controls, run to the end */ memset(&debug, 0, sizeof(debug)); vcpu_guest_debug_set(vcpu, &debug); + vm_install_exception_handler(vm, DB_VECTOR, guest_db_handler); + vm_install_exception_handler(vm, IRQ_VECTOR, guest_irq_handler); + vcpu_run(vcpu); TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO); cmd = get_ucall(vcpu, &uc); diff --git a/tools/testing/selftests/kvm/x86/hwcr_msr_test.c b/tools/testing/selftests/kvm/x86/hwcr_msr_test.c index 8e20a03b3329..53b7971aa072 100644 --- a/tools/testing/selftests/kvm/x86/hwcr_msr_test.c +++ b/tools/testing/selftests/kvm/x86/hwcr_msr_test.c @@ -11,12 +11,17 @@ void test_hwcr_bit(struct kvm_vcpu *vcpu, unsigned int bit) { const u64 ignored = BIT_ULL(3) | BIT_ULL(6) | BIT_ULL(8); - const u64 valid = BIT_ULL(18) | BIT_ULL(24); - const u64 legal = ignored | valid; + u64 valid = BIT_ULL(18) | BIT_ULL(24); u64 val = BIT_ULL(bit); u64 actual; + u64 legal; int r; + if (kvm_cpu_has(X86_FEATURE_GP_ON_USER_CPUID)) + valid |= BIT_ULL(35); + + legal = ignored | valid; + r = _vcpu_set_msr(vcpu, MSR_K7_HWCR, val); TEST_ASSERT(val & ~legal ? !r : r == 1, "Expected KVM_SET_MSRS(MSR_K7_HWCR) = 0x%lx to %s", diff --git a/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c b/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c new file mode 100644 index 000000000000..fa95568f55ff --- /dev/null +++ b/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025, Google, Inc. + */ + +#include "test_util.h" +#include "kvm_util.h" +#include "processor.h" +#include "svm_util.h" +#include "vmx.h" + +#define L2_GUEST_STACK_SIZE 64 + +enum test_type { + TEST_FINAL_PAGE_UNMAPPED, /* Final data page not present */ + TEST_PT_PAGE_UNMAPPED, /* Page table page not present */ + TEST_FINAL_PAGE_WRITE_PROTECTED, /* Final data page read-only */ + TEST_PT_PAGE_WRITE_PROTECTED, /* Page table page read-only */ +}; + +static gva_t l2_test_page; +static void (*l2_entry)(void); + +#define TEST_IO_PORT 0x80 +#define TEST1_VADDR 0x8000000ULL +#define TEST2_VADDR 0x10000000ULL +#define TEST3_VADDR 0x18000000ULL +#define TEST4_VADDR 0x20000000ULL + +/* + * L2 executes OUTS reading from l2_test_page, triggering a nested page + * fault on the read access. + */ +static void l2_guest_code_outs(void) +{ + asm volatile("outsb" ::"S"(l2_test_page), "d"(TEST_IO_PORT) : "memory"); + GUEST_FAIL("L2 should not reach here"); +} + +/* + * L2 executes INS writing to l2_test_page, triggering a nested page + * fault on the write access. + */ +static void l2_guest_code_ins(void) +{ + asm volatile("insb" ::"D"(l2_test_page), "d"(TEST_IO_PORT) : "memory"); + GUEST_FAIL("L2 should not reach here"); +} + +#define GUEST_ASSERT_EXIT_QUAL(ac_eq, ex_eq) \ + __GUEST_ASSERT((ac_eq) == (ex_eq), \ + "Wanted EXIT_QUAL '0x%lx', got '0x%lx'", ex_eq, ac_eq) + +static void l1_vmx_code(struct vmx_pages *vmx, u64 expected_fault_gpa, + u64 test_type) +{ + unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; + u64 exit_qual; + + GUEST_ASSERT(vmx->vmcs_gpa); + GUEST_ASSERT(prepare_for_vmx_operation(vmx)); + GUEST_ASSERT(load_vmcs(vmx)); + + prepare_vmcs(vmx, l2_entry, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + + GUEST_ASSERT(!vmlaunch()); + + /* Verify we got an EPT violation exit */ + __GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_EPT_VIOLATION, + "Expected EPT violation (0x%x), got 0x%lx", + EXIT_REASON_EPT_VIOLATION, + vmreadz(VM_EXIT_REASON)); + + __GUEST_ASSERT(vmreadz(GUEST_PHYSICAL_ADDRESS) == expected_fault_gpa, + "Expected guest_physical_address = 0x%lx, got 0x%lx", + expected_fault_gpa, + vmreadz(GUEST_PHYSICAL_ADDRESS)); + + exit_qual = vmreadz(EXIT_QUALIFICATION); + + /* + * Note, EPT page table accesses are always read+write, e.g. so that + * the CPU can do A/D updates at-will. + */ + switch (test_type) { + case TEST_FINAL_PAGE_UNMAPPED: + GUEST_ASSERT_EXIT_QUAL(exit_qual, EPT_VIOLATION_ACC_READ | + EPT_VIOLATION_GVA_IS_VALID | + EPT_VIOLATION_GVA_TRANSLATED); + break; + case TEST_PT_PAGE_UNMAPPED: + GUEST_ASSERT_EXIT_QUAL(exit_qual, EPT_VIOLATION_ACC_READ | + EPT_VIOLATION_ACC_WRITE | + EPT_VIOLATION_GVA_IS_VALID); + break; + case TEST_FINAL_PAGE_WRITE_PROTECTED: + GUEST_ASSERT_EXIT_QUAL(exit_qual, EPT_VIOLATION_ACC_WRITE | + EPT_VIOLATION_PROT_READ | + EPT_VIOLATION_PROT_EXEC | + EPT_VIOLATION_GVA_IS_VALID | + EPT_VIOLATION_GVA_TRANSLATED); + break; + case TEST_PT_PAGE_WRITE_PROTECTED: + GUEST_ASSERT_EXIT_QUAL(exit_qual, EPT_VIOLATION_ACC_READ | + EPT_VIOLATION_ACC_WRITE | + EPT_VIOLATION_PROT_READ | + EPT_VIOLATION_PROT_EXEC | + EPT_VIOLATION_GVA_IS_VALID); + break; + } + + GUEST_DONE(); +} + +#define GUEST_ASSERT_NPF_EC(ac_ec, ex_ec) \ + __GUEST_ASSERT((ac_ec) == (ex_ec), \ + "Wanted NPF error code '0x%lx', got '0x%lx'", (u64)(ex_ec), ac_ec) + + +static void l1_svm_code(struct svm_test_data *svm, u64 expected_fault_gpa, + u64 test_type) +{ + unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; + struct vmcb *vmcb = svm->vmcb; + u64 exit_info_1; + + generic_svm_setup(svm, l2_entry, + &l2_guest_stack[L2_GUEST_STACK_SIZE]); + + run_guest(vmcb, svm->vmcb_gpa); + + /* Verify we got an NPF exit */ + __GUEST_ASSERT(vmcb->control.exit_code == SVM_EXIT_NPF, + "Expected NPF exit (0x%x), got 0x%lx", SVM_EXIT_NPF, + vmcb->control.exit_code); + + __GUEST_ASSERT(vmcb->control.exit_info_2 == expected_fault_gpa, + "Expected exit_info_2 = 0x%lx, got 0x%lx", + expected_fault_gpa, + vmcb->control.exit_info_2); + + exit_info_1 = vmcb->control.exit_info_1; + + /* + * Note, without GMET enabled, NPT walks are always user accesses. And + * like EPT, page table accesses are always read+write. + */ + switch (test_type) { + case TEST_FINAL_PAGE_UNMAPPED: + GUEST_ASSERT_NPF_EC(exit_info_1, PFERR_USER_MASK | + PFERR_GUEST_FINAL_MASK); + break; + case TEST_PT_PAGE_UNMAPPED: + GUEST_ASSERT_NPF_EC(exit_info_1, PFERR_WRITE_MASK | + PFERR_USER_MASK | + PFERR_GUEST_PAGE_MASK); + break; + case TEST_FINAL_PAGE_WRITE_PROTECTED: + GUEST_ASSERT_NPF_EC(exit_info_1, PFERR_PRESENT_MASK | + PFERR_WRITE_MASK | + PFERR_USER_MASK | + PFERR_GUEST_FINAL_MASK); + break; + case TEST_PT_PAGE_WRITE_PROTECTED: + GUEST_ASSERT_NPF_EC(exit_info_1, PFERR_PRESENT_MASK | + PFERR_WRITE_MASK | + PFERR_USER_MASK | + PFERR_GUEST_PAGE_MASK); + break; + } + + GUEST_DONE(); +} + +static void l1_guest_code(void *data, u64 expected_fault_gpa, + u64 test_type) +{ + if (this_cpu_has(X86_FEATURE_VMX)) + l1_vmx_code(data, expected_fault_gpa, test_type); + else + l1_svm_code(data, expected_fault_gpa, test_type); +} + +/* Returns the GPA of the PT page that maps @vaddr. */ +static u64 get_pt_gpa_for_vaddr(struct kvm_vm *vm, u64 vaddr) +{ + u64 *pte; + + pte = vm_get_pte(vm, vaddr); + TEST_ASSERT(pte && (*pte & 0x1), "PTE not present for vaddr 0x%lx", + (unsigned long)vaddr); + + return addr_hva2gpa(vm, (void *)((u64)pte & ~0xFFFULL)); +} + +static void run_test(enum test_type type) +{ + gpa_t expected_fault_gpa; + gva_t nested_gva; + + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct ucall uc; + + vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); + vm_enable_tdp(vm); + + if (kvm_cpu_has(X86_FEATURE_VMX)) + vcpu_alloc_vmx(vm, &nested_gva); + else + vcpu_alloc_svm(vm, &nested_gva); + + switch (type) { + case TEST_FINAL_PAGE_UNMAPPED: + /* + * Unmap the final data page from NPT/EPT. The guest page + * table walk succeeds, but the final GPA->HPA translation + * fails. L2 reads from the page via OUTS. + */ + l2_entry = l2_guest_code_outs; + l2_test_page = vm_alloc(vm, vm->page_size, TEST1_VADDR); + expected_fault_gpa = addr_gva2gpa(vm, l2_test_page); + break; + case TEST_PT_PAGE_UNMAPPED: + /* + * Unmap a page table page from NPT/EPT. The hardware page + * table walk fails when translating the PT page's GPA + * through NPT/EPT. L2 reads from the page via OUTS. + */ + l2_entry = l2_guest_code_outs; + l2_test_page = vm_alloc(vm, vm->page_size, TEST2_VADDR); + expected_fault_gpa = get_pt_gpa_for_vaddr(vm, l2_test_page); + break; + case TEST_FINAL_PAGE_WRITE_PROTECTED: + /* + * Write-protect the final data page in NPT/EPT. The page + * is present and readable, but not writable. L2 writes to + * the page via INS, triggering a protection violation. + */ + l2_entry = l2_guest_code_ins; + l2_test_page = vm_alloc(vm, vm->page_size, TEST3_VADDR); + expected_fault_gpa = addr_gva2gpa(vm, l2_test_page); + break; + case TEST_PT_PAGE_WRITE_PROTECTED: + /* + * Write-protect a page table page in NPT/EPT. The page is + * present and readable, but not writable. The guest page + * table walk needs write access to set A/D bits, so it + * triggers a protection violation on the PT page. + * L2 reads from the page via OUTS. + */ + l2_entry = l2_guest_code_outs; + l2_test_page = vm_alloc(vm, vm->page_size, TEST4_VADDR); + expected_fault_gpa = get_pt_gpa_for_vaddr(vm, l2_test_page); + break; + } + + tdp_identity_map_default_memslots(vm); + + if (type == TEST_FINAL_PAGE_WRITE_PROTECTED || + type == TEST_PT_PAGE_WRITE_PROTECTED) + *tdp_get_pte(vm, expected_fault_gpa) &= ~PTE_WRITABLE_MASK(&vm->stage2_mmu); + else + *tdp_get_pte(vm, expected_fault_gpa) &= ~(PTE_PRESENT_MASK(&vm->stage2_mmu) | + PTE_READABLE_MASK(&vm->stage2_mmu) | + PTE_WRITABLE_MASK(&vm->stage2_mmu) | + PTE_EXECUTABLE_MASK(&vm->stage2_mmu)); + + sync_global_to_guest(vm, l2_entry); + sync_global_to_guest(vm, l2_test_page); + vcpu_args_set(vcpu, 3, nested_gva, expected_fault_gpa, (u64)type); + + /* + * For the INS-based write test, KVM emulates the instruction and + * first reads from the I/O port, which exits to userspace. + * Re-enter the guest so emulation can proceed to the memory + * write, where the nested page fault is triggered. + */ + for (;;) { + vcpu_run(vcpu); + + if (vcpu->run->exit_reason == KVM_EXIT_IO && + vcpu->run->io.port == TEST_IO_PORT && + vcpu->run->io.direction == KVM_EXIT_IO_IN) { + continue; + } + break; + } + + switch (get_ucall(vcpu, &uc)) { + case UCALL_DONE: + break; + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + default: + TEST_FAIL("Unexpected exit reason: %d", vcpu->run->exit_reason); + } + + kvm_vm_free(vm); +} + +int main(int argc, char *argv[]) +{ + TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) || kvm_cpu_has(X86_FEATURE_SVM)); + TEST_REQUIRE(kvm_cpu_has_tdp()); + + run_test(TEST_FINAL_PAGE_UNMAPPED); + run_test(TEST_PT_PAGE_UNMAPPED); + run_test(TEST_FINAL_PAGE_WRITE_PROTECTED); + run_test(TEST_PT_PAGE_WRITE_PROTECTED); + + return 0; +} diff --git a/tools/testing/selftests/kvm/x86/pmu_event_filter_test.c b/tools/testing/selftests/kvm/x86/pmu_event_filter_test.c index c1232344fda8..84e4c6ca67a3 100644 --- a/tools/testing/selftests/kvm/x86/pmu_event_filter_test.c +++ b/tools/testing/selftests/kvm/x86/pmu_event_filter_test.c @@ -731,6 +731,8 @@ static void test_filter_ioctl(struct kvm_vcpu *vcpu) static void intel_run_fixed_counter_guest_code(u8 idx) { + u8 nr_fixed_counters = this_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS); + for (;;) { wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0); wrmsr(MSR_CORE_PERF_FIXED_CTR0 + idx, 0); @@ -738,6 +740,10 @@ static void intel_run_fixed_counter_guest_code(u8 idx) /* Only OS_EN bit is enabled for fixed counter[idx]. */ wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL, FIXED_PMC_CTRL(idx, FIXED_PMC_KERNEL)); wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, FIXED_PMC_GLOBAL_CTRL_ENABLE(idx)); + if (nr_fixed_counters > 1) + wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL, + FIXED_PMC_CTRL(idx, FIXED_PMC_KERNEL) | + FIXED_PMC_CTRL((idx + 1) % nr_fixed_counters, FIXED_PMC_KERNEL)); __asm__ __volatile__("loop ." : "+c"((int){NUM_BRANCHES})); wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0); |