diff options
Diffstat (limited to 'tools/testing/selftests/kvm/include')
| -rw-r--r-- | tools/testing/selftests/kvm/include/arm64/gic_v5.h | 105 | ||||
| -rw-r--r-- | tools/testing/selftests/kvm/include/arm64/processor.h | 4 | ||||
| -rw-r--r-- | tools/testing/selftests/kvm/include/kvm_util.h | 271 | ||||
| -rw-r--r-- | tools/testing/selftests/kvm/include/loongarch/processor.h | 3 | ||||
| -rw-r--r-- | tools/testing/selftests/kvm/include/numaif.h | 43 | ||||
| -rw-r--r-- | tools/testing/selftests/kvm/include/powerpc/hcall.h | 17 | ||||
| -rw-r--r-- | tools/testing/selftests/kvm/include/powerpc/kvm_util_arch.h | 22 | ||||
| -rw-r--r-- | tools/testing/selftests/kvm/include/powerpc/ppc_asm.h | 32 | ||||
| -rw-r--r-- | tools/testing/selftests/kvm/include/powerpc/processor.h | 39 | ||||
| -rw-r--r-- | tools/testing/selftests/kvm/include/powerpc/ucall.h | 21 | ||||
| -rw-r--r-- | tools/testing/selftests/kvm/include/test_util.h | 40 | ||||
| -rw-r--r-- | tools/testing/selftests/kvm/include/x86/evmcs.h | 78 | ||||
| -rw-r--r-- | tools/testing/selftests/kvm/include/x86/processor.h | 27 | ||||
| -rw-r--r-- | tools/testing/selftests/kvm/include/x86/smm.h | 2 | ||||
| -rw-r--r-- | tools/testing/selftests/kvm/include/x86/vmx.h | 180 |
15 files changed, 644 insertions, 240 deletions
diff --git a/tools/testing/selftests/kvm/include/arm64/gic_v5.h b/tools/testing/selftests/kvm/include/arm64/gic_v5.h index eb523d9277cf..2d455515563e 100644 --- a/tools/testing/selftests/kvm/include/arm64/gic_v5.h +++ b/tools/testing/selftests/kvm/include/arm64/gic_v5.h @@ -10,6 +10,106 @@ #include "processor.h" +/* GIC component base address is guest PA space */ +#define GICV5_IRS_CONFIG_BASE_GPA 0x8000000ULL +#define GICV5_IRS_CONFIG_BASE_GVA ((volatile void *)GICV5_IRS_CONFIG_BASE_GPA) + +#define GICV5_IRS_SIZE 0x20000 +#define GICV5_IRS_ALIGN 0x10000 + +/* + * IRS register offsets. Keep this selftest copy scoped to the offsets and + * fields exercised by the userspace API tests. + */ +#define GICV5_IRS_IDR0 0x0000 +#define GICV5_IRS_IDR1 0x0004 +#define GICV5_IRS_IDR2 0x0008 +#define GICV5_IRS_IDR3 0x000c +#define GICV5_IRS_IDR4 0x0010 +#define GICV5_IRS_IDR5 0x0014 +#define GICV5_IRS_IDR6 0x0018 +#define GICV5_IRS_IDR7 0x001c +#define GICV5_IRS_IIDR 0x0040 +#define GICV5_IRS_AIDR 0x0044 +#define GICV5_IRS_CR0 0x0080 +#define GICV5_IRS_CR1 0x0084 +#define GICV5_IRS_SYNCR 0x00c0 +#define GICV5_IRS_SYNC_STATUSR 0x00c4 +#define GICV5_IRS_SPI_VMR 0x0100 +#define GICV5_IRS_SPI_SELR 0x0108 +#define GICV5_IRS_SPI_DOMAINR 0x010c +#define GICV5_IRS_SPI_RESAMPLER 0x0110 +#define GICV5_IRS_SPI_CFGR 0x0114 +#define GICV5_IRS_SPI_STATUSR 0x0118 +#define GICV5_IRS_PE_SELR 0x0140 +#define GICV5_IRS_PE_STATUSR 0x0144 +#define GICV5_IRS_PE_CR0 0x0148 +#define GICV5_IRS_IST_BASER 0x0180 +#define GICV5_IRS_IST_CFGR 0x0190 +#define GICV5_IRS_IST_STATUSR 0x0194 +#define GICV5_IRS_MAP_L2_ISTR 0x01c0 +#define GICV5_IRS_VMT_BASER 0x0200 +#define GICV5_IRS_VMT_CFGR 0x0210 +#define GICV5_IRS_VMT_STATUSR 0x0214 +#define GICV5_IRS_VPE_SELR 0x0240 +#define GICV5_IRS_VPE_DBR 0x0248 +#define GICV5_IRS_VPE_HPPIR 0x0250 +#define GICV5_IRS_VPE_CR0 0x0258 +#define GICV5_IRS_VPE_STATUSR 0x025c +#define GICV5_IRS_VM_DBR 0x0280 +#define GICV5_IRS_VM_SELR 0x0288 +#define GICV5_IRS_VM_STATUSR 0x028c +#define GICV5_IRS_VMAP_L2_VMTR 0x02c0 +#define GICV5_IRS_VMAP_VMR 0x02c8 +#define GICV5_IRS_VMAP_VISTR 0x02d0 +#define GICV5_IRS_VMAP_L2_VISTR 0x02d8 +#define GICV5_IRS_VMAP_VPER 0x02e0 +#define GICV5_IRS_SAVE_VMR 0x0300 +#define GICV5_IRS_SAVE_VM_STATUSR 0x0308 +#define GICV5_IRS_MEC_IDR 0x0340 +#define GICV5_IRS_MEC_MECID_R 0x0344 +#define GICV5_IRS_MPAM_IDR 0x0380 +#define GICV5_IRS_MPAM_PARTID_R 0x0384 +#define GICV5_IRS_SWERR_STATUSR 0x03c0 +#define GICV5_IRS_SWERR_SYNDROMER0 0x03c8 +#define GICV5_IRS_SWERR_SYNDROMER1 0x03d0 + +#define GICV5_IRS_IDR0_SETLPI BIT(9) +#define GICV5_IRS_IDR0_INT_DOM GENMASK(1, 0) +#define GICV5_IRS_IDR0_INT_DOM_SECURE 0b00 + +#define GICV5_IRS_IDR1_PRIORITY_BITS GENMASK(22, 20) + +#define GICV5_IRS_IDR2_ISTMD_SZ GENMASK(19, 15) +#define GICV5_IRS_IDR2_ISTMD BIT(14) +#define GICV5_IRS_IDR2_IST_L2SZ GENMASK(13, 11) +#define GICV5_IRS_IDR2_IST_LEVELS BIT(10) +#define GICV5_IRS_IDR2_MIN_LPI_ID_BITS GENMASK(9, 6) +#define GICV5_IRS_IDR2_LPI BIT(5) +#define GICV5_IRS_IDR2_ID_BITS GENMASK(4, 0) + +#define GICV5_IRS_CR0_IDLE BIT(1) +#define GICV5_IRS_CR0_IRSEN BIT(0) + +#define GICV5_IRS_SPI_SELR_ID GENMASK(23, 0) +#define GICV5_IRS_SPI_CFGR_TM BIT(0) + +#define GICV5_IRS_SYNC_STATUSR_IDLE BIT(0) +#define GICV5_IRS_SPI_STATUSR_IDLE BIT(0) +#define GICV5_IRS_PE_STATUSR_IDLE BIT(0) +#define GICV5_IRS_IST_STATUSR_IDLE BIT(0) + +#define GICV5_IRS_IST_BASER_ADDR_MASK GENMASK_ULL(55, 6) +#define GICV5_IRS_IST_BASER_VALID BIT_ULL(0) +#define GICV5_IRS_IST_BASER_ADDR_SHIFT 6ULL +#define GICV5_IRS_IST_CFGR_STRUCTURE BIT(16) +#define GICV5_IRS_IST_CFGR_ISTSZ GENMASK(8, 7) +#define GICV5_IRS_IST_CFGR_L2SZ GENMASK(6, 5) +#define GICV5_IRS_IST_CFGR_LPI_ID_BITS GENMASK(4, 0) +#define GICV5_IRS_IST_CFGR_ISTSZ_4 0b00 +#define GICV5_IRS_IST_CFGR_ISTSZ_8 0b01 +#define GICV5_IRS_IST_CFGR_ISTSZ_16 0b10 + /* * Definitions for GICv5 instructions for the Current Domain */ @@ -77,6 +177,11 @@ #define GICV5_GICR_CDNMIA_TYPE_MASK GENMASK_ULL(31, 29) #define GICV5_GICR_CDNMIA_ID_MASK GENMASK_ULL(23, 0) +#define GICV5_HWIRQ_TYPE GENMASK_ULL(31, 29) +#define GICV5_HWIRQ_ID GENMASK_ULL(23, 0) +#define GICV5_HWIRQ_TYPE_LPI 0x2 +#define GICV5_HWIRQ_TYPE_SPI 0x3 + #define gicr_insn(insn) read_sysreg_s(GICV5_OP_GICR_##insn) #define gic_insn(v, insn) write_sysreg_s(v, GICV5_OP_GIC_##insn) diff --git a/tools/testing/selftests/kvm/include/arm64/processor.h b/tools/testing/selftests/kvm/include/arm64/processor.h index b8a902ba8573..5878a6abdf20 100644 --- a/tools/testing/selftests/kvm/include/arm64/processor.h +++ b/tools/testing/selftests/kvm/include/arm64/processor.h @@ -12,6 +12,7 @@ #include <linux/stringify.h> #include <linux/types.h> +#include <asm/barrier.h> #include <asm/brk-imm.h> #include <asm/esr.h> #include <asm/sysreg.h> @@ -191,11 +192,8 @@ static inline void cpu_relax(void) #define dsb(opt) asm volatile("dsb " #opt : : : "memory") #define dmb(opt) asm volatile("dmb " #opt : : : "memory") -#define dma_wmb() dmb(oshst) #define __iowmb() dma_wmb() -#define dma_rmb() dmb(oshld) - #define __iormb(v) \ ({ \ unsigned long tmp; \ diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 5a7a455b5387..e2618877619e 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -14,6 +14,7 @@ #include <linux/kvm.h> #include "linux/rbtree.h" #include <linux/types.h> +#include <unistd.h> #include <asm/atomic.h> #include <asm/kvm.h> @@ -33,6 +34,8 @@ #define NSEC_PER_SEC 1000000000L +#define KVM_INVALID_MEMSLOT UINT_MAX + struct userspace_mem_region { struct kvm_userspace_memory_region2 region; struct sparsebit *unused_phy_pages; @@ -80,11 +83,45 @@ struct userspace_mem_regions { DECLARE_HASHTABLE(slot_hash, 9); }; +/* + * Memory region types are passed to various page allocators to communicate + * various properties and metadata related to the allocation. Note, the + * descriptions below described the primary usage of each type. Individual + * tests may allocate memory for other purposes. + * + * By default, all regions except TEST_EXTRA are mapped to memslot '0'. The + * TEST_EXTRA region is left unmapped as it's intended to be used only for test + * specific allocations, i.e. should never be used by core/infrastructure code. + * Tests can override the memslot for any or all types, e.g. so that all test + * data is allocated from a curated memslot. + */ enum kvm_mem_region_type { + /* + * The CODE region is used by lib/elf when loading the test's code into + * guest memory. + */ MEM_REGION_CODE, + /* + * The DATA region is used to allocate core data structures, e.g. vCPU + * stacks, VM exception tables, x86's TSS, etc. + */ MEM_REGION_DATA, + /* + * The PT region, a.k.a. Page Table region, is used to allocate page + * table pages. + */ MEM_REGION_PT, + /* + * The TEST_DATA region is used for allocating test data that is either + * test specific, and/or isn't considered a "core" data structure. + */ MEM_REGION_TEST_DATA, + /* + * The TEST_EXTRA region is for special snowflakes, where a test wants + * to create and use a one-off memslot, without impacting "normal" test + * data allocations. + */ + MEM_REGION_TEST_EXTRA, NR_MEM_REGIONS, }; @@ -127,11 +164,6 @@ struct kvm_vm { struct kvm_binary_stats stats; - /* - * KVM region slots. These are the default memslots used by page - * allocators, e.g., lib/elf uses the memslots[MEM_REGION_CODE] - * memslot. - */ u32 memslots[NR_MEM_REGIONS]; }; @@ -169,7 +201,8 @@ memslot2region(struct kvm_vm *vm, u32 memslot); static inline struct userspace_mem_region *vm_get_mem_region(struct kvm_vm *vm, enum kvm_mem_region_type type) { - assert(type < NR_MEM_REGIONS); + TEST_ASSERT(type < NR_MEM_REGIONS, + "Invalid memory region type '%u'", type); return memslot2region(vm, vm->memslots[type]); } @@ -209,6 +242,9 @@ enum vm_guest_mode { VM_MODE_P41V48_4K, VM_MODE_P41V39_4K, + VM_MODE_P52V52_4K, /* For powerpc64 */ + VM_MODE_P52V52_64K, + NUM_VM_MODES, }; @@ -268,6 +304,12 @@ extern enum vm_guest_mode vm_mode_default; #define MIN_PAGE_SHIFT 12U #define ptes_per_page(page_size) ((page_size) / 8) +#elif defined(__powerpc64__) + +#define VM_MODE_DEFAULT vm_mode_default +#define MIN_PAGE_SHIFT 12U +#define ptes_per_page(page_size) ((page_size) / 8) + #endif #define VM_SHAPE_DEFAULT VM_SHAPE(VM_MODE_DEFAULT) @@ -392,6 +434,16 @@ static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { } __TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, (vcpu)->vm); \ }) +#define __gmem_ioctl(gmem_fd, cmd, arg) \ + kvm_do_ioctl(gmem_fd, cmd, arg) + +#define gmem_ioctl(gmem_fd, cmd, arg) \ +({ \ + int ret = __gmem_ioctl(gmem_fd, cmd, arg); \ + \ + TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \ +}) + /* * Looks up and returns the value corresponding to the capability * (KVM_CAP_*) given by cap. @@ -418,8 +470,19 @@ static inline void vm_enable_cap(struct kvm_vm *vm, u32 cap, u64 arg0) vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap); } +int kvm_gpa_to_guest_memfd(struct kvm_vm *vm, gpa_t gpa, off_t *fd_offset, + size_t *nr_bytes); + +/* + * KVM_SET_MEMORY_ATTRIBUTES{,2} overwrites _all_ attributes. These + * flows need significant enhancements to support multiple attributes. + */ +#define TEST_ASSERT_SUPPORTED_ATTRIBUTES(attributes) \ + TEST_ASSERT(!(attributes) || (attributes) == KVM_MEMORY_ATTRIBUTE_PRIVATE, \ + "Update me to support multiple attributes!") + static inline void vm_set_memory_attributes(struct kvm_vm *vm, gpa_t gpa, - u64 size, u64 attributes) + size_t size, u64 attributes) { struct kvm_memory_attributes attr = { .attributes = attributes, @@ -428,27 +491,109 @@ static inline void vm_set_memory_attributes(struct kvm_vm *vm, gpa_t gpa, .flags = 0, }; - /* - * KVM_SET_MEMORY_ATTRIBUTES overwrites _all_ attributes. These flows - * need significant enhancements to support multiple attributes. - */ - TEST_ASSERT(!attributes || attributes == KVM_MEMORY_ATTRIBUTE_PRIVATE, - "Update me to support multiple attributes!"); + TEST_ASSERT_SUPPORTED_ATTRIBUTES(attributes); vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr); } +static inline int __gmem_set_memory_attributes(int fd, u64 offset, + size_t size, u64 attributes, + u64 *error_offset) +{ + struct kvm_memory_attributes2 attr = { + .attributes = attributes, + .offset = offset, + .size = size, + .flags = 0, + .error_offset = 0, + }; + int r; + + r = __gmem_ioctl(fd, KVM_SET_MEMORY_ATTRIBUTES2, &attr); + + /* Copy error_offset regardless of r so caller can check. */ + if (error_offset) + *error_offset = attr.error_offset; + + return r; +} + +static inline int __gmem_set_private(int fd, u64 offset, size_t size, + u64 *error_offset) +{ + return __gmem_set_memory_attributes(fd, offset, size, + KVM_MEMORY_ATTRIBUTE_PRIVATE, + error_offset); +} + +static inline int __gmem_set_shared(int fd, u64 offset, size_t size, + u64 *error_offset) +{ + return __gmem_set_memory_attributes(fd, offset, size, 0, + error_offset); +} + +static inline void gmem_set_memory_attributes(int fd, u64 offset, + size_t size, u64 attributes) +{ + struct kvm_memory_attributes2 attr = { + .attributes = attributes, + .offset = offset, + .size = size, + .flags = 0, + }; + + TEST_ASSERT_SUPPORTED_ATTRIBUTES(attributes); + + __TEST_REQUIRE(kvm_check_cap(KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES) > 0, + "No valid attributes for guest_memfd ioctl!"); + + gmem_ioctl(fd, KVM_SET_MEMORY_ATTRIBUTES2, &attr); +} + +static inline void gmem_set_private(int fd, u64 offset, size_t size) +{ + gmem_set_memory_attributes(fd, offset, size, + KVM_MEMORY_ATTRIBUTE_PRIVATE); +} + +static inline void gmem_set_shared(int fd, u64 offset, size_t size) +{ + gmem_set_memory_attributes(fd, offset, size, 0); +} + +static inline void vm_mem_set_memory_attributes(struct kvm_vm *vm, gpa_t gpa, + size_t size, u64 attrs) +{ + if (kvm_has_gmem_attributes) { + gpa_t end = gpa + size; + off_t fd_offset; + gpa_t addr; + size_t len; + int fd; + + for (addr = gpa; addr < end; addr += len) { + fd = kvm_gpa_to_guest_memfd(vm, addr, &fd_offset, &len); + len = min(end - addr, len); + + gmem_set_memory_attributes(fd, fd_offset, len, attrs); + } + } else { + vm_set_memory_attributes(vm, gpa, size, attrs); + } +} static inline void vm_mem_set_private(struct kvm_vm *vm, gpa_t gpa, - u64 size) + size_t size) { - vm_set_memory_attributes(vm, gpa, size, KVM_MEMORY_ATTRIBUTE_PRIVATE); + vm_mem_set_memory_attributes(vm, gpa, size, + KVM_MEMORY_ATTRIBUTE_PRIVATE); } static inline void vm_mem_set_shared(struct kvm_vm *vm, gpa_t gpa, - u64 size) + size_t size) { - vm_set_memory_attributes(vm, gpa, size, 0); + vm_mem_set_memory_attributes(vm, gpa, size, 0); } void vm_guest_mem_fallocate(struct kvm_vm *vm, gpa_t gpa, u64 size, @@ -690,17 +835,47 @@ int __vm_set_user_memory_region(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva); void vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva, - u32 guest_memfd, u64 guest_memfd_offset); + u32 gmem_fd, u64 gmem_offset); int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva, - u32 guest_memfd, u64 guest_memfd_offset); + u32 gmem_fd, u64 gmem_offset); void vm_userspace_mem_region_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, gpa_t gpa, u32 slot, u64 npages, u32 flags); void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, gpa_t gpa, u32 slot, u64 npages, u32 flags, - int guest_memfd_fd, u64 guest_memfd_offset); + int gmem_fd, u64 gmem_offset, u64 gmem_flags); + + +static inline void ____vm_override_mem_region(struct kvm_vm *vm, + enum kvm_mem_region_type type, + u32 slot) +{ + TEST_ASSERT(vm->memslots[type] == KVM_INVALID_MEMSLOT, + "Memory region type '%u' was already overridden with slot=%u", + type, vm->memslots[type]); + + vm->memslots[type] = slot; +} + +static inline void __vm_override_mem_region(struct kvm_vm *vm, + enum kvm_mem_region_type type, + enum vm_mem_backing_src_type src_type, + gpa_t gpa, u32 slot, u64 npages, + u32 flags) +{ + ____vm_override_mem_region(vm, type, slot); + vm_userspace_mem_region_add(vm, src_type, gpa, slot, npages, flags); +} + +static inline void vm_override_mem_region(struct kvm_vm *vm, + enum kvm_mem_region_type type, + enum vm_mem_backing_src_type src_type, + gpa_t gpa, u32 slot, u64 npages) +{ + __vm_override_mem_region(vm, type, src_type, gpa, slot, npages, 0); +} #ifndef vm_arch_has_protected_memory static inline bool vm_arch_has_protected_memory(struct kvm_vm *vm) @@ -732,6 +907,11 @@ void *addr_gva2hva(struct kvm_vm *vm, gva_t gva); gpa_t addr_hva2gpa(struct kvm_vm *vm, void *hva); void *addr_gpa2alias(struct kvm_vm *vm, gpa_t gpa); +static inline int kvm_slot_to_fd(struct kvm_vm *vm, u32 slot) +{ + return memslot2region(vm, slot)->fd; +} + #ifndef vcpu_arch_put_guest #define vcpu_arch_put_guest(mem, val) do { (mem) = (val); } while (0) #endif @@ -990,23 +1170,57 @@ void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing); const char *exit_reason_str(unsigned int exit_reason); -gpa_t vm_phy_page_alloc(struct kvm_vm *vm, gpa_t min_gpa, u32 memslot); -gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, gpa_t min_gpa, - u32 memslot, bool protected); -gpa_t vm_alloc_page_table(struct kvm_vm *vm); +bool kvm_arch_needs_naturally_aligned_page_tables(void); -static inline gpa_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, - gpa_t min_gpa, u32 memslot) +/* + * True on powerpc when this host is a pseries LPAR (nested case). + * PowerPC's PAPR L0 does not guarantee H_GUEST_CREATE / H_GUEST_CREATE_VCPU up + * to MAX_VCPUS or creating hundreds of VMs - due to limited resouces at L0 for + * guest management space. This helper allows us to skip such tests when we are + * running nested. + */ +static inline bool kvm_is_ppc_pseries_lpar(void) +{ +#ifdef __powerpc64__ + return !access("/proc/device-tree/rtas/ibm,hypertas-functions", F_OK); +#else + return false; +#endif +} + +gpa_t ____vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, + u32 memslot, bool protected, bool naturally_aligned); +gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, + enum kvm_mem_region_type type, bool protected); + +static inline gpa_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, + enum kvm_mem_region_type type) { /* * By default, allocate memory as protected for VMs that support * protected memory, as the majority of memory for such VMs is * protected, i.e. using shared memory is effectively opt-in. */ - return __vm_phy_pages_alloc(vm, num, min_gpa, memslot, + return __vm_phy_pages_alloc(vm, nr_pages, type, vm_arch_has_protected_memory(vm)); } +static inline gpa_t vm_phy_page_alloc(struct kvm_vm *vm, + enum kvm_mem_region_type type) +{ + return vm_phy_pages_alloc(vm, 1, type); +} + +static inline gpa_t vm_alloc_page_table_pages(struct kvm_vm *vm, size_t nr_pages) +{ + return vm_phy_pages_alloc(vm, nr_pages, MEM_REGION_PT); +} + +static inline gpa_t vm_alloc_page_table(struct kvm_vm *vm) +{ + return vm_alloc_page_table_pages(vm, 1); +} + /* * ____vm_create() does KVM_CREATE_VM and little else. __vm_create() also * loads the test binary into guest memory and creates an IRQ chip (x86 only). @@ -1123,6 +1337,9 @@ static inline int pin_self_to_any_cpu(void) return pin_task_to_any_cpu(pthread_self()); } +void pin_pages(void *vaddr, uint64_t size); +void unpin_pages(void); + void kvm_print_vcpu_pinning_help(void); void kvm_parse_vcpu_pinning(const char *pcpus_string, u32 vcpu_to_pcpu[], int nr_vcpus); diff --git a/tools/testing/selftests/kvm/include/loongarch/processor.h b/tools/testing/selftests/kvm/include/loongarch/processor.h index 0e59daa8024e..8d9fabdc89e3 100644 --- a/tools/testing/selftests/kvm/include/loongarch/processor.h +++ b/tools/testing/selftests/kvm/include/loongarch/processor.h @@ -79,6 +79,7 @@ #define CPUCFG2_LASX BIT(7) #define CPUCFG2_LLFTP BIT(14) #define CPUCFG2_LLFTPREV GENMASK(17, 15) +#define CPUCFG2_PTW BIT(24) /* LoongArch Basic CSR registers */ #define LOONGARCH_CSR_CRMD 0x0 /* Current mode info */ @@ -128,6 +129,8 @@ #define LOONGARCH_CSR_PGD 0x1b #define LOONGARCH_CSR_PWCTL0 0x1c #define LOONGARCH_CSR_PWCTL1 0x1d +#define CSR_PWCTL1_PTW_SHIFT 24 +#define CSR_PWCTL1_PTW BIT_ULL(CSR_PWCTL1_PTW_SHIFT) #define LOONGARCH_CSR_STLBPGSIZE 0x1e #define LOONGARCH_CSR_CPUID 0x20 #define LOONGARCH_CSR_KS0 0x30 diff --git a/tools/testing/selftests/kvm/include/numaif.h b/tools/testing/selftests/kvm/include/numaif.h index 29572a6d789c..0945500c6322 100644 --- a/tools/testing/selftests/kvm/include/numaif.h +++ b/tools/testing/selftests/kvm/include/numaif.h @@ -6,6 +6,7 @@ #include <dirent.h> +#include <linux/bitmap.h> #include <linux/mempolicy.h> #include "kvm_syscalls.h" @@ -30,6 +31,48 @@ KVM_SYSCALL_DEFINE(mbind, 6, void *, addr, unsigned long, size, int, mode, const unsigned long *, nodemask, unsigned long, maxnode, unsigned int, flags); +/* + * Calculate the @maxnode param for the above syscalls given the mask that will + * be passed to the kernel, to account for a longstanding off-by-one bug in the + * kernel that isn't properly documented in the manpages. The manpages say + * that @maxnode is "the maximum node ID plus one", but the kernel's actual + * behavior is "the number of bits in the mask plus one", i.e. "the maximum + * node ID plus two". + */ +#define MAXNODE_FOR_MASK(mask) (BITS_PER_TYPE(mask) + 1) + +static inline int kvm_get_numa_memory_nodes(unsigned long *nodemask) +{ + int r; + + *nodemask = 0; + + r = get_mempolicy(NULL, nodemask, MAXNODE_FOR_MASK(*nodemask), 0, + MPOL_F_MEMS_ALLOWED); + TEST_ASSERT(!r || errno == ENOSYS || errno == EPERM, + "Unexpected get_mempolicy() failure"); + return __builtin_popcountl(*nodemask); +} + +/* + * Return the node ID of the next NUMA node in the mask, starting at @from+1. + * Guarantees a node is found, and that the found node is not @from. Pass -1 + * to find the first node in the mask. + */ +static inline int kvm_get_next_numa_node(unsigned long nodemask, int from) +{ + const unsigned long nr_bits = BITS_PER_TYPE(nodemask); + int to; + + to = find_next_bit(&nodemask, nr_bits, from + 1); + if (to == nr_bits) + to = find_next_bit(&nodemask, nr_bits, 0); + + TEST_ASSERT(to != nr_bits && to != from, + "Unabled to find second NUMA node (from = %d, to = %d)", from, to); + return to; +} + static inline int get_max_numa_node(void) { struct dirent *de; diff --git a/tools/testing/selftests/kvm/include/powerpc/hcall.h b/tools/testing/selftests/kvm/include/powerpc/hcall.h new file mode 100644 index 000000000000..21b2bf9180f1 --- /dev/null +++ b/tools/testing/selftests/kvm/include/powerpc/hcall.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * powerpc hcall defines + */ +#ifndef SELFTEST_KVM_HCALL_H +#define SELFTEST_KVM_HCALL_H + +#include <linux/compiler.h> + +/* Ucalls use unimplemented PAPR hcall 0 which exits KVM */ +#define H_UCALL 0 + +s64 hcall0(u64 token); +s64 hcall1(u64 token, u64 arg1); +s64 hcall2(u64 token, u64 arg1, u64 arg2); + +#endif diff --git a/tools/testing/selftests/kvm/include/powerpc/kvm_util_arch.h b/tools/testing/selftests/kvm/include/powerpc/kvm_util_arch.h new file mode 100644 index 000000000000..5d45c25cd299 --- /dev/null +++ b/tools/testing/selftests/kvm/include/powerpc/kvm_util_arch.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef SELFTEST_KVM_UTIL_ARCH_H +#define SELFTEST_KVM_UTIL_ARCH_H + +#include <stdint.h> + +#include "kvm_util_types.h" + +struct kvm_mmu_arch {}; + +/* Page table fragment cache for guest page tables < page size */ +struct vm_pt_frag_cache { + gpa_t page; + size_t page_nr_used; +}; + +struct kvm_vm_arch { + gpa_t prtb; /* process table */ + struct vm_pt_frag_cache pt_frag_cache[2]; /* 256B and 4KB PT caches */ +}; + +#endif /* SELFTEST_KVM_UTIL_ARCH_H */ diff --git a/tools/testing/selftests/kvm/include/powerpc/ppc_asm.h b/tools/testing/selftests/kvm/include/powerpc/ppc_asm.h new file mode 100644 index 000000000000..b9df64659792 --- /dev/null +++ b/tools/testing/selftests/kvm/include/powerpc/ppc_asm.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * powerpc asm specific defines + */ +#ifndef SELFTEST_KVM_PPC_ASM_H +#define SELFTEST_KVM_PPC_ASM_H + +#define STACK_FRAME_MIN_SIZE 112 /* Could be 32 on ELFv2 */ +#define STACK_REDZONE_SIZE 512 + +#define INT_FRAME_SIZE (STACK_FRAME_MIN_SIZE + STACK_REDZONE_SIZE) + +#define SPR_SRR0 0x01a +#define SPR_SRR1 0x01b +#define SPR_CFAR 0x01c + +#define MSR_SF 0x8000000000000000ULL +#define MSR_HV 0x1000000000000000ULL +#define MSR_VEC 0x0000000002000000ULL +#define MSR_VSX 0x0000000000800000ULL +#define MSR_EE 0x0000000000008000ULL +#define MSR_PR 0x0000000000004000ULL +#define MSR_FP 0x0000000000002000ULL +#define MSR_ME 0x0000000000001000ULL +#define MSR_IR 0x0000000000000020ULL +#define MSR_DR 0x0000000000000010ULL +#define MSR_RI 0x0000000000000002ULL +#define MSR_LE 0x0000000000000001ULL + +#define LPCR_ILE 0x0000000002000000ULL + +#endif diff --git a/tools/testing/selftests/kvm/include/powerpc/processor.h b/tools/testing/selftests/kvm/include/powerpc/processor.h new file mode 100644 index 000000000000..f0ee21a8e2d4 --- /dev/null +++ b/tools/testing/selftests/kvm/include/powerpc/processor.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * powerpc processor specific defines + */ +#ifndef SELFTEST_KVM_PROCESSOR_H +#define SELFTEST_KVM_PROCESSOR_H + +#include <linux/compiler.h> +#include "ppc_asm.h" +#include "kvm_util_types.h" + +extern unsigned char __interrupts_start[]; +extern unsigned char __interrupts_end[]; + +struct kvm_vm; +struct kvm_vcpu; + +struct ex_regs { + u64 gprs[32]; + u64 nia; + u64 msr; + u64 cfar; + u64 lr; + u64 ctr; + u64 xer; + u32 cr; + u32 trap; + gva_t gva; /* gva of this struct */ +}; + +void vm_install_exception_handler(struct kvm_vm *vm, int vector, + void (*handler)(struct ex_regs *)); + +static inline void cpu_relax(void) +{ + asm volatile("" ::: "memory"); +} + +#endif diff --git a/tools/testing/selftests/kvm/include/powerpc/ucall.h b/tools/testing/selftests/kvm/include/powerpc/ucall.h new file mode 100644 index 000000000000..e0dbe91e8848 --- /dev/null +++ b/tools/testing/selftests/kvm/include/powerpc/ucall.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef SELFTEST_KVM_UCALL_H +#define SELFTEST_KVM_UCALL_H + +#include "hcall.h" + +#define UCALL_EXIT_REASON KVM_EXIT_PAPR_HCALL + +#define UCALL_R4_UCALL 0x5715 /* regular ucall, r5 contains ucall pointer */ +#define UCALL_R4_SIMPLE 0x0000 /* simple exit usable by asm with no ucall data */ + +static inline void ucall_arch_init(struct kvm_vm *vm, gpa_t mmio_gpa) +{ +} + +static inline void ucall_arch_do_ucall(gva_t uc) +{ + hcall2(H_UCALL, UCALL_R4_UCALL, (uintptr_t)(uc)); +} + +#endif diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h index a6a3e1657895..c80019e24b2d 100644 --- a/tools/testing/selftests/kvm/include/test_util.h +++ b/tools/testing/selftests/kvm/include/test_util.h @@ -49,10 +49,6 @@ do { \ #define TEST_REQUIRE(f) __TEST_REQUIRE(f, "Requirement not met: %s", #f) -ssize_t test_write(int fd, const void *buf, size_t count); -ssize_t test_read(int fd, void *buf, size_t count); -int test_seq_read(const char *path, char **bufp, size_t *sizep); - void __printf(5, 6) test_assert(bool exp, const char *exp_str, const char *file, unsigned int line, const char *fmt, ...); @@ -83,21 +79,23 @@ do { \ __builtin_unreachable(); \ } while (0) -extern sigjmp_buf expect_sigbus_jmpbuf; -void expect_sigbus_handler(int signum); - -#define TEST_EXPECT_SIGBUS(action) \ -do { \ - struct sigaction sa_old, sa_new = { \ - .sa_handler = expect_sigbus_handler, \ - }; \ - \ - sigaction(SIGBUS, &sa_new, &sa_old); \ - if (sigsetjmp(expect_sigbus_jmpbuf, 1) == 0) { \ - action; \ - TEST_FAIL("'%s' should have triggered SIGBUS", #action); \ - } \ - sigaction(SIGBUS, &sa_old, NULL); \ +extern __thread sigjmp_buf expect_sigbus_jmpbuf; +extern __thread volatile sig_atomic_t expecting_sigbus; +void catchall_signal_handler(int signum); + +#define TEST_EXPECT_SIGBUS(action) \ +do { \ + struct sigaction __sa = {}; \ + \ + TEST_ASSERT_EQ(sigaction(SIGBUS, NULL, &__sa), 0); \ + TEST_ASSERT_EQ(__sa.sa_handler, &catchall_signal_handler); \ + \ + expecting_sigbus = true; \ + if (sigsetjmp(expect_sigbus_jmpbuf, 1) == 0) { \ + action; \ + TEST_FAIL("'%s' should have triggered SIGBUS", #action);\ + } \ + expecting_sigbus = false; \ } while (0) size_t parse_size(const char *size); @@ -116,6 +114,8 @@ struct kvm_random_state { extern u32 kvm_random_seed; extern struct kvm_random_state kvm_rng; +extern bool kvm_has_gmem_attributes; + struct kvm_random_state new_kvm_random_state(u32 seed); u32 kvm_random_u32(struct kvm_random_state *state); @@ -172,6 +172,7 @@ struct vm_mem_backing_src_alias { bool thp_configured(void); size_t get_trans_hugepagesz(void); size_t get_def_hugetlb_pagesz(void); +size_t get_free_hugepages(size_t page_size); const struct vm_mem_backing_src_alias *vm_mem_backing_src_alias(u32 i); size_t get_backing_src_pagesz(u32 i); bool is_backing_src_hugetlb(u32 i); @@ -238,7 +239,6 @@ static inline u32 atoi_non_negative(const char *name, const char *num_str) } int guest_vsnprintf(char *buf, int n, const char *fmt, va_list args); -__printf(3, 4) int guest_snprintf(char *buf, int n, const char *fmt, ...); char *strdup_printf(const char *fmt, ...) __attribute__((format(printf, 1, 2), nonnull(1))); diff --git a/tools/testing/selftests/kvm/include/x86/evmcs.h b/tools/testing/selftests/kvm/include/x86/evmcs.h index 82a8ea6b661f..491fb8107473 100644 --- a/tools/testing/selftests/kvm/include/x86/evmcs.h +++ b/tools/testing/selftests/kvm/include/x86/evmcs.h @@ -245,32 +245,18 @@ static inline void evmcs_enable(void) enable_evmcs = true; } -static inline int evmcs_vmptrld(u64 vmcs_pa, void *vmcs) +static inline void evmcs_vmptrld(u64 vmcs_pa, void *vmcs) { current_vp_assist->current_nested_vmcs = vmcs_pa; current_vp_assist->enlighten_vmentry = 1; current_evmcs = vmcs; - - return 0; } -static inline bool load_evmcs(struct hyperv_test_pages *hv) +static inline void load_evmcs(struct hyperv_test_pages *hv) { - if (evmcs_vmptrld(hv->enlightened_vmcs_gpa, hv->enlightened_vmcs)) - return false; - + evmcs_vmptrld(hv->enlightened_vmcs_gpa, hv->enlightened_vmcs); current_evmcs->revision_id = EVMCS_VERSION; - - return true; -} - -static inline int evmcs_vmptrst(u64 *value) -{ - *value = current_vp_assist->current_nested_vmcs & - ~HV_X64_MSR_VP_ASSIST_PAGE_ENABLE; - - return 0; } static inline int evmcs_vmread(u64 encoding, u64 *value) @@ -1201,62 +1187,4 @@ static inline int evmcs_vmwrite(u64 encoding, u64 value) return 0; } -static inline int evmcs_vmlaunch(void) -{ - int ret; - - current_evmcs->hv_clean_fields = 0; - - __asm__ __volatile__("push $0;" - "mov %%rsp, (%[host_rsp]);" - "lea 1f(%%rip), %%rax;" - "mov %%rax, (%[host_rip]);" - VMX_SWITCH_GPRS_ASM - "vmlaunch;" - "incq (%%rsp);" - "1: ;" - VMX_SWITCH_GPRS_ASM - "pop %%rax;" - : [ret]"=&a"(ret) - : [host_rsp]"r" - ((u64)¤t_evmcs->host_rsp), - [host_rip]"r" - ((u64)¤t_evmcs->host_rip), - GUEST_REGS_OFFSETS - : "memory", "cc"); - return ret; -} - -/* - * No guest state (e.g. GPRs) is established by this vmresume. - */ -static inline int evmcs_vmresume(void) -{ - int ret; - - /* HOST_RIP */ - current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1; - /* HOST_RSP */ - current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER; - - __asm__ __volatile__("push $0;" - "mov %%rsp, (%[host_rsp]);" - "lea 1f(%%rip), %%rax;" - "mov %%rax, (%[host_rip]);" - VMX_SWITCH_GPRS_ASM - "vmresume;" - "incq (%%rsp);" - "1: ;" - VMX_SWITCH_GPRS_ASM - "pop %%rax;" - : [ret]"=&a"(ret) - : [host_rsp]"r" - ((u64)¤t_evmcs->host_rsp), - [host_rip]"r" - ((u64)¤t_evmcs->host_rip), - GUEST_REGS_OFFSETS - : "memory", "cc"); - return ret; -} - #endif /* !SELFTEST_KVM_EVMCS_H */ diff --git a/tools/testing/selftests/kvm/include/x86/processor.h b/tools/testing/selftests/kvm/include/x86/processor.h index 6e6f70035508..5898a277a2e0 100644 --- a/tools/testing/selftests/kvm/include/x86/processor.h +++ b/tools/testing/selftests/kvm/include/x86/processor.h @@ -23,6 +23,8 @@ extern bool host_cpu_is_intel; extern bool host_cpu_is_amd; extern bool host_cpu_is_hygon; extern bool host_cpu_is_amd_compatible; +extern bool host_cpu_is_zhaoxin; +extern bool host_cpu_is_intel_compatible; extern u64 guest_tsc_khz; extern struct kvm_mmu guest_mmu; @@ -216,6 +218,7 @@ struct kvm_x86_cpu_feature { #define X86_FEATURE_INVTSC KVM_X86_CPU_FEATURE(0x80000007, 0, EDX, 8) #define X86_FEATURE_RDPRU KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 4) #define X86_FEATURE_AMD_IBPB KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 12) +#define X86_FEATURE_EFER_LMSLE_MBZ KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 20) #define X86_FEATURE_NPT KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 0) #define X86_FEATURE_LBRV KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 1) #define X86_FEATURE_NRIPS KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 3) @@ -782,6 +785,12 @@ static inline bool this_cpu_is_hygon(void) return this_cpu_vendor_string_is("HygonGenuine"); } +static inline bool this_cpu_is_zhaoxin(void) +{ + return this_cpu_vendor_string_is("CentaurHauls") || + this_cpu_vendor_string_is(" Shanghai "); +} + static inline u32 __this_cpu_has(u32 function, u32 index, u8 reg, u8 lo, u8 hi) { u32 gprs[4]; @@ -1172,7 +1181,6 @@ static inline void vcpu_set_cpuid(struct kvm_vcpu *vcpu) void vcpu_set_cpuid_property(struct kvm_vcpu *vcpu, struct kvm_x86_cpu_property property, u32 value); -void vcpu_set_cpuid_maxphyaddr(struct kvm_vcpu *vcpu, u8 maxphyaddr); void vcpu_clear_cpuid_entry(struct kvm_vcpu *vcpu, u32 function); @@ -1318,7 +1326,7 @@ gva_t vm_alloc_stack(struct kvm_vm *vm, int nr_pages); "lea 1f(%%rip), %%r10\n\t" \ "lea 2f(%%rip), %%r11\n\t" \ fep "1: " insn "\n\t" \ - "xor %%r9, %%r9\n\t" \ + "mov $0, %%r9d\n\t" \ "2:\n\t" \ "mov %%r9b, %[vector]\n\t" \ "mov %%r10, %[error_code]\n\t" @@ -1446,7 +1454,7 @@ static inline bool kvm_is_pmu_enabled(void) static inline bool kvm_is_mediated_pmu_enabled(void) { - if (host_cpu_is_intel) + if (host_cpu_is_intel_compatible) return get_kvm_intel_param_bool("enable_mediated_pmu"); return get_kvm_amd_param_bool("enable_mediated_pmu"); @@ -1472,6 +1480,19 @@ static inline bool kvm_is_lbrv_enabled(void) return !!get_kvm_amd_param_integer("lbrv"); } +/* + * Do NOT use this to check for nVMX or nSVM support. Querying kvm_cpu_has() + * for either of X86_FEATURE_{VMX,SVM} is the idiomatic way to check for nested + * virtualization support. Use this *only* to validate KVM's own enumeration. + */ +static inline bool kvm_is_nested_virtualization_enabled(void) +{ + if (host_cpu_is_intel) + return get_kvm_intel_param_bool("nested"); + + return get_kvm_amd_param_integer("nested"); +} + u64 *vm_get_pte(struct kvm_vm *vm, gva_t gva); u64 kvm_hypercall(u64 nr, u64 a0, u64 a1, u64 a2, u64 a3); diff --git a/tools/testing/selftests/kvm/include/x86/smm.h b/tools/testing/selftests/kvm/include/x86/smm.h index 2d1afa09819b..15faaa060126 100644 --- a/tools/testing/selftests/kvm/include/x86/smm.h +++ b/tools/testing/selftests/kvm/include/x86/smm.h @@ -8,7 +8,7 @@ #define SMRAM_MEMSLOT ((1 << 16) | 1) #define SMRAM_PAGES (SMRAM_SIZE / PAGE_SIZE) -void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, u64 smram_gpa, +void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, gpa_t smram_gpa, const void *smi_handler, size_t handler_size); void inject_smi(struct kvm_vcpu *vcpu); diff --git a/tools/testing/selftests/kvm/include/x86/vmx.h b/tools/testing/selftests/kvm/include/x86/vmx.h index 04f5e34dea3a..8fd97434eee5 100644 --- a/tools/testing/selftests/kvm/include/x86/vmx.h +++ b/tools/testing/selftests/kvm/include/x86/vmx.h @@ -309,125 +309,67 @@ struct vmx_msr_entry { #include "evmcs.h" -static inline int vmxon(u64 phys) -{ - u8 ret; - - __asm__ __volatile__ ("vmxon %[pa]; setna %[ret]" - : [ret]"=rm"(ret) - : [pa]"m"(phys) - : "cc", "memory"); - - return ret; +#define GUEST_ASSERT_VMX_INSN_SUCCEEDED(insn, __r, __pa) \ + __GUEST_ASSERT(!__r, __stringify(insn) "[0x%lx] hit %s", \ + __pa, __r < 0 ? "VM-Fail" : ex_str(__r)) + +#define BUILD_VMCS_ASM_HELPERS(insn) \ +static inline int __##insn(u64 vmcs_pa) \ +{ \ + u64 error_code; \ + u8 vector; \ + u8 failed; \ + \ + asm volatile(KVM_ASM_SAFE(__stringify(insn) " %[pa]") \ + "\n\tsetna %[failed]" \ + : KVM_ASM_SAFE_OUTPUTS(vector, error_code), \ + [failed]"=qm"(failed) \ + : [pa]"m"(vmcs_pa) \ + : "cc", "memory", KVM_ASM_SAFE_CLOBBERS); \ + \ + return vector ? vector : failed ? -EINVAL : 0; \ +} \ + \ +static inline void insn(u64 vmcs_pa) \ +{ \ + int ret = __##insn(vmcs_pa); \ + \ + GUEST_ASSERT_VMX_INSN_SUCCEEDED(insn, ret, vmcs_pa); \ } +BUILD_VMCS_ASM_HELPERS(vmxon) +BUILD_VMCS_ASM_HELPERS(vmptrld) +BUILD_VMCS_ASM_HELPERS(vmclear) + static inline void vmxoff(void) { __asm__ __volatile__("vmxoff"); } -static inline int vmclear(u64 vmcs_pa) -{ - u8 ret; - - __asm__ __volatile__ ("vmclear %[pa]; setna %[ret]" - : [ret]"=rm"(ret) - : [pa]"m"(vmcs_pa) - : "cc", "memory"); - - return ret; -} - -static inline int vmptrld(u64 vmcs_pa) +static inline u64 vmptrst(void) { + u64 value = 0; u8 ret; - if (enable_evmcs) - return -1; - - __asm__ __volatile__ ("vmptrld %[pa]; setna %[ret]" - : [ret]"=rm"(ret) - : [pa]"m"(vmcs_pa) - : "cc", "memory"); - - return ret; -} - -static inline int vmptrst(u64 *value) -{ - u64 tmp; - u8 ret; - - if (enable_evmcs) - return evmcs_vmptrst(value); - __asm__ __volatile__("vmptrst %[value]; setna %[ret]" - : [value]"=m"(tmp), [ret]"=rm"(ret) + : [value]"=m"(value), [ret]"=rm"(ret) : : "cc", "memory"); - *value = tmp; - return ret; -} - -/* - * A wrapper around vmptrst that ignores errors and returns zero if the - * vmptrst instruction fails. - */ -static inline u64 vmptrstz(void) -{ - u64 value = 0; - vmptrst(&value); + __GUEST_ASSERT(!ret, "vmptrst failed"); return value; } -static inline int vmlaunch(void) -{ - int ret; +int __vmlaunch(void); +int __vmresume(void); - if (enable_evmcs) - return evmcs_vmlaunch(); - - __asm__ __volatile__("push $0;" - "vmwrite %%rsp, %[host_rsp];" - "lea 1f(%%rip), %%rax;" - "vmwrite %%rax, %[host_rip];" - VMX_SWITCH_GPRS_ASM - "vmlaunch;" - "incq (%%rsp);" - "1: ;" - VMX_SWITCH_GPRS_ASM - "pop %%rax;" - : [ret]"=&a"(ret) - : [host_rsp]"r"((u64)HOST_RSP), - [host_rip]"r"((u64)HOST_RIP), - GUEST_REGS_OFFSETS - : "memory", "cc"); - return ret; +static inline void vmlaunch(void) +{ + __GUEST_ASSERT(!__vmlaunch(), "vmlaunch hit VM-Fail"); } -static inline int vmresume(void) +static inline void vmresume(void) { - int ret; - - if (enable_evmcs) - return evmcs_vmresume(); - - __asm__ __volatile__("push $0;" - "vmwrite %%rsp, %[host_rsp];" - "lea 1f(%%rip), %%rax;" - "vmwrite %%rax, %[host_rip];" - VMX_SWITCH_GPRS_ASM - "vmresume;" - "incq (%%rsp);" - "1: ;" - VMX_SWITCH_GPRS_ASM - "pop %%rax;" - : [ret]"=&a"(ret) - : [host_rsp]"r"((u64)HOST_RSP), - [host_rip]"r"((u64)HOST_RIP), - GUEST_REGS_OFFSETS - : "memory", "cc"); - return ret; + __GUEST_ASSERT(!__vmresume(), "vmresume hit VM-Fail"); } static inline void vmcall(void) @@ -444,7 +386,7 @@ static inline void vmcall(void) "r10", "r11", "r12", "r13", "r14", "r15"); } -static inline int vmread(u64 encoding, u64 *value) +static inline int __vmread(u64 encoding, u64 *value) { u64 tmp; u8 ret; @@ -457,22 +399,21 @@ static inline int vmread(u64 encoding, u64 *value) : [encoding]"r"(encoding) : "cc", "memory"); - *value = tmp; + if (!ret) + *value = tmp; return ret; } -/* - * A wrapper around vmread that ignores errors and returns zero if the - * vmread instruction fails. - */ -static inline u64 vmreadz(u64 encoding) +static inline u64 vmread(u64 encoding) { u64 value = 0; - vmread(encoding, &value); + + __GUEST_ASSERT(!__vmread(encoding, &value), + "vmread[0x%lx] hit VM-Fail", encoding); return value; } -static inline int vmwrite(u64 encoding, u64 value) +static __always_inline int __vmwrite(u64 encoding, u64 value) { u8 ret; @@ -487,6 +428,12 @@ static inline int vmwrite(u64 encoding, u64 value) return ret; } +static inline void vmwrite(u64 encoding, u64 value) +{ + __GUEST_ASSERT(!__vmwrite(encoding, value), + "vmwrite[0x%lx] = 0x%lx hit VM-Fail", encoding, value); +} + static inline u32 vmcs_revision(void) { return rdmsr(MSR_IA32_VMX_BASIC); @@ -550,13 +497,24 @@ union vmx_ctrl_msr { }; struct vmx_pages *vcpu_alloc_vmx(struct kvm_vm *vm, gva_t *p_vmx_gva); -bool prepare_for_vmx_operation(struct vmx_pages *vmx); +void prepare_for_vmx_operation(struct vmx_pages *vmx); void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip); -bool load_vmcs(struct vmx_pages *vmx); +void load_vmcs(struct vmx_pages *vmx); bool ept_1g_pages_supported(void); -bool kvm_cpu_has_ept(void); +bool kvm_cpu_has_secondary_exec_control(u32 ctrl); + +static inline bool kvm_cpu_has_ept(void) +{ + return kvm_cpu_has_secondary_exec_control(SECONDARY_EXEC_ENABLE_EPT); +} + +static inline bool kvm_cpu_has_vmx_virtualize_apic_accesses(void) +{ + return kvm_cpu_has_secondary_exec_control(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES); +} + void vm_enable_ept(struct kvm_vm *vm); void prepare_virtualize_apic_accesses(struct vmx_pages *vmx, struct kvm_vm *vm); |