diff options
Diffstat (limited to 'arch/arm64/include/asm')
| -rw-r--r-- | arch/arm64/include/asm/cpufeature.h | 1 | ||||
| -rw-r--r-- | arch/arm64/include/asm/kvm_arm.h | 27 | ||||
| -rw-r--r-- | arch/arm64/include/asm/kvm_asm.h | 4 | ||||
| -rw-r--r-- | arch/arm64/include/asm/kvm_emulate.h | 107 | ||||
| -rw-r--r-- | arch/arm64/include/asm/kvm_host.h | 14 | ||||
| -rw-r--r-- | arch/arm64/include/asm/kvm_nested.h | 20 | ||||
| -rw-r--r-- | arch/arm64/include/asm/kvm_pkvm.h | 8 | ||||
| -rw-r--r-- | arch/arm64/include/asm/vncr_mapping.h | 2 |
8 files changed, 159 insertions, 24 deletions
diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index a57870fa96db..a42683af79fb 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -968,6 +968,7 @@ struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id); extern struct arm64_ftr_override id_aa64mmfr0_override; extern struct arm64_ftr_override id_aa64mmfr1_override; extern struct arm64_ftr_override id_aa64mmfr2_override; +extern struct arm64_ftr_override id_aa64mmfr4_override; extern struct arm64_ftr_override id_aa64pfr0_override; extern struct arm64_ftr_override id_aa64pfr1_override; extern struct arm64_ftr_override id_aa64zfr0_override; diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 3f9233b5a130..4bfbd827c5aa 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -287,21 +287,6 @@ GENMASK(19, 18) | \ GENMASK(15, 0)) -/* - * Polarity masks for HCRX_EL2, limited to the bits that we know about - * at this point in time. It doesn't mean that we actually *handle* - * them, but that at least those that are not advertised to a guest - * will be RES0 for that guest. - */ -#define __HCRX_EL2_MASK (BIT_ULL(6)) -#define __HCRX_EL2_nMASK (GENMASK_ULL(24, 14) | \ - GENMASK_ULL(11, 7) | \ - GENMASK_ULL(5, 0)) -#define __HCRX_EL2_RES0 ~(__HCRX_EL2_nMASK | __HCRX_EL2_MASK) -#define __HCRX_EL2_RES1 ~(__HCRX_EL2_nMASK | \ - __HCRX_EL2_MASK | \ - __HCRX_EL2_RES0) - /* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */ #define HPFAR_MASK (~UL(0xf)) /* @@ -348,4 +333,16 @@ { PSR_AA32_MODE_UND, "32-bit UND" }, \ { PSR_AA32_MODE_SYS, "32-bit SYS" } +/* + * ARMv8 Reset Values + */ +#define VCPU_RESET_PSTATE_EL1 (PSR_MODE_EL1h | PSR_A_BIT | PSR_I_BIT | \ + PSR_F_BIT | PSR_D_BIT) + +#define VCPU_RESET_PSTATE_EL2 (PSR_MODE_EL2h | PSR_A_BIT | PSR_I_BIT | \ + PSR_F_BIT | PSR_D_BIT) + +#define VCPU_RESET_PSTATE_SVC (PSR_AA32_MODE_SVC | PSR_AA32_A_BIT | \ + PSR_AA32_I_BIT | PSR_AA32_F_BIT) + #endif /* __ARM64_KVM_ARM_H__ */ diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 043495f7fc78..e5b92ac09e69 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -113,6 +113,7 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___pkvm_finalize_teardown_vm, __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_load, __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_put, + __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_sync_state, __KVM_HOST_SMCCC_FUNC___pkvm_tlb_flush_vmid, MARKER(__KVM_HOST_SMCCC_FUNC_MAX) @@ -214,7 +215,6 @@ struct kvm_nvhe_init_params { unsigned long hcr_el2; unsigned long vttbr; unsigned long vtcr; - unsigned long tmp; }; /* @@ -281,7 +281,7 @@ extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu); extern void __kvm_adjust_pc(struct kvm_vcpu *vcpu); -extern u64 __vgic_v3_get_gic_config(void); +extern bool __vgic_v3_get_gic_config(void); extern void __vgic_v3_init_lrs(void); #define __KVM_EXTABLE(from, to) \ diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 5bf3d7e1d92c..a3c1928bdf74 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -266,6 +266,25 @@ static inline bool vserror_state_is_nested(struct kvm_vcpu *vcpu) (__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TMEA); } +static inline bool kvm_has_nv2(struct kvm *kvm) +{ + return (cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) && + kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)); +} + +static inline bool kvm_has_nv3(struct kvm *kvm) +{ + return (cpus_have_final_cap(ARM64_HAS_NV3) && + kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV3)); +} + +static inline bool is_nested_nv3_ctxt(struct kvm_vcpu *vcpu) +{ + return (has_vhe() && kvm_has_nv3(vcpu->kvm) && is_nested_ctxt(vcpu) && + (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_NV) && + (__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_NVTGE)); +} + /* * The layout of SPSR for an AArch32 state is different when observed from an * AArch64 SPSR_ELx or an AArch32 SPSR_*. This function generates the AArch32 @@ -506,6 +525,12 @@ static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu) return __vcpu_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK; } +/* In nVHE hyp code, registers are always in memory: use the raw accessors. */ +#if defined(__KVM_NVHE_HYPERVISOR__) +#define vcpu_read_sys_reg(v, r) __vcpu_sys_reg(v, r) +#define vcpu_write_sys_reg(v, x, r) __vcpu_assign_sys_reg(v, r, x) +#endif + static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu) { if (vcpu_mode_is_32bit(vcpu)) { @@ -617,7 +642,7 @@ static __always_inline void kvm_incr_pc(struct kvm_vcpu *vcpu) */ static inline u64 vcpu_sanitised_cptr_el2(const struct kvm_vcpu *vcpu) { - u64 cptr = __vcpu_sys_reg(vcpu, CPTR_EL2); + u64 cptr = vcpu_read_sys_reg(vcpu, CPTR_EL2); if (!vcpu_el2_e2h_is_set(vcpu)) cptr = translate_cptr_el2_to_cpacr_el1(cptr); @@ -686,6 +711,86 @@ static inline void vcpu_set_hcrx(struct kvm_vcpu *vcpu) if (kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_V)) vcpu->arch.hcrx_el2 |= HCRX_EL2_EnASR; + + /* + * NV3 is a host-specific extension, and we always use + * it when present and that the guest uses NV. It may + * be hidden from the guest though. + */ + if (cpus_have_final_cap(ARM64_HAS_NV3) && + vcpu_has_nv(vcpu) && vcpu_el2_e2h_is_set(vcpu)) { + vcpu->arch.hcrx_el2 |= HCRX_EL2_NVTGE; + + /* + * If the guest is NV2-capable, then we need to see + * all the TLBIs, as configured in HCR_EL2. + * Otherwise, relax the TLBI traps to only TGE=0. + */ + if (!kvm_has_nv2(vcpu->kvm)) { + vcpu->arch.hcrx_el2 |= (HCRX_EL2_NVnTTLB | + HCRX_EL2_NVnTTLBIS); + + if (kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS)) + vcpu->arch.hcrx_el2 |= HCRX_EL2_NVnTTLBOS; + } + } + } +} + +/* Reset a vcpu's core registers. */ +static inline void kvm_reset_vcpu_core(struct kvm_vcpu *vcpu) +{ + u32 pstate; + + if (vcpu_el1_is_32bit(vcpu)) + pstate = VCPU_RESET_PSTATE_SVC; + else if (vcpu_has_nv(vcpu)) + pstate = VCPU_RESET_PSTATE_EL2; + else + pstate = VCPU_RESET_PSTATE_EL1; + + /* Reset core registers */ + memset(vcpu_gp_regs(vcpu), 0, sizeof(*vcpu_gp_regs(vcpu))); + memset(&vcpu->arch.ctxt.fp_regs, 0, sizeof(vcpu->arch.ctxt.fp_regs)); + vcpu->arch.ctxt.spsr_abt = 0; + vcpu->arch.ctxt.spsr_und = 0; + vcpu->arch.ctxt.spsr_irq = 0; + vcpu->arch.ctxt.spsr_fiq = 0; + vcpu_gp_regs(vcpu)->pstate = pstate; +} + +/* PSCI reset handling for a vcpu. */ +static inline void kvm_reset_vcpu_psci(struct kvm_vcpu *vcpu, + struct vcpu_reset_state *reset_state) +{ + unsigned long target_pc = reset_state->pc; + + /* Gracefully handle Thumb2 entry point */ + if (vcpu_mode_is_32bit(vcpu) && (target_pc & 1)) { + target_pc &= ~1UL; + vcpu_set_thumb(vcpu); } + + /* Propagate caller endianness */ + if (reset_state->be) + kvm_vcpu_set_be(vcpu); + + *vcpu_pc(vcpu) = target_pc; + + /* + * We may come from a state where either a PC update was + * pending (SMC call resulting in PC being increpented to + * skip the SMC) or a pending exception. Make sure we get + * rid of all that, as this cannot be valid out of reset. + * + * Note that clearing the exception mask also clears PC + * updates, but that's an implementation detail, and we + * really want to make it explicit. + */ + vcpu_clear_flag(vcpu, PENDING_EXCEPTION); + vcpu_clear_flag(vcpu, EXCEPT_MASK); + vcpu_clear_flag(vcpu, INCREMENT_PC); + vcpu_set_reg(vcpu, 0, reset_state->r0); } + #endif /* __ARM64_KVM_EMULATE_H__ */ diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index bae2c4f92ef5..27fe0cd5b2d7 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -39,7 +39,7 @@ #define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS -#define KVM_VCPU_MAX_FEATURES 9 +#define KVM_VCPU_MAX_FEATURES 10 #define KVM_VCPU_VALID_FEATURES (BIT(KVM_VCPU_MAX_FEATURES) - 1) #define KVM_REQ_SLEEP \ @@ -387,6 +387,9 @@ struct kvm_arch { /* Maximum number of counters for the guest */ u8 nr_pmu_counters; + /* PMMIR_EL1.SLOTS value exposed to the guest. */ + u8 pmmir_slots; + /* Hypercall features firmware registers' descriptor */ struct kvm_smccc_features smccc_feat; struct maple_tree smccc_filter; @@ -411,8 +414,8 @@ struct kvm_arch { /* Masks for VNCR-backed and general EL2 sysregs */ struct kvm_sysreg_masks *sysreg_masks; - /* Count the number of VNCR_EL2 currently mapped */ - atomic_t vncr_map_count; + /* Count the number of VNCR_EL2 TLBs */ + atomic_t vncr_tlb_count; /* * For an untrusted host VM, 'pkvm.handle' is used to lookup @@ -543,6 +546,7 @@ enum vcpu_sysreg { MDCR_EL2, /* Monitor Debug Configuration Register (EL2) */ CNTHCTL_EL2, /* Counter-timer Hypervisor Control register */ ZCR_EL2, /* SVE Control Register (EL2) */ + HCR_EL2, /* Hypervisor Control Register */ /* Any VNCR-capable reg goes after this point */ MARKER(__VNCR_START__), @@ -571,7 +575,7 @@ enum vcpu_sysreg { VNCR(TFSR_EL1), /* Tag Fault Status Register (EL1) */ VNCR(VPIDR_EL2),/* Virtualization Processor ID Register */ VNCR(VMPIDR_EL2),/* Virtualization Multiprocessor ID Register */ - VNCR(HCR_EL2), /* Hypervisor Configuration Register */ + VNCR(NVHCR_EL2),/* NV Hypervisor Configuration Register */ VNCR(HSTR_EL2), /* Hypervisor System Trap Register */ VNCR(VTTBR_EL2),/* Virtualization Translation Table Base Register */ VNCR(VTCR_EL2), /* Virtualization Translation Control Register */ @@ -1051,6 +1055,8 @@ struct kvm_vcpu_arch { #define INCREMENT_PC __vcpu_single_flag(iflags, BIT(1)) /* Target EL/MODE (not a single flag, but let's abuse the macro) */ #define EXCEPT_MASK __vcpu_single_flag(iflags, GENMASK(3, 1)) +/* Host-set: the hyp flushes the non-protected vCPU state in on entry */ +#define PKVM_HOST_STATE_DIRTY __vcpu_single_flag(iflags, BIT(4)) /* Helpers to encode exceptions with minimum fuss */ #define __EXCEPT_MASK_VAL unpack_vcpu_flag(EXCEPT_MASK) diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index 012d711034d1..1ed708335809 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -291,6 +291,13 @@ static inline u64 decode_range_tlbi(u64 val, u64 *range, u16 *asid) base = (val & GENMASK(36, 0)) << shift; + /* + * We only deal with at most 48bit VA/IPA, so 48 is where we + * sign-extend from. Should we support FEAT_L{VP}A* at some point, + * this will need to be revisited. + */ + base = (u64)sign_extend64(base, 48); + if (asid) *asid = FIELD_GET(TLBIR_ASID_MASK, val); @@ -298,6 +305,12 @@ static inline u64 decode_range_tlbi(u64 val, u64 *range, u16 *asid) num = FIELD_GET(GENMASK(43, 39), val); *range = __TLBI_RANGE_PAGES(num, scale) << shift; + /* Cap the range to the correct half of the address space */ + if (!(base & BIT(48))) + *range = min(*range, (BIT(48) - base)); + else + *range = min(*range, ~base + 1); + return base; } @@ -388,6 +401,8 @@ struct s1_walk_result { bool failed; }; +#define S1_MMU_DISABLED (-127) + static inline void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) { wr->fst = fst; @@ -396,6 +411,11 @@ static inline void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) wr->failed = true; } +static inline bool s1_walk_translated(struct s1_walk_result *wr) +{ + return wr->level != S1_MMU_DISABLED; +} + int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, struct s1_walk_result *wr, u64 va); int __kvm_find_s1_desc_level(struct kvm_vcpu *vcpu, u64 va, u64 ipa, diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index 74fedd9c5ff0..beea00e693a0 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -45,6 +45,9 @@ static inline bool kvm_pkvm_ext_allowed(struct kvm *kvm, long ext) return true; case KVM_CAP_ARM_MTE: return false; + case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE: + case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES: + return false; default: return !kvm || !kvm_vm_is_protected(kvm); } @@ -195,7 +198,10 @@ struct pkvm_mapping { struct rb_node node; u64 gfn; u64 pfn; - u64 nr_pages; + struct { + u64 nr_pages:48; + u64 nc:1; + }; u64 __subtree_last; /* Internal member for interval tree */ }; diff --git a/arch/arm64/include/asm/vncr_mapping.h b/arch/arm64/include/asm/vncr_mapping.h index 14366d35ce82..9e8a49fa8b63 100644 --- a/arch/arm64/include/asm/vncr_mapping.h +++ b/arch/arm64/include/asm/vncr_mapping.h @@ -11,7 +11,7 @@ #define VNCR_VTCR_EL2 0x040 #define VNCR_VMPIDR_EL2 0x050 #define VNCR_CNTVOFF_EL2 0x060 -#define VNCR_HCR_EL2 0x078 +#define VNCR_NVHCR_EL2 0x078 #define VNCR_HSTR_EL2 0x080 #define VNCR_VPIDR_EL2 0x088 #define VNCR_TPIDR_EL2 0x090 |