From d4236f1ef270347b26ed34bcd0eb28d285b86bcb Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 23 Jan 2026 19:16:33 +0000 Subject: arm64: Add MT_S2{,_FWB}_AS_S1 encodings pKVM usage of S2 translation on the host is purely for isolation purposes, not translation. To that effect, the memory attributes being used must be that of S1. With FWB=0, this is easily achieved by using the Normal Cacheable type (which is the weakest possible memory type) at S2, and let S1 pick something stronger as required. With FWB=1, the attributes are combined in a different way, and we cannot arbitrarily use Normal Cacheable. We can, however, use a memattr encoding that indicates that the final attributes are that of Stage-1. Add these encoding and a few pointers to the relevant parts of the specification. It might come handy some day. Reviewed-by: Joey Gouly Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260123191637.715429-2-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/memory.h | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index 9d54b2ea49d6..a2b7a33966ff 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -175,19 +175,24 @@ #define MT_DEVICE_nGnRE 4 /* - * Memory types for Stage-2 translation + * Memory types for Stage-2 translation when HCR_EL2.FWB=0. See R_HMNDG, + * R_TNHFM, R_GQFSF and I_MCQKW for the details on how these attributes get + * combined with Stage-1. */ #define MT_S2_NORMAL 0xf #define MT_S2_NORMAL_NC 0x5 #define MT_S2_DEVICE_nGnRE 0x1 +#define MT_S2_AS_S1 MT_S2_NORMAL /* - * Memory types for Stage-2 translation when ID_AA64MMFR2_EL1.FWB is 0001 - * Stage-2 enforces Normal-WB and Device-nGnRE + * Memory types for Stage-2 translation when HCR_EL2.FWB=1. Stage-2 enforces + * Normal-WB and Device-nGnRE, unless we actively say that S1 wins. See + * R_VRJSW and R_RHWZM for details. */ #define MT_S2_FWB_NORMAL 6 #define MT_S2_FWB_NORMAL_NC 5 #define MT_S2_FWB_DEVICE_nGnRE 1 +#define MT_S2_FWB_AS_S1 7 #ifdef CONFIG_ARM64_4K_PAGES #define IOREMAP_MAX_ORDER (PUD_SHIFT) -- cgit v1.2.3 From 17d7b15131f3ed7fff29f2a43b6ead547e495653 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 23 Jan 2026 19:16:34 +0000 Subject: KVM: arm64: Add KVM_PGTABLE_S2_AS_S1 flag Plumb the MT_S2{,_FWB}_AS_S1 memory types into the KVM_S2_MEMATTR() macro with a new KVM_PGTABLE_S2_AS_S1 flag. Nobody selects it yet. Reviewed-by: Joey Gouly Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260123191637.715429-3-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_pgtable.h | 2 ++ arch/arm64/kvm/hyp/pgtable.c | 14 +++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index fc02de43c68d..9ce51a637da0 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -232,10 +232,12 @@ struct kvm_pgtable_mm_ops { * @KVM_PGTABLE_S2_NOFWB: Don't enforce Normal-WB even if the CPUs have * ARM64_HAS_STAGE2_FWB. * @KVM_PGTABLE_S2_IDMAP: Only use identity mappings. + * @KVM_PGTABLE_S2_AS_S1: Final memory attributes are that of Stage-1. */ enum kvm_pgtable_stage2_flags { KVM_PGTABLE_S2_NOFWB = BIT(0), KVM_PGTABLE_S2_IDMAP = BIT(1), + KVM_PGTABLE_S2_AS_S1 = BIT(2), }; /** diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 947ac1a951a5..c52a24c15ff2 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -659,7 +659,19 @@ void kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu, } } -#define KVM_S2_MEMATTR(pgt, attr) PAGE_S2_MEMATTR(attr, stage2_has_fwb(pgt)) +#define KVM_S2_MEMATTR(pgt, attr) \ + ({ \ + kvm_pte_t __attr; \ + \ + if ((pgt)->flags & KVM_PGTABLE_S2_AS_S1) \ + __attr = PAGE_S2_MEMATTR(AS_S1, \ + stage2_has_fwb(pgt)); \ + else \ + __attr = PAGE_S2_MEMATTR(attr, \ + stage2_has_fwb(pgt)); \ + \ + __attr; \ + }) static int stage2_set_xn_attr(enum kvm_pgtable_prot prot, kvm_pte_t *attr) { -- cgit v1.2.3 From a373930ec9406aeb9f82b5b78db56ca8d0919d99 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 23 Jan 2026 19:16:35 +0000 Subject: KVM: arm64: Switch pKVM host S2 over to KVM_PGTABLE_S2_AS_S1 Since we have the basics to use the S1 memory attributes as the final ones with FWB, flip the host over to that when FWB is present. Reviewed-by: Joey Gouly Tested-by: Fuad Tabba Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20260123191637.715429-4-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/mem_protect.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 49db32f3ddf7..38f66a56a766 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -19,7 +19,7 @@ #include #include -#define KVM_HOST_S2_FLAGS (KVM_PGTABLE_S2_NOFWB | KVM_PGTABLE_S2_IDMAP) +#define KVM_HOST_S2_FLAGS (KVM_PGTABLE_S2_AS_S1 | KVM_PGTABLE_S2_IDMAP) struct host_mmu host_mmu; @@ -324,6 +324,8 @@ int __pkvm_prot_finalize(void) params->vttbr = kvm_get_vttbr(mmu); params->vtcr = mmu->vtcr; params->hcr_el2 |= HCR_VM; + if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) + params->hcr_el2 |= HCR_FWB; /* * The CMO below not only cleans the updated params to the -- cgit v1.2.3 From 4f27fe82aa304c50b24f92da72b4895cf73b54ba Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 23 Jan 2026 19:16:36 +0000 Subject: KVM: arm64: Kill KVM_PGTABLE_S2_NOFWB Nobody is using this flag anymore, so remove it. This allows some cleanup by removing stage2_has_fwb(), which is can be replaced by a direct check on the capability. Reviewed-by: Joey Gouly Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260123191637.715429-5-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_pgtable.h | 7 ++----- arch/arm64/kvm/hyp/pgtable.c | 21 ++++++--------------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index 9ce51a637da0..2198b6242883 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -229,15 +229,12 @@ struct kvm_pgtable_mm_ops { /** * enum kvm_pgtable_stage2_flags - Stage-2 page-table flags. - * @KVM_PGTABLE_S2_NOFWB: Don't enforce Normal-WB even if the CPUs have - * ARM64_HAS_STAGE2_FWB. * @KVM_PGTABLE_S2_IDMAP: Only use identity mappings. * @KVM_PGTABLE_S2_AS_S1: Final memory attributes are that of Stage-1. */ enum kvm_pgtable_stage2_flags { - KVM_PGTABLE_S2_NOFWB = BIT(0), - KVM_PGTABLE_S2_IDMAP = BIT(1), - KVM_PGTABLE_S2_AS_S1 = BIT(2), + KVM_PGTABLE_S2_IDMAP = BIT(0), + KVM_PGTABLE_S2_AS_S1 = BIT(1), }; /** diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index c52a24c15ff2..00e33a16494b 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -631,14 +631,6 @@ u64 kvm_get_vtcr(u64 mmfr0, u64 mmfr1, u32 phys_shift) return vtcr; } -static bool stage2_has_fwb(struct kvm_pgtable *pgt) -{ - if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) - return false; - - return !(pgt->flags & KVM_PGTABLE_S2_NOFWB); -} - void kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, size_t size) { @@ -661,14 +653,13 @@ void kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu, #define KVM_S2_MEMATTR(pgt, attr) \ ({ \ + bool __fwb = cpus_have_final_cap(ARM64_HAS_STAGE2_FWB); \ kvm_pte_t __attr; \ \ if ((pgt)->flags & KVM_PGTABLE_S2_AS_S1) \ - __attr = PAGE_S2_MEMATTR(AS_S1, \ - stage2_has_fwb(pgt)); \ + __attr = PAGE_S2_MEMATTR(AS_S1, __fwb); \ else \ - __attr = PAGE_S2_MEMATTR(attr, \ - stage2_has_fwb(pgt)); \ + __attr = PAGE_S2_MEMATTR(attr, __fwb); \ \ __attr; \ }) @@ -880,7 +871,7 @@ static bool stage2_unmap_defer_tlb_flush(struct kvm_pgtable *pgt) * system supporting FWB as the optimization is entirely * pointless when the unmap walker needs to perform CMOs. */ - return system_supports_tlb_range() && stage2_has_fwb(pgt); + return system_supports_tlb_range() && cpus_have_final_cap(ARM64_HAS_STAGE2_FWB); } static void stage2_unmap_put_pte(const struct kvm_pgtable_visit_ctx *ctx, @@ -1160,7 +1151,7 @@ static int stage2_unmap_walker(const struct kvm_pgtable_visit_ctx *ctx, if (mm_ops->page_count(childp) != 1) return 0; } else if (stage2_pte_cacheable(pgt, ctx->old)) { - need_flush = !stage2_has_fwb(pgt); + need_flush = !cpus_have_final_cap(ARM64_HAS_STAGE2_FWB); } /* @@ -1390,7 +1381,7 @@ int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size) .arg = pgt, }; - if (stage2_has_fwb(pgt)) + if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) return 0; return kvm_pgtable_walk(pgt, addr, size, &walker); -- cgit v1.2.3 From 65d00e37b17f46a21e7a25e0fb211a4c33274a83 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 23 Jan 2026 19:16:37 +0000 Subject: KVM: arm64: Simplify PAGE_S2_MEMATTR Restore PAGE_S2_MEMATTR() to its former glory, keeping the use of FWB as an implementation detail. Reviewed-by: Joey Gouly Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260123191637.715429-6-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/pgtable-prot.h | 4 ++-- arch/arm64/kvm/hyp/pgtable.c | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/arch/arm64/include/asm/pgtable-prot.h b/arch/arm64/include/asm/pgtable-prot.h index 161e8660eddd..d27e8872fe3c 100644 --- a/arch/arm64/include/asm/pgtable-prot.h +++ b/arch/arm64/include/asm/pgtable-prot.h @@ -109,10 +109,10 @@ static inline bool __pure lpa2_is_enabled(void) #define PAGE_KERNEL_EXEC __pgprot(_PAGE_KERNEL_EXEC) #define PAGE_KERNEL_EXEC_CONT __pgprot(_PAGE_KERNEL_EXEC_CONT) -#define PAGE_S2_MEMATTR(attr, has_fwb) \ +#define PAGE_S2_MEMATTR(attr) \ ({ \ u64 __val; \ - if (has_fwb) \ + if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) \ __val = PTE_S2_MEMATTR(MT_S2_FWB_ ## attr); \ else \ __val = PTE_S2_MEMATTR(MT_S2_ ## attr); \ diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 00e33a16494b..a6f7533ce497 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -653,13 +653,12 @@ void kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu, #define KVM_S2_MEMATTR(pgt, attr) \ ({ \ - bool __fwb = cpus_have_final_cap(ARM64_HAS_STAGE2_FWB); \ kvm_pte_t __attr; \ \ if ((pgt)->flags & KVM_PGTABLE_S2_AS_S1) \ - __attr = PAGE_S2_MEMATTR(AS_S1, __fwb); \ + __attr = PAGE_S2_MEMATTR(AS_S1); \ else \ - __attr = PAGE_S2_MEMATTR(attr, __fwb); \ + __attr = PAGE_S2_MEMATTR(attr); \ \ __attr; \ }) -- cgit v1.2.3