From 6bfbf574a39139da11af9fdf6e8d56fe1989cd3e Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Tue, 7 Apr 2026 11:28:41 +0100 Subject: arm64: tlb: Introduce __tlbi_sync_s1ish_{kernel,batch}() for TLB maintenance Add __tlbi_sync_s1ish_kernel() similar to __tlbi_sync_s1ish() and use it for kernel TLB maintenance. Also use this function in flush_tlb_all() which is only used in relation to kernel mappings. Subsequent patches can differentiate between workarounds that apply to user only or both user and kernel. A subsequent patch will add mm_struct to __tlbi_sync_s1ish(). Since arch_tlbbatch_flush() is not specific to an mm, add a corresponding __tlbi_sync_s1ish_batch() helper. Acked-by: Mark Rutland Cc: Will Deacon Reviewed-by: Will Deacon Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/tlbflush.h | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index 1416e652612b..f41eebf00990 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -191,6 +191,18 @@ static inline void __tlbi_sync_s1ish(void) __repeat_tlbi_sync(vale1is, 0); } +static inline void __tlbi_sync_s1ish_batch(void) +{ + dsb(ish); + __repeat_tlbi_sync(vale1is, 0); +} + +static inline void __tlbi_sync_s1ish_kernel(void) +{ + dsb(ish); + __repeat_tlbi_sync(vale1is, 0); +} + /* * Complete broadcast TLB maintenance issued by hyp code which invalidates * stage 1 translation information in any translation regime. @@ -299,7 +311,7 @@ static inline void flush_tlb_all(void) { dsb(ishst); __tlbi(vmalle1is); - __tlbi_sync_s1ish(); + __tlbi_sync_s1ish_kernel(); isb(); } @@ -385,7 +397,7 @@ static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm) */ static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) { - __tlbi_sync_s1ish(); + __tlbi_sync_s1ish_batch(); } /* @@ -568,7 +580,7 @@ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end dsb(ishst); __flush_tlb_range_op(vaale1is, start, pages, stride, 0, TLBI_TTL_UNKNOWN, false, lpa2_is_enabled()); - __tlbi_sync_s1ish(); + __tlbi_sync_s1ish_kernel(); isb(); } @@ -582,7 +594,7 @@ static inline void __flush_tlb_kernel_pgtable(unsigned long kaddr) dsb(ishst); __tlbi(vaae1is, addr); - __tlbi_sync_s1ish(); + __tlbi_sync_s1ish_kernel(); isb(); } -- cgit v1.2.3 From d9fb08ba946a6190c371dcd9f9e465d0d52c5021 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Tue, 7 Apr 2026 11:28:42 +0100 Subject: arm64: tlb: Pass the corresponding mm to __tlbi_sync_s1ish() The mm structure will be used for workarounds that need limiting to specific tasks. Acked-by: Mark Rutland Cc: Will Deacon Reviewed-by: Will Deacon Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/tlbflush.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index f41eebf00990..262791191935 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -185,7 +185,7 @@ do { \ * Complete broadcast TLB maintenance issued by the host which invalidates * stage 1 information in the host's own translation regime. */ -static inline void __tlbi_sync_s1ish(void) +static inline void __tlbi_sync_s1ish(struct mm_struct *mm) { dsb(ish); __repeat_tlbi_sync(vale1is, 0); @@ -323,7 +323,7 @@ static inline void flush_tlb_mm(struct mm_struct *mm) asid = __TLBI_VADDR(0, ASID(mm)); __tlbi(aside1is, asid); __tlbi_user(aside1is, asid); - __tlbi_sync_s1ish(); + __tlbi_sync_s1ish(mm); mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL); } @@ -377,7 +377,7 @@ static inline void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) { flush_tlb_page_nosync(vma, uaddr); - __tlbi_sync_s1ish(); + __tlbi_sync_s1ish(vma->vm_mm); } static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm) @@ -532,7 +532,7 @@ static inline void __flush_tlb_range(struct vm_area_struct *vma, { __flush_tlb_range_nosync(vma->vm_mm, start, end, stride, last_level, tlb_level); - __tlbi_sync_s1ish(); + __tlbi_sync_s1ish(vma->vm_mm); } static inline void local_flush_tlb_contpte(struct vm_area_struct *vma, -- cgit v1.2.3 From 2c99561016c591f4c3d5ad7d22a61b8726e79735 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Tue, 7 Apr 2026 11:28:43 +0100 Subject: arm64: cputype: Add C1-Pro definitions Add cputype definitions for C1-Pro. These will be used for errata detection in subsequent patches. These values can be found in "Table A-303: MIDR_EL1 bit descriptions" in issue 07 of the C1-Pro TRM: https://documentation-service.arm.com/static/6930126730f8f55a656570af Acked-by: Mark Rutland Cc: Will Deacon Cc: James Morse Reviewed-by: Will Deacon Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/cputype.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 08860d482e60..7b518e81dd15 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -98,6 +98,7 @@ #define ARM_CPU_PART_CORTEX_A725 0xD87 #define ARM_CPU_PART_CORTEX_A720AE 0xD89 #define ARM_CPU_PART_NEOVERSE_N3 0xD8E +#define ARM_CPU_PART_C1_PRO 0xD8B #define APM_CPU_PART_XGENE 0x000 #define APM_CPU_VAR_POTENZA 0x00 @@ -189,6 +190,7 @@ #define MIDR_CORTEX_A725 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A725) #define MIDR_CORTEX_A720AE MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A720AE) #define MIDR_NEOVERSE_N3 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N3) +#define MIDR_C1_PRO MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PRO) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) #define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX) -- cgit v1.2.3 From 0baba94a9779c13c857f6efc55807e6a45b1d4e4 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Tue, 7 Apr 2026 11:28:44 +0100 Subject: arm64: errata: Work around early CME DVMSync acknowledgement C1-Pro acknowledges DVMSync messages before completing the SME/CME memory accesses. Work around this by issuing an IPI to the affected CPUs if they are running in EL0 with SME enabled. Note that we avoid the local DSB in the IPI handler as the kernel runs with SCTLR_EL1.IESB=1. This is sufficient to complete SME memory accesses at EL0 on taking an exception to EL1. On the return to user path, no barrier is necessary either. See the comment in sme_set_active() and the more detailed explanation in the link below. To avoid a potential IPI flood from malicious applications (e.g. madvise(MADV_PAGEOUT) in a tight loop), track where a process is active via mm_cpumask() and only interrupt those CPUs. Link: https://lore.kernel.org/r/ablEXwhfKyJW1i7l@J2N7QTR9R3 Cc: Will Deacon Cc: Mark Rutland Cc: James Morse Cc: Mark Brown Reviewed-by: Will Deacon Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/cpucaps.h | 2 ++ arch/arm64/include/asm/fpsimd.h | 21 ++++++++++++ arch/arm64/include/asm/tlbbatch.h | 10 ++++-- arch/arm64/include/asm/tlbflush.h | 72 +++++++++++++++++++++++++++++++++++++-- 4 files changed, 101 insertions(+), 4 deletions(-) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index 177c691914f8..0b1b78a4c03e 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -64,6 +64,8 @@ cpucap_is_possible(const unsigned int cap) return IS_ENABLED(CONFIG_ARM64_WORKAROUND_REPEAT_TLBI); case ARM64_WORKAROUND_SPECULATIVE_SSBS: return IS_ENABLED(CONFIG_ARM64_ERRATUM_3194386); + case ARM64_WORKAROUND_4193714: + return IS_ENABLED(CONFIG_ARM64_ERRATUM_4193714); case ARM64_MPAM: /* * KVM MPAM support doesn't rely on the host kernel supporting MPAM. diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 1d2e33559bd5..d9d00b45ab11 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -428,6 +428,24 @@ static inline size_t sme_state_size(struct task_struct const *task) return __sme_state_size(task_get_sme_vl(task)); } +void sme_enable_dvmsync(void); +void sme_set_active(void); +void sme_clear_active(void); + +static inline void sme_enter_from_user_mode(void) +{ + if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714) && + test_thread_flag(TIF_SME)) + sme_clear_active(); +} + +static inline void sme_exit_to_user_mode(void) +{ + if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714) && + test_thread_flag(TIF_SME)) + sme_set_active(); +} + #else static inline void sme_user_disable(void) { BUILD_BUG(); } @@ -456,6 +474,9 @@ static inline size_t sme_state_size(struct task_struct const *task) return 0; } +static inline void sme_enter_from_user_mode(void) { } +static inline void sme_exit_to_user_mode(void) { } + #endif /* ! CONFIG_ARM64_SME */ /* For use by EFI runtime services calls only */ diff --git a/arch/arm64/include/asm/tlbbatch.h b/arch/arm64/include/asm/tlbbatch.h index fedb0b87b8db..6297631532e5 100644 --- a/arch/arm64/include/asm/tlbbatch.h +++ b/arch/arm64/include/asm/tlbbatch.h @@ -2,11 +2,17 @@ #ifndef _ARCH_ARM64_TLBBATCH_H #define _ARCH_ARM64_TLBBATCH_H +#include + struct arch_tlbflush_unmap_batch { +#ifdef CONFIG_ARM64_ERRATUM_4193714 /* - * For arm64, HW can do tlb shootdown, so we don't - * need to record cpumask for sending IPI + * Track CPUs that need SME DVMSync on completion of this batch. + * Otherwise, the arm64 HW can do tlb shootdown, so we don't need to + * record cpumask for sending IPI */ + cpumask_var_t cpumask; +#endif }; #endif /* _ARCH_ARM64_TLBBATCH_H */ diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index 262791191935..4aae42b83049 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -80,6 +80,71 @@ static inline unsigned long get_trans_granule(void) } } +#ifdef CONFIG_ARM64_ERRATUM_4193714 + +void sme_do_dvmsync(const struct cpumask *mask); + +static inline void sme_dvmsync(struct mm_struct *mm) +{ + if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) + return; + + sme_do_dvmsync(mm_cpumask(mm)); +} + +static inline void sme_dvmsync_add_pending(struct arch_tlbflush_unmap_batch *batch, + struct mm_struct *mm) +{ + if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) + return; + + /* + * Order the mm_cpumask() read after the hardware DVMSync. + */ + dsb(ish); + if (cpumask_empty(mm_cpumask(mm))) + return; + + /* + * Allocate the batch cpumask on first use. Fall back to an immediate + * IPI for this mm in case of failure. + */ + if (!cpumask_available(batch->cpumask) && + !zalloc_cpumask_var(&batch->cpumask, GFP_ATOMIC)) { + sme_do_dvmsync(mm_cpumask(mm)); + return; + } + + cpumask_or(batch->cpumask, batch->cpumask, mm_cpumask(mm)); +} + +static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch) +{ + if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) + return; + + if (!cpumask_available(batch->cpumask)) + return; + + sme_do_dvmsync(batch->cpumask); + cpumask_clear(batch->cpumask); +} + +#else + +static inline void sme_dvmsync(struct mm_struct *mm) +{ +} +static inline void sme_dvmsync_add_pending(struct arch_tlbflush_unmap_batch *batch, + struct mm_struct *mm) +{ +} +static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch) +{ +} + +#endif /* CONFIG_ARM64_ERRATUM_4193714 */ + /* * Level-based TLBI operations. * @@ -189,12 +254,14 @@ static inline void __tlbi_sync_s1ish(struct mm_struct *mm) { dsb(ish); __repeat_tlbi_sync(vale1is, 0); + sme_dvmsync(mm); } -static inline void __tlbi_sync_s1ish_batch(void) +static inline void __tlbi_sync_s1ish_batch(struct arch_tlbflush_unmap_batch *batch) { dsb(ish); __repeat_tlbi_sync(vale1is, 0); + sme_dvmsync_batch(batch); } static inline void __tlbi_sync_s1ish_kernel(void) @@ -397,7 +464,7 @@ static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm) */ static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) { - __tlbi_sync_s1ish_batch(); + __tlbi_sync_s1ish_batch(batch); } /* @@ -602,6 +669,7 @@ static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *b struct mm_struct *mm, unsigned long start, unsigned long end) { __flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, true, 3); + sme_dvmsync_add_pending(batch, mm); } static inline bool __pte_flags_need_flush(ptdesc_t oldval, ptdesc_t newval) -- cgit v1.2.3