summaryrefslogtreecommitdiff
path: root/arch/x86/mm/tlb.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/mm/tlb.c')
-rw-r--r--arch/x86/mm/tlb.c112
1 files changed, 44 insertions, 68 deletions
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 1023acadd8f8..0c4da320831c 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -162,7 +162,7 @@ static inline unsigned long build_cr3(pgd_t *pgd, u16 asid, unsigned long lam)
{
unsigned long cr3 = __sme_pa(pgd) | lam;
- if (static_cpu_has(X86_FEATURE_PCID)) {
+ if (cpu_feature_enabled(X86_FEATURE_PCID)) {
cr3 |= kern_pcid(asid);
} else {
VM_WARN_ON_ONCE(asid != 0);
@@ -197,7 +197,7 @@ static void clear_asid_other(void)
* This is only expected to be set if we have disabled
* kernel _PAGE_GLOBAL pages.
*/
- if (!static_cpu_has(X86_FEATURE_PTI)) {
+ if (!cpu_feature_enabled(X86_FEATURE_PTI)) {
WARN_ON_ONCE(1);
return;
}
@@ -227,7 +227,7 @@ static struct new_asid choose_new_asid(struct mm_struct *next, u64 next_tlb_gen)
struct new_asid ns;
u16 asid;
- if (!static_cpu_has(X86_FEATURE_PCID)) {
+ if (!cpu_feature_enabled(X86_FEATURE_PCID)) {
ns.asid = 0;
ns.need_flush = 1;
return ns;
@@ -555,7 +555,7 @@ static inline void invalidate_user_asid(u16 asid)
if (!cpu_feature_enabled(X86_FEATURE_PCID))
return;
- if (!static_cpu_has(X86_FEATURE_PTI))
+ if (!cpu_feature_enabled(X86_FEATURE_PTI))
return;
__set_bit(kern_pcid(asid),
@@ -1373,35 +1373,19 @@ void flush_tlb_multi(const struct cpumask *cpumask,
*/
unsigned long tlb_single_page_flush_ceiling __read_mostly = 33;
-static DEFINE_PER_CPU_SHARED_ALIGNED(struct flush_tlb_info, flush_tlb_info);
-
-#ifdef CONFIG_DEBUG_VM
-static DEFINE_PER_CPU(unsigned int, flush_tlb_info_idx);
-#endif
-
-static struct flush_tlb_info *get_flush_tlb_info(struct mm_struct *mm,
- unsigned long start, unsigned long end,
- unsigned int stride_shift, bool freed_tables,
- u64 new_tlb_gen)
+static void init_flush_tlb_info(struct flush_tlb_info *info,
+ struct mm_struct *mm,
+ unsigned long start, unsigned long end,
+ unsigned int stride_shift, bool freed_tables,
+ u64 new_tlb_gen)
{
- struct flush_tlb_info *info = this_cpu_ptr(&flush_tlb_info);
-
-#ifdef CONFIG_DEBUG_VM
- /*
- * Ensure that the following code is non-reentrant and flush_tlb_info
- * is not overwritten. This means no TLB flushing is initiated by
- * interrupt handlers and machine-check exception handlers.
- */
- BUG_ON(this_cpu_inc_return(flush_tlb_info_idx) != 1);
-#endif
-
/*
* If the number of flushes is so large that a full flush
* would be faster, do a full flush.
*/
if ((end - start) >> stride_shift > tlb_single_page_flush_ceiling) {
- start = 0;
- end = TLB_FLUSH_ALL;
+ start = 0;
+ end = TLB_FLUSH_ALL;
}
info->start = start;
@@ -1412,32 +1396,21 @@ static struct flush_tlb_info *get_flush_tlb_info(struct mm_struct *mm,
info->new_tlb_gen = new_tlb_gen;
info->initiating_cpu = smp_processor_id();
info->trim_cpumask = 0;
-
- return info;
-}
-
-static void put_flush_tlb_info(void)
-{
-#ifdef CONFIG_DEBUG_VM
- /* Complete reentrancy prevention checks */
- barrier();
- this_cpu_dec(flush_tlb_info_idx);
-#endif
}
void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
unsigned long end, unsigned int stride_shift,
bool freed_tables)
{
- struct flush_tlb_info *info;
+ struct flush_tlb_info info;
+ bool remote_flush = false;
int cpu = get_cpu();
u64 new_tlb_gen;
/* This is also a barrier that synchronizes with switch_mm(). */
new_tlb_gen = inc_mm_tlb_gen(mm);
- info = get_flush_tlb_info(mm, start, end, stride_shift, freed_tables,
- new_tlb_gen);
+ init_flush_tlb_info(&info, mm, start, end, stride_shift, freed_tables, new_tlb_gen);
/*
* flush_tlb_multi() is not optimized for the common case in which only
@@ -1445,20 +1418,24 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
* flush_tlb_func_local() directly in this case.
*/
if (mm_global_asid(mm)) {
- broadcast_tlb_flush(info);
+ broadcast_tlb_flush(&info);
} else if (cpumask_any_but(mm_cpumask(mm), cpu) < nr_cpu_ids) {
- info->trim_cpumask = should_trim_cpumask(mm);
- flush_tlb_multi(mm_cpumask(mm), info);
- consider_global_asid(mm);
+ remote_flush = true;
} else if (mm == this_cpu_read(cpu_tlbstate.loaded_mm)) {
lockdep_assert_irqs_enabled();
local_irq_disable();
- flush_tlb_func(info);
+ flush_tlb_func(&info);
local_irq_enable();
}
- put_flush_tlb_info();
put_cpu();
+
+ if (remote_flush) {
+ info.trim_cpumask = should_trim_cpumask(mm);
+ flush_tlb_multi(mm_cpumask(mm), &info);
+ consider_global_asid(mm);
+ }
+
mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end);
}
@@ -1527,19 +1504,16 @@ static void kernel_tlb_flush_range(struct flush_tlb_info *info)
void flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
- struct flush_tlb_info *info;
+ struct flush_tlb_info info;
guard(preempt)();
+ init_flush_tlb_info(&info, NULL, start, end, PAGE_SHIFT, false,
+ TLB_GENERATION_INVALID);
- info = get_flush_tlb_info(NULL, start, end, PAGE_SHIFT, false,
- TLB_GENERATION_INVALID);
-
- if (info->end == TLB_FLUSH_ALL)
- kernel_tlb_flush_all(info);
+ if (info.end == TLB_FLUSH_ALL)
+ kernel_tlb_flush_all(&info);
else
- kernel_tlb_flush_range(info);
-
- put_flush_tlb_info();
+ kernel_tlb_flush_range(&info);
}
/*
@@ -1584,7 +1558,7 @@ void flush_tlb_one_kernel(unsigned long addr)
*/
flush_tlb_one_user(addr);
- if (!static_cpu_has(X86_FEATURE_PTI))
+ if (!cpu_feature_enabled(X86_FEATURE_PTI))
return;
/*
@@ -1608,7 +1582,7 @@ STATIC_NOPV void native_flush_tlb_one_user(unsigned long addr)
invlpg(addr);
/* If PTI is off there is no user PCID and nothing to flush. */
- if (!static_cpu_has(X86_FEATURE_PTI))
+ if (!cpu_feature_enabled(X86_FEATURE_PTI))
return;
loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
@@ -1637,7 +1611,7 @@ STATIC_NOPV void native_flush_tlb_global(void)
{
unsigned long flags;
- if (static_cpu_has(X86_FEATURE_INVPCID)) {
+ if (cpu_feature_enabled(X86_FEATURE_INVPCID)) {
/*
* Using INVPCID is considerably faster than a pair of writes
* to CR4 sandwiched inside an IRQ flag save/restore.
@@ -1707,12 +1681,12 @@ EXPORT_SYMBOL_FOR_KVM(__flush_tlb_all);
void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
{
- struct flush_tlb_info *info;
-
+ struct flush_tlb_info info;
+ bool remote_flush = false;
int cpu = get_cpu();
- info = get_flush_tlb_info(NULL, 0, TLB_FLUSH_ALL, 0, false,
- TLB_GENERATION_INVALID);
+ init_flush_tlb_info(&info, NULL, 0, TLB_FLUSH_ALL, 0, false,
+ TLB_GENERATION_INVALID);
/*
* flush_tlb_multi() is not optimized for the common case in which only
* a local TLB flush is needed. Optimize this use-case by calling
@@ -1722,18 +1696,20 @@ void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
invlpgb_flush_all_nonglobals();
batch->unmapped_pages = false;
} else if (cpumask_any_but(&batch->cpumask, cpu) < nr_cpu_ids) {
- flush_tlb_multi(&batch->cpumask, info);
+ remote_flush = true;
} else if (cpumask_test_cpu(cpu, &batch->cpumask)) {
lockdep_assert_irqs_enabled();
local_irq_disable();
- flush_tlb_func(info);
+ flush_tlb_func(&info);
local_irq_enable();
}
- cpumask_clear(&batch->cpumask);
-
- put_flush_tlb_info();
put_cpu();
+
+ if (remote_flush)
+ flush_tlb_multi(&batch->cpumask, &info);
+
+ cpumask_clear(&batch->cpumask);
}
/*
generated by cgit v1.2.3 (git 2.0.4) at 2026-09-25 23:27:48 +0000