27 files changed, 18 insertions, 117 deletions

diff --git a/arch/Kconfig b/arch/Kconfig
index 45c657772362..6f7516916797 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -526,13 +526,6 @@ config HAVE_ARCH_JUMP_LABEL
 config HAVE_ARCH_JUMP_LABEL_RELATIVE
 	bool
 
-config MMU_GATHER_TABLE_FREE
-	bool
-
-config MMU_GATHER_RCU_TABLE_FREE
-	bool
-	select MMU_GATHER_TABLE_FREE
-
 config MMU_GATHER_PAGE_SIZE
 	bool
 
@@ -548,7 +541,6 @@ config MMU_GATHER_MERGE_VMAS
 
 config MMU_GATHER_NO_GATHER
 	bool
-	depends on MMU_GATHER_TABLE_FREE
 
 config ARCH_WANT_IRQS_OFF_ACTIVATE_MM
 	bool
diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig
index e53ef2d88463..9063c7bda4e4 100644
--- a/arch/alpha/Kconfig
+++ b/arch/alpha/Kconfig
@@ -42,7 +42,6 @@ config ALPHA
 	select ARCH_STACKWALK
 	select CPU_NO_EFFICIENT_FFS if !ALPHA_EV67
 	select MMU_GATHER_NO_RANGE
-	select MMU_GATHER_RCU_TABLE_FREE
 	select SPARSEMEM_EXTREME if SPARSEMEM
 	select ZONE_DMA
 	select TRACE_IRQFLAGS_SUPPORT
diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig
index 0c1679f4d386..80e61175bf52 100644
--- a/arch/arc/Kconfig
+++ b/arch/arc/Kconfig
@@ -47,7 +47,6 @@ config ARC
 	select HAVE_SYSCALL_TRACEPOINTS
 	select IRQ_DOMAIN
 	select LOCK_MM_AND_FIND_VMA
-	select MMU_GATHER_RCU_TABLE_FREE
 	select MODULES_USE_ELF_RELA
 	select OF
 	select OF_EARLY_FLATTREE
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 72b9afc6ae10..0cc289a7184a 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -134,7 +134,6 @@ config ARM
 	select HAVE_PERF_REGS
 	select HAVE_PERF_USER_STACK_DUMP
 	select HAVE_POSIX_CPU_TIMERS_TASK_WORK
-	select MMU_GATHER_RCU_TABLE_FREE if MMU
 	select HAVE_REGS_AND_STACK_ACCESS_API
 	select HAVE_RSEQ
 	select HAVE_RUST if CPU_LITTLE_ENDIAN && CPU_32v7 && !KASAN
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 2bbeded33da0..b6c2dd8b2612 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -221,7 +221,6 @@ config ARM64
 	select HAVE_RELIABLE_STACKTRACE
 	select HAVE_POSIX_CPU_TIMERS_TASK_WORK
 	select HAVE_FUNCTION_ARG_ACCESS_API
-	select MMU_GATHER_RCU_TABLE_FREE
 	select HAVE_RSEQ
 	select HAVE_RUST if RUSTC_SUPPORTS_ARM64
 	select HAVE_STACKPROTECTOR
diff --git a/arch/csky/Kconfig b/arch/csky/Kconfig
index 80f89ef1d962..4331313a42ff 100644
--- a/arch/csky/Kconfig
+++ b/arch/csky/Kconfig
@@ -96,7 +96,6 @@ config CSKY
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HOTPLUG_CORE_SYNC_DEAD if HOTPLUG_CPU
 	select LOCK_MM_AND_FIND_VMA
-	select MMU_GATHER_RCU_TABLE_FREE
 	select MAY_HAVE_SPARSE_IRQ
 	select MODULES_USE_ELF_RELA if MODULES
 	select OF
diff --git a/arch/hexagon/Kconfig b/arch/hexagon/Kconfig
index d9b3fb86556b..b48491140013 100644
--- a/arch/hexagon/Kconfig
+++ b/arch/hexagon/Kconfig
@@ -23,7 +23,6 @@ config HEXAGON
 	# select HAVE_CLK
 	select GENERIC_ATOMIC64
 	select HAVE_PERF_EVENTS
-	select MMU_GATHER_RCU_TABLE_FREE
 	# GENERIC_ALLOCATOR is used by dma_alloc_coherent()
 	select GENERIC_ALLOCATOR
 	select GENERIC_IRQ_PROBE
diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig
index 1d8fb1e456d6..0bd8503fd5c4 100644
--- a/arch/loongarch/Kconfig
+++ b/arch/loongarch/Kconfig
@@ -188,7 +188,6 @@ config LOONGARCH
 	select IRQ_LOONGARCH_CPU
 	select LOCK_MM_AND_FIND_VMA
 	select MMU_GATHER_MERGE_VMAS if MMU
-	select MMU_GATHER_RCU_TABLE_FREE
 	select MODULES_USE_ELF_RELA if MODULES
 	select NEED_PER_CPU_EMBED_FIRST_CHUNK
 	select NEED_PER_CPU_PAGE_FIRST_CHUNK
diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig
index fa5d39549da9..eb84c3af92c0 100644
--- a/arch/m68k/Kconfig
+++ b/arch/m68k/Kconfig
@@ -37,7 +37,6 @@ config M68K
 	select HAVE_MOD_ARCH_SPECIFIC
 	select HAVE_UID16
 	select MMU_GATHER_NO_RANGE if MMU
-	select MMU_GATHER_RCU_TABLE_FREE if MMU
 	select MODULES_USE_ELF_REL
 	select MODULES_USE_ELF_RELA
 	select NO_DMA if !MMU && !COLDFIRE
diff --git a/arch/microblaze/Kconfig b/arch/microblaze/Kconfig
index af7e821e96c1..484ebb3baedf 100644
--- a/arch/microblaze/Kconfig
+++ b/arch/microblaze/Kconfig
@@ -41,7 +41,6 @@ config MICROBLAZE
 	select PCI_SYSCALL if PCI
 	select CPU_NO_EFFICIENT_FFS
 	select MMU_GATHER_NO_RANGE
-	select MMU_GATHER_RCU_TABLE_FREE
 	select SPARSE_IRQ
 	select ZONE_DMA
 	select TRACE_IRQFLAGS_SUPPORT
diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index d7c67cebe065..ed17c9b1624c 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -97,7 +97,6 @@ config MIPS
 	select IRQ_FORCED_THREADING
 	select ISA if EISA
 	select LOCK_MM_AND_FIND_VMA
-	select MMU_GATHER_RCU_TABLE_FREE
 	select MODULES_USE_ELF_REL if MODULES
 	select MODULES_USE_ELF_RELA if MODULES && 64BIT
 	select PERF_USE_VMALLOC
diff --git a/arch/nios2/Kconfig b/arch/nios2/Kconfig
index b0ccfc3b7a7e..9c0e6eaeb005 100644
--- a/arch/nios2/Kconfig
+++ b/arch/nios2/Kconfig
@@ -19,7 +19,6 @@ config NIOS2
 	select HAVE_PAGE_SIZE_4KB
 	select IRQ_DOMAIN
 	select LOCK_MM_AND_FIND_VMA
-	select MMU_GATHER_RCU_TABLE_FREE
 	select MODULES_USE_ELF_RELA
 	select OF
 	select OF_EARLY_FLATTREE
diff --git a/arch/openrisc/Kconfig b/arch/openrisc/Kconfig
index d90b24dd3bce..5eb995c13074 100644
--- a/arch/openrisc/Kconfig
+++ b/arch/openrisc/Kconfig
@@ -35,7 +35,6 @@ config OPENRISC
 	select GENERIC_ATOMIC64
 	select GENERIC_CLOCKEVENTS_BROADCAST
 	select GENERIC_SMP_IDLE_THREAD
-	select MMU_GATHER_RCU_TABLE_FREE
 	select MODULES_USE_ELF_RELA
 	select HAVE_DEBUG_STACKOVERFLOW
 	select OR1K_PIC
diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig
index d3afac2f0d9b..77f67028ad89 100644
--- a/arch/parisc/Kconfig
+++ b/arch/parisc/Kconfig
@@ -80,7 +80,6 @@ config PARISC
 	select GENERIC_CLOCKEVENTS
 	select CPU_NO_EFFICIENT_FFS
 	select THREAD_INFO_IN_TASK
-	select MMU_GATHER_RCU_TABLE_FREE
 	select NEED_DMA_MAP_STATE
 	select NEED_SG_DMA_LENGTH
 	select HAVE_ARCH_KGDB
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 2580e27e4328..0767cfcbaa42 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -307,7 +307,6 @@ config PPC
 	select KASAN_VMALLOC			if KASAN && EXECMEM
 	select LOCK_MM_AND_FIND_VMA
 	select MMU_GATHER_PAGE_SIZE
-	select MMU_GATHER_RCU_TABLE_FREE
 	select HAVE_ARCH_TLB_REMOVE_TABLE
 	select MMU_GATHER_MERGE_VMAS
 	select MMU_LAZY_TLB_SHOOTDOWN		if PPC_BOOK3S_64
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index c13ef9cd688d..0db108ea1466 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -208,7 +208,6 @@ config RISCV
 	select IRQ_FORCED_THREADING
 	select KASAN_VMALLOC if KASAN
 	select LOCK_MM_AND_FIND_VMA
-	select MMU_GATHER_RCU_TABLE_FREE if MMU
 	select MODULES_USE_ELF_RELA if MODULES
 	select OF
 	select OF_EARLY_FLATTREE
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index b88b85042136..a34376c05f6e 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -267,7 +267,6 @@ config S390
 	select LOCK_MM_AND_FIND_VMA
 	select MMU_GATHER_MERGE_VMAS
 	select MMU_GATHER_NO_GATHER
-	select MMU_GATHER_RCU_TABLE_FREE
 	select MODULES_USE_ELF_RELA
 	select NEED_DMA_MAP_STATE	if PCI
 	select NEED_PER_CPU_EMBED_FIRST_CHUNK
diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig
index 75236bef6f16..fe859def918c 100644
--- a/arch/sh/Kconfig
+++ b/arch/sh/Kconfig
@@ -62,7 +62,6 @@ config SUPERH
 	select HAVE_SYSCALL_TRACEPOINTS
 	select IRQ_FORCED_THREADING
 	select LOCK_MM_AND_FIND_VMA
-	select MMU_GATHER_RCU_TABLE_FREE if MMU
 	select MODULES_USE_ELF_RELA
 	select NEED_SG_DMA_LENGTH
 	select NO_DMA if !MMU && !DMA_COHERENT
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index 79c09d6ee466..742ffff8c37f 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -64,7 +64,6 @@ config SPARC32
 	select HAVE_UID16
 	select HAVE_PAGE_SIZE_4KB
 	select LOCK_MM_AND_FIND_VMA
-	select MMU_GATHER_RCU_TABLE_FREE
 	select HAVE_ARCH_TLB_REMOVE_TABLE
 	select OLD_SIGACTION
 	select ZONE_DMA
@@ -77,7 +76,6 @@ config SPARC64
 	select HAVE_FUNCTION_GRAPH_TRACER
 	select HAVE_KRETPROBES
 	select HAVE_KPROBES
-	select MMU_GATHER_RCU_TABLE_FREE
 	select HAVE_ARCH_TLB_REMOVE_TABLE
 	select MMU_GATHER_MERGE_VMAS
 	select MMU_GATHER_NO_FLUSH_CACHE
diff --git a/arch/sparc/include/asm/tlb_64.h b/arch/sparc/include/asm/tlb_64.h
index 3037187482db..f5f9631685d5 100644
--- a/arch/sparc/include/asm/tlb_64.h
+++ b/arch/sparc/include/asm/tlb_64.h
@@ -29,9 +29,7 @@ void flush_tlb_pending(void);
  * and therefore we don't need a TLBI when freeing page-table pages.
  */
 
-#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
 #define tlb_needs_table_invalidate()	(false)
-#endif
 
 #include <asm-generic/tlb.h>
 
diff --git a/arch/um/Kconfig b/arch/um/Kconfig
index d9541d13d9eb..94b8ff70f578 100644
--- a/arch/um/Kconfig
+++ b/arch/um/Kconfig
@@ -44,7 +44,6 @@ config UML
 	select HAVE_SYSCALL_TRACEPOINTS
 	select THREAD_INFO_IN_TASK
 	select SPARSE_IRQ
-	select MMU_GATHER_RCU_TABLE_FREE
 
 config MMU
 	bool
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index a8c3b3d31a27..6e5e462ec059 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -283,7 +283,6 @@ config X86
 	select HAVE_PERF_REGS
 	select HAVE_PERF_USER_STACK_DUMP
 	select ASYNC_KERNEL_PGTABLE_FREE	if IOMMU_SVA
-	select MMU_GATHER_RCU_TABLE_FREE
 	select MMU_GATHER_MERGE_VMAS
 	select HAVE_POSIX_CPU_TIMERS_TASK_WORK
 	select HAVE_REGS_AND_STACK_ACCESS_API
diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig
index 33c4caee30e2..f2f9cd9cde50 100644
--- a/arch/xtensa/Kconfig
+++ b/arch/xtensa/Kconfig
@@ -55,7 +55,6 @@ config XTENSA
 	select HAVE_VIRT_CPU_ACCOUNTING_GEN
 	select IRQ_DOMAIN
 	select LOCK_MM_AND_FIND_VMA
-	select MMU_GATHER_RCU_TABLE_FREE if MMU
 	select MODULES_USE_ELF_RELA
 	select PERF_USE_VMALLOC
 	select TRACE_IRQFLAGS_SUPPORT
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index bdcc2778ac64..9d827076db19 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -67,11 +67,8 @@
  *  - tlb_remove_table()
  *
  *    tlb_remove_table() is the basic primitive to free page-table directories
- *    (__p*_free_tlb()).  In it's most primitive form it is an alias for
- *    tlb_remove_page() below, for when page directories are pages and have no
- *    additional constraints.
- *
- *    See also MMU_GATHER_TABLE_FREE and MMU_GATHER_RCU_TABLE_FREE.
+ *    (__p*_free_tlb()).  Page directories are freed after an RCU grace
+ *    period - see the comment in mm/mmu_gather.c.
  *
  *  - tlb_remove_page() / tlb_remove_page_size()
  *  - __tlb_remove_folio_pages() / __tlb_remove_page_size()
@@ -151,24 +148,15 @@
  *  This might be useful if your architecture has size specific TLB
  *  invalidation instructions.
  *
- *  MMU_GATHER_TABLE_FREE
- *
- *  This provides tlb_remove_table(), to be used instead of tlb_remove_page()
- *  for page directores (__p*_free_tlb()).
- *
- *  Useful if your architecture has non-page page directories.
+ *  Page directories (__p*_free_tlb()) are always freed via tlb_remove_table(),
+ *  after an RCU grace period (see mm/mmu_gather.c).
  *
- *  When used, an architecture is expected to provide __tlb_remove_table() or
- *  use the generic __tlb_remove_table(), which does the actual freeing of these
- *  pages.
+ *  This serialises against software page-table walkers, including architectures
+ *  which do not use IPIs for remote TLB invalidates.
  *
- *  MMU_GATHER_RCU_TABLE_FREE
- *
- *  Like MMU_GATHER_TABLE_FREE, and adds semi-RCU semantics to the free (see
- *  comment below).
- *
- *  Useful if your architecture doesn't use IPIs for remote TLB invalidates
- *  and therefore doesn't naturally serialize with software page-table walkers.
+ *  An architecture is expected to provide __tlb_remove_table() (see
+ *  HAVE_ARCH_TLB_REMOVE_TABLE) or use the generic __tlb_remove_table(), which
+ *  does the actual freeing of these pages.
  *
  *  MMU_GATHER_NO_FLUSH_CACHE
  *
@@ -200,12 +188,8 @@
  *  various ptep_get_and_clear() functions.
  */
 
-#ifdef CONFIG_MMU_GATHER_TABLE_FREE
-
 struct mmu_table_batch {
-#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
 	struct rcu_head		rcu;
-#endif
 	unsigned int		nr;
 	void			*tables[];
 };
@@ -224,23 +208,6 @@ static inline void __tlb_remove_table(void *table)
 
 extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
 
-#else /* !CONFIG_MMU_GATHER_TABLE_FREE */
-
-static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page);
-/*
- * Without MMU_GATHER_TABLE_FREE the architecture is assumed to have page based
- * page directories and we can use the normal page batching to free them.
- */
-static inline void tlb_remove_table(struct mmu_gather *tlb, void *table)
-{
-	struct ptdesc *ptdesc = (struct ptdesc *)table;
-
-	pagetable_dtor(ptdesc);
-	tlb_remove_page(tlb, ptdesc_page(ptdesc));
-}
-#endif /* CONFIG_MMU_GATHER_TABLE_FREE */
-
-#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
 /*
  * This allows an architecture that does not use the linux page-tables for
  * hardware to skip the TLBI when freeing page tables.
@@ -253,19 +220,6 @@ void tlb_remove_table_sync_one(void);
 
 void tlb_remove_table_sync_rcu(void);
 
-#else
-
-#ifdef tlb_needs_table_invalidate
-#error tlb_needs_table_invalidate() requires MMU_GATHER_RCU_TABLE_FREE
-#endif
-
-static inline void tlb_remove_table_sync_one(void) { }
-
-static inline void tlb_remove_table_sync_rcu(void) { }
-
-#endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */
-
-
 #ifndef CONFIG_MMU_GATHER_NO_GATHER
 /*
  * If we can't allocate a page to make a big batch of page pointers
@@ -325,9 +279,7 @@ static inline void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct
 struct mmu_gather {
 	struct mm_struct	*mm;
 
-#ifdef CONFIG_MMU_GATHER_TABLE_FREE
 	struct mmu_table_batch	*batch;
-#endif
 
 	unsigned long		start;
 	unsigned long		end;
diff --git a/mm/Kconfig b/mm/Kconfig
index c1ddf59c0d71..bc7befafb47b 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -1465,7 +1465,7 @@ config HAVE_ARCH_TLB_REMOVE_TABLE
 
 config PT_RECLAIM
 	def_bool y
-	depends on MMU_GATHER_RCU_TABLE_FREE && !HAVE_ARCH_TLB_REMOVE_TABLE
+	depends on MMU && !HAVE_ARCH_TLB_REMOVE_TABLE
 	help
 	  Try to reclaim empty user page table pages in paths other than munmap
 	  and exit_mmap path.
diff --git a/mm/gup.c b/mm/gup.c
index a4036c02e213..c2dfcb4744bc 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -2700,8 +2700,9 @@ EXPORT_SYMBOL(get_user_pages_unlocked);
  * Before activating this code, please be aware that the following assumptions
  * are currently made:
  *
- *  *) Either MMU_GATHER_RCU_TABLE_FREE is enabled, and tlb_remove_table() is used to
- *  free pages containing page tables or TLB flushing requires IPI broadcast.
+ *  *) tlb_remove_table() is used to free pages containing page tables, with
+ *  the free deferred until an RCU grace period has elapsed (see
+ *  mm/mmu_gather.c).
  *
  *  *) ptes can be read atomically by the architecture.
  *
diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c
index 3985d856de7f..2a72a9686773 100644
--- a/mm/mmu_gather.c
+++ b/mm/mmu_gather.c
@@ -218,8 +218,6 @@ bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_
 
 #endif /* MMU_GATHER_NO_GATHER */
 
-#ifdef CONFIG_MMU_GATHER_TABLE_FREE
-
 static void __tlb_remove_table_free(struct mmu_table_batch *batch)
 {
 	int i;
@@ -230,10 +228,8 @@ static void __tlb_remove_table_free(struct mmu_table_batch *batch)
 	free_page((unsigned long)batch);
 }
 
-#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
-
 /*
- * Semi RCU freeing of the page directories.
+ * RCU freeing of the page directories.
  *
  * This is needed by some architectures to implement software pagetable walkers.
  *
@@ -259,13 +255,13 @@ static void __tlb_remove_table_free(struct mmu_table_batch *batch)
  * means.
  *
  * What we do is batch the freed directory pages (tables) and RCU free them.
- * We use the sched RCU variant, as that guarantees that IRQ/preempt disabling
- * holds off grace periods.
+ * Disabling IRQs or preemption holds off RCU grace periods, so this protects
+ * both rcu_read_lock() and IRQ-disabling walkers.
  *
  * However, in order to batch these pages we need to allocate storage, this
  * allocation is deep inside the MM code and can thus easily fail on memory
- * pressure. To guarantee progress we fall back to single table freeing, see
- * the implementation of tlb_remove_table_one().
+ * pressure. To guarantee progress we fall back to single table freeing, which
+ * is also RCU-deferred - see the implementation of tlb_remove_table_one().
  *
  */
 
@@ -315,15 +311,6 @@ void tlb_remove_table_sync_rcu(void)
 	synchronize_rcu();
 }
 
-#else /* !CONFIG_MMU_GATHER_RCU_TABLE_FREE */
-
-static void tlb_remove_table_free(struct mmu_table_batch *batch)
-{
-	__tlb_remove_table_free(batch);
-}
-
-#endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */
-
 /*
  * If we want tlb_remove_table() to imply TLB invalidates.
  */
@@ -403,13 +390,6 @@ static inline void tlb_table_init(struct mmu_gather *tlb)
 	tlb->batch = NULL;
 }
 
-#else /* !CONFIG_MMU_GATHER_TABLE_FREE */
-
-static inline void tlb_table_flush(struct mmu_gather *tlb) { }
-static inline void tlb_table_init(struct mmu_gather *tlb) { }
-
-#endif /* CONFIG_MMU_GATHER_TABLE_FREE */
-
 static void tlb_flush_mmu_free(struct mmu_gather *tlb)
 {
 	tlb_table_flush(tlb);