Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 updates from Will Deacon:
 "There's a reasonable amount of stuff here, including a bunch of
  updates to the perf PMU drivers and some MPAM updates to expose the
  memory bandwidth counters via resctrl.

  On the architecture side, some highlights include support for BBML3
  and steps towards support for an architectural NMI solution, all
  wrapped up in a web of fixes for latent issues identified by Sashiko.

  ACPI:

   - Combine reads of AMU counters into a single FFH feedback counter op

  Confidential computing:

   - Fix smp_processor_id() in preemptible context when retrieving an
     attestation token inside a realm

   - Convert pKVM over to a "CC platform"

   - Clean-up our SWIOTLB configuration in preparation for reworking the
     handling of encrypted/decryped DMA buffers in the dma-mapping tree

  CPU errata handling:

   - Work around broken device memory ordering on NVIDIA Olympus cores

   - Fix broken 'nospectre_bhb' command-line option

   - Select the idle loop backend instruction on the command-line

  CPU features:

   - Replace our BBML2-noabort feature with the new architectural BBML3
     feature

   - Disable in-kernel BTI for recent versions of Clang due to issues
     with livepatch that are still being investigated

   - Clean-up documentation describing which ID register fields are
     exposed to userspace

  Interrupts:

   - Preliminary work towards supporting FEAT_NMI, which cleans up our
     IRQ entry code and fixes some latent issues with pseudo-NMI

   - Support for an SDEI backend to trigger an NMI backtrace

  Memory management:

   - Treat all devices as coherent when CLIDR_EL1.LoC == 0

   - Fix no-map handling of sub-page-sized regions

   - Second attempt at unmapping the linear aliases of the kernel data
     and bss sections

   - Fix EFI runtime calls when software-PAN is enabled

  Miscellaneous:

   - Add Mark Rutland as a reviewer!

   - Tidy-up our futex cmpxchg logic when using the new LSUI
     instructions

   - Drop the requirement on DYNAMIC_FTRACE_WITH_CALL_OPS when
     selecting HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS

   - Fix a false-positive KCSCAN splat in the delay loop

   - Use a portable typedef for 128-bit scalar types in our UAPI headers

   - Non-critical fixes for Sashiko reports all over

  MPAM:

   - Hook MPAM memory bandwidth counters into resctrl's counter
     assignment interface

   - Fix a quirk in the MPAM bandwidth counting on Nvidia T241 so that
     it also applies to 63 bit counters

  Perf:

   - Workarounds for hardware issues in the CMN-S3 PMU (Graviton 5) and
     CPU PMU (NVIDIA Olympus again!)

   - Add support for the DDR PMU on Marvell CN20K SoCs

   - Add support for Picoheart implementations of the DCW PCIe PMU

   - Add support for Channel/Rank/Bank filtering in the CXL PMU driver

   - Add support for 64-bit counters in the CSPMU device

   - Add support for revision 2 of the CMN S3 PMU

  Ptrace:

   - Fix a decade-old bug in our handling of seccomp and tracing on
     syscall entry

   - Fix regset handling for inactive SVE and SSVE registers

  Selftests

   - Add some tests for the decade-old bug that we just tried to fix in
     our syscall entry path

   - Fix SVE test crash on SME-only CPUs"

* tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (95 commits)
  arm64/efi: Avoid voluntary preemption with efi_mm installed
  arm64: bti: Disable in-kernel BTI with recent versions of Clang
  arm64: entry: Avoid unnecessary local_irq_disable() on kernel exit
  irqchip/gic-v3: make the unmasking of pseudo-NMIs explicit when handling IRQs
  arm64: Disable KCSAN instrumentation in delay.o
  arm_mpam: Disable driver unbind to avoid UAF
  arm_mpam: Fix a NULL pointer dereference on unbinding after an error interrupt
  perf: arm_pmuv3: Zero initialize hw_id branch stack field
  arm64: mm: Unmap kernel data/bss entirely from the linear map
  iommu/arm-smmu-v3-sva: Use system_supports_bbml3() to detect CPU feature
  perf/arm-cmn: Support CMN S3 r2
  perf/arm-cmn: Plumb in new filter types
  perf/arm-cmn: Refactor event filter data
  perf/arm-cmn: Refactor event filter programming
  perf/arm-cmn: Rename filter variables for clarity
  arm64: mm: fix accidental linear mapping of no-map reserved memory
  tools: Ensure tools copy of linux/filter.h exports the UAPI
  kselftest/arm64: Fix abi test compilation errors
  arch: arm64: add early_param idle=<wfi|yield|nop>
  arm64: entry: mask DAIF before returning from C EL1 handlers
  ...

42 files changed, 682 insertions, 267 deletions

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index b3afe0688919..8b863b9f2124 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -187,7 +187,7 @@ config ARM64
 		if (GCC_SUPPORTS_DYNAMIC_FTRACE_WITH_ARGS || \
 		    CLANG_SUPPORTS_DYNAMIC_FTRACE_WITH_ARGS)
 	select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS \
-		if DYNAMIC_FTRACE_WITH_ARGS && DYNAMIC_FTRACE_WITH_CALL_OPS
+		if DYNAMIC_FTRACE_WITH_ARGS
 	select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS \
 		if (DYNAMIC_FTRACE_WITH_ARGS && !CFI && \
 		    (CC_IS_CLANG || !CC_OPTIMIZE_FOR_SIZE))
@@ -701,12 +701,34 @@ config ARM64_ERRATUM_1530923
 
 	  If unsure, say Y.
 
-config ARM64_WORKAROUND_REPEAT_TLBI
+config ARM64_WORKAROUND_REPEAT_TLBI_SYNC
 	bool
+	# This workaround is (only) suitable for TLB invalidation errata where
+	# all of the following conditions are true:
+	#
+	# - The effects of the errata are only a loss of ordering/completion
+	#   for explicit memory accesses when the TLBI is completed with a DSB.
+	#   The removal of TLB entries is not affected.
+	#
+	#   Note that architecturally, S2-only invalidation does not remove
+	#   combined S1+S2 entries, and does not complete accesses translated
+	#   via those S1+S2 entries. Consequently, where this condition holds,
+	#   the errata do not affect S2-only invalidation.
+	#
+	# - The errata only affect broadcast TLB invalidation operations (e.g.
+	#   TLBI VMALLE1IS), and do not affect local TLB invalidation
+	#   operations (e.g. TLBI VMALLE1).
+	#
+	# - After any number of affected TLBI operations are completed with a
+	#   DSB, the errata can be mitigated by executing a single arbitrary
+	#   broadcast TLBI (which targets an arbitrary translation regime),
+	#   followed by a DSB.
+	#
+	# For more rationale, see commit a8f78680ee6bf795.
 
 config ARM64_ERRATUM_2441007
 	bool "Cortex-A55: Completion of affected memory accesses might not be guaranteed by completion of a TLBI (rare)"
-	select ARM64_WORKAROUND_REPEAT_TLBI
+	select ARM64_WORKAROUND_REPEAT_TLBI_SYNC
 	help
 	  This option adds a workaround for ARM Cortex-A55 erratum #2441007.
 
@@ -722,7 +744,7 @@ config ARM64_ERRATUM_2441007
 
 config ARM64_ERRATUM_1286807
 	bool "Cortex-A76: Modification of the translation table for a virtual address might lead to read-after-read ordering violation (rare)"
-	select ARM64_WORKAROUND_REPEAT_TLBI
+	select ARM64_WORKAROUND_REPEAT_TLBI_SYNC
 	help
 	  This option adds a workaround for ARM Cortex-A76 erratum 1286807.
 
@@ -944,7 +966,7 @@ config ARM64_ERRATUM_2224489
 
 config ARM64_ERRATUM_2441009
 	bool "Cortex-A510: Completion of affected memory accesses might not be guaranteed by completion of a TLBI (rare)"
-	select ARM64_WORKAROUND_REPEAT_TLBI
+	select ARM64_WORKAROUND_REPEAT_TLBI_SYNC
 	help
 	  This option adds a workaround for ARM Cortex-A510 erratum #2441009.
 
@@ -1156,7 +1178,7 @@ config ARM64_ERRATUM_4193714
 config ARM64_ERRATUM_4118414
 	bool "Various: Completion of affected memory accesses might not be guaranteed by completion of a TLBI"
 	default y
-	select ARM64_WORKAROUND_REPEAT_TLBI
+	select ARM64_WORKAROUND_REPEAT_TLBI_SYNC
 	help
 	  This option adds a workaround for the following errata:
 
@@ -1340,7 +1362,7 @@ config QCOM_FALKOR_ERRATUM_1003
 config QCOM_FALKOR_ERRATUM_1009
 	bool "Falkor E1009: Prematurely complete a DSB after a TLBI"
 	default y
-	select ARM64_WORKAROUND_REPEAT_TLBI
+	select ARM64_WORKAROUND_REPEAT_TLBI_SYNC
 	help
 	  On Falkor v1, the CPU may prematurely complete a DSB following a
 	  TLBI xxIS invalidate maintenance operation. Repeat the TLBI operation
@@ -1382,6 +1404,28 @@ config NVIDIA_CARMEL_CNP_ERRATUM
 
 	  If unsure, say Y.
 
+config NVIDIA_OLYMPUS_1027_ERRATUM
+	bool "NVIDIA Olympus: device store/load ordering erratum"
+	default y
+	help
+	  This option adds an alternative code sequence to work around an
+	  NVIDIA Olympus core erratum where a Device-nGnR* store can be
+	  observed by a peripheral after a younger Device-nGnR* load to the
+	  same peripheral. This breaks the program order that drivers rely
+	  on for MMIO and can leave a device in an incorrect state.
+
+	  The workaround inserts a DMB OSH immediately before raw MMIO loads.
+	  The erratum cannot occur when a DMB that orders loads appears
+	  between the store and load, preventing the younger load from being
+	  observed before the older store.
+
+	  The alternatives framework patches in DMB OSH only when an affected
+	  CPU is detected. Other CPUs execute a NOP in its place. Disabling
+	  this option leaves the original MMIO read instruction stream
+	  unchanged.
+
+	  If unsure, say Y.
+
 config ROCKCHIP_ERRATUM_3568002
 	bool "Rockchip 3568002: GIC600 can not access physical addresses higher than 4GB"
 	default y
@@ -2116,6 +2160,8 @@ config ARM64_BTI_KERNEL
 	depends on !CC_IS_GCC || GCC_VERSION >= 100100
 	# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=106671
 	depends on !CC_IS_GCC
+	# https://github.com/llvm/llvm-project/issues/215547
+	depends on !CC_IS_CLANG || CLANG_VERSION < 210000
 	depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_ARGS)
 	help
 	  Build the kernel with Branch Target Identification annotations
diff --git a/arch/arm64/include/asm/arch_gicv3.h b/arch/arm64/include/asm/arch_gicv3.h
index d20b03931a8d..3dcb7b8309d9 100644
--- a/arch/arm64/include/asm/arch_gicv3.h
+++ b/arch/arm64/include/asm/arch_gicv3.h
@@ -178,9 +178,12 @@ static inline void gic_pmr_mask_irqs(void)
 	gic_write_pmr(GIC_PRIO_IRQOFF);
 }
 
-static inline void gic_arch_enable_irqs(void)
+static inline void gic_unmask_pnmis(void)
 {
-	asm volatile ("msr daifclr, #3" : : : "memory");
+	if (gic_prio_masking_enabled()) {
+		gic_pmr_mask_irqs();
+		asm volatile ("msr daifclr, #3" : : : "memory");
+	}
 }
 
 static inline bool gic_has_relaxed_pmr_sync(void)
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index effae53e9739..0b58b550e8dc 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -34,10 +34,6 @@
 	wx\n	.req	w\n
 	.endr
 
-	.macro disable_daif
-	msr	daifset, #0xf
-	.endm
-
 /*
  * Save/restore interrupts.
  */
diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h
index 25c61cda901c..76350b38f0d7 100644
--- a/arch/arm64/include/asm/cpucaps.h
+++ b/arch/arm64/include/asm/cpucaps.h
@@ -60,8 +60,8 @@ cpucap_is_possible(const unsigned int cap)
 		return IS_ENABLED(CONFIG_CAVIUM_ERRATUM_23154);
 	case ARM64_WORKAROUND_DISABLE_CNP:
 		return IS_ENABLED(CONFIG_ARM64_WORKAROUND_DISABLE_CNP);
-	case ARM64_WORKAROUND_REPEAT_TLBI:
-		return IS_ENABLED(CONFIG_ARM64_WORKAROUND_REPEAT_TLBI);
+	case ARM64_WORKAROUND_REPEAT_TLBI_SYNC:
+		return IS_ENABLED(CONFIG_ARM64_WORKAROUND_REPEAT_TLBI_SYNC);
 	case ARM64_WORKAROUND_SPECULATIVE_SSBS:
 		return IS_ENABLED(CONFIG_ARM64_ERRATUM_3194386);
 	case ARM64_WORKAROUND_4193714:
diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index a57870fa96db..d90040fb9de6 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -878,11 +878,11 @@ static inline bool system_supports_pmuv3(void)
 	return cpus_have_final_cap(ARM64_HAS_PMUV3);
 }
 
-bool cpu_supports_bbml2_noabort(void);
+bool cpu_supports_bbml3(void);
 
-static inline bool system_supports_bbml2_noabort(void)
+static inline bool system_supports_bbml3(void)
 {
-	return alternative_has_cap_unlikely(ARM64_HAS_BBML2_NOABORT);
+	return alternative_has_cap_unlikely(ARM64_HAS_BBML3);
 }
 
 int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt);
diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h
index 1b9f0cda1336..1fa29616e586 100644
--- a/arch/arm64/include/asm/cputype.h
+++ b/arch/arm64/include/asm/cputype.h
@@ -82,6 +82,7 @@
 #define ARM_CPU_PART_CORTEX_X1		0xD44
 #define ARM_CPU_PART_CORTEX_A510	0xD46
 #define ARM_CPU_PART_CORTEX_A520	0xD80
+#define ARM_CPU_PART_CORTEX_A520AE	0xD88
 #define ARM_CPU_PART_CORTEX_A710	0xD47
 #define ARM_CPU_PART_CORTEX_A715	0xD4D
 #define ARM_CPU_PART_CORTEX_X2		0xD48
@@ -99,6 +100,7 @@
 #define ARM_CPU_PART_CORTEX_A720AE	0xD89
 #define ARM_CPU_PART_C1_ULTRA		0xD8C
 #define ARM_CPU_PART_NEOVERSE_N3	0xD8E
+#define ARM_CPU_PART_C1_NANO		0xD8A
 #define ARM_CPU_PART_C1_PRO		0xD8B
 #define ARM_CPU_PART_C1_PREMIUM		0xD90
 
@@ -176,6 +178,7 @@
 #define MIDR_CORTEX_X1	MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X1)
 #define MIDR_CORTEX_A510 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A510)
 #define MIDR_CORTEX_A520 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A520)
+#define MIDR_CORTEX_A520AE MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A520AE)
 #define MIDR_CORTEX_A710 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A710)
 #define MIDR_CORTEX_A715 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A715)
 #define MIDR_CORTEX_X2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X2)
@@ -193,6 +196,7 @@
 #define MIDR_CORTEX_A720AE MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A720AE)
 #define MIDR_C1_ULTRA MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_ULTRA)
 #define MIDR_NEOVERSE_N3 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N3)
+#define MIDR_C1_NANO MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_NANO)
 #define MIDR_C1_PRO MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PRO)
 #define MIDR_C1_PREMIUM MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PREMIUM)
 #define MIDR_THUNDERX	MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
diff --git a/arch/arm64/include/asm/entry-common.h b/arch/arm64/include/asm/entry-common.h
index cab8cd78f693..1905765159aa 100644
--- a/arch/arm64/include/asm/entry-common.h
+++ b/arch/arm64/include/asm/entry-common.h
@@ -32,7 +32,7 @@ static inline bool arch_irqentry_exit_need_resched(void)
 	/*
 	 * DAIF.DA are cleared at the start of IRQ/FIQ handling, and when GIC
 	 * priority masking is used the GIC irqchip driver will clear DAIF.IF
-	 * using gic_arch_enable_irqs() for normal IRQs. If anything is set in
+	 * in gic_unmask_pnmis() for normal IRQs. If anything is set in
 	 * DAIF we must have handled an NMI, so skip preemption.
 	 */
 	if (system_uses_irq_prio_masking() && read_sysreg(daif))
diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h
index d1d2ff9d323a..79c6d86c38a9 100644
--- a/arch/arm64/include/asm/futex.h
+++ b/arch/arm64/include/asm/futex.h
@@ -151,42 +151,31 @@ __lsui_cmpxchg64(u64 __user *uaddr, u64 *oldval, u64 newval)
 }
 
 static __always_inline int
-__lsui_cmpxchg32(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)
+__lsui_cmpxchg32(u32 __user *uaddr, u32 *oldval, u32 newval)
 {
 	u64 __user *uaddr64;
 	bool futex_pos, other_pos;
-	u32 other, orig_other;
 	union {
 		u32 futex[2];
 		u64 raw;
-	} oval64, orig64, nval64;
+	} orig64, oval64, nval64;
 
 	uaddr64 = (u64 __user *)PTR_ALIGN_DOWN(uaddr, sizeof(u64));
 	futex_pos = !IS_ALIGNED((unsigned long)uaddr, sizeof(u64));
 	other_pos = !futex_pos;
 
-	oval64.futex[futex_pos] = oldval;
-	if (get_user(oval64.futex[other_pos], (u32 __user *)uaddr64 + other_pos))
+	orig64.futex[futex_pos] = *oldval;
+	if (get_user(orig64.futex[other_pos], (u32 __user *)uaddr64 + other_pos))
 		return -EFAULT;
 
-	orig64.raw = oval64.raw;
-
+	nval64 = oval64 = orig64;
 	nval64.futex[futex_pos] = newval;
-	nval64.futex[other_pos] = oval64.futex[other_pos];
 
 	if (__lsui_cmpxchg64(uaddr64, &oval64.raw, nval64.raw))
 		return -EFAULT;
 
-	oldval = oval64.futex[futex_pos];
-	other = oval64.futex[other_pos];
-	orig_other = orig64.futex[other_pos];
-
-	if (other != orig_other)
-		return -EAGAIN;
-
-	*oval = oldval;
-
-	return 0;
+	*oldval = oval64.futex[futex_pos];
+	return oval64.raw == orig64.raw ? 0 : -EAGAIN;
 }
 
 static __always_inline int
@@ -202,7 +191,7 @@ __lsui_futex_atomic_and(int oparg, u32 __user *uaddr, int *oval)
 static __always_inline int
 __lsui_futex_atomic_eor(int oparg, u32 __user *uaddr, int *oval)
 {
-	u32 oldval, newval, val;
+	u32 oldval, newval;
 	int ret, i;
 
 	if (get_user(oldval, uaddr))
@@ -214,33 +203,27 @@ __lsui_futex_atomic_eor(int oparg, u32 __user *uaddr, int *oval)
 	for (i = 0; i < FUTEX_MAX_LOOPS; i++) {
 		newval = oldval ^ oparg;
 
-		ret = __lsui_cmpxchg32(uaddr, oldval, newval, &val);
-		switch (ret) {
-		case -EFAULT:
-			return ret;
-		case -EAGAIN:
-			continue;
-		}
-
-		if (val == oldval) {
-			*oval = val;
-			return 0;
-		}
-
-		oldval = val;
+		ret = __lsui_cmpxchg32(uaddr, &oldval, newval);
+		if (ret != -EAGAIN)
+			break;
 	}
 
-	return -EAGAIN;
+	*oval = oldval;
+	return ret;
 }
 
 static __always_inline int
 __lsui_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)
 {
-	/*
-	 * Callers of futex_atomic_cmpxchg_inatomic() already retry on
-	 * -EAGAIN, no need for another loop of max retries.
-	 */
-	return __lsui_cmpxchg32(uaddr, oldval, newval, oval);
+	u32 curval = oldval;
+	int ret;
+
+	ret = __lsui_cmpxchg32(uaddr, &curval, newval);
+	if (ret == -EAGAIN && curval != oldval)
+		ret = 0;
+
+	*oval = curval;
+	return ret;
 }
 #endif	/* CONFIG_ARM64_LSUI */
 
diff --git a/arch/arm64/include/asm/hypervisor.h b/arch/arm64/include/asm/hypervisor.h
index a12fd897c877..8889a0ba1ec5 100644
--- a/arch/arm64/include/asm/hypervisor.h
+++ b/arch/arm64/include/asm/hypervisor.h
@@ -3,6 +3,9 @@
 #define _ASM_ARM64_HYPERVISOR_H
 
 #include <asm/xen/hypervisor.h>
+#include <linux/jump_label.h>
+
+DECLARE_STATIC_KEY_FALSE(pkvm_guest);
 
 void kvm_init_hyp_services(void);
 bool kvm_arm_hyp_service_available(u32 func_id);
@@ -10,8 +13,18 @@ void kvm_arm_target_impl_cpu_init(void);
 
 #ifdef CONFIG_ARM_PKVM_GUEST
 void pkvm_init_hyp_services(void);
+
+static inline bool is_protected_kvm_guest(void)
+{
+	return static_branch_unlikely(&pkvm_guest);
+}
 #else
 static inline void pkvm_init_hyp_services(void) { };
+
+static inline bool is_protected_kvm_guest(void)
+{
+	return false;
+}
 #endif
 
 static inline void kvm_arch_init_hyp_services(void)
diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h
index 21c8e400107c..49a7002661a9 100644
--- a/arch/arm64/include/asm/io.h
+++ b/arch/arm64/include/asm/io.h
@@ -54,7 +54,9 @@ static __always_inline void __raw_writeq(u64 val, volatile void __iomem *addr)
 static __always_inline u8 __raw_readb(const volatile void __iomem *addr)
 {
 	u8 val;
-	asm volatile(ALTERNATIVE("ldrb %w0, [%1]",
+	asm volatile(ALTERNATIVE("nop", "dmb osh",
+				 ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027)
+		     ALTERNATIVE("ldrb %w0, [%1]",
 				 "ldarb %w0, [%1]",
 				 ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE)
 		     : "=r" (val) : "r" (addr));
@@ -66,7 +68,9 @@ static __always_inline u16 __raw_readw(const volatile void __iomem *addr)
 {
 	u16 val;
 
-	asm volatile(ALTERNATIVE("ldrh %w0, [%1]",
+	asm volatile(ALTERNATIVE("nop", "dmb osh",
+				 ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027)
+		     ALTERNATIVE("ldrh %w0, [%1]",
 				 "ldarh %w0, [%1]",
 				 ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE)
 		     : "=r" (val) : "r" (addr));
@@ -77,7 +81,9 @@ static __always_inline u16 __raw_readw(const volatile void __iomem *addr)
 static __always_inline u32 __raw_readl(const volatile void __iomem *addr)
 {
 	u32 val;
-	asm volatile(ALTERNATIVE("ldr %w0, [%1]",
+	asm volatile(ALTERNATIVE("nop", "dmb osh",
+				 ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027)
+		     ALTERNATIVE("ldr %w0, [%1]",
 				 "ldar %w0, [%1]",
 				 ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE)
 		     : "=r" (val) : "r" (addr));
@@ -88,7 +94,9 @@ static __always_inline u32 __raw_readl(const volatile void __iomem *addr)
 static __always_inline u64 __raw_readq(const volatile void __iomem *addr)
 {
 	u64 val;
-	asm volatile(ALTERNATIVE("ldr %0, [%1]",
+	asm volatile(ALTERNATIVE("nop", "dmb osh",
+				 ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027)
+		     ALTERNATIVE("ldr %0, [%1]",
 				 "ldar %0, [%1]",
 				 ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE)
 		     : "=r" (val) : "r" (addr));
diff --git a/arch/arm64/include/asm/mem_encrypt.h b/arch/arm64/include/asm/mem_encrypt.h
index 314b2b52025f..636f45b4d8af 100644
--- a/arch/arm64/include/asm/mem_encrypt.h
+++ b/arch/arm64/include/asm/mem_encrypt.h
@@ -2,6 +2,7 @@
 #ifndef __ASM_MEM_ENCRYPT_H
 #define __ASM_MEM_ENCRYPT_H
 
+#include <asm/hypervisor.h>
 #include <asm/rsi.h>
 
 struct device;
@@ -20,7 +21,7 @@ int realm_register_memory_enc_ops(void);
 
 static inline bool force_dma_unencrypted(struct device *dev)
 {
-	return is_realm_world();
+	return is_realm_world() || is_protected_kvm_guest();
 }
 
 /*
diff --git a/arch/arm64/include/asm/nmi.h b/arch/arm64/include/asm/nmi.h
new file mode 100644
index 000000000000..2e8974ff8d63
--- /dev/null
+++ b/arch/arm64/include/asm/nmi.h
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __ASM_NMI_H
+#define __ASM_NMI_H
+
+#include <linux/cpumask.h>
+
+struct pt_regs;
+
+/*
+ * Cross-CPU NMI provider hooks, consulted by the arm64 arch code before
+ * its regular-IRQ / pseudo-NMI IPI paths. The SDEI provider in
+ * drivers/firmware/arm_sdei_nmi.c implements them when active; a future
+ * FEAT_NMI provider could slot in here too. The stubs let callers stay
+ * unconditional when ARM_SDEI_NMI is off.
+ *
+ * sdei_nmi_active() lets a caller test for the service before committing
+ * to (and waiting on) the SDEI stop rung; sdei_nmi_stop_cpus() then signals
+ * the targets, which ack by going offline.
+ */
+#ifdef CONFIG_ARM_SDEI_NMI
+bool sdei_nmi_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu);
+bool sdei_nmi_active(void);
+void sdei_nmi_stop_cpus(const cpumask_t *mask);
+#else
+static inline bool sdei_nmi_trigger_cpumask_backtrace(const cpumask_t *mask,
+						      int exclude_cpu)
+{
+	return false;
+}
+
+static inline bool sdei_nmi_active(void)
+{
+	return false;
+}
+
+static inline void sdei_nmi_stop_cpus(const cpumask_t *mask) { }
+#endif
+
+/*
+ * The common "stop this CPU" entry every arm64 stop path funnels through:
+ * the regular/pseudo-NMI stop IPI handlers, panic_smp_self_stop(), and the
+ * SDEI cross-CPU NMI handler. @die_on_crash powers the CPU off on the kdump
+ * crash path (IPI handlers) instead of parking it (SDEI / self-stop).
+ * Defined in arch/arm64/kernel/smp.c.
+ */
+void __noreturn arm64_nmi_cpu_stop(struct pt_regs *regs, bool die_on_crash);
+
+#endif /* __ASM_NMI_H */
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 27689c62bd25..25001694ae8e 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -140,10 +140,17 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys)
 #define pte_none(pte)		(!pte_val(pte))
 #define pte_page(pte)		(pfn_to_page(pte_pfn(pte)))
 
+#define pte_valid(pte)		(!!(pte_val(pte) & PTE_VALID))
+#define pte_present_invalid(pte) \
+	((pte_val(pte) & (PTE_VALID | PTE_PRESENT_INVALID)) == PTE_PRESENT_INVALID)
+
 /*
  * The following only work if pte_present(). Undefined behaviour otherwise.
  */
-#define pte_present(pte)	(pte_valid(pte) || pte_present_invalid(pte))
+static __always_inline bool pte_present(pte_t pte)
+{
+	return pte_valid(pte) || pte_present_invalid(pte);
+}
 #define pte_young(pte)		(!!(pte_val(pte) & PTE_AF))
 #define pte_special(pte)	(!!(pte_val(pte) & PTE_SPECIAL))
 #define pte_write(pte)		(!!(pte_val(pte) & PTE_WRITE))
@@ -168,9 +175,6 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys)
 #define pte_sw_dirty(pte)	(!!(pte_val(pte) & PTE_DIRTY))
 #define pte_dirty(pte)		(pte_sw_dirty(pte) || pte_hw_dirty(pte))
 
-#define pte_valid(pte)		(!!(pte_val(pte) & PTE_VALID))
-#define pte_present_invalid(pte) \
-	((pte_val(pte) & (PTE_VALID | PTE_PRESENT_INVALID)) == PTE_PRESENT_INVALID)
 /*
  * Execute-only user mappings do not have the PTE_USER bit set. All valid
  * kernel mappings have the PTE_UXN bit set.
diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h
index 39582511ad72..f7dc5fb9427d 100644
--- a/arch/arm64/include/asm/ptrace.h
+++ b/arch/arm64/include/asm/ptrace.h
@@ -18,8 +18,6 @@
 
 #define INIT_PSTATE_EL1 \
 	(PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | PSR_MODE_EL1h)
-#define INIT_PSTATE_EL2 \
-	(PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | PSR_MODE_EL2h)
 
 #include <linux/irqchip/arm-gic-v3-prio.h>
 
diff --git a/arch/arm64/include/asm/rsi_cmds.h b/arch/arm64/include/asm/rsi_cmds.h
index 2c8763876dfb..c1fab41f671e 100644
--- a/arch/arm64/include/asm/rsi_cmds.h
+++ b/arch/arm64/include/asm/rsi_cmds.h
@@ -88,6 +88,14 @@ static inline long rsi_set_addr_range_state(phys_addr_t start,
 	return res.a0;
 }
 
+#define RSI_ATTEST_CHALLENGE_MIN_SIZE	32
+#define RSI_ATTEST_CHALLENGE_MAX_SIZE	64
+
+struct rsi_attestation_token_init_args {
+	unsigned long fid;
+	u8 challenge[RSI_ATTEST_CHALLENGE_MAX_SIZE];
+};
+
 /**
  * rsi_attestation_token_init - Initialise the operation to retrieve an
  * attestation token.
@@ -109,18 +117,21 @@ static inline long rsi_set_addr_range_state(phys_addr_t start,
 static inline long
 rsi_attestation_token_init(const u8 *challenge, unsigned long size)
 {
-	struct arm_smccc_1_2_regs regs = { 0 };
+	union {
+		struct arm_smccc_1_2_regs regs;
+		struct rsi_attestation_token_init_args init;
+	} args = { 0 };
 
-	/* The challenge must be at least 32bytes and at most 64bytes */
-	if (!challenge || size < 32 || size > 64)
+	if (!challenge || size < RSI_ATTEST_CHALLENGE_MIN_SIZE ||
+	    size > RSI_ATTEST_CHALLENGE_MAX_SIZE)
 		return -EINVAL;
 
-	regs.a0 = SMC_RSI_ATTESTATION_TOKEN_INIT;
-	memcpy(&regs.a1, challenge, size);
-	arm_smccc_1_2_smc(&regs, &regs);
+	args.init.fid = SMC_RSI_ATTESTATION_TOKEN_INIT;
+	memcpy(args.init.challenge, challenge, size);
+	arm_smccc_1_2_smc(&args.regs, &args.regs);
 
-	if (regs.a0 == RSI_SUCCESS)
-		return regs.a1;
+	if (args.regs.a0 == RSI_SUCCESS)
+		return args.regs.a1;
 
 	return -EINVAL;
 }
diff --git a/arch/arm64/include/asm/syscall_wrapper.h b/arch/arm64/include/asm/syscall_wrapper.h
index abb57bc54305..395152ef5372 100644
--- a/arch/arm64/include/asm/syscall_wrapper.h
+++ b/arch/arm64/include/asm/syscall_wrapper.h
@@ -10,13 +10,13 @@
 
 #include <asm/ptrace.h>
 
-#define SC_ARM64_REGS_TO_ARGS(x, ...)				\
+#ifdef CONFIG_COMPAT
+
+#define COMPAT_SC_ARM64_REGS_TO_ARGS(x, ...)			\
 	__MAP(x,__SC_ARGS					\
 	      ,,regs->regs[0],,regs->regs[1],,regs->regs[2]	\
 	      ,,regs->regs[3],,regs->regs[4],,regs->regs[5])
 
-#ifdef CONFIG_COMPAT
-
 #define COMPAT_SYSCALL_DEFINEx(x, name, ...)						\
 	asmlinkage long __arm64_compat_sys##name(const struct pt_regs *regs);		\
 	ALLOW_ERROR_INJECTION(__arm64_compat_sys##name, ERRNO);				\
@@ -24,7 +24,7 @@
 	static inline long __do_compat_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__));	\
 	asmlinkage long __arm64_compat_sys##name(const struct pt_regs *regs)		\
 	{										\
-		return __se_compat_sys##name(SC_ARM64_REGS_TO_ARGS(x,__VA_ARGS__));	\
+		return __se_compat_sys##name(COMPAT_SC_ARM64_REGS_TO_ARGS(x,__VA_ARGS__));	\
 	}										\
 	static long __se_compat_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__))		\
 	{										\
@@ -46,6 +46,11 @@
 
 #endif /* CONFIG_COMPAT */
 
+#define SC_ARM64_REGS_TO_ARGS(x, ...)				\
+	__MAP(x,__SC_ARGS					\
+	      ,,regs->orig_x0,,regs->regs[1],,regs->regs[2]	\
+	      ,,regs->regs[3],,regs->regs[4],,regs->regs[5])
+
 #define __SYSCALL_DEFINEx(x, name, ...)						\
 	asmlinkage long __arm64_sys##name(const struct pt_regs *regs);		\
 	ALLOW_ERROR_INJECTION(__arm64_sys##name, ERRNO);			\
diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index e0e84332f51b..14a78ac0f800 100644
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h
@@ -236,7 +236,7 @@ static inline void __tlbi_level(tlbi_op op, u64 addr, u32 level)
 
 #define __repeat_tlbi_sync(op, arg...)						\
 do {										\
-	if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_REPEAT_TLBI))	\
+	if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_REPEAT_TLBI_SYNC))	\
 		break;								\
 	__tlbi(op, ##arg);							\
 	dsb(ish);								\
diff --git a/arch/arm64/include/uapi/asm/ptrace.h b/arch/arm64/include/uapi/asm/ptrace.h
index 6fed93fb2536..15649a253a57 100644
--- a/arch/arm64/include/uapi/asm/ptrace.h
+++ b/arch/arm64/include/uapi/asm/ptrace.h
@@ -93,7 +93,7 @@ struct user_pt_regs {
 };
 
 struct user_fpsimd_state {
-	__uint128_t	vregs[32];
+	__u128		vregs[32];
 	__u32		fpsr;
 	__u32		fpcr;
 	__u32		__reserved[2];
@@ -258,14 +258,14 @@ struct user_pac_mask {
 /* pointer authentication keys (NT_ARM_PACA_KEYS, NT_ARM_PACG_KEYS) */
 
 struct user_pac_address_keys {
-	__uint128_t	apiakey;
-	__uint128_t	apibkey;
-	__uint128_t	apdakey;
-	__uint128_t	apdbkey;
+	__u128	apiakey;
+	__u128	apibkey;
+	__u128	apdakey;
+	__u128	apdbkey;
 };
 
 struct user_pac_generic_keys {
-	__uint128_t	apgakey;
+	__u128	apgakey;
 };
 
 /* ZA state (NT_ARM_ZA) */
diff --git a/arch/arm64/include/uapi/asm/sigcontext.h b/arch/arm64/include/uapi/asm/sigcontext.h
index e29bf3e2d0cc..d250ca7a1d46 100644
--- a/arch/arm64/include/uapi/asm/sigcontext.h
+++ b/arch/arm64/include/uapi/asm/sigcontext.h
@@ -78,7 +78,7 @@ struct fpsimd_context {
 	struct _aarch64_ctx head;
 	__u32 fpsr;
 	__u32 fpcr;
-	__uint128_t vregs[32];
+	__u128 vregs[32];
 };
 
 /*
@@ -266,8 +266,8 @@ struct gcs_context {
  *	-	----				-----------
  *	REGS					the entire SVE context
  *
- *	ZREGS	__uint128_t[SVE_NUM_ZREGS][vq]	all Z-registers
- *	ZREG	__uint128_t[vq]			individual Z-register Zn
+ *	ZREGS	__u128[SVE_NUM_ZREGS][vq]	all Z-registers
+ *	ZREG	__u128[vq]			individual Z-register Zn
  *
  *	PREGS	uint16_t[SVE_NUM_PREGS][vq]	all P-registers
  *	PREG	uint16_t[vq]			individual P-register Pn
diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index 1995e1198648..b5ba9c455aef 100644
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -309,7 +309,7 @@ static void cpu_enable_impdef_pmuv3_traps(const struct arm64_cpu_capabilities *_
 	sysreg_clear_set_s(SYS_HACR_EL2, 0, BIT(56));
 }
 
-#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI
+#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI_SYNC
 static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = {
 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1009
 	{
@@ -733,10 +733,10 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
 		.match_list = qcom_erratum_1003_list,
 	},
 #endif
-#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI
+#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI_SYNC
 	{
 		.desc = "Broken broadcast TLBI completion",
-		.capability = ARM64_WORKAROUND_REPEAT_TLBI,
+		.capability = ARM64_WORKAROUND_REPEAT_TLBI_SYNC,
 		.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 		.matches = cpucap_multi_entry_cap_matches,
 		.match_list = arm64_repeat_tlbi_list,
@@ -850,6 +850,14 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
 		ERRATA_MIDR_RANGE_LIST(cnp_erratum_cpus),
 	},
 #endif
+#ifdef CONFIG_NVIDIA_OLYMPUS_1027_ERRATUM
+	{
+		/* NVIDIA Olympus core */
+		.desc = "NVIDIA Olympus device store/load ordering erratum",
+		.capability = ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027,
+		ERRATA_MIDR_ALL_VERSIONS(MIDR_NVIDIA_OLYMPUS),
+	},
+#endif
 #ifdef CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE
 	{
 		/*
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 9a22df0c5120..4025d51d2d93 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -2131,45 +2131,39 @@ static bool hvhe_possible(const struct arm64_cpu_capabilities *entry,
 	return arm64_test_sw_feature_override(ARM64_SW_FEATURE_OVERRIDE_HVHE);
 }
 
-bool cpu_supports_bbml2_noabort(void)
+bool cpu_supports_bbml3(void)
 {
-	/*
-	 * We want to allow usage of BBML2 in as wide a range of kernel contexts
-	 * as possible. This list is therefore an allow-list of known-good
-	 * implementations that both support BBML2 and additionally, fulfill the
-	 * extra constraint of never generating TLB conflict aborts when using
-	 * the relaxed BBML2 semantics (such aborts make use of BBML2 in certain
-	 * kernel contexts difficult to prove safe against recursive aborts).
-	 *
-	 * Note that implementations can only be considered "known-good" if their
-	 * implementors attest to the fact that the implementation never raises
-	 * TLB conflict aborts for BBML2 mapping granularity changes.
-	 */
-	static const struct midr_range supports_bbml2_noabort_list[] = {
+	/* CPUs that support BBML3 but dont advertise through ID_AA64MMFR2_EL1 */
+	static const struct midr_range supports_bbml3_list[] = {
 		MIDR_REV_RANGE(MIDR_CORTEX_X4, 0, 3, 0xf),
 		MIDR_REV_RANGE(MIDR_NEOVERSE_V3, 0, 2, 0xf),
 		MIDR_REV_RANGE(MIDR_NEOVERSE_V3AE, 0, 2, 0xf),
 		MIDR_ALL_VERSIONS(MIDR_NVIDIA_OLYMPUS),
 		MIDR_ALL_VERSIONS(MIDR_AMPERE1),
 		MIDR_ALL_VERSIONS(MIDR_AMPERE1A),
+		MIDR_ALL_VERSIONS(MIDR_CORTEX_A520AE),
+		MIDR_ALL_VERSIONS(MIDR_CORTEX_A715),
+		MIDR_ALL_VERSIONS(MIDR_CORTEX_A720AE),
+		MIDR_ALL_VERSIONS(MIDR_CORTEX_A725),
+		MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N3),
+		MIDR_ALL_VERSIONS(MIDR_C1_NANO),
+		MIDR_ALL_VERSIONS(MIDR_C1_PRO),
+		/* Erratum 3683289 fixed in r1p1 */
+		MIDR_RANGE(MIDR_C1_ULTRA, 1, 1, 0xf, 0xf),
+		MIDR_RANGE(MIDR_C1_PREMIUM, 1, 1, 0xf, 0xf),
 		{}
 	};
+	u64 mmfr2 = __read_sysreg_by_encoding(SYS_ID_AA64MMFR2_EL1);
 
-	/* Does our cpu guarantee to never raise TLB conflict aborts? */
-	if (!is_midr_in_range_list(supports_bbml2_noabort_list))
-		return false;
-
-	/*
-	 * We currently ignore the ID_AA64MMFR2_EL1 register, and only care
-	 * about whether the MIDR check passes.
-	 */
+	if (SYS_FIELD_GET(ID_AA64MMFR2_EL1, BBM, mmfr2) >= ID_AA64MMFR2_EL1_BBM_3)
+		return true;
 
-	return true;
+	return is_midr_in_range_list(supports_bbml3_list);
 }
 
-static bool has_bbml2_noabort(const struct arm64_cpu_capabilities *caps, int scope)
+static bool has_bbml3(const struct arm64_cpu_capabilities *caps, int scope)
 {
-	return cpu_supports_bbml2_noabort();
+	return cpu_supports_bbml3();
 }
 
 static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused)
@@ -3062,10 +3056,10 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, EVT, IMP)
 	},
 	{
-		.desc = "BBM Level 2 without TLB conflict abort",
-		.capability = ARM64_HAS_BBML2_NOABORT,
+		.desc = "BBM Level 3",
+		.capability = ARM64_HAS_BBML3,
 		.type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE,
-		.matches = has_bbml2_noabort,
+		.matches = has_bbml3,
 	},
 	{
 		.desc = "52-bit Virtual Addressing for KVM (LPA2)",
diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c
index 29307642f4c9..e271fbac5f82 100644
--- a/arch/arm64/kernel/debug-monitors.c
+++ b/arch/arm64/kernel/debug-monitors.c
@@ -40,10 +40,7 @@ u8 debug_monitors_arch(void)
  */
 static void mdscr_write(u64 mdscr)
 {
-	unsigned long flags;
-	flags = local_daif_save();
 	write_sysreg(mdscr, mdscr_el1);
-	local_daif_restore(flags);
 }
 NOKPROBE_SYMBOL(mdscr_write);
 
diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c
index 30cd7f804398..0ec90fd1754e 100644
--- a/arch/arm64/kernel/efi.c
+++ b/arch/arm64/kernel/efi.c
@@ -184,6 +184,8 @@ void arch_efi_call_virt_setup(void)
 		efi_virtmap_load();
 	}
 
+	__efi_fpsimd_begin();
+
 	/*
 	 * Enable access to the valid TTBR0_EL1 and invoke the errata
 	 * workaround directly since there is no return from exception when
@@ -191,8 +193,6 @@ void arch_efi_call_virt_setup(void)
 	 */
 	uaccess_ttbr0_enable();
 	post_ttbr_update_workaround();
-
-	__efi_fpsimd_begin();
 }
 
 void arch_efi_call_virt_teardown(void)
diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index ceb4eb11232a..72c03ccea59f 100644
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -52,16 +52,36 @@ static noinstr irqentry_state_t arm64_enter_from_kernel_mode(struct pt_regs *reg
  * After this function returns it is not safe to call regular kernel code,
  * instrumentable code, or any code which may trigger an exception.
  */
-static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs,
-					      irqentry_state_t state)
+static void noinstr __arm64_exit_to_kernel_mode(struct pt_regs *regs,
+						irqentry_state_t state)
 {
-	local_irq_disable();
-	irqentry_exit_to_kernel_mode_preempt(regs, state);
 	local_daif_mask();
 	mte_check_tfsr_exit();
 	irqentry_exit_to_kernel_mode_after_preempt(regs, state);
 }
 
+/*
+ * We are returning from the context which allows involuntary kernel preemption
+ */
+static void noinstr arm64_exit_to_kernel_mode_preempt(struct pt_regs *regs,
+						      irqentry_state_t state)
+{
+	irqentry_exit_to_kernel_mode_preempt(regs, state);
+	__arm64_exit_to_kernel_mode(regs, state);
+}
+
+static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs,
+					      irqentry_state_t state)
+{
+	if (!regs_irqs_disabled(regs)) {
+		local_irq_disable();
+		arm64_exit_to_kernel_mode_preempt(regs, state);
+		return;
+	}
+
+	__arm64_exit_to_kernel_mode(regs, state);
+}
+
 static __always_inline void arm64_syscall_enter_from_user_mode(struct pt_regs *regs)
 {
 	enter_from_user_mode(regs);
@@ -495,6 +515,7 @@ static __always_inline void __el1_pnmi(struct pt_regs *regs,
 
 	state = irqentry_nmi_enter(regs);
 	do_interrupt_handler(regs, handler);
+	local_daif_mask();
 	irqentry_nmi_exit(regs, state);
 }
 
@@ -509,7 +530,7 @@ static __always_inline void __el1_irq(struct pt_regs *regs,
 	do_interrupt_handler(regs, handler);
 	irq_exit_rcu();
 
-	arm64_exit_to_kernel_mode(regs, state);
+	arm64_exit_to_kernel_mode_preempt(regs, state);
 }
 static void noinstr el1_interrupt(struct pt_regs *regs,
 				  void (*handler)(struct pt_regs *))
@@ -540,6 +561,7 @@ asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)
 	local_daif_restore(DAIF_ERRCTX);
 	state = irqentry_nmi_enter(regs);
 	do_serror(regs, esr);
+	local_daif_mask();
 	irqentry_nmi_exit(regs, state);
 }
 
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index e0db14e9c843..f63049ac32dc 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -333,10 +333,6 @@ alternative_else_nop_endif
 	.endm
 
 	.macro	kernel_exit, el
-	.if	\el != 0
-	disable_daif
-	.endif
-
 #ifdef CONFIG_ARM64_PSEUDO_NMI
 alternative_if_not ARM64_HAS_GIC_PRIO_MASKING
 	b	.Lskip_pmr_restore\@
diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c
index 5a1554a44162..e1a3c0b3a051 100644
--- a/arch/arm64/kernel/ftrace.c
+++ b/arch/arm64/kernel/ftrace.c
@@ -409,7 +409,8 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
 	return ftrace_modify_code(pc, old, new, true);
 }
 
-#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
+#if defined(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS) || \
+	defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS)
 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
 		       unsigned long addr)
 {
@@ -417,7 +418,7 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
 	u32 old, new;
 	int ret;
 
-	ret = ftrace_rec_set_ops(rec, arm64_rec_get_ops(rec));
+	ret = ftrace_rec_update_ops(rec);
 	if (ret)
 		return ret;
 
diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c
index 9717568518ba..7bf117427777 100644
--- a/arch/arm64/kernel/hibernate.c
+++ b/arch/arm64/kernel/hibernate.c
@@ -348,8 +348,10 @@ int swsusp_arch_suspend(void)
 		crash_prepare_suspend();
 
 		ret = swsusp_mte_save_tags();
-		if (ret)
+		if (ret) {
+			local_daif_restore(flags);
 			return ret;
+		}
 
 		sleep_cpu = smp_processor_id();
 		ret = swsusp_save();
@@ -465,9 +467,21 @@ int __nocfi swsusp_arch_resume(void)
 	if (el2_reset_needed())
 		__hyp_set_vectors(el2_vectors);
 
+	/*
+	 * It is necessary to mask all DAIF exceptions here as:
+	 *
+	 * - The copy of swsusp_arch_suspend_exit() in the hibernation
+	 *   text cannot handle taking any exceptions.
+	 *
+	 * - The suspended kernel masked all DAIF exceptions in
+	 *   swsusp_arch_resume(), and expects to be re-entered in the
+	 *   same state : with all DAIF exceptions masked.
+	 */
+	local_daif_save();
 	hibernate_exit(virt_to_phys(tmp_pg_dir), resume_hdr.ttbr1_el1,
 		       resume_hdr.reenter_kernel, restore_pblist,
 		       resume_hdr.__hyp_stub_vectors, virt_to_phys(zero_page));
+	unreachable();
 
 	return 0;
 }
diff --git a/arch/arm64/kernel/idle.c b/arch/arm64/kernel/idle.c
index 05cfb347ec26..42c5543f9589 100644
--- a/arch/arm64/kernel/idle.c
+++ b/arch/arm64/kernel/idle.c
@@ -11,6 +11,29 @@
 #include <asm/cpufeature.h>
 #include <asm/sysreg.h>
 
+enum {
+    ARM64_IDLE_WFI,
+    ARM64_IDLE_YIELD,
+    ARM64_IDLE_NOP,
+} idle = ARM64_IDLE_WFI;
+
+static int __init setup_idle(char *arg)
+{
+	if (!arg)
+		return -1;
+	else if (!strcmp(arg, "wfi"))
+		idle = ARM64_IDLE_WFI;
+	else if (!strcmp(arg, "yield"))
+		idle = ARM64_IDLE_YIELD;
+	else if (!strcmp(arg, "nop"))
+		idle = ARM64_IDLE_NOP;
+	else
+		return -1;
+
+	return 0;
+}
+early_param("idle", setup_idle);
+
 /*
  *	cpu_do_idle()
  *
@@ -26,8 +49,13 @@ void __cpuidle cpu_do_idle(void)
 
 	arm_cpuidle_save_irq_context(&context);
 
-	dsb(sy);
-	wfi();
+	if (likely(idle == ARM64_IDLE_WFI)) {
+		dsb(sy);
+		wfi();
+	} else if (idle == ARM64_IDLE_YIELD) {
+		dsb(sy);
+		asm volatile("yield" ::: "memory");
+	}
 
 	arm_cpuidle_restore_irq_context(&context);
 }
diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c
index 7bb6553fec08..3bcf86154d95 100644
--- a/arch/arm64/kernel/proton-pack.c
+++ b/arch/arm64/kernel/proton-pack.c
@@ -1023,6 +1023,11 @@ static int __init parse_spectre_bhb_param(char *str)
 }
 early_param("nospectre_bhb", parse_spectre_bhb_param);
 
+static bool spectre_bhb_mitigations_off(void)
+{
+	return __nospectre_bhb || cpu_mitigations_off();
+}
+
 void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry)
 {
 	bp_hardening_cb_t cpu_cb;
@@ -1036,6 +1041,8 @@ void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry)
 		/* No point mitigating Spectre-BHB alone. */
 	} else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) {
 		/* Do nothing */
+	} else if (spectre_bhb_mitigations_off()) {
+		/* Mitigation disabled on the command line */
 	} else if (supports_ecbhb(SCOPE_LOCAL_CPU)) {
 		state = SPECTRE_MITIGATED;
 		set_bit(BHB_HW, &system_bhb_mitigations);
@@ -1201,6 +1208,6 @@ void spectre_print_disabled_mitigations(void)
 	if (spectre_v4_mitigations_off())
 		pr_info("spectre-v4 %s", spectre_disabled_suffix);
 
-	if (__nospectre_bhb || cpu_mitigations_off())
+	if (spectre_bhb_mitigations_off())
 		pr_info("spectre-bhb %s", spectre_disabled_suffix);
 }
diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c
index 4d08598e2891..4955166723b8 100644
--- a/arch/arm64/kernel/ptrace.c
+++ b/arch/arm64/kernel/ptrace.c
@@ -560,6 +560,42 @@ static int gpr_get(struct task_struct *target,
 	return membuf_write(&to, uregs, sizeof(*uregs));
 }
 
+static void update_syscall_orig_x0_after_ptrace(struct task_struct *target)
+{
+	struct pt_regs *regs = task_pt_regs(target);
+	struct kernel_siginfo *info = target->last_siginfo;
+
+	/*
+	 * Skip the update for NO_SYSCALL (set either by the user or the
+	 * tracer), as regs[0] holds the return value (see the comment in
+	 * el0_svc_common()) and can be unwound using syscall_rollback().
+	 */
+	if (regs->syscallno == NO_SYSCALL)
+		return;
+
+	/* We should only be called when target is in a ptrace stop */
+	if (WARN_ON_ONCE(!info))
+		return;
+
+	/*
+	 * For compat tasks, orig_r0 is provided directly through GPR index
+	 * 17.
+	 */
+	if (is_compat_thread(task_thread_info(target)))
+		return;
+
+	/*
+	 * Don't update orig_x0 for a syscall-exit-stop, as x0 now contains the
+	 * return value of the system call.
+	 */
+	if ((info->si_code & ~0x80) == SIGTRAP &&
+	    target->ptrace_message == PTRACE_EVENTMSG_SYSCALL_EXIT) {
+		return;
+	}
+
+	regs->orig_x0 = regs->regs[0];
+}
+
 static int gpr_set(struct task_struct *target, const struct user_regset *regset,
 		   unsigned int pos, unsigned int count,
 		   const void *kbuf, const void __user *ubuf)
@@ -575,6 +611,14 @@ static int gpr_set(struct task_struct *target, const struct user_regset *regset,
 		return -EINVAL;
 
 	task_pt_regs(target)->user_regs = newregs;
+
+	/*
+	 * Keep orig_x0 authoritative so that seccomp (via
+	 * syscall_get_arguments()), audit and the restart path all see the same
+	 * first argument the syscall is dispatched with, even if it has been
+	 * updated by a tracer.
+	 */
+	update_syscall_orig_x0_after_ptrace(target);
 	return 0;
 }
 
@@ -753,6 +797,12 @@ static int system_call_set(struct task_struct *target,
 		return ret;
 
 	task_pt_regs(target)->syscallno = syscallno;
+
+	/*
+	 * Re-sync orig_x0 in case the syscall number has been changed
+	 * from NO_SYSCALL.
+	 */
+	update_syscall_orig_x0_after_ptrace(target);
 	return ret;
 }
 
@@ -801,7 +851,7 @@ static void sve_init_header_from_task(struct user_sve_header *header,
 	if (active)
 		header->size = SVE_PT_SIZE(vq, header->flags);
 	else
-		header->size = sizeof(header);
+		header->size = sizeof(*header);
 	header->max_size = SVE_PT_SIZE(sve_vq_from_vl(header->max_vl),
 				      SVE_PT_REGS_SVE);
 }
@@ -837,7 +887,7 @@ static int sve_get_common(struct task_struct *target,
 	 * from the other mode to userspace.
 	 */
 	if (header.size == sizeof(header))
-		return 0;
+		return to.left;
 
 	switch ((header.flags & SVE_PT_REGS_MASK)) {
 	case SVE_PT_REGS_FPSIMD:
diff --git a/arch/arm64/kernel/rsi.c b/arch/arm64/kernel/rsi.c
index 92160f2e57ff..25ca75ce1a4d 100644
--- a/arch/arm64/kernel/rsi.c
+++ b/arch/arm64/kernel/rsi.c
@@ -7,7 +7,6 @@
 #include <linux/memblock.h>
 #include <linux/psci.h>
 #include <linux/swiotlb.h>
-#include <linux/cc_platform.h>
 #include <linux/platform_device.h>
 
 #include <asm/io.h>
@@ -23,17 +22,6 @@ EXPORT_SYMBOL(prot_ns_shared);
 DEFINE_STATIC_KEY_FALSE_RO(rsi_present);
 EXPORT_SYMBOL(rsi_present);
 
-bool cc_platform_has(enum cc_attr attr)
-{
-	switch (attr) {
-	case CC_ATTR_MEM_ENCRYPT:
-		return is_realm_world();
-	default:
-		return false;
-	}
-}
-EXPORT_SYMBOL_GPL(cc_platform_has);
-
 static bool rsi_version_matches(void)
 {
 	unsigned long ver_lower, ver_higher;
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index cdcdd160e5b6..a61dc3016a11 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -33,6 +33,7 @@
 #include <linux/kernel_stat.h>
 #include <linux/kexec.h>
 #include <linux/kgdb.h>
+#include <linux/kprobes.h>
 #include <linux/kvm_host.h>
 #include <linux/nmi.h>
 
@@ -45,6 +46,7 @@
 #include <asm/daifflags.h>
 #include <asm/kvm_mmu.h>
 #include <asm/mmu_context.h>
+#include <asm/nmi.h>
 #include <asm/numa.h>
 #include <asm/processor.h>
 #include <asm/smp_plat.h>
@@ -866,52 +868,71 @@ void arch_irq_work_raise(void)
 }
 #endif
 
-static void __noreturn local_cpu_stop(unsigned int cpu)
-{
-	set_cpu_online(cpu, false);
-
-	local_daif_mask();
-	sdei_mask_local_cpu();
-	cpu_park_loop();
-}
-
-/*
- * We need to implement panic_smp_self_stop() for parallel panic() calls, so
- * that cpu_online_mask gets correctly updated and smp_send_stop() can skip
- * CPUs that have already stopped themselves.
+/**
+ * arm64_nmi_cpu_stop() - stop the local CPU after it is told to stop.
+ * @regs: register state to record in the vmcore on a crash stop, or NULL for
+ *        panic_smp_self_stop(), which has no interrupted context to save.
+ * @die_on_crash: on the kdump crash path, power the CPU off via PSCI CPU_OFF
+ *                (so a capture kernel can reclaim it) rather than parking it.
+ *
+ * The single point every arm64 stop path funnels through, keeping the
+ * bookkeeping (mask interrupts, save the crash context, mark offline, mask
+ * SDEI, optionally power off) in one place:
+ *
+ *   - the regular IPI_CPU_STOP and pseudo-NMI IPI_CPU_STOP_NMI handlers;
+ *   - panic_smp_self_stop(), a CPU parking itself on a parallel panic();
+ *   - the SDEI cross-CPU NMI handler (drivers/firmware/arm_sdei_nmi.c),
+ *     which reaches CPUs the stop IPIs could not.
+ *
+ * The IPI stop handlers pass @die_on_crash true. The SDEI handler and
+ * panic_smp_self_stop() pass false and only park. For SDEI that is required,
+ * not just conservative: it runs inside an SDEI event that is deliberately
+ * never completed (completing it has firmware resume the wedged context), and
+ * a CPU_OFF from that not-yet-completed context wedges EL3 on some firmware --
+ * a documented follow-up. Parking also matches this path's own fallback when
+ * CPU_OFF is unavailable.
  */
-void __noreturn panic_smp_self_stop(void)
+void __noreturn arm64_nmi_cpu_stop(struct pt_regs *regs, bool die_on_crash)
 {
-	local_cpu_stop(smp_processor_id());
-}
+	unsigned int cpu = smp_processor_id();
+	bool crash = IS_ENABLED(CONFIG_KEXEC_CORE) && crash_stop;
 
-static void __noreturn ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs)
-{
-#ifdef CONFIG_KEXEC_CORE
 	/*
 	 * Use local_daif_mask() instead of local_irq_disable() to make sure
-	 * that pseudo-NMIs are disabled. The "crash stop" code starts with
-	 * an IRQ and falls back to NMI (which might be pseudo). If the IRQ
-	 * finally goes through right as we're timing out then the NMI could
-	 * interrupt us. It's better to prevent the NMI and let the IRQ
-	 * finish since the pt_regs will be better.
+	 * that pseudo-NMIs are disabled. The "stop" code starts with an IRQ
+	 * and falls back to NMI (which might be pseudo). If the IRQ finally
+	 * goes through right as we're timing out then the NMI could interrupt
+	 * us. It's better to prevent the NMI and let the IRQ finish since the
+	 * pt_regs will be better.
 	 */
 	local_daif_mask();
 
-	crash_save_cpu(regs, cpu);
+#ifdef CONFIG_KEXEC_CORE
+	if (crash && regs)
+		crash_save_cpu(regs, cpu);
+#endif
 
+	/* the ack a stop requester (e.g. smp_send_stop()) polls for */
 	set_cpu_online(cpu, false);
 
 	sdei_mask_local_cpu();
 
-	if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
+	if (crash && die_on_crash)
 		__cpu_try_die(cpu);
 
 	/* just in case */
 	cpu_park_loop();
-#else
-	BUG();
-#endif
+}
+NOKPROBE_SYMBOL(arm64_nmi_cpu_stop);
+
+/*
+ * We need to implement panic_smp_self_stop() for parallel panic() calls, so
+ * that cpu_online_mask gets correctly updated and smp_send_stop() can skip
+ * CPUs that have already stopped themselves.
+ */
+void __noreturn panic_smp_self_stop(void)
+{
+	arm64_nmi_cpu_stop(NULL, false);
 }
 
 static void arm64_send_ipi(const cpumask_t *mask, unsigned int nr)
@@ -933,6 +954,16 @@ static void arm64_backtrace_ipi(cpumask_t *mask)
 void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu)
 {
 	/*
+	 * Prefer the SDEI cross-CPU NMI provider when active: firmware
+	 * dispatches the event out of EL3 and reaches CPUs that have
+	 * interrupts locally masked, without the per-IRQ-mask cost that
+	 * pseudo-NMI pays for the same reach. The plain IPI path below
+	 * can't reach such a CPU unless pseudo-NMI is enabled.
+	 */
+	if (sdei_nmi_trigger_cpumask_backtrace(mask, exclude_cpu))
+		return;
+
+	/*
 	 * NOTE: though nmi_trigger_cpumask_backtrace() has "nmi_" in the name,
 	 * nothing about it truly needs to be implemented using an NMI, it's
 	 * just that it's _allowed_ to work with NMIs. If ipi_should_be_nmi()
@@ -978,12 +1009,7 @@ static void do_handle_IPI(int ipinr)
 
 	case IPI_CPU_STOP:
 	case IPI_CPU_STOP_NMI:
-		if (IS_ENABLED(CONFIG_KEXEC_CORE) && crash_stop) {
-			ipi_cpu_crash_stop(cpu, get_irq_regs());
-			unreachable();
-		} else {
-			local_cpu_stop(cpu);
-		}
+		arm64_nmi_cpu_stop(get_irq_regs(), true);
 		break;
 
 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
@@ -1086,7 +1112,7 @@ static void ipi_teardown(int cpu)
 				disable_percpu_irq(ipi_irq_base + i);
 			}
 		} else {
-			disable_irq(irq_desc_get_irq(get_ipi_desc(cpu, i)));
+			disable_irq_nosync(irq_desc_get_irq(get_ipi_desc(cpu, i)));
 		}
 	}
 }
@@ -1257,6 +1283,28 @@ void smp_send_stop(void)
 			udelay(1);
 	}
 
+	/*
+	 * If CPUs are *still* online, try the SDEI cross-CPU NMI. Firmware
+	 * delivers it regardless of the target's DAIF state, so it reaches
+	 * a CPU spinning with interrupts masked, which neither rung above
+	 * could (without pseudo-NMI there is no NMI rung at all). Allow
+	 * 100ms: a firmware round-trip per CPU, with headroom.
+	 */
+	if (num_other_online_cpus() && sdei_nmi_active()) {
+		/* re-snapshot after the rungs above took CPUs offline */
+		smp_rmb();
+		cpumask_copy(&mask, cpu_online_mask);
+		cpumask_clear_cpu(smp_processor_id(), &mask);
+
+		pr_info("SMP: retry stop with SDEI NMI for CPUs %*pbl\n",
+			cpumask_pr_args(&mask));
+
+		sdei_nmi_stop_cpus(&mask);
+		timeout = USEC_PER_MSEC * 100;
+		while (num_other_online_cpus() && timeout--)
+			udelay(1);
+	}
+
 	if (num_other_online_cpus()) {
 		smp_rmb();
 		cpumask_copy(&mask, cpu_online_mask);
diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c
index eaaff94329cd..c41724a40b75 100644
--- a/arch/arm64/kernel/suspend.c
+++ b/arch/arm64/kernel/suspend.c
@@ -99,7 +99,6 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
 	int ret = 0;
 	unsigned long flags;
 	struct sleep_stack_data state;
-	struct arm_cpuidle_irq_context context;
 
 	/*
 	 * Some portions of CPU state (e.g. PSTATE.{PAN,DIT}) are initialized
@@ -121,6 +120,9 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
 	 * Strictly speaking the trace_hardirqs_off() here is superfluous,
 	 * hardirqs should be firmly off by now. This really ought to use
 	 * something like raw_local_daif_save().
+	 *
+	 * This also unmasks interrupts in PMR in order to reliably
+	 * resume if we're using pseudo-NMIs.
 	 */
 	flags = local_daif_save();
 
@@ -131,12 +133,6 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
 	 */
 	pause_graph_tracing();
 
-	/*
-	 * Switch to using DAIF.IF instead of PMR in order to reliably
-	 * resume if we're using pseudo-NMIs.
-	 */
-	arm_cpuidle_save_irq_context(&context);
-
 	ct_cpuidle_enter();
 
 	if (__cpu_suspend_enter(&state)) {
@@ -159,8 +155,6 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
 		__cpu_suspend_exit();
 	}
 
-	arm_cpuidle_restore_irq_context(&context);
-
 	unpause_graph_tracing();
 
 	/*
diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c
index b32f13358fbb..d28438f8b83f 100644
--- a/arch/arm64/kernel/topology.c
+++ b/arch/arm64/kernel/topology.c
@@ -373,6 +373,16 @@ core_initcall(init_amu_fie);
 #ifdef CONFIG_ACPI_CPPC_LIB
 #include <acpi/cppc_acpi.h>
 
+struct amu_ffh_ctrs {
+	u64 corecnt;
+	u64 constcnt;
+};
+
+enum cpc_ffh_ctr_id {
+	CPC_FFH_CTR_CORE  = 0x0,
+	CPC_FFH_CTR_CONST = 0x1,
+};
+
 static void cpu_read_corecnt(void *val)
 {
 	/*
@@ -397,7 +407,7 @@ static void cpu_read_constcnt(void *val)
 }
 
 static inline
-int counters_read_on_cpu(int cpu, smp_call_func_t func, u64 *val)
+int counters_read_on_cpu(int cpu, smp_call_func_t func, void *val)
 {
 	/*
 	 * Abort call on counterless CPU.
@@ -447,24 +457,90 @@ bool cpc_ffh_supported(void)
 	return true;
 }
 
+static void amu_read_core_const_ctrs(void *val)
+{
+	struct amu_ffh_ctrs *ctrs = val;
+
+	/*
+	 * cpu_read_constcnt() incurs slight latency due to the
+	 * ARM64_WORKAROUND_2457168 check. Read it first to minimize
+	 * the sampling skew between the const and core counters.
+	 */
+	cpu_read_constcnt(&ctrs->constcnt);
+	cpu_read_corecnt(&ctrs->corecnt);
+}
+
+static u64 cpc_ffh_extract_bits(const struct cpc_reg *reg, u64 val)
+{
+	val &= GENMASK_ULL(reg->bit_offset + reg->bit_width - 1,
+			   reg->bit_offset);
+	val >>= reg->bit_offset;
+
+	return val;
+}
+
+static void cpc_ffh_ctr_value(const struct cpc_reg *reg,
+			      const struct amu_ffh_ctrs *ctrs, u64 *val)
+{
+	switch ((u64)reg->address) {
+	case CPC_FFH_CTR_CORE:
+		*val = ctrs->corecnt;
+		break;
+	case CPC_FFH_CTR_CONST:
+		*val = ctrs->constcnt;
+		break;
+	}
+
+	*val = cpc_ffh_extract_bits(reg, *val);
+}
+
+static bool is_amu_ctr_reg(const struct cpc_reg *reg)
+{
+	return reg->address == CPC_FFH_CTR_CORE ||
+		reg->address == CPC_FFH_CTR_CONST;
+}
+
+int cpc_read_ffh_fb_ctrs(int cpu, struct cpc_reg *reg1, u64 *val1,
+			 struct cpc_reg *reg2, u64 *val2)
+{
+	struct amu_ffh_ctrs ctrs;
+	int ret;
+
+	if (!is_amu_ctr_reg(reg1) || !is_amu_ctr_reg(reg2))
+		return -EINVAL;
+
+	ret = counters_read_on_cpu(cpu, amu_read_core_const_ctrs, &ctrs);
+	if (ret) {
+		/*
+		 * If AMU is unsupported (-EOPNOTSUPP), translate the error
+		 * to -ENODEV. This explicitly tells the generic CPPC layer
+		 * to abort immediately and avoid falling back to pointless
+		 * single-counter reads.
+		 */
+		return ret == -EOPNOTSUPP ? -ENODEV : ret;
+	}
+
+	cpc_ffh_ctr_value(reg1, &ctrs, val1);
+	cpc_ffh_ctr_value(reg2, &ctrs, val2);
+
+	return 0;
+}
+
 int cpc_read_ffh(int cpu, struct cpc_reg *reg, u64 *val)
 {
 	int ret = -EOPNOTSUPP;
 
 	switch ((u64)reg->address) {
-	case 0x0:
+	case CPC_FFH_CTR_CORE:
 		ret = counters_read_on_cpu(cpu, cpu_read_corecnt, val);
 		break;
-	case 0x1:
+	case CPC_FFH_CTR_CONST:
 		ret = counters_read_on_cpu(cpu, cpu_read_constcnt, val);
 		break;
 	}
 
-	if (!ret) {
-		*val &= GENMASK_ULL(reg->bit_offset + reg->bit_width - 1,
-				    reg->bit_offset);
-		*val >>= reg->bit_offset;
-	}
+	if (!ret)
+		*val = cpc_ffh_extract_bits(reg, *val);
 
 	return ret;
 }
diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile
index 448c917494f3..b33e1ca4a781 100644
--- a/arch/arm64/lib/Makefile
+++ b/arch/arm64/lib/Makefile
@@ -1,4 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0
+
+# KCSAN uses udelay for introducing watchpoint delay; avoid recursion.
+KCSAN_SANITIZE_delay.o := n
+
 lib-y		:= clear_user.o delay.o copy_from_user.o		\
 		   copy_to_user.o copy_page.o				\
 		   clear_page.o csum.o insn.o memchr.o memcpy.o		\
diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c
index 2de12656b4d8..0acab179fc1a 100644
--- a/arch/arm64/mm/contpte.c
+++ b/arch/arm64/mm/contpte.c
@@ -89,7 +89,7 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr,
 	}
 
 	/*
-	 * On eliding the __tlb_flush_range() under BBML2+noabort:
+	 * On eliding the __tlb_flush_range() under BBML3:
 	 *
 	 * NOTE: Instead of using N=16 as the contiguous block length, we use
 	 *       N=4 for clarity.
@@ -135,7 +135,7 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr,
 	 * contiguous TLB entry, which is a micro-optimisation opportunity,
 	 * but does not affect correctness.
 	 *
-	 * In the BBML2 case, the change is avoiding the intermediate tlbi+dsb.
+	 * In the BBML3 case, the change is avoiding the intermediate tlbi+dsb.
 	 * This means a few things, but notably other PEs will still "see" any
 	 * stale cached TLB entries. This could lead to a "contiguous bit
 	 * misprogramming" issue until the final tlbi+dsb of the changed page,
@@ -158,21 +158,16 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr,
 	 *  are present, and a write is made to this address, do we fault or
 	 *  is the write permitted (via amalgamation)?
 	 *
-	 * The relevant Arm ARM DDI 0487L.a requirements are RNGLXZ and RJQQTC,
-	 * and together state that when BBML1 or BBML2 are implemented, either
-	 * a TLB conflict abort is raised (which we expressly forbid), or will
-	 * "produce an OA, access permissions, and memory attributes that are
-	 * consistent with any of the programmed translation table values".
-	 *
-	 * That is to say, will either raise a TLB conflict, or produce one of
-	 * the cached TLB entries, but never amalgamate.
+	 * With BBML3 implemented, no TLB conflict abort is raised and the OA,
+	 * access permissions and memory attributes produced is one of the cached
+	 * TLB entries, but never amalgamate.
 	 *
 	 * Thus, as the page tables are only considered "consistent" after
 	 * the final tlbi+dsb (which evicts both the single stale (RW,n) TLB
 	 * entry as well as the new contiguous (RO,c) TLB entry), omitting the
 	 * initial tlbi+dsb is correct.
 	 *
-	 * It is also important to note that at the end of the BBML2 folding
+	 * It is also important to note that at the end of the BBML3 folding
 	 * case, we are still left with potentially all N TLB entries still
 	 * cached (the N-1 non-contiguous ptes, and the single contiguous
 	 * block). However, over time, natural TLB pressure will cause the
@@ -214,7 +209,7 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr,
 	 *
 	 *                  |____| <--- tlbi + dsb
 	 *
-	 * For BBML2, we again remove the intermediate tlbi+dsb. Here, there
+	 * For BBML3, we again remove the intermediate tlbi+dsb. Here, there
 	 * are no issues, as the final tlbi+dsb covering the changed page is
 	 * guaranteed to remove the original large contiguous (RW,c) TLB entry,
 	 * as well as the intermediate (RW,n) TLB entry; the next access will
@@ -224,7 +219,7 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr,
 	 * regardless.
 	 */
 
-	if (!system_supports_bbml2_noabort())
+	if (!system_supports_bbml3())
 		__flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, 3,
 				  TLBF_NOWALKCACHE);
 
diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
index 994b7b36e2b9..7f6f2f40dba2 100644
--- a/arch/arm64/mm/dma-mapping.c
+++ b/arch/arm64/mm/dma-mapping.c
@@ -42,6 +42,11 @@ void arch_setup_dma_ops(struct device *dev, bool coherent)
 {
 	int cls = cache_line_size_of_cpu();
 
+	if (!coherent && !CLIDR_LOC(read_sysreg(clidr_el1))) {
+		dev_warn(dev, "CLIDR_EL1.LoC == 0, treating as coherent\n");
+		coherent = true;
+	}
+
 	WARN_TAINT(!coherent && cls > ARCH_DMA_MINALIGN,
 		   TAINT_CPU_OUT_OF_SPEC,
 		   "%s %s: ARCH_DMA_MINALIGN smaller than CTR_EL0.CWG (%d < %d)",
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 227f58522dad..52282d561a0d 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -11,6 +11,7 @@
 #include <linux/errno.h>
 #include <linux/swap.h>
 #include <linux/init.h>
+#include <linux/cc_platform.h>
 #include <linux/cache.h>
 #include <linux/mman.h>
 #include <linux/nodemask.h>
@@ -36,6 +37,7 @@
 
 #include <asm/boot.h>
 #include <asm/fixmap.h>
+#include <asm/hypervisor.h>
 #include <asm/kasan.h>
 #include <asm/kernel-pgtable.h>
 #include <asm/kvm_host.h>
@@ -337,25 +339,21 @@ void __init arch_setup_zero_pages(void)
 void __init arch_mm_preinit(void)
 {
 	unsigned int flags = SWIOTLB_VERBOSE;
-	bool swiotlb = max_pfn > PFN_DOWN(arm64_dma_phys_limit);
 
-	if (is_realm_world()) {
-		swiotlb = true;
+	if (is_realm_world() || is_protected_kvm_guest()) {
 		flags |= SWIOTLB_FORCE;
-	}
-
-	if (IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && !swiotlb) {
+	} else if (max_pfn <= PFN_DOWN(arm64_dma_phys_limit)) {
 		/*
 		 * If no bouncing needed for ZONE_DMA, reduce the swiotlb
 		 * buffer for kmalloc() bouncing to 1MB per 1GB of RAM.
 		 */
 		unsigned long size =
 			DIV_ROUND_UP(memblock_phys_mem_size(), 1024);
+
 		swiotlb_adjust_size(min(swiotlb_size_or_default(), size));
-		swiotlb = true;
 	}
 
-	swiotlb_init(swiotlb, flags);
+	swiotlb_init(true, flags);
 
 	/*
 	 * Check boundaries twice: Some fundamental inconsistencies can be
@@ -417,6 +415,17 @@ void dump_mem_limit(void)
 	}
 }
 
+bool cc_platform_has(enum cc_attr attr)
+{
+	switch (attr) {
+	case CC_ATTR_MEM_ENCRYPT:
+		return is_realm_world() || is_protected_kvm_guest();
+	default:
+		return false;
+	}
+}
+EXPORT_SYMBOL_GPL(cc_platform_has);
+
 #ifdef CONFIG_EXECMEM
 static u64 module_direct_base __ro_after_init = 0;
 static u64 module_plt_base __ro_after_init = 0;
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index d4de88770ecf..79d90226fd5d 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -24,6 +24,7 @@
 #include <linux/mm.h>
 #include <linux/vmalloc.h>
 #include <linux/set_memory.h>
+#include <linux/suspend.h>
 #include <linux/kfence.h>
 #include <linux/pkeys.h>
 #include <linux/mm_inline.h>
@@ -777,18 +778,18 @@ out:
 
 static inline bool force_pte_mapping(void)
 {
-	const bool bbml2 = system_capabilities_finalized() ?
-		system_supports_bbml2_noabort() : cpu_supports_bbml2_noabort();
+	const bool bbml3 = system_capabilities_finalized() ?
+		system_supports_bbml3() : cpu_supports_bbml3();
 
 	if (debug_pagealloc_enabled())
 		return true;
-	if (bbml2)
+	if (bbml3)
 		return false;
 	return rodata_full || arm64_kfence_can_set_direct_map() || is_realm_world();
 }
 
 static DEFINE_MUTEX(pgtable_split_lock);
-static bool linear_map_requires_bbml2;
+static bool linear_map_requires_bbml3;
 
 int split_kernel_leaf_mapping(unsigned long start, unsigned long end)
 {
@@ -801,15 +802,15 @@ int split_kernel_leaf_mapping(unsigned long start, unsigned long end)
 	 * always pte-mapped), we must not go any further because taking the
 	 * mutex below may sleep. Do not call force_pte_mapping() here because
 	 * it could return a confusing result if called from a secondary cpu
-	 * prior to finalizing caps. Instead, linear_map_requires_bbml2 gives us
+	 * prior to finalizing caps. Instead, linear_map_requires_bbml3 gives us
 	 * what we need.
 	 */
-	if (!linear_map_requires_bbml2 || is_kfence_address((void *)start))
+	if (!linear_map_requires_bbml3 || is_kfence_address((void *)start))
 		return 0;
 
-	if (!system_supports_bbml2_noabort()) {
+	if (!system_supports_bbml3()) {
 		/*
-		 * !BBML2_NOABORT systems should not be trying to change
+		 * BBML3 systems should not be trying to change
 		 * permissions on anything that is not pte-mapped in the first
 		 * place. Just return early and let the permission change code
 		 * raise a warning if not already pte-mapped.
@@ -826,8 +827,8 @@ int split_kernel_leaf_mapping(unsigned long start, unsigned long end)
 
 		/*
 		 * Boot-time: Started secondary cpus but don't know if they
-		 * support BBML2_NOABORT yet. Can't allow splitting in this
-		 * window in case they don't.
+		 * support BBML3 yet. Can't allow splitting in this window
+		 * in case they don't.
 		 */
 		if (WARN_ON(num_online_cpus() > 1))
 			return -EBUSY;
@@ -932,11 +933,11 @@ static int range_split_to_ptes(unsigned long start, unsigned long end, gfp_t gfp
 	return ret;
 }
 
-u32 idmap_kpti_bbml2_flag;
+u32 idmap_kpti_bbml3_flag;
 
-static void __init init_idmap_kpti_bbml2_flag(void)
+static void __init init_idmap_kpti_bbml3_flag(void)
 {
-	WRITE_ONCE(idmap_kpti_bbml2_flag, 1);
+	WRITE_ONCE(idmap_kpti_bbml3_flag, 1);
 	/* Must be visible to other CPUs before stop_machine() is called. */
 	smp_mb();
 }
@@ -945,7 +946,7 @@ static int __init linear_map_split_to_ptes(void *__unused)
 {
 	/*
 	 * Repainting the linear map must be done by CPU0 (the boot CPU) because
-	 * that's the only CPU that we know supports BBML2. The other CPUs will
+	 * that's the only CPU that we know supports BBML3. The other CPUs will
 	 * be held in a waiting area with the idmap active.
 	 */
 	if (!smp_processor_id()) {
@@ -958,7 +959,7 @@ static int __init linear_map_split_to_ptes(void *__unused)
 		/*
 		 * Wait for all secondary CPUs to be put into the waiting area.
 		 */
-		smp_cond_load_acquire(&idmap_kpti_bbml2_flag, VAL == num_online_cpus());
+		smp_cond_load_acquire(&idmap_kpti_bbml3_flag, VAL == num_online_cpus());
 
 		/*
 		 * Walk all of the linear map [lstart, lend), except the kernel
@@ -977,7 +978,7 @@ static int __init linear_map_split_to_ptes(void *__unused)
 		 * Relies on dsb in flush_tlb_kernel_range() to avoid reordering
 		 * before any page table split operations.
 		 */
-		WRITE_ONCE(idmap_kpti_bbml2_flag, 0);
+		WRITE_ONCE(idmap_kpti_bbml3_flag, 0);
 	} else {
 		typedef void (wait_split_fn)(void);
 		extern wait_split_fn wait_linear_map_split_to_ptes;
@@ -986,7 +987,7 @@ static int __init linear_map_split_to_ptes(void *__unused)
 		wait_fn = (void *)__pa_symbol(wait_linear_map_split_to_ptes);
 
 		/*
-		 * At least one secondary CPU doesn't support BBML2 so cannot
+		 * At least one secondary CPU doesn't support BBML3 so cannot
 		 * tolerate the size of the live mappings changing. So have the
 		 * secondary CPUs wait for the boot CPU to make the changes
 		 * with the idmap active and init_mm inactive.
@@ -1001,8 +1002,8 @@ static int __init linear_map_split_to_ptes(void *__unused)
 
 void __init linear_map_maybe_split_to_ptes(void)
 {
-	if (linear_map_requires_bbml2 && !system_supports_bbml2_noabort()) {
-		init_idmap_kpti_bbml2_flag();
+	if (linear_map_requires_bbml3 && !system_supports_bbml3()) {
+		init_idmap_kpti_bbml3_flag();
 		stop_machine(linear_map_split_to_ptes, NULL, cpu_online_mask);
 	}
 }
@@ -1060,6 +1061,29 @@ static void __init __map_memblock(phys_addr_t start, phys_addr_t end,
 				 end - start, prot, early_pgtable_alloc, flags);
 }
 
+static void mark_linear_data_alias_valid(bool valid)
+{
+	set_memory_valid((unsigned long)lm_alias(__init_end),
+			 (unsigned long)(__bss_stop - __init_end) / PAGE_SIZE,
+			 valid);
+}
+
+static int arm64_hibernate_pm_notify(struct notifier_block *nb,
+				     unsigned long mode, void *unused)
+{
+	switch (mode) {
+	default:
+		break;
+	case PM_POST_HIBERNATION:
+		mark_linear_data_alias_valid(false);
+		break;
+	case PM_HIBERNATION_PREPARE:
+		mark_linear_data_alias_valid(true);
+		break;
+	}
+	return 0;
+}
+
 void __init mark_linear_text_alias_ro(void)
 {
 	/*
@@ -1068,6 +1092,21 @@ void __init mark_linear_text_alias_ro(void)
 	update_mapping_prot(__pa_symbol(_text), (unsigned long)lm_alias(_text),
 			    (unsigned long)__init_begin - (unsigned long)_text,
 			    PAGE_KERNEL_RO);
+
+	/*
+	 * Register a PM notifier to remap the linear alias of data/bss as
+	 * valid read/write before hibernation. This is needed because the
+	 * snapshot logic disregards PageReserved pages (such as the ones
+	 * covering the kernel image) unless they are mapped in the linear
+	 * map.
+	 */
+	if (IS_ENABLED(CONFIG_HIBERNATION) && rodata_enabled) {
+		static struct notifier_block nb = {
+			.notifier_call = arm64_hibernate_pm_notify
+		};
+
+		register_pm_notifier(&nb);
+	}
 }
 
 #ifdef CONFIG_KFENCE
@@ -1125,7 +1164,7 @@ bool arch_kfence_init_pool(void)
 	mutex_unlock(&pgtable_split_lock);
 
 	/*
-	 * Since the system supports bbml2_noabort, tlb invalidation is not
+	 * Since the system supports bbml3, tlb invalidation is not
 	 * required here; the pgtable mappings have been split to pte but larger
 	 * entries may safely linger in the TLB.
 	 */
@@ -1164,7 +1203,7 @@ static void __init map_mem(void)
 
 	arm64_kfence_map_pool();
 
-	linear_map_requires_bbml2 = !force_pte_mapping() && can_set_direct_map();
+	linear_map_requires_bbml3 = !force_pte_mapping() && can_set_direct_map();
 
 	if (force_pte_mapping())
 		flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
@@ -1189,6 +1228,20 @@ static void __init map_mem(void)
 	/* map all the memory banks */
 	for_each_mem_range(i, &start, &end) {
 		/*
+		 * for_each_mem_range may return sub-page-aligned boundaries
+		 * after memblock_mark_nomap() splits regions at byte precision.
+		 * __create_pgd_mapping_locked aligns phys down to PAGE_MASK,
+		 * which could accidentally map no-map memory on the boundary.
+		 * Round the mappable range inward: start UP, end DOWN, so
+		 * that the mapped area never overlaps with adjacent no-map
+		 * regions. The cost is at most one page of unmapped gap at
+		 * each boundary.
+		 */
+		start = PAGE_ALIGN(start);
+		end = end & PAGE_MASK;
+		if (start >= end)
+			continue;
+		/*
 		 * The linear map must allow allocation tags reading/writing
 		 * if MTE is present. Otherwise, it has the same attributes as
 		 * PAGE_KERNEL.
@@ -1215,11 +1268,8 @@ void mark_rodata_ro(void)
 			    (unsigned long)_stext - (unsigned long)_text,
 			    PAGE_KERNEL_RO);
 
-	/* Map the kernel data/bss read-only in the linear map */
-	update_mapping_prot(__pa_symbol(__init_end),
-			    (unsigned long)lm_alias(__init_end),
-			    (unsigned long)__bss_stop - (unsigned long)__init_end,
-			    PAGE_KERNEL_RO);
+	/* Map the kernel data/bss as invalid in the linear map */
+	mark_linear_data_alias_valid(false);
 }
 
 static void __init declare_vma(struct vm_struct *vma,
@@ -1331,7 +1381,7 @@ void __init kpti_install_ng_mappings(void)
 	if (arm64_use_ng_mappings)
 		return;
 
-	init_idmap_kpti_bbml2_flag();
+	init_idmap_kpti_bbml3_flag();
 	stop_machine(__kpti_install_ng_mappings, NULL, cpu_online_mask);
 }
 
@@ -1392,7 +1442,7 @@ void __pi_map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa,
 		    u64 va_offset);
 
 static u8 idmap_ptes[IDMAP_LEVELS - 1][PAGE_SIZE] __aligned(PAGE_SIZE) __ro_after_init,
-	  kpti_bbml2_ptes[IDMAP_LEVELS - 1][PAGE_SIZE] __aligned(PAGE_SIZE) __ro_after_init;
+	  kpti_bbml3_ptes[IDMAP_LEVELS - 1][PAGE_SIZE] __aligned(PAGE_SIZE) __ro_after_init;
 
 static void __init create_idmap(void)
 {
@@ -1404,17 +1454,17 @@ static void __init create_idmap(void)
 		       IDMAP_ROOT_LEVEL, (pte_t *)idmap_pg_dir, false,
 		       __phys_to_virt(ptep) - ptep);
 
-	if (linear_map_requires_bbml2 ||
+	if (linear_map_requires_bbml3 ||
 	    (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0) && !arm64_use_ng_mappings)) {
-		phys_addr_t pa = __pa_symbol(&idmap_kpti_bbml2_flag);
+		phys_addr_t pa = __pa_symbol(&idmap_kpti_bbml3_flag);
 
 		/*
 		 * The KPTI G-to-nG conversion code needs a read-write mapping
 		 * of its synchronization flag in the ID map. This is also used
 		 * when splitting the linear map to ptes if a secondary CPU
-		 * doesn't support bbml2.
+		 * doesn't support bbml3.
 		 */
-		ptep = __pa_symbol(kpti_bbml2_ptes);
+		ptep = __pa_symbol(kpti_bbml3_ptes);
 		__pi_map_range(&ptep, pa, pa + sizeof(u32), pa, PAGE_KERNEL,
 			       IDMAP_ROOT_LEVEL, (pte_t *)idmap_pg_dir, false,
 			       __phys_to_virt(ptep) - ptep);
diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index 22866b49be37..12aacc74f764 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -169,6 +169,13 @@ alternative_if ARM64_HAS_RAS_EXTN
 	msr_s	SYS_DISR_EL1, xzr
 alternative_else_nop_endif
 
+#ifdef CONFIG_ARM64_PSEUDO_NMI
+alternative_if ARM64_HAS_GIC_PRIO_MASKING
+	mov	x1, #GIC_PRIO_IRQON
+	msr_s	SYS_ICC_PMR_EL1, x1
+alternative_else_nop_endif
+#endif
+
 	ptrauth_keys_install_kernel_nosync x14, x1, x2, x3
 	isb
 	ret
@@ -287,7 +294,7 @@ SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings)
 
 	mov	x5, x3				// preserve temp_pte arg
 	mrs	swapper_ttb, ttbr1_el1
-	adr_l	flag_ptr, idmap_kpti_bbml2_flag
+	adr_l	flag_ptr, idmap_kpti_bbml3_flag
 
 	cbnz	cpu, __idmap_kpti_secondary
 
@@ -445,7 +452,7 @@ SYM_TYPED_FUNC_START(wait_linear_map_split_to_ptes)
 	flag_ptr	.req	x4
 
 	mrs     swapper_ttb, ttbr1_el1
-	adr_l   flag_ptr, idmap_kpti_bbml2_flag
+	adr_l   flag_ptr, idmap_kpti_bbml3_flag
 	__idmap_cpu_set_reserved_ttbr1 x16, x17
 
 scondary_cpu_wait:
diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps
index 9b85a84f6fd4..0ee42b0cc8d3 100644
--- a/arch/arm64/tools/cpucaps
+++ b/arch/arm64/tools/cpucaps
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 #
-# Internal CPU capabilities constants, keep this list sorted
+# Internal CPU capabilities constants
 
 ALWAYS_BOOT
 ALWAYS_SYSTEM
@@ -14,6 +14,7 @@ HAS_ADDRESS_AUTH_ARCH_QARMA5
 HAS_ADDRESS_AUTH_IMP_DEF
 HAS_AMU_EXTN
 HAS_ARMv8_4_TTL
+HAS_BBML3
 HAS_CACHE_DIC
 HAS_CACHE_IDC
 HAS_CNP
@@ -51,7 +52,6 @@ HAS_LS64_V
 HAS_LSUI
 HAS_MOPS
 HAS_NESTED_VIRT
-HAS_BBML2_NOABORT
 HAS_PAN
 HAS_PMUV3
 HAS_S1PIE
@@ -121,10 +121,11 @@ WORKAROUND_CAVIUM_TX2_219_TVM
 WORKAROUND_CLEAN_CACHE
 WORKAROUND_DEVICE_LOAD_ACQUIRE
 WORKAROUND_DISABLE_CNP
+WORKAROUND_NVIDIA_OLYMPUS_1027
 WORKAROUND_PMUV3_IMPDEF_TRAPS
 WORKAROUND_QCOM_FALKOR_E1003
 WORKAROUND_QCOM_ORYON_CNTVOFF
-WORKAROUND_REPEAT_TLBI
+WORKAROUND_REPEAT_TLBI_SYNC
 WORKAROUND_SPECULATIVE_AT
 WORKAROUND_SPECULATIVE_SSBS
 WORKAROUND_SPECULATIVE_UNPRIV_LOAD
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index 7cb61aca3797..e070ada06196 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -2259,6 +2259,7 @@ UnsignedEnum	55:52	BBM
 	0b0000	0
 	0b0001	1
 	0b0010	2
+	0b0011	3
 EndEnum
 UnsignedEnum	51:48	TTL
 	0b0000	NI