diff options
Diffstat (limited to 'arch/arm64/kernel')
57 files changed, 1373 insertions, 588 deletions
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 76f32e424065..d2690c3ec528 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -27,7 +27,7 @@ KCOV_INSTRUMENT_idle.o := n # Object file lists. obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ - entry-common.o entry-fpsimd.o process.o ptrace.o \ + entry-common.o process.o ptrace.o \ setup.o signal.o sys.o stacktrace.o time.o traps.o \ io.o vdso.o hyp-stub.o psci.o cpu_ops.o \ return_address.o cpuinfo.o cpu_errata.o \ @@ -46,6 +46,7 @@ obj-$(CONFIG_MODULES) += module.o module-plts.o obj-$(CONFIG_PERF_EVENTS) += perf_regs.o perf_callchain.o obj-$(CONFIG_HARDLOCKUP_DETECTOR_PERF) += watchdog_hld.o obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o +obj-$(CONFIG_HAVE_STATIC_CALL) += static_call.o obj-$(CONFIG_CPU_PM) += sleep.o suspend.o obj-$(CONFIG_KGDB) += kgdb.o obj-$(CONFIG_EFI) += efi.o efi-rt-wrapper.o @@ -67,6 +68,7 @@ obj-$(CONFIG_CRASH_DUMP) += crash_dump.o obj-$(CONFIG_VMCORE_INFO) += vmcore_info.o obj-$(CONFIG_ARM_SDE_INTERFACE) += sdei.o obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o +obj-$(CONFIG_ARM64_MPAM) += mpam.o obj-$(CONFIG_ARM64_MTE) += mte.o obj-y += vdso-wrap.o obj-$(CONFIG_COMPAT_VDSO) += vdso32-wrap.o diff --git a/arch/arm64/kernel/acpi.c b/arch/arm64/kernel/acpi.c index af90128cfed5..681aa2bbc399 100644 --- a/arch/arm64/kernel/acpi.c +++ b/arch/arm64/kernel/acpi.c @@ -377,7 +377,7 @@ void __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size) prot = __acpi_get_writethrough_mem_attribute(); } } - return ioremap_prot(phys, size, prot); + return __ioremap_prot(phys, size, prot); } /* @@ -448,13 +448,45 @@ int acpi_map_cpu(acpi_handle handle, phys_cpuid_t physid, u32 apci_id, return *pcpu; } + set_cpu_present(*pcpu, true); return 0; } EXPORT_SYMBOL(acpi_map_cpu); int acpi_unmap_cpu(int cpu) { + set_cpu_present(cpu, false); return 0; } EXPORT_SYMBOL(acpi_unmap_cpu); #endif /* CONFIG_ACPI_HOTPLUG_CPU */ + +int acpi_get_cpu_uid(unsigned int cpu, u32 *uid) +{ + struct acpi_madt_generic_interrupt *gicc; + + if (cpu >= nr_cpu_ids) + return -EINVAL; + + gicc = acpi_cpu_get_madt_gicc(cpu); + if (!gicc) + return -ENODEV; + + *uid = gicc->uid; + return 0; +} +EXPORT_SYMBOL_GPL(acpi_get_cpu_uid); + +int get_cpu_for_acpi_id(u32 uid) +{ + u32 cpu_uid; + int ret; + + for (int cpu = 0; cpu < nr_cpu_ids; cpu++) { + ret = acpi_get_cpu_uid(cpu, &cpu_uid); + if (ret == 0 && uid == cpu_uid) + return cpu; + } + + return -EINVAL; +} diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c index e737c6295ec7..b7a1f8b788bb 100644 --- a/arch/arm64/kernel/armv8_deprecated.c +++ b/arch/arm64/kernel/armv8_deprecated.c @@ -610,6 +610,20 @@ static int __init armv8_deprecated_init(void) } #endif + +#ifdef CONFIG_SWP_EMULATION + /* + * The purpose of supporting LSUI is to eliminate PAN toggling. CPUs + * that support LSUI are unlikely to support a 32-bit runtime. Rather + * than emulating the SWP instruction using LSUI instructions, simply + * disable SWP emulation. + */ + if (cpus_have_final_cap(ARM64_HAS_LSUI)) { + insn_swp.status = INSN_UNAVAILABLE; + pr_info("swp/swpb instruction emulation is not supported on this system\n"); + } +#endif + for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) { struct insn_emulation *ie = insn_emulations[i]; diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index b6367ff3a49c..9c853ed3ceab 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -124,7 +124,6 @@ int main(void) DEFINE(NVHE_INIT_HCR_EL2, offsetof(struct kvm_nvhe_init_params, hcr_el2)); DEFINE(NVHE_INIT_VTTBR, offsetof(struct kvm_nvhe_init_params, vttbr)); DEFINE(NVHE_INIT_VTCR, offsetof(struct kvm_nvhe_init_params, vtcr)); - DEFINE(NVHE_INIT_TMP, offsetof(struct kvm_nvhe_init_params, tmp)); #endif #ifdef CONFIG_CPU_PM DEFINE(CPU_CTX_SP, offsetof(struct cpu_suspend_ctx, sp)); diff --git a/arch/arm64/kernel/compat_alignment.c b/arch/arm64/kernel/compat_alignment.c index b68e1d328d4c..9b58d0aa38d2 100644 --- a/arch/arm64/kernel/compat_alignment.c +++ b/arch/arm64/kernel/compat_alignment.c @@ -114,8 +114,8 @@ do_alignment_ldrdstrd(unsigned long addr, u32 instr, struct pt_regs *regs) static int do_alignment_ldmstm(unsigned long addr, u32 instr, struct pt_regs *regs) { - unsigned int rd, rn, nr_regs, regbits; - unsigned long eaddr, newaddr; + unsigned int rd, rn, regbits; + unsigned long eaddr, newaddr, nr_regs; unsigned int val; /* count the number of registers in the mask to be transferred */ diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 8cb3b575a031..8ec47d89b45b 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -11,6 +11,7 @@ #include <asm/cpu.h> #include <asm/cputype.h> #include <asm/cpufeature.h> +#include <asm/fpsimd.h> #include <asm/kvm_asm.h> #include <asm/smp_plat.h> @@ -27,18 +28,21 @@ bool cpu_errata_set_target_impl(u64 num, void *impl_cpus) return true; } +static inline bool __is_midr_in_range(u32 midr, struct midr_range const *range) +{ + return midr_is_cpu_model_range(midr, range->model, + range->rv_min, range->rv_max); +} + static inline bool is_midr_in_range(struct midr_range const *range) { int i; if (!target_impl_cpu_num) - return midr_is_cpu_model_range(read_cpuid_id(), range->model, - range->rv_min, range->rv_max); + return __is_midr_in_range(read_cpuid_id(), range); for (i = 0; i < target_impl_cpu_num; i++) { - if (midr_is_cpu_model_range(target_impl_cpus[i].midr, - range->model, - range->rv_min, range->rv_max)) + if (__is_midr_in_range(target_impl_cpus[i].midr, range)) return true; } return false; @@ -58,7 +62,7 @@ __is_affected_midr_range(const struct arm64_cpu_capabilities *entry, u32 midr, u32 revidr) { const struct arm64_midr_revidr *fix; - if (!is_midr_in_range(&entry->midr_range)) + if (!__is_midr_in_range(midr, &entry->midr_range)) return false; midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK; @@ -81,7 +85,7 @@ is_affected_midr_range(const struct arm64_cpu_capabilities *entry, int scope) for (i = 0; i < target_impl_cpu_num; i++) { if (__is_affected_midr_range(entry, target_impl_cpus[i].midr, - target_impl_cpus[i].midr)) + target_impl_cpus[i].revidr)) return true; } return false; @@ -141,6 +145,30 @@ has_mismatched_cache_type(const struct arm64_cpu_capabilities *entry, return (ctr_real != sys) && (ctr_raw != sys); } +#ifdef CONFIG_ARM64_ERRATUM_4311569 +static DEFINE_STATIC_KEY_FALSE(arm_si_l1_workaround_4311569); +static int __init early_arm_si_l1_workaround_4311569_cfg(char *arg) +{ + static_branch_enable(&arm_si_l1_workaround_4311569); + pr_info("Enabling cache maintenance workaround for ARM SI-L1 erratum 4311569\n"); + + return 0; +} +early_param("arm_si_l1_workaround_4311569", early_arm_si_l1_workaround_4311569_cfg); + +/* + * We have some earlier use cases to call cache maintenance operation functions, for example, + * dcache_inval_poc() and dcache_clean_poc() in head.S, before making decision to turn on this + * workaround. Since the scope of this workaround is limited to non-coherent DMA agents, its + * safe to have the workaround off by default. + */ +static bool +need_arm_si_l1_workaround_4311569(const struct arm64_cpu_capabilities *entry, int scope) +{ + return static_branch_unlikely(&arm_si_l1_workaround_4311569); +} +#endif + static void cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *cap) { @@ -247,7 +275,7 @@ has_neoverse_n1_erratum_1542419(const struct arm64_cpu_capabilities *entry, return is_midr_in_range(&range) && has_dic; } -static const struct midr_range impdef_pmuv3_cpus[] = { +static const struct midr_range apple_cpus[] = { MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM), MIDR_ALL_VERSIONS(MIDR_APPLE_M1_FIRESTORM), MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM_PRO), @@ -276,7 +304,14 @@ static bool has_impdef_pmuv3(const struct arm64_cpu_capabilities *entry, int sco if (pmuver != ID_AA64DFR0_EL1_PMUVer_IMP_DEF) return false; - return is_midr_in_range_list(impdef_pmuv3_cpus); + return is_midr_in_range_list(apple_cpus); +} + +static bool has_broken_gic_v3_seis(const struct arm64_cpu_capabilities *entry, int scope) +{ + return (is_kernel_in_hyp_mode() && + is_midr_in_range_list(apple_cpus) && + (read_sysreg_s(SYS_ICH_VTR_EL2) & ICH_VTR_EL2_SEIS)); } static void cpu_enable_impdef_pmuv3_traps(const struct arm64_cpu_capabilities *__unused) @@ -284,7 +319,7 @@ static void cpu_enable_impdef_pmuv3_traps(const struct arm64_cpu_capabilities *_ sysreg_clear_set_s(SYS_HACR_EL2, 0, BIT(56)); } -#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI +#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI_SYNC static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = { #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1009 { @@ -315,7 +350,37 @@ static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = { ERRATA_MIDR_RANGE(MIDR_CORTEX_A510, 0, 0, 1, 1), }, #endif - {}, +#ifdef CONFIG_ARM64_ERRATUM_4118414 + { + ERRATA_MIDR_RANGE_LIST(((const struct midr_range[]) { + MIDR_ALL_VERSIONS(MIDR_C1_PREMIUM), + MIDR_ALL_VERSIONS(MIDR_C1_ULTRA), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A76), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A76AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A77), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A710), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X2), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X3), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X4), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X925), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3AE), + MIDR_ALL_VERSIONS(MIDR_NVIDIA_OLYMPUS), + MIDR_ALL_VERSIONS(MIDR_MICROSOFT_AZURE_COBALT_100), + {} + })), + }, +#endif + {} }; #endif @@ -551,6 +616,23 @@ static const struct midr_range erratum_spec_ssbs_list[] = { }; #endif +#ifdef CONFIG_ARM64_ERRATUM_4193714 +static bool has_sme_dvmsync_erratum(const struct arm64_cpu_capabilities *entry, + int scope) +{ + if (!id_aa64pfr1_sme(read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1))) + return false; + + return is_affected_midr_range(entry, scope); +} + +static void cpu_enable_sme_dvmsync(const struct arm64_cpu_capabilities *__unused) +{ + if (this_cpu_has_cap(ARM64_WORKAROUND_4193714)) + sme_enable_dvmsync(); +} +#endif + #ifdef CONFIG_AMPERE_ERRATUM_AC03_CPU_38 static const struct midr_range erratum_ac03_cpu_38_list[] = { MIDR_ALL_VERSIONS(MIDR_AMPERE1), @@ -566,6 +648,18 @@ static const struct midr_range erratum_ac04_cpu_23_list[] = { }; #endif +#ifdef CONFIG_ARM64_WORKAROUND_DISABLE_CNP +static const struct midr_range cnp_erratum_cpus[] = { +#ifdef CONFIG_NVIDIA_CARMEL_CNP_ERRATUM + MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL), +#endif +#ifdef CONFIG_HISILICON_ERRATUM_162100125 + MIDR_ALL_VERSIONS(MIDR_HISI_HIP09), +#endif + {}, +}; +#endif + const struct arm64_cpu_capabilities arm64_errata[] = { #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE { @@ -649,10 +743,10 @@ const struct arm64_cpu_capabilities arm64_errata[] = { .match_list = qcom_erratum_1003_list, }, #endif -#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI +#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI_SYNC { - .desc = "Qualcomm erratum 1009, or ARM erratum 1286807, 2441009", - .capability = ARM64_WORKAROUND_REPEAT_TLBI, + .desc = "Broken broadcast TLBI completion", + .capability = ARM64_WORKAROUND_REPEAT_TLBI_SYNC, .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, .matches = cpucap_multi_entry_cap_matches, .match_list = arm64_repeat_tlbi_list, @@ -759,12 +853,19 @@ const struct arm64_cpu_capabilities arm64_errata[] = { 1, 0), }, #endif -#ifdef CONFIG_NVIDIA_CARMEL_CNP_ERRATUM +#ifdef CONFIG_ARM64_WORKAROUND_DISABLE_CNP + { + .desc = "NVIDIA Carmel CNP erratum, or Hisilicon erratum 162100125", + .capability = ARM64_WORKAROUND_DISABLE_CNP, + ERRATA_MIDR_RANGE_LIST(cnp_erratum_cpus), + }, +#endif +#ifdef CONFIG_NVIDIA_OLYMPUS_1027_ERRATUM { - /* NVIDIA Carmel */ - .desc = "NVIDIA Carmel CNP erratum", - .capability = ARM64_WORKAROUND_NVIDIA_CARMEL_CNP, - ERRATA_MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL), + /* NVIDIA Olympus core */ + .desc = "NVIDIA Olympus device store/load ordering erratum", + .capability = ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027, + ERRATA_MIDR_ALL_VERSIONS(MIDR_NVIDIA_OLYMPUS), }, #endif #ifdef CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE @@ -870,6 +971,25 @@ const struct arm64_cpu_capabilities arm64_errata[] = { ERRATA_MIDR_RANGE_LIST(erratum_spec_ssbs_list), }, #endif +#ifdef CONFIG_ARM64_ERRATUM_4311569 + { + .capability = ARM64_WORKAROUND_4311569, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = need_arm_si_l1_workaround_4311569, + }, +#endif +#ifdef CONFIG_ARM64_ERRATUM_4193714 + { + .desc = "C1-Pro SME DVMSync early acknowledgement", + .capability = ARM64_WORKAROUND_4193714, + .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, + .matches = has_sme_dvmsync_erratum, + .cpu_enable = cpu_enable_sme_dvmsync, + /* C1-Pro r0p0 - r1p2 (the latter only when REVIDR_EL1[0]==0) */ + .midr_range = MIDR_RANGE(MIDR_C1_PRO, 0, 0, 1, 2), + MIDR_FIXED(MIDR_CPU_VAR_REV(1, 2), BIT(0)), + }, +#endif #ifdef CONFIG_ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD { .desc = "ARM errata 2966298, 3117295", @@ -908,5 +1028,11 @@ const struct arm64_cpu_capabilities arm64_errata[] = { .cpu_enable = cpu_enable_impdef_pmuv3_traps, }, { + .desc = "Known broken GICv3 SEIS implementation", + .capability = ARM64_WORKAROUND_GICv3_BROKEN_SEIS, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_broken_gic_v3_seis, + }, + { } }; diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index c840a93b9ef9..32102c3912fa 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -77,6 +77,7 @@ #include <linux/percpu.h> #include <linux/sched/isolation.h> +#include <asm/arm_pmuv3.h> #include <asm/cpu.h> #include <asm/cpufeature.h> #include <asm/cpu_ops.h> @@ -86,6 +87,7 @@ #include <asm/kvm_host.h> #include <asm/mmu.h> #include <asm/mmu_context.h> +#include <asm/mpam.h> #include <asm/mte.h> #include <asm/hypervisor.h> #include <asm/processor.h> @@ -240,6 +242,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar0[] = { }; static const struct arm64_ftr_bits ftr_id_aa64isar1[] = { + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_LS64_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_XS_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_I8MM_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_DGH_SHIFT, 4, 0), @@ -263,6 +266,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = { }; static const struct arm64_ftr_bits ftr_id_aa64isar2[] = { + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_ATS1A_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_LUT_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_CSSC_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_RPRFM_SHIFT, 4, 0), @@ -280,6 +284,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar2[] = { static const struct arm64_ftr_bits ftr_id_aa64isar3[] = { ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR3_EL1_FPRCVT_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR3_EL1_LSUI_SHIFT, 4, ID_AA64ISAR3_EL1_LSUI_NI), ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR3_EL1_LSFE_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR3_EL1_FAMINMAX_SHIFT, 4, 0), ARM64_FTR_END, @@ -324,6 +329,7 @@ static const struct arm64_ftr_bits ftr_id_aa64pfr1[] = { static const struct arm64_ftr_bits ftr_id_aa64pfr2[] = { ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR2_EL1_FPMR_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR2_EL1_GCIE_SHIFT, 4, ID_AA64PFR2_EL1_GCIE_NI), ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR2_EL1_MTEFAR_SHIFT, 4, ID_AA64PFR2_EL1_MTEFAR_NI), ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR2_EL1_MTESTOREONLY_SHIFT, 4, ID_AA64PFR2_EL1_MTESTOREONLY_NI), ARM64_FTR_END, @@ -361,6 +367,8 @@ static const struct arm64_ftr_bits ftr_id_aa64smfr0[] = { ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_FA64_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), + FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_LUT6_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_LUTv2_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SMEver_SHIFT, 4, 0), @@ -414,6 +422,7 @@ static const struct arm64_ftr_bits ftr_id_aa64fpfr0[] = { ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8DP2_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8MM8_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8MM4_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F16MM2_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8E4M3_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8E5M2_SHIFT, 1, 0), ARM64_FTR_END, @@ -564,7 +573,7 @@ static const struct arm64_ftr_bits ftr_id_aa64dfr0[] = { * We can instantiate multiple PMU instances with different levels * of support. */ - S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64DFR0_EL1_PMUVer_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64DFR0_EL1_PMUVer_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64DFR0_EL1_DebugVer_SHIFT, 4, 0x6), ARM64_FTR_END, }; @@ -708,7 +717,7 @@ static const struct arm64_ftr_bits ftr_id_pfr2[] = { static const struct arm64_ftr_bits ftr_id_dfr0[] = { /* [31:28] TraceFilt */ - S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_DFR0_EL1_PerfMon_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_DFR0_EL1_PerfMon_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MProfDbg_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MMapTrc_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopTrc_SHIFT, 4, 0), @@ -776,6 +785,7 @@ static const struct arm64_ftr_bits ftr_raz[] = { struct arm64_ftr_override __read_mostly id_aa64mmfr0_override; struct arm64_ftr_override __read_mostly id_aa64mmfr1_override; struct arm64_ftr_override __read_mostly id_aa64mmfr2_override; +struct arm64_ftr_override __read_mostly id_aa64mmfr4_override; struct arm64_ftr_override __read_mostly id_aa64pfr0_override; struct arm64_ftr_override __read_mostly id_aa64pfr1_override; struct arm64_ftr_override __read_mostly id_aa64zfr0_override; @@ -849,7 +859,8 @@ static const struct __ftr_reg_entry { ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2, &id_aa64mmfr2_override), ARM64_FTR_REG(SYS_ID_AA64MMFR3_EL1, ftr_id_aa64mmfr3), - ARM64_FTR_REG(SYS_ID_AA64MMFR4_EL1, ftr_id_aa64mmfr4), + ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR4_EL1, ftr_id_aa64mmfr4, + &id_aa64mmfr4_override), /* Op1 = 0, CRn = 10, CRm = 4 */ ARM64_FTR_REG(SYS_MPAMIDR_EL1, ftr_mpamidr), @@ -1159,6 +1170,41 @@ static __init void detect_system_supports_pseudo_nmi(void) static inline void detect_system_supports_pseudo_nmi(void) { } #endif +static bool detect_ftr_has_mpam(void) +{ + u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); + u64 pfr1 = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1); + + return id_aa64pfr0_mpam(pfr0) || id_aa64pfr1_mpamfrac(pfr1); +} + +bool gmid_el1_accessible(const struct cpuinfo_arm64 *info) +{ + const struct arm64_ftr_bits *ftrp; + s64 mte, ovr; + u64 ftr_mask; + + /* No ID register reflects CONFIG_ARM64_MTE. */ + if (!IS_ENABLED(CONFIG_ARM64_MTE)) + return false; + + for (ftrp = ftr_id_aa64pfr1; ftrp->width; ftrp++) { + if (ftrp->shift == ID_AA64PFR1_EL1_MTE_SHIFT) + break; + } + + ftr_mask = arm64_ftr_mask(ftrp); + mte = arm64_ftr_value(ftrp, info->reg_id_aa64pfr1); + + /* The boot CPU runs before init_cpu_ftr_reg() strips unsafe overrides. */ + if ((id_aa64pfr1_override.mask & ftr_mask) == ftr_mask) { + ovr = arm64_ftr_value(ftrp, id_aa64pfr1_override.val); + mte = arm64_ftr_safe_value(ftrp, ovr, mte); + } + + return mte >= ID_AA64PFR1_EL1_MTE_MTE2; +} + void __init init_cpu_features(struct cpuinfo_arm64 *info) { /* Before we start using the tables, make sure it is sorted */ @@ -1206,12 +1252,12 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) cpacr_restore(cpacr); } - if (id_aa64pfr0_mpam(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) { + if (detect_ftr_has_mpam()) { info->reg_mpamidr = read_cpuid(MPAMIDR_EL1); init_cpu_ftr_reg(SYS_MPAMIDR_EL1, info->reg_mpamidr); } - if (id_aa64pfr1_mte(info->reg_id_aa64pfr1)) + if (gmid_el1_accessible(info)) init_cpu_ftr_reg(SYS_GMID_EL1, info->reg_gmid); } @@ -1462,7 +1508,7 @@ void update_cpu_features(int cpu, cpacr_restore(cpacr); } - if (id_aa64pfr0_mpam(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) { + if (detect_ftr_has_mpam()) { info->reg_mpamidr = read_cpuid(MPAMIDR_EL1); taint |= check_update_ftr_reg(SYS_MPAMIDR_EL1, cpu, info->reg_mpamidr, boot->reg_mpamidr); @@ -1473,11 +1519,9 @@ void update_cpu_features(int cpu, * they read/write depends on the GMID_EL1.BS field. Check that the * value is the same on all CPUs. */ - if (IS_ENABLED(CONFIG_ARM64_MTE) && - id_aa64pfr1_mte(info->reg_id_aa64pfr1)) { + if (gmid_el1_accessible(info)) taint |= check_update_ftr_reg(SYS_GMID_EL1, cpu, info->reg_gmid, boot->reg_gmid); - } /* * If we don't have AArch32 at all then skip the checks entirely @@ -1669,7 +1713,7 @@ const struct cpumask *system_32bit_el0_cpumask(void) const struct cpumask *task_cpu_fallback_mask(struct task_struct *p) { - return __task_cpu_possible_mask(p, housekeeping_cpumask(HK_TYPE_TICK)); + return __task_cpu_possible_mask(p, housekeeping_cpumask(HK_TYPE_DOMAIN)); } static int __init parse_32bit_el0_param(char *str) @@ -1780,7 +1824,7 @@ has_useable_cnp(const struct arm64_cpu_capabilities *entry, int scope) if (is_kdump_kernel()) return false; - if (cpus_have_cap(ARM64_WORKAROUND_NVIDIA_CARMEL_CNP)) + if (cpus_have_cap(ARM64_WORKAROUND_DISABLE_CNP)) return false; return has_cpuid_feature(entry, scope); @@ -1926,19 +1970,10 @@ static bool has_pmuv3(const struct arm64_cpu_capabilities *entry, int scope) u64 dfr0 = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1); unsigned int pmuver; - /* - * PMUVer follows the standard ID scheme for an unsigned field with the - * exception of 0xF (IMP_DEF) which is treated specially and implies - * FEAT_PMUv3 is not implemented. - * - * See DDI0487L.a D24.1.3.2 for more details. - */ pmuver = cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_PMUVer_SHIFT); - if (pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF) - return false; - return pmuver >= ID_AA64DFR0_EL1_PMUVer_IMP; + return pmuv3_implemented(pmuver); } #endif @@ -2123,48 +2158,41 @@ static bool hvhe_possible(const struct arm64_cpu_capabilities *entry, return arm64_test_sw_feature_override(ARM64_SW_FEATURE_OVERRIDE_HVHE); } -bool cpu_supports_bbml2_noabort(void) +bool cpu_supports_bbml3(void) { - /* - * We want to allow usage of BBML2 in as wide a range of kernel contexts - * as possible. This list is therefore an allow-list of known-good - * implementations that both support BBML2 and additionally, fulfill the - * extra constraint of never generating TLB conflict aborts when using - * the relaxed BBML2 semantics (such aborts make use of BBML2 in certain - * kernel contexts difficult to prove safe against recursive aborts). - * - * Note that implementations can only be considered "known-good" if their - * implementors attest to the fact that the implementation never raises - * TLB conflict aborts for BBML2 mapping granularity changes. - */ - static const struct midr_range supports_bbml2_noabort_list[] = { + /* CPUs that support BBML3 but dont advertise through ID_AA64MMFR2_EL1 */ + static const struct midr_range supports_bbml3_list[] = { MIDR_REV_RANGE(MIDR_CORTEX_X4, 0, 3, 0xf), MIDR_REV_RANGE(MIDR_NEOVERSE_V3, 0, 2, 0xf), MIDR_REV_RANGE(MIDR_NEOVERSE_V3AE, 0, 2, 0xf), MIDR_ALL_VERSIONS(MIDR_NVIDIA_OLYMPUS), MIDR_ALL_VERSIONS(MIDR_AMPERE1), MIDR_ALL_VERSIONS(MIDR_AMPERE1A), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A520AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A715), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A720AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A725), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N3), + MIDR_ALL_VERSIONS(MIDR_C1_NANO), + MIDR_ALL_VERSIONS(MIDR_C1_PRO), + /* Erratum 3683289 fixed in r1p1 */ + MIDR_RANGE(MIDR_C1_ULTRA, 1, 1, 0xf, 0xf), + MIDR_RANGE(MIDR_C1_PREMIUM, 1, 1, 0xf, 0xf), {} }; + u64 mmfr2 = __read_sysreg_by_encoding(SYS_ID_AA64MMFR2_EL1); - /* Does our cpu guarantee to never raise TLB conflict aborts? */ - if (!is_midr_in_range_list(supports_bbml2_noabort_list)) - return false; - - /* - * We currently ignore the ID_AA64MMFR2_EL1 register, and only care - * about whether the MIDR check passes. - */ + if (SYS_FIELD_GET(ID_AA64MMFR2_EL1, BBM, mmfr2) >= ID_AA64MMFR2_EL1_BBM_3) + return true; - return true; + return is_midr_in_range_list(supports_bbml3_list); } -static bool has_bbml2_noabort(const struct arm64_cpu_capabilities *caps, int scope) +static bool has_bbml3(const struct arm64_cpu_capabilities *caps, int scope) { - return cpu_supports_bbml2_noabort(); + return cpu_supports_bbml3(); } -#ifdef CONFIG_ARM64_PAN static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused) { /* @@ -2176,7 +2204,6 @@ static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused) sysreg_clear_set(sctlr_el1, SCTLR_EL1_SPAN, 0); set_pstate_pan(1); } -#endif /* CONFIG_ARM64_PAN */ #ifdef CONFIG_ARM64_RAS_EXTN static void cpu_clear_disr(const struct arm64_cpu_capabilities *__unused) @@ -2260,6 +2287,16 @@ static void cpu_enable_e0pd(struct arm64_cpu_capabilities const *cap) } #endif /* CONFIG_ARM64_E0PD */ +static void cpu_enable_ls64(struct arm64_cpu_capabilities const *cap) +{ + sysreg_clear_set(sctlr_el1, SCTLR_EL1_EnALS, SCTLR_EL1_EnALS); +} + +static void cpu_enable_ls64_v(struct arm64_cpu_capabilities const *cap) +{ + sysreg_clear_set(sctlr_el1, SCTLR_EL1_EnASR, 0); +} + #ifdef CONFIG_ARM64_PSEUDO_NMI static bool can_use_gic_priorities(const struct arm64_cpu_capabilities *entry, int scope) @@ -2326,16 +2363,25 @@ static bool can_trap_icv_dir_el1(const struct arm64_cpu_capabilities *entry, BUILD_BUG_ON(ARM64_HAS_ICH_HCR_EL2_TDIR <= ARM64_HAS_GICV3_CPUIF); BUILD_BUG_ON(ARM64_HAS_ICH_HCR_EL2_TDIR <= ARM64_HAS_GICV5_LEGACY); - if (!this_cpu_has_cap(ARM64_HAS_GICV3_CPUIF) && - !is_midr_in_range_list(has_vgic_v3)) - return false; - if (!is_hyp_mode_available()) return false; if (this_cpu_has_cap(ARM64_HAS_GICV5_LEGACY)) return true; + if (!this_cpu_has_cap(ARM64_HAS_GICV3_CPUIF) && + !is_midr_in_range_list(has_vgic_v3)) + return false; + + /* + * pKVM prevents late onlining of CPUs. This means that whatever + * state the capability is in after deprivilege cannot be affected + * by a new CPU booting -- this is garanteed to be a CPU we have + * already seen, and the cap is therefore unchanged. + */ + if (system_capabilities_finalized() && is_protected_kvm_enabled()) + return cpus_have_final_cap(ARM64_HAS_ICH_HCR_EL2_TDIR); + if (is_kernel_in_hyp_mode()) res.a1 = read_sysreg_s(SYS_ICH_VTR_EL2); else @@ -2473,7 +2519,7 @@ cpucap_panic_on_conflict(const struct arm64_cpu_capabilities *cap) static bool test_has_mpam(const struct arm64_cpu_capabilities *entry, int scope) { - if (!has_cpuid_feature(entry, scope)) + if (!detect_ftr_has_mpam()) return false; /* Check firmware actually enabled MPAM on this cpu. */ @@ -2483,13 +2529,19 @@ test_has_mpam(const struct arm64_cpu_capabilities *entry, int scope) static void cpu_enable_mpam(const struct arm64_cpu_capabilities *entry) { - /* - * Access by the kernel (at EL1) should use the reserved PARTID - * which is configured unrestricted. This avoids priority-inversion - * where latency sensitive tasks have to wait for a task that has - * been throttled to release the lock. - */ - write_sysreg_s(0, SYS_MPAM1_EL1); + int cpu = smp_processor_id(); + u64 regval = 0; + + if (IS_ENABLED(CONFIG_ARM64_MPAM) && static_branch_likely(&mpam_enabled)) + regval = READ_ONCE(per_cpu(arm64_mpam_current, cpu)); + + write_sysreg_s(regval | MPAM1_EL1_MPAMEN, SYS_MPAM1_EL1); + if (cpus_have_cap(ARM64_SME)) + write_sysreg_s(regval & (MPAMSM_EL1_PARTID_D | MPAMSM_EL1_PMG_D), SYS_MPAMSM_EL1); + isb(); + + /* Synchronising the EL0 write is left until the ERET to EL0 */ + write_sysreg_s(regval, SYS_MPAM0_EL1); } static bool @@ -2541,7 +2593,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, ECV, CNTPOFF) }, -#ifdef CONFIG_ARM64_PAN { .desc = "Privileged Access Never", .capability = ARM64_HAS_PAN, @@ -2550,7 +2601,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .cpu_enable = cpu_enable_pan, ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, PAN, IMP) }, -#endif /* CONFIG_ARM64_PAN */ #ifdef CONFIG_ARM64_EPAN { .desc = "Enhanced Privileged Access Never", @@ -2560,7 +2610,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = { ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, PAN, PAN3) }, #endif /* CONFIG_ARM64_EPAN */ -#ifdef CONFIG_ARM64_LSE_ATOMICS { .desc = "LSE atomic instructions", .capability = ARM64_HAS_LSE_ATOMICS, @@ -2568,7 +2617,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, ARM64_CPUID_FIELDS(ID_AA64ISAR0_EL1, ATOMIC, IMP) }, -#endif /* CONFIG_ARM64_LSE_ATOMICS */ { .desc = "Virtualization Host Extensions", .capability = ARM64_HAS_VIRT_HOST_EXTN, @@ -2594,6 +2642,20 @@ static const struct arm64_cpu_capabilities arm64_features[] = { }, }, { + .desc = "FEAT_NV2p1", + .capability = ARM64_HAS_NV2P1, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_cpuid_feature, + ARM64_CPUID_FIELDS(ID_AA64MMFR4_EL1, NV_frac, NV2P1) + }, + { + .desc = "FEAT_NV3", + .capability = ARM64_HAS_NV3, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_cpuid_feature, + ARM64_CPUID_FIELDS(ID_AA64MMFR4_EL1, NV_frac, NV3) + }, + { .capability = ARM64_HAS_32BIT_EL0_DO_NOT_USE, .type = ARM64_CPUCAP_SYSTEM_FEATURE, .matches = has_32bit_el0, @@ -3035,10 +3097,10 @@ static const struct arm64_cpu_capabilities arm64_features[] = { ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, EVT, IMP) }, { - .desc = "BBM Level 2 without TLB conflict abort", - .capability = ARM64_HAS_BBML2_NOABORT, + .desc = "BBM Level 3", + .capability = ARM64_HAS_BBML3, .type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE, - .matches = has_bbml2_noabort, + .matches = has_bbml3, }, { .desc = "52-bit Virtual Addressing for KVM (LPA2)", @@ -3078,7 +3140,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .capability = ARM64_MPAM, .matches = test_has_mpam, .cpu_enable = cpu_enable_mpam, - ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, MPAM, 1) }, { .desc = "Memory Partitioning And Monitoring Virtualisation", @@ -3148,6 +3209,31 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, XNX, IMP) }, + { + .desc = "LS64", + .capability = ARM64_HAS_LS64, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_cpuid_feature, + .cpu_enable = cpu_enable_ls64, + ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, LS64, LS64) + }, + { + .desc = "LS64_V", + .capability = ARM64_HAS_LS64_V, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_cpuid_feature, + .cpu_enable = cpu_enable_ls64_v, + ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, LS64, LS64_V) + }, +#ifdef CONFIG_ARM64_LSUI + { + .desc = "Unprivileged Load Store Instructions (LSUI)", + .capability = ARM64_HAS_LSUI, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_cpuid_feature, + ARM64_CPUID_FIELDS(ID_AA64ISAR3_EL1, LSUI, IMP) + }, +#endif {}, }; @@ -3244,6 +3330,8 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(ID_AA64ISAR0_EL1, SM4, IMP, CAP_HWCAP, KERNEL_HWCAP_SM4), HWCAP_CAP(ID_AA64ISAR0_EL1, DP, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMDDP), HWCAP_CAP(ID_AA64ISAR0_EL1, FHM, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMDFHM), + HWCAP_CAP(ID_AA64ISAR0_EL1, FHM, F16F32DOT, CAP_HWCAP, KERNEL_HWCAP_F16F32DOT), + HWCAP_CAP(ID_AA64ISAR0_EL1, FHM, F16F32MM, CAP_HWCAP, KERNEL_HWCAP_F16F32MM), HWCAP_CAP(ID_AA64ISAR0_EL1, TS, FLAGM, CAP_HWCAP, KERNEL_HWCAP_FLAGM), HWCAP_CAP(ID_AA64ISAR0_EL1, TS, FLAGM2, CAP_HWCAP, KERNEL_HWCAP_FLAGM2), HWCAP_CAP(ID_AA64ISAR0_EL1, RNDR, IMP, CAP_HWCAP, KERNEL_HWCAP_RNG), @@ -3267,12 +3355,15 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(ID_AA64ISAR1_EL1, BF16, EBF16, CAP_HWCAP, KERNEL_HWCAP_EBF16), HWCAP_CAP(ID_AA64ISAR1_EL1, DGH, IMP, CAP_HWCAP, KERNEL_HWCAP_DGH), HWCAP_CAP(ID_AA64ISAR1_EL1, I8MM, IMP, CAP_HWCAP, KERNEL_HWCAP_I8MM), + HWCAP_CAP(ID_AA64ISAR1_EL1, LS64, LS64, CAP_HWCAP, KERNEL_HWCAP_LS64), HWCAP_CAP(ID_AA64ISAR2_EL1, LUT, IMP, CAP_HWCAP, KERNEL_HWCAP_LUT), HWCAP_CAP(ID_AA64ISAR3_EL1, FAMINMAX, IMP, CAP_HWCAP, KERNEL_HWCAP_FAMINMAX), HWCAP_CAP(ID_AA64ISAR3_EL1, LSFE, IMP, CAP_HWCAP, KERNEL_HWCAP_LSFE), HWCAP_CAP(ID_AA64MMFR2_EL1, AT, IMP, CAP_HWCAP, KERNEL_HWCAP_USCAT), #ifdef CONFIG_ARM64_SVE + HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ISAR2_EL1, LUT, LUT6, CAP_HWCAP, KERNEL_HWCAP_SVE_LUT6), HWCAP_CAP(ID_AA64PFR0_EL1, SVE, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE), + HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p3, CAP_HWCAP, KERNEL_HWCAP_SVE2P3), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p2, CAP_HWCAP, KERNEL_HWCAP_SVE2P2), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p1, CAP_HWCAP, KERNEL_HWCAP_SVE2P1), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2, CAP_HWCAP, KERNEL_HWCAP_SVE2), @@ -3282,6 +3373,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BitPerm, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBITPERM), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE_B16B16), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, BFSCALE, CAP_HWCAP, KERNEL_HWCAP_SVE_BFSCALE), + HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, B16MM, CAP_HWCAP, KERNEL_HWCAP_SVE_B16MM), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BF16, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBF16), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BF16, EBF16, CAP_HWCAP, KERNEL_HWCAP_SVE_EBF16), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SHA3, IMP, CAP_HWCAP, KERNEL_HWCAP_SVESHA3), @@ -3321,7 +3413,9 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { #ifdef CONFIG_ARM64_SME HWCAP_CAP(ID_AA64PFR1_EL1, SME, IMP, CAP_HWCAP, KERNEL_HWCAP_SME), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, FA64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_FA64), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, LUT6, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_LUT6), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, LUTv2, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_LUTV2), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p3, CAP_HWCAP, KERNEL_HWCAP_SME2P3), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p2, CAP_HWCAP, KERNEL_HWCAP_SME2P2), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p1, CAP_HWCAP, KERNEL_HWCAP_SME2P1), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2, CAP_HWCAP, KERNEL_HWCAP_SME2), @@ -3352,6 +3446,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(ID_AA64FPFR0_EL1, F8DP2, IMP, CAP_HWCAP, KERNEL_HWCAP_F8DP2), HWCAP_CAP(ID_AA64FPFR0_EL1, F8MM8, IMP, CAP_HWCAP, KERNEL_HWCAP_F8MM8), HWCAP_CAP(ID_AA64FPFR0_EL1, F8MM4, IMP, CAP_HWCAP, KERNEL_HWCAP_F8MM4), + HWCAP_CAP(ID_AA64FPFR0_EL1, F16MM2, IMP, CAP_HWCAP, KERNEL_HWCAP_F16MM), HWCAP_CAP(ID_AA64FPFR0_EL1, F8E4M3, IMP, CAP_HWCAP, KERNEL_HWCAP_F8E4M3), HWCAP_CAP(ID_AA64FPFR0_EL1, F8E5M2, IMP, CAP_HWCAP, KERNEL_HWCAP_F8E5M2), #ifdef CONFIG_ARM64_POE @@ -3987,8 +4082,8 @@ static int enable_mismatched_32bit_el0(unsigned int cpu) bool cpu_32bit = false; if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) { - if (!housekeeping_cpu(cpu, HK_TYPE_TICK)) - pr_info("Treating adaptive-ticks CPU %u as 64-bit only\n", cpu); + if (!housekeeping_cpu(cpu, HK_TYPE_DOMAIN)) + pr_info("Treating domain isolated CPU %u as 64-bit only\n", cpu); else cpu_32bit = true; } @@ -4011,7 +4106,7 @@ static int enable_mismatched_32bit_el0(unsigned int cpu) */ lucky_winner = cpu_32bit ? cpu : cpumask_any_and(cpu_32bit_el0_mask, cpu_active_mask); - get_cpu_device(lucky_winner)->offline_disabled = true; + dev_set_offline_disabled(get_cpu_device(lucky_winner)); setup_elf_hwcaps(compat_elf_hwcaps); elf_hwcap_fixup(); pr_info("Asymmetric 32-bit EL0 support detected on CPU %u; CPU hot-unplug disabled on CPU %u\n", diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index c44e6d94f5de..45c63f3d75c5 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -81,6 +81,7 @@ static const char *const hwcap_str[] = { [KERNEL_HWCAP_PACA] = "paca", [KERNEL_HWCAP_PACG] = "pacg", [KERNEL_HWCAP_GCS] = "gcs", + [KERNEL_HWCAP_LS64] = "ls64", [KERNEL_HWCAP_DCPODP] = "dcpodp", [KERNEL_HWCAP_SVE2] = "sve2", [KERNEL_HWCAP_SVEAES] = "sveaes", @@ -163,6 +164,14 @@ static const char *const hwcap_str[] = { [KERNEL_HWCAP_MTE_FAR] = "mtefar", [KERNEL_HWCAP_MTE_STORE_ONLY] = "mtestoreonly", [KERNEL_HWCAP_LSFE] = "lsfe", + [KERNEL_HWCAP_SVE_B16MM] = "sveb16mm", + [KERNEL_HWCAP_SVE2P3] = "sve2p3", + [KERNEL_HWCAP_SME_LUT6] = "smelut6", + [KERNEL_HWCAP_SME2P3] = "sme2p3", + [KERNEL_HWCAP_F16MM] = "f16mm", + [KERNEL_HWCAP_F16F32DOT] = "f16f32dot", + [KERNEL_HWCAP_F16F32MM] = "f16f32mm", + [KERNEL_HWCAP_SVE_LUT6] = "svelut6", }; #ifdef CONFIG_COMPAT @@ -493,7 +502,7 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) info->reg_id_aa64smfr0 = read_cpuid(ID_AA64SMFR0_EL1); info->reg_id_aa64fpfr0 = read_cpuid(ID_AA64FPFR0_EL1); - if (id_aa64pfr1_mte(info->reg_id_aa64pfr1)) + if (gmid_el1_accessible(info)) info->reg_gmid = read_cpuid(GMID_EL1); if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c index 29307642f4c9..e271fbac5f82 100644 --- a/arch/arm64/kernel/debug-monitors.c +++ b/arch/arm64/kernel/debug-monitors.c @@ -40,10 +40,7 @@ u8 debug_monitors_arch(void) */ static void mdscr_write(u64 mdscr) { - unsigned long flags; - flags = local_daif_save(); write_sysreg(mdscr, mdscr_el1); - local_daif_restore(flags); } NOKPROBE_SYMBOL(mdscr_write); diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index a81cb4aa4738..0ec90fd1754e 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -31,7 +31,7 @@ static bool region_is_misaligned(const efi_memory_desc_t *md) * executable, everything else can be mapped with the XN bits * set. Also take the new (optional) RO/XP bits into account. */ -static __init ptdesc_t create_mapping_protection(efi_memory_desc_t *md) +static __init ptval_t create_mapping_protection(efi_memory_desc_t *md) { u64 attr = md->attribute; u32 type = md->type; @@ -85,7 +85,7 @@ static __init ptdesc_t create_mapping_protection(efi_memory_desc_t *md) int __init efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md) { - ptdesc_t prot_val = create_mapping_protection(md); + ptval_t prot_val = create_mapping_protection(md); bool page_mappings_only = (md->type == EFI_RUNTIME_SERVICES_CODE || md->type == EFI_RUNTIME_SERVICES_DATA); @@ -184,6 +184,8 @@ void arch_efi_call_virt_setup(void) efi_virtmap_load(); } + __efi_fpsimd_begin(); + /* * Enable access to the valid TTBR0_EL1 and invoke the errata * workaround directly since there is no return from exception when @@ -191,8 +193,6 @@ void arch_efi_call_virt_setup(void) */ uaccess_ttbr0_enable(); post_ttbr_update_workaround(); - - __efi_fpsimd_begin(); } void arch_efi_call_virt_teardown(void) diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 3625797e9ee8..72c03ccea59f 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -21,6 +21,7 @@ #include <asm/daifflags.h> #include <asm/esr.h> #include <asm/exception.h> +#include <asm/fpsimd.h> #include <asm/irq_regs.h> #include <asm/kprobes.h> #include <asm/mmu.h> @@ -35,11 +36,11 @@ * Before this function is called it is not safe to call regular kernel code, * instrumentable code, or any code which may trigger an exception. */ -static noinstr irqentry_state_t enter_from_kernel_mode(struct pt_regs *regs) +static noinstr irqentry_state_t arm64_enter_from_kernel_mode(struct pt_regs *regs) { irqentry_state_t state; - state = irqentry_enter(regs); + state = irqentry_enter_from_kernel_mode(regs); mte_check_tfsr_entry(); mte_disable_tco_entry(current); @@ -51,11 +52,41 @@ static noinstr irqentry_state_t enter_from_kernel_mode(struct pt_regs *regs) * After this function returns it is not safe to call regular kernel code, * instrumentable code, or any code which may trigger an exception. */ -static void noinstr exit_to_kernel_mode(struct pt_regs *regs, - irqentry_state_t state) +static void noinstr __arm64_exit_to_kernel_mode(struct pt_regs *regs, + irqentry_state_t state) { + local_daif_mask(); mte_check_tfsr_exit(); - irqentry_exit(regs, state); + irqentry_exit_to_kernel_mode_after_preempt(regs, state); +} + +/* + * We are returning from the context which allows involuntary kernel preemption + */ +static void noinstr arm64_exit_to_kernel_mode_preempt(struct pt_regs *regs, + irqentry_state_t state) +{ + irqentry_exit_to_kernel_mode_preempt(regs, state); + __arm64_exit_to_kernel_mode(regs, state); +} + +static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs, + irqentry_state_t state) +{ + if (!regs_irqs_disabled(regs)) { + local_irq_disable(); + arm64_exit_to_kernel_mode_preempt(regs, state); + return; + } + + __arm64_exit_to_kernel_mode(regs, state); +} + +static __always_inline void arm64_syscall_enter_from_user_mode(struct pt_regs *regs) +{ + enter_from_user_mode(regs); + mte_disable_tco_entry(current); + sme_enter_from_user_mode(); } /* @@ -66,7 +97,19 @@ static void noinstr exit_to_kernel_mode(struct pt_regs *regs, static __always_inline void arm64_enter_from_user_mode(struct pt_regs *regs) { enter_from_user_mode(regs); + rseq_note_user_irq_entry(); mte_disable_tco_entry(current); + sme_enter_from_user_mode(); +} + +static __always_inline void arm64_syscall_exit_to_user_mode(struct pt_regs *regs) +{ + local_irq_disable(); + syscall_exit_to_user_mode_prepare(regs); + local_daif_mask(); + sme_exit_to_user_mode(); + mte_check_tfsr_exit(); + exit_to_user_mode(); } /* @@ -74,19 +117,19 @@ static __always_inline void arm64_enter_from_user_mode(struct pt_regs *regs) * After this function returns it is not safe to call regular kernel code, * instrumentable code, or any code which may trigger an exception. */ - static __always_inline void arm64_exit_to_user_mode(struct pt_regs *regs) { local_irq_disable(); - exit_to_user_mode_prepare_legacy(regs); + irqentry_exit_to_user_mode_prepare(regs); local_daif_mask(); + sme_exit_to_user_mode(); mte_check_tfsr_exit(); exit_to_user_mode(); } asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs) { - arm64_exit_to_user_mode(regs); + arm64_syscall_exit_to_user_mode(regs); } /* @@ -231,12 +274,8 @@ static inline void fpsimd_syscall_enter(void) if (!system_supports_sve()) return; - if (test_thread_flag(TIF_SVE)) { - unsigned int sve_vq_minus_one; - - sve_vq_minus_one = sve_vq_from_vl(task_get_sve_vl(current)) - 1; - sve_flush_live(true, sve_vq_minus_one); - } + if (test_thread_flag(TIF_SVE)) + sve_flush_live(); /* * Any live non-FPSIMD SVE state has been zeroed. Allow @@ -298,11 +337,10 @@ static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr) unsigned long far = read_sysreg(far_el1); irqentry_state_t state; - state = enter_from_kernel_mode(regs); + state = arm64_enter_from_kernel_mode(regs); local_daif_inherit(regs); do_mem_abort(far, esr, regs); - local_daif_mask(); - exit_to_kernel_mode(regs, state); + arm64_exit_to_kernel_mode(regs, state); } static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr) @@ -310,55 +348,50 @@ static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr) unsigned long far = read_sysreg(far_el1); irqentry_state_t state; - state = enter_from_kernel_mode(regs); + state = arm64_enter_from_kernel_mode(regs); local_daif_inherit(regs); do_sp_pc_abort(far, esr, regs); - local_daif_mask(); - exit_to_kernel_mode(regs, state); + arm64_exit_to_kernel_mode(regs, state); } static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr) { irqentry_state_t state; - state = enter_from_kernel_mode(regs); + state = arm64_enter_from_kernel_mode(regs); local_daif_inherit(regs); do_el1_undef(regs, esr); - local_daif_mask(); - exit_to_kernel_mode(regs, state); + arm64_exit_to_kernel_mode(regs, state); } static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr) { irqentry_state_t state; - state = enter_from_kernel_mode(regs); + state = arm64_enter_from_kernel_mode(regs); local_daif_inherit(regs); do_el1_bti(regs, esr); - local_daif_mask(); - exit_to_kernel_mode(regs, state); + arm64_exit_to_kernel_mode(regs, state); } static void noinstr el1_gcs(struct pt_regs *regs, unsigned long esr) { irqentry_state_t state; - state = enter_from_kernel_mode(regs); + state = arm64_enter_from_kernel_mode(regs); local_daif_inherit(regs); do_el1_gcs(regs, esr); - local_daif_mask(); - exit_to_kernel_mode(regs, state); + arm64_exit_to_kernel_mode(regs, state); } static void noinstr el1_mops(struct pt_regs *regs, unsigned long esr) { irqentry_state_t state; - state = enter_from_kernel_mode(regs); + state = arm64_enter_from_kernel_mode(regs); local_daif_inherit(regs); do_el1_mops(regs, esr); - local_daif_mask(); - exit_to_kernel_mode(regs, state); + arm64_exit_to_kernel_mode(regs, state); } static void noinstr el1_breakpt(struct pt_regs *regs, unsigned long esr) @@ -420,11 +453,10 @@ static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr) { irqentry_state_t state; - state = enter_from_kernel_mode(regs); + state = arm64_enter_from_kernel_mode(regs); local_daif_inherit(regs); do_el1_fpac(regs, esr); - local_daif_mask(); - exit_to_kernel_mode(regs, state); + arm64_exit_to_kernel_mode(regs, state); } asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs) @@ -483,6 +515,7 @@ static __always_inline void __el1_pnmi(struct pt_regs *regs, state = irqentry_nmi_enter(regs); do_interrupt_handler(regs, handler); + local_daif_mask(); irqentry_nmi_exit(regs, state); } @@ -491,13 +524,13 @@ static __always_inline void __el1_irq(struct pt_regs *regs, { irqentry_state_t state; - state = enter_from_kernel_mode(regs); + state = arm64_enter_from_kernel_mode(regs); irq_enter_rcu(); do_interrupt_handler(regs, handler); irq_exit_rcu(); - exit_to_kernel_mode(regs, state); + arm64_exit_to_kernel_mode_preempt(regs, state); } static void noinstr el1_interrupt(struct pt_regs *regs, void (*handler)(struct pt_regs *)) @@ -528,6 +561,7 @@ asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs) local_daif_restore(DAIF_ERRCTX); state = irqentry_nmi_enter(regs); do_serror(regs, esr); + local_daif_mask(); irqentry_nmi_exit(regs, state); } @@ -717,12 +751,12 @@ static void noinstr el0_brk64(struct pt_regs *regs, unsigned long esr) static void noinstr el0_svc(struct pt_regs *regs) { - arm64_enter_from_user_mode(regs); + arm64_syscall_enter_from_user_mode(regs); cortex_a76_erratum_1463225_svc_handler(); fpsimd_syscall_enter(); local_daif_restore(DAIF_PROCCTX); do_el0_svc(regs); - arm64_exit_to_user_mode(regs); + arm64_syscall_exit_to_user_mode(regs); fpsimd_syscall_exit(); } @@ -869,11 +903,11 @@ static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr) static void noinstr el0_svc_compat(struct pt_regs *regs) { - arm64_enter_from_user_mode(regs); + arm64_syscall_enter_from_user_mode(regs); cortex_a76_erratum_1463225_svc_handler(); local_daif_restore(DAIF_PROCCTX); do_el0_svc_compat(regs); - arm64_exit_to_user_mode(regs); + arm64_syscall_exit_to_user_mode(regs); } static void noinstr el0_bkpt32(struct pt_regs *regs, unsigned long esr) diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S deleted file mode 100644 index 6325db1a2179..000000000000 --- a/arch/arm64/kernel/entry-fpsimd.S +++ /dev/null @@ -1,134 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * FP/SIMD state saving and restoring - * - * Copyright (C) 2012 ARM Ltd. - * Author: Catalin Marinas <catalin.marinas@arm.com> - */ - -#include <linux/linkage.h> - -#include <asm/assembler.h> -#include <asm/fpsimdmacros.h> - -/* - * Save the FP registers. - * - * x0 - pointer to struct fpsimd_state - */ -SYM_FUNC_START(fpsimd_save_state) - fpsimd_save x0, 8 - ret -SYM_FUNC_END(fpsimd_save_state) - -/* - * Load the FP registers. - * - * x0 - pointer to struct fpsimd_state - */ -SYM_FUNC_START(fpsimd_load_state) - fpsimd_restore x0, 8 - ret -SYM_FUNC_END(fpsimd_load_state) - -#ifdef CONFIG_ARM64_SVE - -/* - * Save the SVE state - * - * x0 - pointer to buffer for state - * x1 - pointer to storage for FPSR - * x2 - Save FFR if non-zero - */ -SYM_FUNC_START(sve_save_state) - sve_save 0, x1, x2, 3 - ret -SYM_FUNC_END(sve_save_state) - -/* - * Load the SVE state - * - * x0 - pointer to buffer for state - * x1 - pointer to storage for FPSR - * x2 - Restore FFR if non-zero - */ -SYM_FUNC_START(sve_load_state) - sve_load 0, x1, x2, 4 - ret -SYM_FUNC_END(sve_load_state) - -SYM_FUNC_START(sve_get_vl) - _sve_rdvl 0, 1 - ret -SYM_FUNC_END(sve_get_vl) - -SYM_FUNC_START(sve_set_vq) - sve_load_vq x0, x1, x2 - ret -SYM_FUNC_END(sve_set_vq) - -/* - * Zero all SVE registers but the first 128-bits of each vector - * - * VQ must already be configured by caller, any further updates of VQ - * will need to ensure that the register state remains valid. - * - * x0 = include FFR? - * x1 = VQ - 1 - */ -SYM_FUNC_START(sve_flush_live) - cbz x1, 1f // A VQ-1 of 0 is 128 bits so no extra Z state - sve_flush_z -1: sve_flush_p - tbz x0, #0, 2f - sve_flush_ffr -2: ret -SYM_FUNC_END(sve_flush_live) - -#endif /* CONFIG_ARM64_SVE */ - -#ifdef CONFIG_ARM64_SME - -SYM_FUNC_START(sme_get_vl) - _sme_rdsvl 0, 1 - ret -SYM_FUNC_END(sme_get_vl) - -SYM_FUNC_START(sme_set_vq) - sme_load_vq x0, x1, x2 - ret -SYM_FUNC_END(sme_set_vq) - -/* - * Save the ZA and ZT state - * - * x0 - pointer to buffer for state - * x1 - number of ZT registers to save - */ -SYM_FUNC_START(sme_save_state) - _sme_rdsvl 2, 1 // x2 = VL/8 - sme_save_za 0, x2, 12 // Leaves x0 pointing to the end of ZA - - cbz x1, 1f - _str_zt 0 -1: - ret -SYM_FUNC_END(sme_save_state) - -/* - * Load the ZA and ZT state - * - * x0 - pointer to buffer for state - * x1 - number of ZT registers to save - */ -SYM_FUNC_START(sme_load_state) - _sme_rdsvl 2, 1 // x2 = VL/8 - sme_load_za 0, x2, 12 // Leaves x0 pointing to the end of ZA - - cbz x1, 1f - _ldr_zt 0 -1: - ret -SYM_FUNC_END(sme_load_state) - -#endif /* CONFIG_ARM64_SME */ diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index f8018b5c1f9a..f63049ac32dc 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -273,7 +273,7 @@ alternative_if ARM64_HAS_ADDRESS_AUTH alternative_else_nop_endif 1: - scs_load_current + scs_load_current_base .else add x21, sp, #PT_REGS_SIZE get_current_task tsk @@ -333,10 +333,6 @@ alternative_else_nop_endif .endm .macro kernel_exit, el - .if \el != 0 - disable_daif - .endif - #ifdef CONFIG_ARM64_PSEUDO_NMI alternative_if_not ARM64_HAS_GIC_PRIO_MASKING b .Lskip_pmr_restore\@ @@ -378,8 +374,6 @@ alternative_if ARM64_WORKAROUND_845719 alternative_else_nop_endif #endif 3: - scs_save tsk - /* Ignore asynchronous tag check faults in the uaccess routines */ ldr x0, [tsk, THREAD_SCTLR_USER] clear_mte_async_tcf x0 @@ -473,7 +467,7 @@ alternative_else_nop_endif */ SYM_CODE_START_LOCAL(__swpan_entry_el1) mrs x21, ttbr0_el1 - tst x21, #TTBR_ASID_MASK // Check for the reserved ASID + tst x21, #TTBRx_EL1_ASID_MASK // Check for the reserved ASID orr x23, x23, #PSR_PAN_BIT // Set the emulated PAN in the saved SPSR b.eq 1f // TTBR0 access already disabled and x23, x23, #~PSR_PAN_BIT // Clear the emulated PAN in the saved SPSR diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 9de1d8a604cb..e7f1682a3059 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -15,6 +15,7 @@ #include <linux/compiler.h> #include <linux/cpu.h> #include <linux/cpu_pm.h> +#include <linux/cpumask.h> #include <linux/ctype.h> #include <linux/kernel.h> #include <linux/linkage.h> @@ -28,6 +29,7 @@ #include <linux/sched/task_stack.h> #include <linux/signal.h> #include <linux/slab.h> +#include <linux/smp.h> #include <linux/stddef.h> #include <linux/sysctl.h> #include <linux/swab.h> @@ -375,8 +377,10 @@ static void task_fpsimd_load(void) if (!thread_sm_enabled(¤t->thread)) WARN_ON_ONCE(!test_and_set_thread_flag(TIF_SVE)); - if (test_thread_flag(TIF_SVE)) - sve_set_vq(sve_vq_from_vl(task_get_sve_vl(current)) - 1); + if (test_thread_flag(TIF_SVE)) { + unsigned long vq = sve_vq_from_vl(task_get_sve_vl(current)); + sysreg_clear_set_s(SYS_ZCR_EL1, ZCR_ELx_LEN, vq - 1); + } restore_sve_regs = true; restore_ffr = true; @@ -401,8 +405,10 @@ static void task_fpsimd_load(void) unsigned long sme_vl = task_get_sme_vl(current); /* Ensure VL is set up for restoring data */ - if (test_thread_flag(TIF_SME)) - sme_set_vq(sve_vq_from_vl(sme_vl) - 1); + if (test_thread_flag(TIF_SME)) { + unsigned long vq = sve_vq_from_vl(sme_vl); + sysreg_clear_set_s(SYS_SMCR_EL1, SMCR_ELx_LEN, vq - 1); + } write_sysreg_s(current->thread.svcr, SYS_SVCR); @@ -419,9 +425,8 @@ static void task_fpsimd_load(void) if (restore_sve_regs) { WARN_ON_ONCE(current->thread.fp_type != FP_STATE_SVE); - sve_load_state(sve_pffr(¤t->thread), - ¤t->thread.uw.fpsimd_state.fpsr, - restore_ffr); + sve_load_state(current->thread.sve_state, restore_ffr); + fpsimd_load_common(¤t->thread.uw.fpsimd_state); } else { WARN_ON_ONCE(current->thread.fp_type != FP_STATE_FPSIMD); fpsimd_load_state(¤t->thread.uw.fpsimd_state); @@ -501,9 +506,8 @@ static void fpsimd_save_user_state(void) return; } - sve_save_state((char *)last->sve_state + - sve_ffr_offset(vl), - &last->st->fpsr, save_ffr); + sve_save_state(last->sve_state, save_ffr); + fpsimd_save_common(last->st); *last->fp_type = FP_STATE_SVE; } else { fpsimd_save_state(last->st); @@ -634,7 +638,8 @@ static __uint128_t arm64_cpu_to_le128(__uint128_t x) #define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x) -static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst, +static void __fpsimd_to_sve(struct arm64_sve_state *sst, + struct user_fpsimd_state const *fst, unsigned int vq) { unsigned int i; @@ -661,7 +666,7 @@ static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst, static inline void fpsimd_to_sve(struct task_struct *task) { unsigned int vq; - void *sst = task->thread.sve_state; + struct arm64_sve_state *sst = task->thread.sve_state; struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state; if (!system_supports_sve() && !system_supports_sme()) @@ -685,7 +690,7 @@ static inline void fpsimd_to_sve(struct task_struct *task) static inline void sve_to_fpsimd(struct task_struct *task) { unsigned int vq, vl; - void const *sst = task->thread.sve_state; + const struct arm64_sve_state *sst = task->thread.sve_state; struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state; unsigned int i; __uint128_t const *p; @@ -784,7 +789,7 @@ void fpsimd_sync_from_effective_state(struct task_struct *task) void fpsimd_sync_to_effective_state_zeropad(struct task_struct *task) { unsigned int vq; - void *sst = task->thread.sve_state; + struct arm64_sve_state *sst = task->thread.sve_state; struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state; if (task->thread.fp_type != FP_STATE_SVE) @@ -802,7 +807,8 @@ static int change_live_vector_length(struct task_struct *task, { unsigned int sve_vl = task_get_sve_vl(task); unsigned int sme_vl = task_get_sme_vl(task); - void *sve_state = NULL, *sme_state = NULL; + struct arm64_sve_state *sve_state = NULL; + struct arm64_sme_state *sme_state = NULL; if (type == ARM64_VEC_SME) sme_vl = vl; @@ -1291,31 +1297,6 @@ void sme_suspend_exit(void) #endif /* CONFIG_ARM64_SME */ -static void sve_init_regs(void) -{ - /* - * Convert the FPSIMD state to SVE, zeroing all the state that - * is not shared with FPSIMD. If (as is likely) the current - * state is live in the registers then do this there and - * update our metadata for the current task including - * disabling the trap, otherwise update our in-memory copy. - * We are guaranteed to not be in streaming mode, we can only - * take a SVE trap when not in streaming mode and we can't be - * in streaming mode when taking a SME trap. - */ - if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { - unsigned long vq_minus_one = - sve_vq_from_vl(task_get_sve_vl(current)) - 1; - sve_set_vq(vq_minus_one); - sve_flush_live(true, vq_minus_one); - fpsimd_bind_task_to_cpu(); - } else { - fpsimd_to_sve(current); - current->thread.fp_type = FP_STATE_SVE; - fpsimd_flush_task_state(current); - } -} - /* * Trapped SVE access * @@ -1347,17 +1328,108 @@ void do_sve_acc(unsigned long esr, struct pt_regs *regs) WARN_ON(1); /* SVE access shouldn't have trapped */ /* - * Even if the task can have used streaming mode we can only - * generate SVE access traps in normal SVE mode and - * transitioning out of streaming mode may discard any - * streaming mode state. Always clear the high bits to avoid - * any potential errors tracking what is properly initialised. + * Convert the FPSIMD state to SVE. Stale SVE state can be present in + * registers or memory, so we must zero all state that is not shared + * with FPSIMD. + * + * SVE traps cannot be taken from streaming mode, so there cannot be + * any effective streaming mode SVE state. */ - sve_init_regs(); + if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { + unsigned long vq = sve_vq_from_vl(task_get_sve_vl(current)); + sysreg_clear_set_s(SYS_ZCR_EL1, ZCR_ELx_LEN, vq - 1); + sve_flush_live(); + fpsimd_bind_task_to_cpu(); + } else { + fpsimd_to_sve(current); + current->thread.fp_type = FP_STATE_SVE; + fpsimd_flush_task_state(current); + } put_cpu_fpsimd_context(); } +#ifdef CONFIG_ARM64_ERRATUM_4193714 + +/* + * SME/CME erratum handling. + */ +static cpumask_t sme_dvmsync_cpus; +cpumask_t sme_active_cpus; + +/* + * These helpers are only called from non-preemptible contexts, so + * smp_processor_id() is safe here. + */ +void sme_set_active(void) +{ + unsigned int cpu = smp_processor_id(); + + if (!cpumask_test_cpu(cpu, &sme_dvmsync_cpus)) + return; + + cpumask_set_cpu(cpu, mm_cpumask(current->mm)); + cpumask_set_cpu(cpu, &sme_active_cpus); + + /* + * A subsequent (post ERET) SME access may use a stale address + * translation. On C1-Pro, a TLBI+DSB on a different CPU will wait for + * the completion of the cpumask_set_cpu() operations above as they + * appear in program order before the SME access. The post-TLBI+DSB + * read of mm_cpumask() or sme_active_cpus will lead to the IPI being + * issued. + * + * https://lore.kernel.org/r/ablEXwhfKyJW1i7l@J2N7QTR9R3 + */ +} + +void sme_clear_active(void) +{ + unsigned int cpu = smp_processor_id(); + + if (!cpumask_test_cpu(cpu, &sme_dvmsync_cpus)) + return; + + /* + * With SCTLR_EL1.IESB enabled, the SME memory transactions are + * completed on entering EL1. + */ + cpumask_clear_cpu(cpu, mm_cpumask(current->mm)); + cpumask_clear_cpu(cpu, &sme_active_cpus); +} + +static void sme_dvmsync_ipi(void *unused) +{ + /* + * With SCTLR_EL1.IESB on, taking an exception is sufficient to ensure + * the completion of the SME memory accesses, so no need for an + * explicit DSB. + */ +} + +void sme_do_dvmsync(const struct cpumask *mask) +{ + /* + * This is called from the TLB maintenance functions after the DSB ISH + * to send the hardware DVMSync message. If this CPU sees the mask as + * empty, the remote CPU executing sme_set_active() would have seen + * the DVMSync and no IPI required. + */ + if (cpumask_empty(mask)) + return; + + preempt_disable(); + smp_call_function_many(mask, sme_dvmsync_ipi, NULL, true); + preempt_enable(); +} + +void sme_enable_dvmsync(void) +{ + cpumask_set_cpu(smp_processor_id(), &sme_dvmsync_cpus); +} + +#endif /* CONFIG_ARM64_ERRATUM_4193714 */ + /* * Trapped SME access * @@ -1400,9 +1472,8 @@ void do_sme_acc(unsigned long esr, struct pt_regs *regs) WARN_ON(1); if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { - unsigned long vq_minus_one = - sve_vq_from_vl(task_get_sme_vl(current)) - 1; - sme_set_vq(vq_minus_one); + unsigned long vq = sve_vq_from_vl(task_get_sme_vl(current)); + sysreg_clear_set_s(SYS_SMCR_EL1, SMCR_ELx_LEN, vq - 1); fpsimd_bind_task_to_cpu(); } else { @@ -1577,8 +1648,8 @@ static void fpsimd_flush_thread_vl(enum vec_type type) void fpsimd_flush_thread(void) { - void *sve_state = NULL; - void *sme_state = NULL; + struct arm64_sve_state *sve_state = NULL; + struct arm64_sme_state *sme_state = NULL; if (!system_supports_fpsimd()) return; diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c index 5a1554a44162..e1a3c0b3a051 100644 --- a/arch/arm64/kernel/ftrace.c +++ b/arch/arm64/kernel/ftrace.c @@ -409,7 +409,8 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) return ftrace_modify_code(pc, old, new, true); } -#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS +#if defined(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS) || \ + defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS) int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) { @@ -417,7 +418,7 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, u32 old, new; int ret; - ret = ftrace_rec_set_ops(rec, arm64_rec_get_ops(rec)); + ret = ftrace_rec_update_ops(rec); if (ret) return ret; diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index ca04b338cb0d..87a822e5c4ca 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -299,7 +299,7 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL) isb 0: - init_el2_hcr HCR_HOST_NVHE_FLAGS + init_el2_hcr HCR_HOST_NVHE_FLAGS | HCR_ATA init_el2_state /* Hypervisor stub */ diff --git a/arch/arm64/kernel/hibernate-asm.S b/arch/arm64/kernel/hibernate-asm.S index 0e1d9c3c6a93..2baefe7a82d3 100644 --- a/arch/arm64/kernel/hibernate-asm.S +++ b/arch/arm64/kernel/hibernate-asm.S @@ -89,6 +89,8 @@ alternative_insn "dc cvau, x4", "dc civac, x4", ARM64_WORKAROUND_CLEAN_CACHE isb cbz x24, 3f /* Do we need to re-initialise EL2? */ + mov x1, x24 + mov x0, #HVC_SET_VECTORS hvc #0 3: ret SYM_CODE_END(swsusp_arch_suspend_exit) diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index 9717568518ba..424291c547f0 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -348,8 +348,10 @@ int swsusp_arch_suspend(void) crash_prepare_suspend(); ret = swsusp_mte_save_tags(); - if (ret) + if (ret) { + local_daif_restore(flags); return ret; + } sleep_cpu = smp_processor_id(); ret = swsusp_save(); @@ -421,8 +423,8 @@ int __nocfi swsusp_arch_resume(void) * Create a second copy of just the linear map, and use this when * restoring. */ - rc = trans_pgd_create_copy(&trans_info, &tmp_pg_dir, PAGE_OFFSET, - PAGE_END); + rc = trans_pgd_create_copy(&trans_info, &tmp_pg_dir, + _PAGE_OFFSET(vabits_actual), PAGE_END); if (rc) return rc; @@ -465,9 +467,21 @@ int __nocfi swsusp_arch_resume(void) if (el2_reset_needed()) __hyp_set_vectors(el2_vectors); + /* + * It is necessary to mask all DAIF exceptions here as: + * + * - The copy of swsusp_arch_suspend_exit() in the hibernation + * text cannot handle taking any exceptions. + * + * - The suspended kernel masked all DAIF exceptions in + * swsusp_arch_resume(), and expects to be re-entered in the + * same state : with all DAIF exceptions masked. + */ + local_daif_save(); hibernate_exit(virt_to_phys(tmp_pg_dir), resume_hdr.ttbr1_el1, resume_hdr.reenter_kernel, restore_pblist, resume_hdr.__hyp_stub_vectors, virt_to_phys(zero_page)); + unreachable(); return 0; } diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c index ab76b36dce82..73cce8ac8368 100644 --- a/arch/arm64/kernel/hw_breakpoint.c +++ b/arch/arm64/kernel/hw_breakpoint.c @@ -559,6 +559,15 @@ int hw_breakpoint_arch_parse(struct perf_event *bp, else alignment_mask = 0x7; offset = hw->address & alignment_mask; + + /* + * BAS is an 8-bit field in WCR/BCR; the shift below would + * silently drop the high bits of ctrl.len when offset + len + * exceeds 8, programming hardware to watch fewer bytes than + * the user requested. + */ + if (((u32)hw->ctrl.len << offset) > ARM_BREAKPOINT_LEN_8) + return -EINVAL; } hw->address &= ~alignment_mask; diff --git a/arch/arm64/kernel/hyp-stub.S b/arch/arm64/kernel/hyp-stub.S index 085bc9972f6b..37c6976e44a4 100644 --- a/arch/arm64/kernel/hyp-stub.S +++ b/arch/arm64/kernel/hyp-stub.S @@ -103,13 +103,10 @@ SYM_CODE_START_LOCAL(__finalise_el2) // Engage the VHE magic! mov_q x0, HCR_HOST_VHE_FLAGS msr_hcr_el2 x0 - isb - // Use the EL1 allocated stack, per-cpu offset + // Use the EL1 allocated stack mrs x0, sp_el1 mov sp, x0 - mrs x0, tpidr_el1 - msr tpidr_el2, x0 // FP configuration, vectors mrs_s x0, SYS_CPACR_EL12 diff --git a/arch/arm64/kernel/idle.c b/arch/arm64/kernel/idle.c index 05cfb347ec26..42c5543f9589 100644 --- a/arch/arm64/kernel/idle.c +++ b/arch/arm64/kernel/idle.c @@ -11,6 +11,29 @@ #include <asm/cpufeature.h> #include <asm/sysreg.h> +enum { + ARM64_IDLE_WFI, + ARM64_IDLE_YIELD, + ARM64_IDLE_NOP, +} idle = ARM64_IDLE_WFI; + +static int __init setup_idle(char *arg) +{ + if (!arg) + return -1; + else if (!strcmp(arg, "wfi")) + idle = ARM64_IDLE_WFI; + else if (!strcmp(arg, "yield")) + idle = ARM64_IDLE_YIELD; + else if (!strcmp(arg, "nop")) + idle = ARM64_IDLE_NOP; + else + return -1; + + return 0; +} +early_param("idle", setup_idle); + /* * cpu_do_idle() * @@ -26,8 +49,13 @@ void __cpuidle cpu_do_idle(void) arm_cpuidle_save_irq_context(&context); - dsb(sy); - wfi(); + if (likely(idle == ARM64_IDLE_WFI)) { + dsb(sy); + wfi(); + } else if (idle == ARM64_IDLE_YIELD) { + dsb(sy); + asm volatile("yield" ::: "memory"); + } arm_cpuidle_restore_irq_context(&context); } diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index 211f0e2e55e2..14beb7b9d304 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h @@ -38,7 +38,7 @@ PROVIDE(__efistub__end = _end); PROVIDE(__efistub___inittext_end = __inittext_end); PROVIDE(__efistub__edata = _edata); #if defined(CONFIG_EFI_EARLYCON) || defined(CONFIG_SYSFB) -PROVIDE(__efistub_screen_info = screen_info); +PROVIDE(__efistub_sysfb_primary_display = sysfb_primary_display); #endif PROVIDE(__efistub__ctype = _ctype); @@ -51,6 +51,7 @@ PI_EXPORT_SYM(id_aa64isar2_override); PI_EXPORT_SYM(id_aa64mmfr0_override); PI_EXPORT_SYM(id_aa64mmfr1_override); PI_EXPORT_SYM(id_aa64mmfr2_override); +PI_EXPORT_SYM(id_aa64mmfr4_override); PI_EXPORT_SYM(id_aa64pfr0_override); PI_EXPORT_SYM(id_aa64pfr1_override); PI_EXPORT_SYM(id_aa64smfr0_override); @@ -86,13 +87,13 @@ KVM_NVHE_ALIAS(kvm_patch_vector_branch); KVM_NVHE_ALIAS(kvm_update_va_mask); KVM_NVHE_ALIAS(kvm_get_kimage_voffset); KVM_NVHE_ALIAS(kvm_compute_final_ctr_el0); -KVM_NVHE_ALIAS(kvm_pan_patch_el2_entry); KVM_NVHE_ALIAS(spectre_bhb_patch_loop_iter); KVM_NVHE_ALIAS(spectre_bhb_patch_loop_mitigation_enable); KVM_NVHE_ALIAS(spectre_bhb_patch_wa3); KVM_NVHE_ALIAS(spectre_bhb_patch_clearbhb); KVM_NVHE_ALIAS(alt_cb_patch_nops); KVM_NVHE_ALIAS(kvm_compute_ich_hcr_trap_bits); +KVM_NVHE_ALIAS(kvm_patch_ich_vtr_el2); /* Global kernel state accessed by nVHE hyp code. */ KVM_NVHE_ALIAS(kvm_vgic_global_state); @@ -139,6 +140,10 @@ KVM_NVHE_ALIAS(__hyp_data_start); KVM_NVHE_ALIAS(__hyp_data_end); KVM_NVHE_ALIAS(__hyp_rodata_start); KVM_NVHE_ALIAS(__hyp_rodata_end); +#ifdef CONFIG_NVHE_EL2_TRACING +KVM_NVHE_ALIAS(__hyp_event_ids_start); +KVM_NVHE_ALIAS(__hyp_event_ids_end); +#endif /* pKVM static key */ KVM_NVHE_ALIAS(kvm_protected_mode_initialized); diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c index 15dedb385b9e..9fafd826002b 100644 --- a/arch/arm64/kernel/irq.c +++ b/arch/arm64/kernel/irq.c @@ -10,6 +10,7 @@ * Copyright (C) 2012 ARM Ltd. */ +#include <linux/errno.h> #include <linux/hardirq.h> #include <linux/init.h> #include <linux/irq.h> @@ -32,34 +33,43 @@ DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts); DEFINE_PER_CPU(unsigned long *, irq_stack_ptr); - DECLARE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr); #ifdef CONFIG_SHADOW_CALL_STACK DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr); #endif -static void init_irq_scs(void) +static int __init init_irq_scs(void) { int cpu; + void *s; if (!scs_is_enabled()) - return; + return 0; + + for_each_possible_cpu(cpu) { + s = scs_alloc(early_cpu_to_node(cpu)); + if (!s) + return -ENOMEM; + per_cpu(irq_shadow_call_stack_ptr, cpu) = s; + } - for_each_possible_cpu(cpu) - per_cpu(irq_shadow_call_stack_ptr, cpu) = - scs_alloc(early_cpu_to_node(cpu)); + return 0; } -static void __init init_irq_stacks(void) +static int __init init_irq_stacks(void) { int cpu; unsigned long *p; for_each_possible_cpu(cpu) { p = arch_alloc_vmap_stack(IRQ_STACK_SIZE, early_cpu_to_node(cpu)); + if (!p) + return -ENOMEM; per_cpu(irq_stack_ptr, cpu) = p; } + + return 0; } #ifdef CONFIG_SOFTIRQ_ON_OWN_STACK @@ -109,8 +119,9 @@ int __init set_handle_fiq(void (*handle_fiq)(struct pt_regs *)) void __init init_IRQ(void) { - init_irq_stacks(); - init_irq_scs(); + if (init_irq_stacks() || init_irq_scs()) + panic("Failed to allocate IRQ stack resources\n"); + irqchip_init(); if (system_uses_irq_prio_masking()) { diff --git a/arch/arm64/kernel/kexec_image.c b/arch/arm64/kernel/kexec_image.c index 532d72ea42ee..b70f4df15a1a 100644 --- a/arch/arm64/kernel/kexec_image.c +++ b/arch/arm64/kernel/kexec_image.c @@ -41,7 +41,7 @@ static void *image_load(struct kimage *image, struct arm64_image_header *h; u64 flags, value; bool be_image, be_kernel; - struct kexec_buf kbuf; + struct kexec_buf kbuf = {}; unsigned long text_offset, kernel_segment_number; struct kexec_segment *kernel_segment; int ret; diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c index 239c16e3d02f..8f9bc2327dc8 100644 --- a/arch/arm64/kernel/machine_kexec.c +++ b/arch/arm64/kernel/machine_kexec.c @@ -129,10 +129,8 @@ int machine_kexec_post_load(struct kimage *kimage) } /* Create a copy of the linear map */ - trans_pgd = kexec_page_alloc(kimage); - if (!trans_pgd) - return -ENOMEM; - rc = trans_pgd_create_copy(&info, &trans_pgd, PAGE_OFFSET, PAGE_END); + rc = trans_pgd_create_copy(&info, &trans_pgd, + _PAGE_OFFSET(vabits_actual), PAGE_END); if (rc) return rc; kimage->arch.ttbr1 = __pa(trans_pgd); diff --git a/arch/arm64/kernel/machine_kexec_file.c b/arch/arm64/kernel/machine_kexec_file.c index 410060ebd86d..854d872dfd0f 100644 --- a/arch/arm64/kernel/machine_kexec_file.c +++ b/arch/arm64/kernel/machine_kexec_file.c @@ -40,46 +40,30 @@ int arch_kimage_file_post_load_cleanup(struct kimage *image) } #ifdef CONFIG_CRASH_DUMP -static int prepare_elf_headers(void **addr, unsigned long *sz) +unsigned int arch_get_system_nr_ranges(void) { - struct crash_mem *cmem; - unsigned int nr_ranges; - int ret; - u64 i; + unsigned int nr_ranges = 2 + crashk_cma_cnt; /* for exclusion of crashkernel region */ phys_addr_t start, end; + u64 i; - nr_ranges = 2; /* for exclusion of crashkernel region */ for_each_mem_range(i, &start, &end) nr_ranges++; - cmem = kmalloc(struct_size(cmem, ranges, nr_ranges), GFP_KERNEL); - if (!cmem) - return -ENOMEM; + return nr_ranges; +} + +int arch_crash_populate_cmem(struct crash_mem *cmem) +{ + phys_addr_t start, end; + u64 i; - cmem->max_nr_ranges = nr_ranges; - cmem->nr_ranges = 0; for_each_mem_range(i, &start, &end) { cmem->ranges[cmem->nr_ranges].start = start; cmem->ranges[cmem->nr_ranges].end = end - 1; cmem->nr_ranges++; } - /* Exclude crashkernel region */ - ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end); - if (ret) - goto out; - - if (crashk_low_res.end) { - ret = crash_exclude_mem_range(cmem, crashk_low_res.start, crashk_low_res.end); - if (ret) - goto out; - } - - ret = crash_prepare_elf64_headers(cmem, true, addr, sz); - -out: - kfree(cmem); - return ret; + return 0; } #endif @@ -109,7 +93,7 @@ int load_other_segments(struct kimage *image, void *headers; unsigned long headers_sz; if (image->type == KEXEC_TYPE_CRASH) { - ret = prepare_elf_headers(&headers, &headers_sz); + ret = crash_prepare_headers(true, &headers, &headers_sz, NULL); if (ret) { pr_err("Preparing elf core header failed\n"); goto out_err; @@ -134,6 +118,10 @@ int load_other_segments(struct kimage *image, kexec_dprintk("Loaded elf core header at 0x%lx bufsz=0x%lx memsz=0x%lx\n", image->elf_load_addr, kbuf.bufsz, kbuf.memsz); + + ret = crash_load_dm_crypt_keys(image); + if (ret) + goto out_err; } #endif diff --git a/arch/arm64/kernel/mpam.c b/arch/arm64/kernel/mpam.c new file mode 100644 index 000000000000..3a490de4fa12 --- /dev/null +++ b/arch/arm64/kernel/mpam.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2025 Arm Ltd. */ + +#include <asm/mpam.h> + +#include <linux/arm_mpam.h> +#include <linux/cpu_pm.h> +#include <linux/jump_label.h> +#include <linux/percpu.h> + +DEFINE_STATIC_KEY_FALSE(mpam_enabled); +DEFINE_PER_CPU(u64, arm64_mpam_default); +DEFINE_PER_CPU(u64, arm64_mpam_current); + +u64 arm64_mpam_global_default; + +static int mpam_pm_notifier(struct notifier_block *self, + unsigned long cmd, void *v) +{ + u64 regval; + int cpu = smp_processor_id(); + + switch (cmd) { + case CPU_PM_EXIT: + /* + * Don't use mpam_thread_switch() as the system register + * value has changed under our feet. + */ + regval = READ_ONCE(per_cpu(arm64_mpam_current, cpu)); + write_sysreg_s(regval | MPAM1_EL1_MPAMEN, SYS_MPAM1_EL1); + if (system_supports_sme()) { + write_sysreg_s(regval & (MPAMSM_EL1_PARTID_D | MPAMSM_EL1_PMG_D), + SYS_MPAMSM_EL1); + } + isb(); + + write_sysreg_s(regval, SYS_MPAM0_EL1); + + return NOTIFY_OK; + default: + return NOTIFY_DONE; + } +} + +static struct notifier_block mpam_pm_nb = { + .notifier_call = mpam_pm_notifier, +}; + +static int __init arm64_mpam_register_cpus(void) +{ + u64 mpamidr = read_sanitised_ftr_reg(SYS_MPAMIDR_EL1); + u16 partid_max = FIELD_GET(MPAMIDR_EL1_PARTID_MAX, mpamidr); + u8 pmg_max = FIELD_GET(MPAMIDR_EL1_PMG_MAX, mpamidr); + + if (!system_supports_mpam()) + return 0; + + cpu_pm_register_notifier(&mpam_pm_nb); + return mpam_register_requestor(partid_max, pmg_max); +} +/* Must occur before mpam_msc_driver_init() from subsys_initcall() */ +arch_initcall(arm64_mpam_register_cpus) diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c index 32148bf09c1d..31f5c6b0510e 100644 --- a/arch/arm64/kernel/mte.c +++ b/arch/arm64/kernel/mte.c @@ -8,6 +8,7 @@ #include <linux/kernel.h> #include <linux/mm.h> #include <linux/prctl.h> +#include <linux/ptrace.h> #include <linux/sched.h> #include <linux/sched/mm.h> #include <linux/string.h> @@ -291,6 +292,9 @@ void mte_thread_switch(struct task_struct *next) /* TCO may not have been disabled on exception entry for the current task. */ mte_disable_tco_entry(next); + if (!system_uses_mte_async_or_asymm_mode()) + return; + /* * Check if an async tag exception occurred at EL1. * @@ -315,8 +319,8 @@ void mte_cpu_setup(void) * CnP is not a boot feature so MTE gets enabled before CnP, but let's * make sure that is the case. */ - BUG_ON(read_sysreg(ttbr0_el1) & TTBR_CNP_BIT); - BUG_ON(read_sysreg(ttbr1_el1) & TTBR_CNP_BIT); + BUG_ON(read_sysreg(ttbr0_el1) & TTBRx_EL1_CnP); + BUG_ON(read_sysreg(ttbr1_el1) & TTBRx_EL1_CnP); /* Normal Tagged memory type at the corresponding MAIR index */ sysreg_clear_set(mair_el1, @@ -350,6 +354,9 @@ void mte_suspend_enter(void) if (!system_supports_mte()) return; + if (!system_uses_mte_async_or_asymm_mode()) + return; + /* * The barriers are required to guarantee that the indirect writes * to TFSR_EL1 are synchronized before we report the state. @@ -469,7 +476,7 @@ static int __access_remote_tags(struct mm_struct *mm, unsigned long addr, * was never mapped with PROT_MTE. */ if (!(vma->vm_flags & VM_MTE)) { - err = -EOPNOTSUPP; + err = -EIO; put_page(page); break; } @@ -531,16 +538,13 @@ static int access_remote_tags(struct task_struct *tsk, unsigned long addr, if (!mm) return -EPERM; - if (!tsk->ptrace || (current != tsk->parent) || - ((get_dumpable(mm) != SUID_DUMP_USER) && - !ptracer_capable(tsk, mm->user_ns))) { + if (!ptracer_access_allowed(tsk)) { mmput(mm); return -EPERM; } ret = __access_remote_tags(mm, addr, kiov, gup_flags); mmput(mm); - return ret; } diff --git a/arch/arm64/kernel/paravirt.c b/arch/arm64/kernel/paravirt.c index aa718d6a9274..572efb96b23f 100644 --- a/arch/arm64/kernel/paravirt.c +++ b/arch/arm64/kernel/paravirt.c @@ -19,21 +19,12 @@ #include <linux/slab.h> #include <linux/types.h> #include <linux/static_call.h> +#include <linux/sched/cputime.h> #include <asm/paravirt.h> #include <asm/pvclock-abi.h> #include <asm/smp_plat.h> -struct static_key paravirt_steal_enabled; -struct static_key paravirt_steal_rq_enabled; - -static u64 native_steal_clock(int cpu) -{ - return 0; -} - -DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); - struct pv_time_stolen_time_region { struct pvclock_vcpu_stolen_time __rcu *kaddr; }; diff --git a/arch/arm64/kernel/patching.c b/arch/arm64/kernel/patching.c index 1041bc67a3ee..09f019c6547a 100644 --- a/arch/arm64/kernel/patching.c +++ b/arch/arm64/kernel/patching.c @@ -116,8 +116,7 @@ static void *__text_poke(text_poke_f func, void *addr, void *src, size_t len) while (patched < len) { ptr = addr + patched; - size = min_t(size_t, PAGE_SIZE - offset_in_page(ptr), - len - patched); + size = min(PAGE_SIZE - offset_in_page(ptr), len - patched); waddr = patch_map(ptr, FIX_TEXT_POKE0); func(waddr, src, patched, size); diff --git a/arch/arm64/kernel/pi/idreg-override.c b/arch/arm64/kernel/pi/idreg-override.c index bc57b290e5e7..acaf56234973 100644 --- a/arch/arm64/kernel/pi/idreg-override.c +++ b/arch/arm64/kernel/pi/idreg-override.c @@ -64,6 +64,7 @@ static const struct ftr_set_desc mmfr1 __prel64_initconst = { .override = &id_aa64mmfr1_override, .fields = { FIELD("vh", ID_AA64MMFR1_EL1_VH_SHIFT, mmfr1_vh_filter), + FIELD("hafdbs", ID_AA64MMFR1_EL1_HAFDBS_SHIFT, NULL), {} }, }; @@ -106,6 +107,15 @@ static const struct ftr_set_desc mmfr2 __prel64_initconst = { }, }; +static const struct ftr_set_desc mmfr4 __prel64_initconst = { + .name = "id_aa64mmfr4", + .override = &id_aa64mmfr4_override, + .fields = { + FIELD("nv_frac", ID_AA64MMFR4_EL1_NV_frac_SHIFT, NULL), + {} + }, +}; + static bool __init pfr0_sve_filter(u64 val) { /* @@ -176,6 +186,7 @@ static const struct ftr_set_desc isar2 __prel64_initconst = { .name = "id_aa64isar2", .override = &id_aa64isar2_override, .fields = { + FIELD("wfxt", ID_AA64ISAR2_EL1_WFxT_SHIFT, NULL), FIELD("gpa3", ID_AA64ISAR2_EL1_GPA3_SHIFT, NULL), FIELD("apa3", ID_AA64ISAR2_EL1_APA3_SHIFT, NULL), FIELD("mops", ID_AA64ISAR2_EL1_MOPS_SHIFT, NULL), @@ -220,6 +231,7 @@ PREL64(const struct ftr_set_desc, reg) regs[] __prel64_initconst = { { &mmfr0 }, { &mmfr1 }, { &mmfr2 }, + { &mmfr4 }, { &pfr0 }, { &pfr1 }, { &isar1 }, @@ -243,9 +255,11 @@ static const struct { "id_aa64isar1.api=0 id_aa64isar1.apa=0 " "id_aa64isar2.gpa3=0 id_aa64isar2.apa3=0" }, { "arm64.nomops", "id_aa64isar2.mops=0" }, + { "arm64.nowfxt", "id_aa64isar2.wfxt=0" }, { "arm64.nomte", "id_aa64pfr1.mte=0" }, { "nokaslr", "arm64_sw.nokaslr=1" }, { "rodata=off", "arm64_sw.rodataoff=1" }, + { "arm64.nohaft", "id_aa64mmfr1.hafdbs=2" }, { "arm64.nolva", "id_aa64mmfr2.varange=0" }, { "arm64.no32bit_el0", "id_aa64pfr0.el0=1" }, { "arm64.nompam", "id_aa64pfr0.mpam=0 id_aa64pfr1.mpam_frac=0" }, diff --git a/arch/arm64/kernel/pi/map_kernel.c b/arch/arm64/kernel/pi/map_kernel.c index a852264958c3..fb44cbdd2f29 100644 --- a/arch/arm64/kernel/pi/map_kernel.c +++ b/arch/arm64/kernel/pi/map_kernel.c @@ -165,7 +165,7 @@ static void noinline __section(".idmap.text") set_ttbr0_for_lpa2(phys_addr_t ttb static void __init remap_idmap_for_lpa2(void) { /* clear the bits that change meaning once LPA2 is turned on */ - ptdesc_t mask = PTE_SHARED; + ptval_t mask = PTE_SHARED; /* * We have to clear bits [9:8] in all block or page descriptors in the diff --git a/arch/arm64/kernel/pi/map_range.c b/arch/arm64/kernel/pi/map_range.c index de52cd85c691..761b14893f74 100644 --- a/arch/arm64/kernel/pi/map_range.c +++ b/arch/arm64/kernel/pi/map_range.c @@ -31,7 +31,7 @@ void __init map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa, u64 va_offset) { u64 cmask = (level == 3) ? CONT_PTE_SIZE - 1 : U64_MAX; - ptdesc_t protval = pgprot_val(prot) & ~PTE_TYPE_MASK; + ptval_t protval = pgprot_val(prot) & ~PTE_TYPE_MASK; int lshift = (3 - level) * PTDESC_TABLE_SHIFT; u64 lmask = (PAGE_SIZE << lshift) - 1; @@ -88,7 +88,7 @@ void __init map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa, } } -asmlinkage phys_addr_t __init create_init_idmap(pgd_t *pg_dir, ptdesc_t clrmask) +asmlinkage phys_addr_t __init create_init_idmap(pgd_t *pg_dir, ptval_t clrmask) { phys_addr_t ptep = (phys_addr_t)pg_dir + PAGE_SIZE; /* MMU is off */ pgprot_t text_prot = PAGE_KERNEL_ROX; diff --git a/arch/arm64/kernel/pi/patch-scs.c b/arch/arm64/kernel/pi/patch-scs.c index bbe7d30ed12b..3944ad899021 100644 --- a/arch/arm64/kernel/pi/patch-scs.c +++ b/arch/arm64/kernel/pi/patch-scs.c @@ -192,6 +192,14 @@ static int scs_handle_fde_frame(const struct eh_frame *frame, size -= 2; break; + case DW_CFA_advance_loc4: + loc += *opcode++ * code_alignment_factor; + loc += (*opcode++ << 8) * code_alignment_factor; + loc += (*opcode++ << 16) * code_alignment_factor; + loc += ((u64)*opcode++ << 24) * code_alignment_factor; + size -= 4; + break; + case DW_CFA_def_cfa: case DW_CFA_offset_extended: size = skip_xleb128(&opcode, size); diff --git a/arch/arm64/kernel/pi/pi.h b/arch/arm64/kernel/pi/pi.h index aec3172d4003..5dfd8484d200 100644 --- a/arch/arm64/kernel/pi/pi.h +++ b/arch/arm64/kernel/pi/pi.h @@ -35,4 +35,4 @@ void map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa, asmlinkage void early_map_kernel(u64 boot_status, phys_addr_t fdt); -asmlinkage phys_addr_t create_init_idmap(pgd_t *pgd, ptdesc_t clrmask); +asmlinkage phys_addr_t create_init_idmap(pgd_t *pgd, ptval_t clrmask); diff --git a/arch/arm64/kernel/pi/relocate.c b/arch/arm64/kernel/pi/relocate.c index 2407d2696398..82592f3a5c1c 100644 --- a/arch/arm64/kernel/pi/relocate.c +++ b/arch/arm64/kernel/pi/relocate.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only // Copyright 2023 Google LLC // Authors: Ard Biesheuvel <ardb@google.com> -// Peter Collingbourne <pcc@google.com> +// Peter Collingbourne <peter@pcc.me.uk> #include <linux/elf.h> #include <linux/init.h> diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c index 43a0361a8bf0..4e0efad5caf2 100644 --- a/arch/arm64/kernel/probes/kprobes.c +++ b/arch/arm64/kernel/probes/kprobes.c @@ -174,12 +174,27 @@ static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) { kcb->prev_kprobe.kp = kprobe_running(); kcb->prev_kprobe.status = kcb->kprobe_status; + + /* + * Save the outer kprobe's original DAIF flags before the nested + * kprobe calls kprobes_save_local_irqflag() and overwrites + * kcb->saved_irqflag. Without this, the outer kprobe will restore + * the wrong DAIF state and leave interrupts permanently masked. + */ + kcb->prev_kprobe.saved_irqflag = kcb->saved_irqflag; } static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) { __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); kcb->kprobe_status = kcb->prev_kprobe.status; + + /* + * Restore the outer kprobe's saved_irqflag so that when its + * single-step completes, kprobes_restore_local_irqflag() uses + * the correct original DAIF value. + */ + kcb->saved_irqflag = kcb->prev_kprobe.saved_irqflag; } static void __kprobes set_current_kprobe(struct kprobe *p) @@ -240,10 +255,16 @@ static int __kprobes reenter_kprobe(struct kprobe *p, switch (kcb->kprobe_status) { case KPROBE_HIT_SSDONE: case KPROBE_HIT_ACTIVE: + case KPROBE_HIT_SS: + /* + * A probe can be hit while another kprobe is preparing or + * executing its XOL single-step instruction. This is still a + * recoverable one-level reentry, so handle it in the same way as + * reentry from KPROBE_HIT_ACTIVE or KPROBE_HIT_SSDONE. + */ kprobes_inc_nmissed_count(p); setup_singlestep(p, regs, kcb, 1); break; - case KPROBE_HIT_SS: case KPROBE_REENTER: pr_warn("Failed to recover from reentered kprobes.\n"); dump_kprobe(p); @@ -282,10 +303,32 @@ int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr) struct kprobe *cur = kprobe_running(); struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + /* + * Simulated kprobes execute in the debug trap context and have no + * XOL slot. Any page fault taken while a simulated kprobe is in + * progress cannot have been caused by kprobe single-stepping and + * must be left alone for the normal page fault handler, including + * fixup_exception. + */ + if (cur && !cur->ainsn.xol_insn) + return 0; + switch (kcb->kprobe_status) { case KPROBE_HIT_SS: case KPROBE_REENTER: /* + * A page fault taken while in KPROBE_HIT_SS or + * KPROBE_REENTER state is only attributable to kprobe + * single-stepping if the faulting PC points to the + * current kprobe's XOL instruction. If the fault occurred + * elsewhere (e.g. in perf or tracing code invoked from the + * debug exception path), leave it for the normal page fault + * handler to process. + */ + if (instruction_pointer(regs) != (unsigned long)cur->ainsn.xol_insn) + break; + + /* * We are here because the instruction being single * stepped caused a page fault. We reset the current * kprobe and the ip points back to the probe address diff --git a/arch/arm64/kernel/probes/uprobes.c b/arch/arm64/kernel/probes/uprobes.c index 941668800aea..2a9b0bc083a3 100644 --- a/arch/arm64/kernel/probes/uprobes.c +++ b/arch/arm64/kernel/probes/uprobes.c @@ -15,7 +15,7 @@ void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, void *src, unsigned long len) { - void *xol_page_kaddr = kmap_atomic(page); + void *xol_page_kaddr = kmap_local_page(page); void *dst = xol_page_kaddr + (vaddr & ~PAGE_MASK); /* @@ -32,7 +32,7 @@ void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, sync_icache_aliases((unsigned long)dst, (unsigned long)dst + len); done: - kunmap_atomic(xol_page_kaddr); + kunmap_local(xol_page_kaddr); } unsigned long uprobe_get_swbp_addr(struct pt_regs *regs) @@ -103,10 +103,7 @@ bool arch_uprobe_xol_was_trapped(struct task_struct *t) * insn itself is trapped, then detect the case with the help of * invalid fault code which is being set in arch_uprobe_pre_xol */ - if (t->thread.fault_code != UPROBE_INV_FAULT_CODE) - return true; - - return false; + return t->thread.fault_code != UPROBE_INV_FAULT_CODE; } bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs) diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 489554931231..581f80e9b9b7 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -26,6 +26,7 @@ #include <linux/reboot.h> #include <linux/interrupt.h> #include <linux/init.h> +#include <linux/cpumask.h> #include <linux/cpu.h> #include <linux/elfcore.h> #include <linux/pm.h> @@ -51,6 +52,7 @@ #include <asm/fpsimd.h> #include <asm/gcs.h> #include <asm/mmu_context.h> +#include <asm/mpam.h> #include <asm/mte.h> #include <asm/processor.h> #include <asm/pointer_auth.h> @@ -699,6 +701,29 @@ void update_sctlr_el1(u64 sctlr) isb(); } +static inline void debug_switch_state(void) +{ + if (system_uses_irq_prio_masking()) { + unsigned long daif_expected = 0; + unsigned long daif_actual = read_sysreg(daif); + unsigned long pmr_expected = GIC_PRIO_IRQOFF; + unsigned long pmr_actual = read_sysreg_s(SYS_ICC_PMR_EL1); + + WARN_ONCE(daif_actual != daif_expected || + pmr_actual != pmr_expected, + "Unexpected DAIF + PMR: 0x%lx + 0x%lx (expected 0x%lx + 0x%lx)\n", + daif_actual, pmr_actual, + daif_expected, pmr_expected); + } else { + unsigned long daif_expected = DAIF_PROCCTX_NOIRQ; + unsigned long daif_actual = read_sysreg(daif); + + WARN_ONCE(daif_actual != daif_expected, + "Unexpected DAIF value: 0x%lx (expected 0x%lx)\n", + daif_actual, daif_expected); + } +} + /* * Thread switching. */ @@ -708,6 +733,8 @@ struct task_struct *__switch_to(struct task_struct *prev, { struct task_struct *last; + debug_switch_state(); + fpsimd_thread_switch(next); tls_thread_switch(next); hw_breakpoint_thread_switch(next); @@ -738,6 +765,12 @@ struct task_struct *__switch_to(struct task_struct *prev, if (prev->thread.sctlr_user != next->thread.sctlr_user) update_sctlr_el1(next->thread.sctlr_user); + /* + * MPAM thread switch happens after the DSB to ensure prev's accesses + * use prev's MPAM settings. + */ + mpam_thread_switch(next); + /* the actual thread switch */ last = cpu_switch_to(prev, next); diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index 80a580e019c5..3bcf86154d95 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -24,6 +24,7 @@ #include <linux/nospec.h> #include <linux/prctl.h> #include <linux/sched/task_stack.h> +#include <linux/sysfs.h> #include <asm/debug-monitors.h> #include <asm/insn.h> @@ -61,7 +62,7 @@ static void update_mitigation_state(enum mitigation_state *oldp, ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf) { - return sprintf(buf, "Mitigation: __user pointer sanitization\n"); + return sysfs_emit(buf, "Mitigation: __user pointer sanitization\n"); } /* @@ -126,7 +127,7 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, switch (spectre_v2_state) { case SPECTRE_UNAFFECTED: if (bhb_state == SPECTRE_UNAFFECTED) - return sprintf(buf, "Not affected\n"); + return sysfs_emit(buf, "Not affected\n"); /* * Platforms affected by Spectre-BHB can't report @@ -136,13 +137,13 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, fallthrough; case SPECTRE_MITIGATED: if (bhb_state == SPECTRE_MITIGATED && _unprivileged_ebpf_enabled()) - return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n"); + return sysfs_emit(buf, "Vulnerable: Unprivileged eBPF enabled\n"); - return sprintf(buf, "Mitigation: %s%s\n", v2_str, bhb_str); + return sysfs_emit(buf, "Mitigation: %s%s\n", v2_str, bhb_str); case SPECTRE_VULNERABLE: fallthrough; default: - return sprintf(buf, "Vulnerable\n"); + return sysfs_emit(buf, "Vulnerable\n"); } } @@ -438,13 +439,13 @@ ssize_t cpu_show_spec_store_bypass(struct device *dev, { switch (spectre_v4_state) { case SPECTRE_UNAFFECTED: - return sprintf(buf, "Not affected\n"); + return sysfs_emit(buf, "Not affected\n"); case SPECTRE_MITIGATED: - return sprintf(buf, "Mitigation: Speculative Store Bypass disabled via prctl\n"); + return sysfs_emit(buf, "Mitigation: Speculative Store Bypass disabled via prctl\n"); case SPECTRE_VULNERABLE: fallthrough; default: - return sprintf(buf, "Vulnerable\n"); + return sysfs_emit(buf, "Vulnerable\n"); } } @@ -887,6 +888,7 @@ static u8 spectre_bhb_loop_affected(void) MIDR_ALL_VERSIONS(MIDR_CORTEX_X2), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), + MIDR_ALL_VERSIONS(MIDR_HISI_TSV110), {}, }; static const struct midr_range spectre_bhb_k24_list[] = { @@ -1021,6 +1023,11 @@ static int __init parse_spectre_bhb_param(char *str) } early_param("nospectre_bhb", parse_spectre_bhb_param); +static bool spectre_bhb_mitigations_off(void) +{ + return __nospectre_bhb || cpu_mitigations_off(); +} + void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry) { bp_hardening_cb_t cpu_cb; @@ -1034,6 +1041,8 @@ void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry) /* No point mitigating Spectre-BHB alone. */ } else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) { /* Do nothing */ + } else if (spectre_bhb_mitigations_off()) { + /* Mitigation disabled on the command line */ } else if (supports_ecbhb(SCOPE_LOCAL_CPU)) { state = SPECTRE_MITIGATED; set_bit(BHB_HW, &system_bhb_mitigations); @@ -1199,6 +1208,6 @@ void spectre_print_disabled_mitigations(void) if (spectre_v4_mitigations_off()) pr_info("spectre-v4 %s", spectre_disabled_suffix); - if (__nospectre_bhb || cpu_mitigations_off()) + if (spectre_bhb_mitigations_off()) pr_info("spectre-bhb %s", spectre_disabled_suffix); } diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 6c5ff6807d4c..f743cbec1c3a 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -560,6 +560,42 @@ static int gpr_get(struct task_struct *target, return membuf_write(&to, uregs, sizeof(*uregs)); } +static void update_syscall_orig_x0_after_ptrace(struct task_struct *target) +{ + struct pt_regs *regs = task_pt_regs(target); + struct kernel_siginfo *info = target->last_siginfo; + + /* + * Skip the update for NO_SYSCALL (set either by the user or the + * tracer), as regs[0] holds the return value (see the comment in + * el0_svc_common()) and can be unwound using syscall_rollback(). + */ + if (regs->syscallno == NO_SYSCALL) + return; + + /* We should only be called when target is in a ptrace stop */ + if (WARN_ON_ONCE(!info)) + return; + + /* + * For compat tasks, orig_r0 is provided directly through GPR index + * 17. + */ + if (is_compat_thread(task_thread_info(target))) + return; + + /* + * Don't update orig_x0 for a syscall-exit-stop, as x0 now contains the + * return value of the system call. + */ + if ((info->si_code & ~0x80) == SIGTRAP && + target->ptrace_message == PTRACE_EVENTMSG_SYSCALL_EXIT) { + return; + } + + regs->orig_x0 = regs->regs[0]; +} + static int gpr_set(struct task_struct *target, const struct user_regset *regset, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) @@ -575,6 +611,14 @@ static int gpr_set(struct task_struct *target, const struct user_regset *regset, return -EINVAL; task_pt_regs(target)->user_regs = newregs; + + /* + * Keep orig_x0 authoritative so that seccomp (via + * syscall_get_arguments()), audit and the restart path all see the same + * first argument the syscall is dispatched with, even if it has been + * updated by a tracer. + */ + update_syscall_orig_x0_after_ptrace(target); return 0; } @@ -753,6 +797,12 @@ static int system_call_set(struct task_struct *target, return ret; task_pt_regs(target)->syscallno = syscallno; + + /* + * Re-sync orig_x0 in case the syscall number has been changed + * from NO_SYSCALL. + */ + update_syscall_orig_x0_after_ptrace(target); return ret; } @@ -801,7 +851,7 @@ static void sve_init_header_from_task(struct user_sve_header *header, if (active) header->size = SVE_PT_SIZE(vq, header->flags); else - header->size = sizeof(header); + header->size = sizeof(*header); header->max_size = SVE_PT_SIZE(sve_vq_from_vl(header->max_vl), SVE_PT_REGS_SVE); } @@ -837,7 +887,7 @@ static int sve_get_common(struct task_struct *target, * from the other mode to userspace. */ if (header.size == sizeof(header)) - return 0; + return to.left; switch ((header.flags & SVE_PT_REGS_MASK)) { case SVE_PT_REGS_FPSIMD: @@ -983,8 +1033,8 @@ static int sve_set_common(struct task_struct *target, } /* Always zero V regs, FPSR, and FPCR */ - memset(¤t->thread.uw.fpsimd_state, 0, - sizeof(current->thread.uw.fpsimd_state)); + memset(&target->thread.uw.fpsimd_state, 0, + sizeof(target->thread.uw.fpsimd_state)); /* Registers: FPSIMD-only case */ @@ -1484,6 +1534,9 @@ static int poe_get(struct task_struct *target, if (!system_supports_poe()) return -EINVAL; + if (target == current) + current->thread.por_el0 = read_sysreg_s(SYS_POR_EL0); + return membuf_write(&to, &target->thread.por_el0, sizeof(target->thread.por_el0)); } @@ -2341,9 +2394,10 @@ enum ptrace_syscall_dir { PTRACE_SYSCALL_EXIT, }; -static void report_syscall(struct pt_regs *regs, enum ptrace_syscall_dir dir) +static __always_inline unsigned long ptrace_save_reg(struct pt_regs *regs, + enum ptrace_syscall_dir dir, + int *regno) { - int regno; unsigned long saved_reg; /* @@ -2362,15 +2416,34 @@ static void report_syscall(struct pt_regs *regs, enum ptrace_syscall_dir dir) * - Syscall stops behave differently to seccomp and pseudo-step traps * (the latter do not nobble any registers). */ - regno = (is_compat_task() ? 12 : 7); - saved_reg = regs->regs[regno]; - regs->regs[regno] = dir; + *regno = (is_compat_task() ? 12 : 7); + saved_reg = regs->regs[*regno]; + regs->regs[*regno] = dir; - if (dir == PTRACE_SYSCALL_ENTER) { - if (ptrace_report_syscall_entry(regs)) - forget_syscall(regs); - regs->regs[regno] = saved_reg; - } else if (!test_thread_flag(TIF_SINGLESTEP)) { + return saved_reg; +} + +static int report_syscall_entry(struct pt_regs *regs) +{ + unsigned long saved_reg; + int regno, ret; + + saved_reg = ptrace_save_reg(regs, PTRACE_SYSCALL_ENTER, ®no); + ret = !ptrace_report_syscall_permit_entry(regs); + if (ret) + forget_syscall(regs); + regs->regs[regno] = saved_reg; + + return ret; +} + +static void report_syscall_exit(struct pt_regs *regs) +{ + unsigned long saved_reg; + int regno; + + saved_reg = ptrace_save_reg(regs, PTRACE_SYSCALL_EXIT, ®no); + if (!test_thread_flag(TIF_SINGLESTEP)) { ptrace_report_syscall_exit(regs, 0); regs->regs[regno] = saved_reg; } else { @@ -2388,15 +2461,16 @@ static void report_syscall(struct pt_regs *regs, enum ptrace_syscall_dir dir) int syscall_trace_enter(struct pt_regs *regs) { unsigned long flags = read_thread_flags(); + int ret; if (flags & (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE)) { - report_syscall(regs, PTRACE_SYSCALL_ENTER); - if (flags & _TIF_SYSCALL_EMU) + ret = report_syscall_entry(regs); + if (ret || (flags & _TIF_SYSCALL_EMU)) return NO_SYSCALL; } /* Do the secure computing after ptrace; failures should be fast. */ - if (secure_computing() == -1) + if (!seccomp_permit_syscall()) return NO_SYSCALL; if (test_thread_flag(TIF_SYSCALL_TRACEPOINT)) @@ -2418,7 +2492,7 @@ void syscall_trace_exit(struct pt_regs *regs) trace_sys_exit(regs, syscall_get_return_value(current, regs)); if (flags & (_TIF_SYSCALL_TRACE | _TIF_SINGLESTEP)) - report_syscall(regs, PTRACE_SYSCALL_EXIT); + report_syscall_exit(regs); rseq_syscall(regs); } diff --git a/arch/arm64/kernel/relocate_kernel.S b/arch/arm64/kernel/relocate_kernel.S index 413f899e4ac6..6cb4209f5dab 100644 --- a/arch/arm64/kernel/relocate_kernel.S +++ b/arch/arm64/kernel/relocate_kernel.S @@ -64,7 +64,8 @@ SYM_CODE_START(arm64_relocate_new_kernel) mov x19, x13 copy_page x13, x12, x1, x2, x3, x4, x5, x6, x7, x8 add x1, x19, #PAGE_SIZE - dcache_by_myline_op civac, sy, x19, x1, x15, x20 + dcache_by_myline_op_nosync civac, x19, x1, x15, x20 + dsb sy b .Lnext .Ltest_indirection: tbz x16, IND_INDIRECTION_BIT, .Ltest_destination diff --git a/arch/arm64/kernel/rsi.c b/arch/arm64/kernel/rsi.c index c64a06f58c0b..25ca75ce1a4d 100644 --- a/arch/arm64/kernel/rsi.c +++ b/arch/arm64/kernel/rsi.c @@ -7,11 +7,11 @@ #include <linux/memblock.h> #include <linux/psci.h> #include <linux/swiotlb.h> -#include <linux/cc_platform.h> #include <linux/platform_device.h> #include <asm/io.h> #include <asm/mem_encrypt.h> +#include <asm/pgtable.h> #include <asm/rsi.h> static struct realm_config config; @@ -22,17 +22,6 @@ EXPORT_SYMBOL(prot_ns_shared); DEFINE_STATIC_KEY_FALSE_RO(rsi_present); EXPORT_SYMBOL(rsi_present); -bool cc_platform_has(enum cc_attr attr) -{ - switch (attr) { - case CC_ATTR_MEM_ENCRYPT: - return is_realm_world(); - default: - return false; - } -} -EXPORT_SYMBOL_GPL(cc_platform_has); - static bool rsi_version_matches(void) { unsigned long ver_lower, ver_higher; @@ -144,9 +133,9 @@ void __init arm64_rsi_init(void) return; if (!rsi_version_matches()) return; - if (WARN_ON(rsi_get_realm_config(&config))) + if (WARN_ON(rsi_get_realm_config(lm_alias(&config)))) return; - prot_ns_shared = BIT(config.ipa_bits - 1); + prot_ns_shared = __phys_to_pte_val(BIT(config.ipa_bits - 1)); if (arm64_ioremap_prot_hook_register(realm_ioremap_hook)) return; diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 23c05dc7a8f2..29c6100f0c50 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -347,8 +347,6 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) request_standard_resources(); - early_ioremap_reset(); - if (acpi_disabled) psci_dt_init(); else diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 08ffc5a5aea4..38e6fa204c17 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -67,6 +67,9 @@ struct rt_sigframe_user_layout { unsigned long end_offset; }; +#define TERMINATOR_SIZE round_up(sizeof(struct _aarch64_ctx), 16) +#define EXTRA_CONTEXT_SIZE round_up(sizeof(struct extra_context), 16) + /* * Holds any EL0-controlled state that influences unprivileged memory accesses. * This includes both accesses done in userspace and uaccess done in the kernel. @@ -74,13 +77,35 @@ struct rt_sigframe_user_layout { * This state needs to be carefully managed to ensure that it doesn't cause * uaccess to fail when setting up the signal frame, and the signal handler * itself also expects a well-defined state when entered. + * + * The struct should be zero-initialised. Its members should only be accessed + * via the accessors below. __valid_fields tracks which of the fields are valid + * (have been set to some value). */ struct user_access_state { - u64 por_el0; + unsigned int __valid_fields; + u64 __por_el0; }; -#define TERMINATOR_SIZE round_up(sizeof(struct _aarch64_ctx), 16) -#define EXTRA_CONTEXT_SIZE round_up(sizeof(struct extra_context), 16) +#define UA_STATE_HAS_POR_EL0 BIT(0) + +static void set_ua_state_por_el0(struct user_access_state *ua_state, + u64 por_el0) +{ + ua_state->__por_el0 = por_el0; + ua_state->__valid_fields |= UA_STATE_HAS_POR_EL0; +} + +static int get_ua_state_por_el0(const struct user_access_state *ua_state, + u64 *por_el0) +{ + if (ua_state->__valid_fields & UA_STATE_HAS_POR_EL0) { + *por_el0 = ua_state->__por_el0; + return 0; + } + + return -ENOENT; +} /* * Save the user access state into ua_state and reset it to disable any @@ -94,7 +119,7 @@ static void save_reset_user_access_state(struct user_access_state *ua_state) for (int pkey = 0; pkey < arch_max_pkey(); pkey++) por_enable_all |= POR_ELx_PERM_PREP(pkey, POE_RWX); - ua_state->por_el0 = read_sysreg_s(SYS_POR_EL0); + set_ua_state_por_el0(ua_state, read_sysreg_s(SYS_POR_EL0)); write_sysreg_s(por_enable_all, SYS_POR_EL0); /* * No ISB required as we can tolerate spurious Overlay faults - @@ -122,8 +147,10 @@ static void set_handler_user_access_state(void) */ static void restore_user_access_state(const struct user_access_state *ua_state) { - if (system_supports_poe()) - write_sysreg_s(ua_state->por_el0, SYS_POR_EL0); + u64 por_el0; + + if (get_ua_state_por_el0(ua_state, &por_el0) == 0) + write_sysreg_s(por_el0, SYS_POR_EL0); } static void init_user_layout(struct rt_sigframe_user_layout *user) @@ -333,11 +360,16 @@ static int restore_fpmr_context(struct user_ctxs *user) static int preserve_poe_context(struct poe_context __user *ctx, const struct user_access_state *ua_state) { - int err = 0; + int err; + u64 por_el0; + + err = get_ua_state_por_el0(ua_state, &por_el0); + if (WARN_ON_ONCE(err)) + return err; __put_user_error(POE_MAGIC, &ctx->head.magic, err); __put_user_error(sizeof(*ctx), &ctx->head.size, err); - __put_user_error(ua_state->por_el0, &ctx->por_el0, err); + __put_user_error(por_el0, &ctx->por_el0, err); return err; } @@ -353,7 +385,7 @@ static int restore_poe_context(struct user_ctxs *user, __get_user_error(por_el0, &(user->poe->por_el0), err); if (!err) - ua_state->por_el0 = por_el0; + set_ua_state_por_el0(ua_state, por_el0); return err; } @@ -1095,7 +1127,7 @@ SYSCALL_DEFINE0(rt_sigreturn) { struct pt_regs *regs = current_pt_regs(); struct rt_sigframe __user *frame; - struct user_access_state ua_state; + struct user_access_state ua_state = {}; /* Always make any pending restarted system calls return -EINTR */ current->restart_block.fn = do_no_restart_syscall; @@ -1507,7 +1539,7 @@ static int setup_rt_frame(int usig, struct ksignal *ksig, sigset_t *set, { struct rt_sigframe_user_layout user; struct rt_sigframe __user *frame; - struct user_access_state ua_state; + struct user_access_state ua_state = {}; int err = 0; fpsimd_save_and_flush_current_state(); diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 1aa324104afb..a61dc3016a11 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -33,6 +33,7 @@ #include <linux/kernel_stat.h> #include <linux/kexec.h> #include <linux/kgdb.h> +#include <linux/kprobes.h> #include <linux/kvm_host.h> #include <linux/nmi.h> @@ -45,6 +46,7 @@ #include <asm/daifflags.h> #include <asm/kvm_mmu.h> #include <asm/mmu_context.h> +#include <asm/nmi.h> #include <asm/numa.h> #include <asm/processor.h> #include <asm/smp_plat.h> @@ -535,23 +537,13 @@ void arch_unregister_cpu(int cpu) { acpi_handle acpi_handle = acpi_get_processor_handle(cpu); struct cpu *c = &per_cpu(cpu_devices, cpu); - acpi_status status; unsigned long long sta; - - if (!acpi_handle) { - pr_err_once("Removing a CPU without associated ACPI handle\n"); - return; - } + acpi_status status; status = acpi_evaluate_integer(acpi_handle, "_STA", NULL, &sta); - if (ACPI_FAILURE(status)) - return; - - /* For now do not allow anything that looks like physical CPU HP */ - if (cpu_present(cpu) && !(sta & ACPI_STA_DEVICE_PRESENT)) { + if (!ACPI_FAILURE(status) && + cpu_present(cpu) && !(sta & ACPI_STA_DEVICE_PRESENT)) pr_err_once("Changing CPU present bit is not supported\n"); - return; - } unregister_cpu(c); } @@ -566,6 +558,11 @@ struct acpi_madt_generic_interrupt *acpi_cpu_get_madt_gicc(int cpu) } EXPORT_SYMBOL_GPL(acpi_cpu_get_madt_gicc); +static bool acpi_cpu_is_present(int cpu) +{ + return acpi_cpu_get_madt_gicc(cpu)->flags & ACPI_MADT_ENABLED; +} + /* * acpi_map_gic_cpu_interface - parse processor MADT entry * @@ -670,6 +667,10 @@ static void __init acpi_parse_and_init_cpus(void) early_map_cpu_to_node(i, acpi_numa_get_nid(i)); } #else +static bool acpi_cpu_is_present(int cpu) +{ + return false; +} #define acpi_parse_and_init_cpus(...) do { } while (0) #endif @@ -745,16 +746,22 @@ void __init smp_init_cpus(void) else acpi_parse_and_init_cpus(); - if (cpu_count > nr_cpu_ids) - pr_warn("Number of cores (%d) exceeds configured maximum of %u - clipping\n", - cpu_count, nr_cpu_ids); - if (!bootcpu_valid) { pr_err("missing boot CPU MPIDR, not enabling secondaries\n"); return; } /* + * For the nosmp/maxcpus=0 case, do not mark the secondary CPUs + * possible. + */ + if (!setup_max_cpus) + return; + + if (cpu_count > nr_cpu_ids) + pr_warn("Number of cores (%d) exceeds configured maximum of %u - clipping\n", + cpu_count, nr_cpu_ids); + /* * We need to set the cpu_logical_map entries before enabling * the cpus so that cpu processor description entries (DT cpu nodes * and ACPI MADT entries) can be retrieved by matching the cpu hwid @@ -808,7 +815,8 @@ void __init smp_prepare_cpus(unsigned int max_cpus) if (err) continue; - set_cpu_present(cpu, true); + if (acpi_disabled || acpi_cpu_is_present(cpu)) + set_cpu_present(cpu, true); numa_store_cpu_info(cpu); } } @@ -833,11 +841,10 @@ int arch_show_interrupts(struct seq_file *p, int prec) unsigned int cpu, i; for (i = 0; i < MAX_IPI; i++) { - seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, - prec >= 4 ? " " : ""); + seq_printf(p, "%*s%u: ", prec - 1, "IPI", i); for_each_online_cpu(cpu) seq_printf(p, "%10u ", irq_desc_kstat_cpu(get_ipi_desc(cpu, i), cpu)); - seq_printf(p, " %s\n", ipi_types[i]); + seq_printf(p, " %s\n", ipi_types[i]); } seq_printf(p, "%*s: %10lu\n", prec, "Err", irq_err_count); @@ -861,52 +868,71 @@ void arch_irq_work_raise(void) } #endif -static void __noreturn local_cpu_stop(unsigned int cpu) -{ - set_cpu_online(cpu, false); - - local_daif_mask(); - sdei_mask_local_cpu(); - cpu_park_loop(); -} - -/* - * We need to implement panic_smp_self_stop() for parallel panic() calls, so - * that cpu_online_mask gets correctly updated and smp_send_stop() can skip - * CPUs that have already stopped themselves. +/** + * arm64_nmi_cpu_stop() - stop the local CPU after it is told to stop. + * @regs: register state to record in the vmcore on a crash stop, or NULL for + * panic_smp_self_stop(), which has no interrupted context to save. + * @die_on_crash: on the kdump crash path, power the CPU off via PSCI CPU_OFF + * (so a capture kernel can reclaim it) rather than parking it. + * + * The single point every arm64 stop path funnels through, keeping the + * bookkeeping (mask interrupts, save the crash context, mark offline, mask + * SDEI, optionally power off) in one place: + * + * - the regular IPI_CPU_STOP and pseudo-NMI IPI_CPU_STOP_NMI handlers; + * - panic_smp_self_stop(), a CPU parking itself on a parallel panic(); + * - the SDEI cross-CPU NMI handler (drivers/firmware/arm_sdei_nmi.c), + * which reaches CPUs the stop IPIs could not. + * + * The IPI stop handlers pass @die_on_crash true. The SDEI handler and + * panic_smp_self_stop() pass false and only park. For SDEI that is required, + * not just conservative: it runs inside an SDEI event that is deliberately + * never completed (completing it has firmware resume the wedged context), and + * a CPU_OFF from that not-yet-completed context wedges EL3 on some firmware -- + * a documented follow-up. Parking also matches this path's own fallback when + * CPU_OFF is unavailable. */ -void __noreturn panic_smp_self_stop(void) +void __noreturn arm64_nmi_cpu_stop(struct pt_regs *regs, bool die_on_crash) { - local_cpu_stop(smp_processor_id()); -} + unsigned int cpu = smp_processor_id(); + bool crash = IS_ENABLED(CONFIG_KEXEC_CORE) && crash_stop; -static void __noreturn ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs) -{ -#ifdef CONFIG_KEXEC_CORE /* * Use local_daif_mask() instead of local_irq_disable() to make sure - * that pseudo-NMIs are disabled. The "crash stop" code starts with - * an IRQ and falls back to NMI (which might be pseudo). If the IRQ - * finally goes through right as we're timing out then the NMI could - * interrupt us. It's better to prevent the NMI and let the IRQ - * finish since the pt_regs will be better. + * that pseudo-NMIs are disabled. The "stop" code starts with an IRQ + * and falls back to NMI (which might be pseudo). If the IRQ finally + * goes through right as we're timing out then the NMI could interrupt + * us. It's better to prevent the NMI and let the IRQ finish since the + * pt_regs will be better. */ local_daif_mask(); - crash_save_cpu(regs, cpu); +#ifdef CONFIG_KEXEC_CORE + if (crash && regs) + crash_save_cpu(regs, cpu); +#endif + /* the ack a stop requester (e.g. smp_send_stop()) polls for */ set_cpu_online(cpu, false); sdei_mask_local_cpu(); - if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) + if (crash && die_on_crash) __cpu_try_die(cpu); /* just in case */ cpu_park_loop(); -#else - BUG(); -#endif +} +NOKPROBE_SYMBOL(arm64_nmi_cpu_stop); + +/* + * We need to implement panic_smp_self_stop() for parallel panic() calls, so + * that cpu_online_mask gets correctly updated and smp_send_stop() can skip + * CPUs that have already stopped themselves. + */ +void __noreturn panic_smp_self_stop(void) +{ + arm64_nmi_cpu_stop(NULL, false); } static void arm64_send_ipi(const cpumask_t *mask, unsigned int nr) @@ -928,6 +954,16 @@ static void arm64_backtrace_ipi(cpumask_t *mask) void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) { /* + * Prefer the SDEI cross-CPU NMI provider when active: firmware + * dispatches the event out of EL3 and reaches CPUs that have + * interrupts locally masked, without the per-IRQ-mask cost that + * pseudo-NMI pays for the same reach. The plain IPI path below + * can't reach such a CPU unless pseudo-NMI is enabled. + */ + if (sdei_nmi_trigger_cpumask_backtrace(mask, exclude_cpu)) + return; + + /* * NOTE: though nmi_trigger_cpumask_backtrace() has "nmi_" in the name, * nothing about it truly needs to be implemented using an NMI, it's * just that it's _allowed_ to work with NMIs. If ipi_should_be_nmi() @@ -973,12 +1009,7 @@ static void do_handle_IPI(int ipinr) case IPI_CPU_STOP: case IPI_CPU_STOP_NMI: - if (IS_ENABLED(CONFIG_KEXEC_CORE) && crash_stop) { - ipi_cpu_crash_stop(cpu, get_irq_regs()); - unreachable(); - } else { - local_cpu_stop(cpu); - } + arm64_nmi_cpu_stop(get_irq_regs(), true); break; #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST @@ -1081,7 +1112,7 @@ static void ipi_teardown(int cpu) disable_percpu_irq(ipi_irq_base + i); } } else { - disable_irq(irq_desc_get_irq(get_ipi_desc(cpu, i))); + disable_irq_nosync(irq_desc_get_irq(get_ipi_desc(cpu, i))); } } } @@ -1252,6 +1283,28 @@ void smp_send_stop(void) udelay(1); } + /* + * If CPUs are *still* online, try the SDEI cross-CPU NMI. Firmware + * delivers it regardless of the target's DAIF state, so it reaches + * a CPU spinning with interrupts masked, which neither rung above + * could (without pseudo-NMI there is no NMI rung at all). Allow + * 100ms: a firmware round-trip per CPU, with headroom. + */ + if (num_other_online_cpus() && sdei_nmi_active()) { + /* re-snapshot after the rungs above took CPUs offline */ + smp_rmb(); + cpumask_copy(&mask, cpu_online_mask); + cpumask_clear_cpu(smp_processor_id(), &mask); + + pr_info("SMP: retry stop with SDEI NMI for CPUs %*pbl\n", + cpumask_pr_args(&mask)); + + sdei_nmi_stop_cpus(&mask); + timeout = USEC_PER_MSEC * 100; + while (num_other_online_cpus() && timeout--) + udelay(1); + } + if (num_other_online_cpus()) { smp_rmb(); cpumask_copy(&mask, cpu_online_mask); diff --git a/arch/arm64/kernel/static_call.c b/arch/arm64/kernel/static_call.c new file mode 100644 index 000000000000..c126edced022 --- /dev/null +++ b/arch/arm64/kernel/static_call.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/static_call.h> +#include <linux/memory.h> +#include <asm/text-patching.h> +#include <asm/insn.h> + +void arch_static_call_transform(void *site, void *tramp, void *func, bool tail) +{ + u64 literal; + int ret; + + if (!func) + func = __static_call_return0; + + /* decode the instructions to discover the literal address */ + literal = ALIGN_DOWN((u64)tramp + 4, SZ_4K) + + aarch64_insn_adrp_get_offset(le32_to_cpup(tramp + 4)) + + 8 * aarch64_insn_decode_immediate(AARCH64_INSN_IMM_12, + le32_to_cpup(tramp + 8)); + + ret = aarch64_insn_write_literal_u64((void *)literal, (u64)func); + WARN_ON_ONCE(ret); +} +EXPORT_SYMBOL_GPL(arch_static_call_transform); diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index eaaff94329cd..c41724a40b75 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -99,7 +99,6 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) int ret = 0; unsigned long flags; struct sleep_stack_data state; - struct arm_cpuidle_irq_context context; /* * Some portions of CPU state (e.g. PSTATE.{PAN,DIT}) are initialized @@ -121,6 +120,9 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) * Strictly speaking the trace_hardirqs_off() here is superfluous, * hardirqs should be firmly off by now. This really ought to use * something like raw_local_daif_save(). + * + * This also unmasks interrupts in PMR in order to reliably + * resume if we're using pseudo-NMIs. */ flags = local_daif_save(); @@ -131,12 +133,6 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) */ pause_graph_tracing(); - /* - * Switch to using DAIF.IF instead of PMR in order to reliably - * resume if we're using pseudo-NMIs. - */ - arm_cpuidle_save_irq_context(&context); - ct_cpuidle_enter(); if (__cpu_suspend_enter(&state)) { @@ -159,8 +155,6 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) __cpu_suspend_exit(); } - arm_cpuidle_restore_irq_context(&context); - unpause_graph_tracing(); /* diff --git a/arch/arm64/kernel/sys32.c b/arch/arm64/kernel/sys32.c index 96bcfb907443..12a948f3a504 100644 --- a/arch/arm64/kernel/sys32.c +++ b/arch/arm64/kernel/sys32.c @@ -89,7 +89,7 @@ COMPAT_SYSCALL_DEFINE4(aarch32_truncate64, const char __user *, pathname, COMPAT_SYSCALL_DEFINE4(aarch32_ftruncate64, unsigned int, fd, u32, __pad, arg_u32p(length)) { - return ksys_ftruncate(fd, arg_u64(length)); + return ksys_ftruncate(fd, arg_u64(length), FTRUNCATE_LFS); } COMPAT_SYSCALL_DEFINE5(aarch32_readahead, int, fd, u32, __pad, diff --git a/arch/arm64/kernel/sys_compat.c b/arch/arm64/kernel/sys_compat.c index 4a609e9b65de..0451f96c2c3f 100644 --- a/arch/arm64/kernel/sys_compat.c +++ b/arch/arm64/kernel/sys_compat.c @@ -36,8 +36,8 @@ __do_compat_cache_op(unsigned long start, unsigned long end) * The workaround requires an inner-shareable tlbi. * We pick the reserved-ASID to minimise the impact. */ - __tlbi(aside1is, __TLBI_VADDR(0, 0)); - dsb(ish); + __tlbi(aside1is, 0UL); + __tlbi_sync_s1ish(current->mm); } ret = caches_clean_inval_user_pou(start, start + chunk); diff --git a/arch/arm64/kernel/syscall.c b/arch/arm64/kernel/syscall.c index c062badd1a56..358ddfbf1401 100644 --- a/arch/arm64/kernel/syscall.c +++ b/arch/arm64/kernel/syscall.c @@ -52,17 +52,6 @@ static void invoke_syscall(struct pt_regs *regs, unsigned int scno, } syscall_set_return_value(current, regs, 0, ret); - - /* - * This value will get limited by KSTACK_OFFSET_MAX(), which is 10 - * bits. The actual entropy will be further reduced by the compiler - * when applying stack alignment constraints: the AAPCS mandates a - * 16-byte aligned SP at function boundaries, which will remove the - * 4 low bits from any entropy chosen here. - * - * The resulting 6 bits of entropy is seen in SP[9:4]. - */ - choose_random_kstack_offset(get_random_u16()); } static inline bool has_syscall_work(unsigned long flags) diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index 5d24dc53799b..d28438f8b83f 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -272,7 +272,7 @@ static void amu_fie_setup(const struct cpumask *cpus) cpumask_or(amu_fie_cpus, amu_fie_cpus, cpus); - topology_set_scale_freq_source(&amu_sfd, amu_fie_cpus); + topology_set_scale_freq_source(&amu_sfd, cpus); pr_debug("CPUs[%*pbl]: counters will be used for FIE.", cpumask_pr_args(cpus)); @@ -284,7 +284,7 @@ static int init_amu_fie_callback(struct notifier_block *nb, unsigned long val, struct cpufreq_policy *policy = data; if (val == CPUFREQ_CREATE_POLICY) - amu_fie_setup(policy->related_cpus); + amu_fie_setup(policy->cpus); /* * We don't need to handle CPUFREQ_REMOVE_POLICY event as the AMU @@ -303,16 +303,86 @@ static struct notifier_block init_amu_fie_notifier = { .notifier_call = init_amu_fie_callback, }; +static int cpuhp_topology_online(unsigned int cpu) +{ + struct cpufreq_policy *policy = cpufreq_cpu_policy(cpu); + + /* Those are cheap checks */ + + /* + * Skip this CPU if: + * - it has no cpufreq policy assigned yet, + * - no policy exists that spans CPUs with AMU counters, or + * - it was already handled. + */ + if (unlikely(!policy) || !cpumask_available(amu_fie_cpus) || + cpumask_test_cpu(cpu, amu_fie_cpus)) + return 0; + + /* + * Only proceed if all already-online CPUs in this policy + * support AMU counters. + */ + if (unlikely(!cpumask_subset(policy->cpus, amu_fie_cpus))) + return 0; + + /* + * If the new online CPU cannot pass this check, all the CPUs related to + * the same policy should be clear from amu_fie_cpus mask, otherwise they + * may use different source of the freq scale. + */ + if (!freq_counters_valid(cpu)) { + topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_ARCH, + policy->related_cpus); + cpumask_andnot(amu_fie_cpus, amu_fie_cpus, policy->related_cpus); + return 0; + } + + cpumask_set_cpu(cpu, amu_fie_cpus); + + topology_set_scale_freq_source(&amu_sfd, cpumask_of(cpu)); + + pr_debug("CPU[%u]: counter will be used for FIE.", cpu); + + return 0; +} + static int __init init_amu_fie(void) { - return cpufreq_register_notifier(&init_amu_fie_notifier, + int ret; + + ret = cpufreq_register_notifier(&init_amu_fie_notifier, CPUFREQ_POLICY_NOTIFIER); + if (ret) + return ret; + + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "arm64/topology:online", + cpuhp_topology_online, + NULL); + if (ret < 0) { + cpufreq_unregister_notifier(&init_amu_fie_notifier, + CPUFREQ_POLICY_NOTIFIER); + return ret; + } + + return 0; } core_initcall(init_amu_fie); #ifdef CONFIG_ACPI_CPPC_LIB #include <acpi/cppc_acpi.h> +struct amu_ffh_ctrs { + u64 corecnt; + u64 constcnt; +}; + +enum cpc_ffh_ctr_id { + CPC_FFH_CTR_CORE = 0x0, + CPC_FFH_CTR_CONST = 0x1, +}; + static void cpu_read_corecnt(void *val) { /* @@ -337,19 +407,28 @@ static void cpu_read_constcnt(void *val) } static inline -int counters_read_on_cpu(int cpu, smp_call_func_t func, u64 *val) +int counters_read_on_cpu(int cpu, smp_call_func_t func, void *val) { /* - * Abort call on counterless CPU or when interrupts are - * disabled - can lead to deadlock in smp sync call. + * Abort call on counterless CPU. */ if (!cpu_has_amu_feat(cpu)) return -EOPNOTSUPP; - if (WARN_ON_ONCE(irqs_disabled())) - return -EPERM; - - smp_call_function_single(cpu, func, val, 1); + if (irqs_disabled()) { + /* + * When IRQs are disabled (tick path: sched_tick -> + * topology_scale_freq_tick or cppc_scale_freq_tick), only local + * CPU counter reads are allowed. Remote CPU counter read would + * require smp_call_function_single() which is unsafe with IRQs + * disabled. + */ + if (WARN_ON_ONCE(cpu != smp_processor_id())) + return -EPERM; + func(val); + } else { + smp_call_function_single(cpu, func, val, 1); + } return 0; } @@ -378,24 +457,90 @@ bool cpc_ffh_supported(void) return true; } +static void amu_read_core_const_ctrs(void *val) +{ + struct amu_ffh_ctrs *ctrs = val; + + /* + * cpu_read_constcnt() incurs slight latency due to the + * ARM64_WORKAROUND_2457168 check. Read it first to minimize + * the sampling skew between the const and core counters. + */ + cpu_read_constcnt(&ctrs->constcnt); + cpu_read_corecnt(&ctrs->corecnt); +} + +static u64 cpc_ffh_extract_bits(const struct cpc_reg *reg, u64 val) +{ + val &= GENMASK_ULL(reg->bit_offset + reg->bit_width - 1, + reg->bit_offset); + val >>= reg->bit_offset; + + return val; +} + +static void cpc_ffh_ctr_value(const struct cpc_reg *reg, + const struct amu_ffh_ctrs *ctrs, u64 *val) +{ + switch ((u64)reg->address) { + case CPC_FFH_CTR_CORE: + *val = ctrs->corecnt; + break; + case CPC_FFH_CTR_CONST: + *val = ctrs->constcnt; + break; + } + + *val = cpc_ffh_extract_bits(reg, *val); +} + +static bool is_amu_ctr_reg(const struct cpc_reg *reg) +{ + return reg->address == CPC_FFH_CTR_CORE || + reg->address == CPC_FFH_CTR_CONST; +} + +int cpc_read_ffh_fb_ctrs(int cpu, struct cpc_reg *reg1, u64 *val1, + struct cpc_reg *reg2, u64 *val2) +{ + struct amu_ffh_ctrs ctrs; + int ret; + + if (!is_amu_ctr_reg(reg1) || !is_amu_ctr_reg(reg2)) + return -EINVAL; + + ret = counters_read_on_cpu(cpu, amu_read_core_const_ctrs, &ctrs); + if (ret) { + /* + * If AMU is unsupported (-EOPNOTSUPP), translate the error + * to -ENODEV. This explicitly tells the generic CPPC layer + * to abort immediately and avoid falling back to pointless + * single-counter reads. + */ + return ret == -EOPNOTSUPP ? -ENODEV : ret; + } + + cpc_ffh_ctr_value(reg1, &ctrs, val1); + cpc_ffh_ctr_value(reg2, &ctrs, val2); + + return 0; +} + int cpc_read_ffh(int cpu, struct cpc_reg *reg, u64 *val) { int ret = -EOPNOTSUPP; switch ((u64)reg->address) { - case 0x0: + case CPC_FFH_CTR_CORE: ret = counters_read_on_cpu(cpu, cpu_read_corecnt, val); break; - case 0x1: + case CPC_FFH_CTR_CONST: ret = counters_read_on_cpu(cpu, cpu_read_constcnt, val); break; } - if (!ret) { - *val &= GENMASK_ULL(reg->bit_offset + reg->bit_width - 1, - reg->bit_offset); - *val >>= reg->bit_offset; - } + if (!ret) + *val = cpc_ffh_extract_bits(reg, *val); return ret; } diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c index 78ddf6bdecad..592dd8668de4 100644 --- a/arch/arm64/kernel/vdso.c +++ b/arch/arm64/kernel/vdso.c @@ -81,9 +81,7 @@ static int __init __vdso_init(enum vdso_abi abi) vdso_info[abi].vdso_code_start) >> PAGE_SHIFT; - vdso_pagelist = kcalloc(vdso_info[abi].vdso_pages, - sizeof(struct page *), - GFP_KERNEL); + vdso_pagelist = kzalloc_objs(struct page *, vdso_info[abi].vdso_pages); if (vdso_pagelist == NULL) return -ENOMEM; diff --git a/arch/arm64/kernel/vdso32/Makefile b/arch/arm64/kernel/vdso32/Makefile index 9d0efed91414..bea3675fa668 100644 --- a/arch/arm64/kernel/vdso32/Makefile +++ b/arch/arm64/kernel/vdso32/Makefile @@ -63,7 +63,7 @@ VDSO_CFLAGS += -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs \ $(filter -Werror,$(KBUILD_CPPFLAGS)) \ -Werror-implicit-function-declaration \ -Wno-format-security \ - -std=gnu11 -fms-extensions + $(CC_FLAGS_DIALECT) VDSO_CFLAGS += -O2 # Some useful compiler-dependent flags from top-level Makefile VDSO_CFLAGS += $(call cc32-option,-Wno-pointer-sign) @@ -71,7 +71,6 @@ VDSO_CFLAGS += -fno-strict-overflow VDSO_CFLAGS += $(call cc32-option,-Werror=strict-prototypes) VDSO_CFLAGS += -Werror=date-time VDSO_CFLAGS += $(call cc32-option,-Werror=incompatible-pointer-types) -VDSO_CFLAGS += $(if $(CONFIG_CC_IS_CLANG),-Wno-microsoft-anon-tag) # Compile as THUMB2 or ARM. Unwinding via frame-pointers in THUMB2 is # unreliable. diff --git a/arch/arm64/kernel/vdso32/vdso.lds.S b/arch/arm64/kernel/vdso32/vdso.lds.S index e02b27487ce8..12bfc39e8aab 100644 --- a/arch/arm64/kernel/vdso32/vdso.lds.S +++ b/arch/arm64/kernel/vdso32/vdso.lds.S @@ -82,10 +82,13 @@ VERSION { LINUX_2.6 { global: +#ifdef CONFIG_COMPAT_32BIT_TIME __vdso_clock_gettime; __vdso_gettimeofday; __vdso_clock_getres; +#endif /* CONFIG_COMPAT_32BIT_TIME */ __vdso_clock_gettime64; + __vdso_clock_getres_time64; local: *; }; } diff --git a/arch/arm64/kernel/vdso32/vgettimeofday.c b/arch/arm64/kernel/vdso32/vgettimeofday.c index 29b4d8f61e39..12d0255cc2cf 100644 --- a/arch/arm64/kernel/vdso32/vgettimeofday.c +++ b/arch/arm64/kernel/vdso32/vgettimeofday.c @@ -8,16 +8,17 @@ #define BUILD_VDSO32_64 #include <vdso/gettime.h> +#ifdef CONFIG_COMPAT_32BIT_TIME int __vdso_clock_gettime(clockid_t clock, struct old_timespec32 *ts) { return __cvdso_clock_gettime32(clock, ts); } -int __vdso_clock_gettime64(clockid_t clock, - struct __kernel_timespec *ts) +int __vdso_clock_getres(clockid_t clock_id, + struct old_timespec32 *res) { - return __cvdso_clock_gettime(clock, ts); + return __cvdso_clock_getres_time32(clock_id, res); } int __vdso_gettimeofday(struct __kernel_old_timeval *tv, @@ -25,11 +26,17 @@ int __vdso_gettimeofday(struct __kernel_old_timeval *tv, { return __cvdso_gettimeofday(tv, tz); } +#endif /* CONFIG_COMPAT_32BIT_TIME */ -int __vdso_clock_getres(clockid_t clock_id, - struct old_timespec32 *res) +int __vdso_clock_gettime64(clockid_t clock, + struct __kernel_timespec *ts) { - return __cvdso_clock_getres_time32(clock_id, res); + return __cvdso_clock_gettime(clock, ts); +} + +int __vdso_clock_getres_time64(clockid_t clock_id, struct __kernel_timespec *res) +{ + return __cvdso_clock_getres(clock_id, res); } /* Avoid unresolved references emitted by GCC */ diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index ad6133b89e7a..af1d72020976 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -13,12 +13,23 @@ *(__kvm_ex_table) \ __stop___kvm_ex_table = .; +#ifdef CONFIG_NVHE_EL2_TRACING +#define HYPERVISOR_EVENT_IDS \ + . = ALIGN(PAGE_SIZE); \ + __hyp_event_ids_start = .; \ + *(HYP_SECTION_NAME(.event_ids)) \ + __hyp_event_ids_end = .; +#else +#define HYPERVISOR_EVENT_IDS +#endif + #define HYPERVISOR_RODATA_SECTIONS \ HYP_SECTION_NAME(.rodata) : { \ . = ALIGN(PAGE_SIZE); \ __hyp_rodata_start = .; \ *(HYP_SECTION_NAME(.data..ro_after_init)) \ *(HYP_SECTION_NAME(.rodata)) \ + HYPERVISOR_EVENT_IDS \ . = ALIGN(PAGE_SIZE); \ __hyp_rodata_end = .; \ } @@ -191,6 +202,7 @@ SECTIONS LOCK_TEXT KPROBES_TEXT HYPERVISOR_TEXT + STATIC_CALL_TEXT *(.gnu.warning) } @@ -307,6 +319,13 @@ SECTIONS HYPERVISOR_DATA_SECTION +#ifdef CONFIG_NVHE_EL2_TRACING + .data.hyp_events : { + __hyp_events_start = .; + *(SORT(_hyp_events.*)) + __hyp_events_end = .; + } +#endif /* * Data written with the MMU off but read with the MMU on requires * cache lines to be invalidated, discarding up to a Cache Writeback @@ -330,9 +349,15 @@ SECTIONS _edata = .; /* start of zero-init region */ - BSS_SECTION(SBSS_ALIGN, 0, 0) + BSS_SECTION(SBSS_ALIGN, 0, PAGE_SIZE) __pi___bss_start = __bss_start; + /* page table BSS starts here - preceding data/BSS is omitted from the linear map */ + .pgtbl : ALIGN(PAGE_SIZE) { + *(.bss..pgtbl) + } + ASSERT(ADDR(.pgtbl) == __bss_stop, ".pgtbl must follow BSS") + . = ALIGN(PAGE_SIZE); __pi_init_pg_dir = .; . += INIT_DIR_SIZE; @@ -349,6 +374,8 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + PROPELLER_DATA + MODINFO ELF_DETAILS HEAD_SYMBOLS |