diff options
Diffstat (limited to 'arch/riscv/kernel')
25 files changed, 376 insertions, 344 deletions
diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index cabb99cadfb6..ebe1c3588177 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -128,3 +128,5 @@ obj-$(CONFIG_ACPI_NUMA) += acpi_numa.o obj-$(CONFIG_GENERIC_CPU_VULNERABILITIES) += bugs.o obj-$(CONFIG_RISCV_USER_CFI) += usercfi.o + +obj-$(CONFIG_RISCV_ISA_SSQOSID) += qos.o diff --git a/arch/riscv/kernel/acpi.c b/arch/riscv/kernel/acpi.c index 068e0b404b6f..efd52a5d05b5 100644 --- a/arch/riscv/kernel/acpi.c +++ b/arch/riscv/kernel/acpi.c @@ -353,3 +353,8 @@ int acpi_get_cpu_uid(unsigned int cpu, u32 *uid) return 0; } EXPORT_SYMBOL_GPL(acpi_get_cpu_uid); + +void __init arch_reserve_mem_area(acpi_physical_address addr, size_t size) +{ + memblock_mark_nomap(addr, size); +} diff --git a/arch/riscv/kernel/alternative.c b/arch/riscv/kernel/alternative.c index 104dc0862c5c..c0c9306022c5 100644 --- a/arch/riscv/kernel/alternative.c +++ b/arch/riscv/kernel/alternative.c @@ -173,24 +173,14 @@ static void __init_or_module _apply_alternatives(struct alt_entry *begin, stage); } -static void __init apply_vdso_alternatives(void *start) +static __always_inline +void __init apply_vdso_alternatives(void *base, size_t alternatives_begin, size_t alternatives_end) { - const Elf_Ehdr *hdr; - const Elf_Shdr *shdr; - const Elf_Shdr *alt; - struct alt_entry *begin, *end; - - hdr = (Elf_Ehdr *)start; - shdr = (void *)hdr + hdr->e_shoff; - alt = find_section(hdr, shdr, ".alternative"); - if (!alt) + if (alternatives_begin == alternatives_end) return; - begin = (void *)hdr + alt->sh_offset, - end = (void *)hdr + alt->sh_offset + alt->sh_size, - - _apply_alternatives((struct alt_entry *)begin, - (struct alt_entry *)end, + _apply_alternatives(base + alternatives_begin, + base + alternatives_end, RISCV_ALTERNATIVES_BOOT); } @@ -204,10 +194,19 @@ void __init apply_boot_alternatives(void) RISCV_ALTERNATIVES_BOOT); if (IS_ENABLED(CONFIG_MMU)) - apply_vdso_alternatives(vdso_start); + apply_vdso_alternatives(vdso_start, + __vdso_alternatives_start_offset, + __vdso_alternatives_end_offset); if (IS_ENABLED(CONFIG_RISCV_USER_CFI)) - apply_vdso_alternatives(vdso_cfi_start); + apply_vdso_alternatives(vdso_cfi_start, + __vdso_alternatives_start_cfi_offset, + __vdso_alternatives_end_cfi_offset); + + if (IS_ENABLED(CONFIG_COMPAT)) + apply_vdso_alternatives(compat_vdso_start, + compat__vdso_alternatives_start_offset, + compat__vdso_alternatives_end_offset); } /* diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index f46aa5602d74..61d21f714830 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -84,7 +84,7 @@ EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); static int riscv_ext_f_depends(const struct riscv_isa_ext_data *data, const unsigned long *isa_bitmap) { - if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_f)) + if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_F)) return 0; return -EPROBE_DEFER; @@ -146,7 +146,7 @@ static int riscv_ext_f_validate(const struct riscv_isa_ext_data *data, * Due to extension ordering, d is checked before f, so no deferral * is required. */ - if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d)) { + if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_D)) { pr_warn_once("This kernel does not support systems with F but not D\n"); return -EINVAL; } @@ -189,7 +189,7 @@ static int riscv_ext_vector_float_validate(const struct riscv_isa_ext_data *data * Since this function validates vector only, and v/Zve* are probed * after f/d, there's no need for a deferral here. */ - if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d)) + if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_D)) return -EINVAL; return 0; @@ -224,7 +224,7 @@ static int riscv_ext_zcd_validate(const struct riscv_isa_ext_data *data, const unsigned long *isa_bitmap) { if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA) && - __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d)) + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_D)) return 0; return -EPROBE_DEFER; @@ -237,7 +237,7 @@ static int riscv_ext_zcf_validate(const struct riscv_isa_ext_data *data, return -EINVAL; if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA) && - __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_f)) + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_F)) return 0; return -EPROBE_DEFER; @@ -330,6 +330,27 @@ static const unsigned int riscv_a_exts[] = { RISCV_ISA_EXT_ZKNE, \ RISCV_ISA_EXT_ZKNH +static int riscv_ext_smcdeleg_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_SSCSRIND) && + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZIHPM) && + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZICNTR)) + return 0; + + return -EPROBE_DEFER; +} + +static int riscv_ext_ssccfg_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (!riscv_ext_smcdeleg_validate(data, isa_bitmap) && + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_SMCDELEG)) + return 0; + + return -EPROBE_DEFER; +} + static const unsigned int riscv_zk_bundled_exts[] = { RISCV_ISA_EXT_ZKN, RISCV_ISA_EXT_ZKR, @@ -391,6 +412,19 @@ static const unsigned int riscv_zvbb_exts[] = { RISCV_ISA_EXT_ZVKB }; +/* + * The RISC-V ISA manual specifies that Zfh implies Zfhmin and Zvfh implies + * Zvfhmin. Report the implied subset extensions whenever the supersets are + * detected (see https://github.com/riscv/riscv-isa-manual/pull/3070). + */ +static const unsigned int riscv_zfh_exts[] = { + RISCV_ISA_EXT_ZFHMIN +}; + +static const unsigned int riscv_zvfh_exts[] = { + RISCV_ISA_EXT_ZVFHMIN +}; + #define RISCV_ISA_EXT_ZVE64F_IMPLY_LIST \ RISCV_ISA_EXT_ZVE64X, \ RISCV_ISA_EXT_ZVE32F, \ @@ -452,7 +486,8 @@ static const unsigned int riscv_c_exts[] = { /* * The canonical order of ISA extension names in the ISA string is defined in - * chapter 27 of the unprivileged specification. + * Chapter 27 of the RISC-V Instruction Set Manual Volume I Unprivileged ISA + * (Document Version 20191213). * * Ordinarily, for in-kernel data structures, this order is unimportant but * isa_ext_arr defines the order of the ISA string in /proc/cpuinfo. @@ -490,18 +525,22 @@ static const unsigned int riscv_c_exts[] = { * New entries to this struct should follow the ordering rules described above. */ const struct riscv_isa_ext_data riscv_isa_ext[] = { - __RISCV_ISA_EXT_DATA(i, RISCV_ISA_EXT_i), - __RISCV_ISA_EXT_DATA(m, RISCV_ISA_EXT_m), - __RISCV_ISA_EXT_SUPERSET(a, RISCV_ISA_EXT_a, riscv_a_exts), - __RISCV_ISA_EXT_DATA_VALIDATE(f, RISCV_ISA_EXT_f, riscv_ext_f_validate), - __RISCV_ISA_EXT_DATA_VALIDATE(d, RISCV_ISA_EXT_d, riscv_ext_d_validate), - __RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_q), - __RISCV_ISA_EXT_SUPERSET(c, RISCV_ISA_EXT_c, riscv_c_exts), - __RISCV_ISA_EXT_SUPERSET_VALIDATE(v, RISCV_ISA_EXT_v, riscv_v_exts, riscv_ext_vector_float_validate), - __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_h), + __RISCV_ISA_EXT_DATA(i, RISCV_ISA_EXT_I), + __RISCV_ISA_EXT_DATA(m, RISCV_ISA_EXT_M), + __RISCV_ISA_EXT_SUPERSET(a, RISCV_ISA_EXT_A, riscv_a_exts), + __RISCV_ISA_EXT_DATA_VALIDATE(f, RISCV_ISA_EXT_F, riscv_ext_f_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(d, RISCV_ISA_EXT_D, riscv_ext_d_validate), + __RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_Q), + __RISCV_ISA_EXT_SUPERSET(c, RISCV_ISA_EXT_C, riscv_c_exts), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(v, RISCV_ISA_EXT_V, riscv_v_exts, + riscv_ext_vector_float_validate), + __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_H), __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicbom, RISCV_ISA_EXT_ZICBOM, riscv_xlinuxenvcfg_exts, riscv_ext_zicbom_validate), __RISCV_ISA_EXT_DATA_VALIDATE(zicbop, RISCV_ISA_EXT_ZICBOP, riscv_ext_zicbop_validate), __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts, riscv_ext_zicboz_validate), + __RISCV_ISA_EXT_DATA(ziccamoa, RISCV_ISA_EXT_ZICCAMOA), + __RISCV_ISA_EXT_DATA(ziccif, RISCV_ISA_EXT_ZICCIF), + __RISCV_ISA_EXT_DATA(zicclsm, RISCV_ISA_EXT_ZICCLSM), __RISCV_ISA_EXT_DATA(ziccrse, RISCV_ISA_EXT_ZICCRSE), __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicfilp, RISCV_ISA_EXT_ZICFILP, riscv_xlinuxenvcfg_exts, riscv_cfilp_validate), @@ -515,6 +554,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM), __RISCV_ISA_EXT_DATA(zimop, RISCV_ISA_EXT_ZIMOP), + __RISCV_ISA_EXT_DATA(za64rs, RISCV_ISA_EXT_ZA64RS), __RISCV_ISA_EXT_DATA(zaamo, RISCV_ISA_EXT_ZAAMO), __RISCV_ISA_EXT_DATA(zabha, RISCV_ISA_EXT_ZABHA), __RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS), @@ -523,7 +563,8 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zawrs, RISCV_ISA_EXT_ZAWRS), __RISCV_ISA_EXT_DATA_VALIDATE(zfa, RISCV_ISA_EXT_ZFA, riscv_ext_f_depends), __RISCV_ISA_EXT_DATA_VALIDATE(zfbfmin, RISCV_ISA_EXT_ZFBFMIN, riscv_ext_f_depends), - __RISCV_ISA_EXT_DATA_VALIDATE(zfh, RISCV_ISA_EXT_ZFH, riscv_ext_f_depends), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zfh, RISCV_ISA_EXT_ZFH, + riscv_zfh_exts, riscv_ext_f_depends), __RISCV_ISA_EXT_DATA_VALIDATE(zfhmin, RISCV_ISA_EXT_ZFHMIN, riscv_ext_f_depends), __RISCV_ISA_EXT_DATA(zca, RISCV_ISA_EXT_ZCA), __RISCV_ISA_EXT_DATA_VALIDATE(zcb, RISCV_ISA_EXT_ZCB, riscv_ext_zca_depends), @@ -559,7 +600,9 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64x, RISCV_ISA_EXT_ZVE64X, riscv_zve64x_exts, riscv_ext_vector_x_validate), __RISCV_ISA_EXT_DATA_VALIDATE(zvfbfmin, RISCV_ISA_EXT_ZVFBFMIN, riscv_vector_f_validate), __RISCV_ISA_EXT_DATA_VALIDATE(zvfbfwma, RISCV_ISA_EXT_ZVFBFWMA, riscv_ext_zvfbfwma_validate), - __RISCV_ISA_EXT_DATA(zvfh, RISCV_ISA_EXT_ZVFH), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zvfh, RISCV_ISA_EXT_ZVFH, + riscv_zvfh_exts, + riscv_ext_vector_float_validate), __RISCV_ISA_EXT_DATA(zvfhmin, RISCV_ISA_EXT_ZVFHMIN), __RISCV_ISA_EXT_DATA_VALIDATE(zvkb, RISCV_ISA_EXT_ZVKB, riscv_ext_vector_crypto_validate), __RISCV_ISA_EXT_DATA_VALIDATE(zvkg, RISCV_ISA_EXT_ZVKG, riscv_ext_vector_crypto_validate), @@ -576,12 +619,19 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvksg, riscv_zvksg_bundled_exts, riscv_ext_vector_crypto_validate), __RISCV_ISA_EXT_DATA_VALIDATE(zvkt, RISCV_ISA_EXT_ZVKT, riscv_ext_vector_crypto_validate), __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA), + __RISCV_ISA_EXT_DATA_VALIDATE(smcdeleg, RISCV_ISA_EXT_SMCDELEG, + riscv_ext_smcdeleg_validate), + __RISCV_ISA_EXT_DATA(smcntrpmf, RISCV_ISA_EXT_SMCNTRPMF), + __RISCV_ISA_EXT_DATA(smcsrind, RISCV_ISA_EXT_SMCSRIND), __RISCV_ISA_EXT_DATA(smmpm, RISCV_ISA_EXT_SMMPM), __RISCV_ISA_EXT_SUPERSET(smnpm, RISCV_ISA_EXT_SMNPM, riscv_xlinuxenvcfg_exts), __RISCV_ISA_EXT_DATA(smstateen, RISCV_ISA_EXT_SMSTATEEN), __RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA), + __RISCV_ISA_EXT_DATA_VALIDATE(ssccfg, RISCV_ISA_EXT_SSCCFG, riscv_ext_ssccfg_validate), __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF), + __RISCV_ISA_EXT_DATA(sscsrind, RISCV_ISA_EXT_SSCSRIND), __RISCV_ISA_EXT_SUPERSET(ssnpm, RISCV_ISA_EXT_SSNPM, riscv_xlinuxenvcfg_exts), + __RISCV_ISA_EXT_DATA(ssqosid, RISCV_ISA_EXT_SSQOSID), __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC), __RISCV_ISA_EXT_DATA(svade, RISCV_ISA_EXT_SVADE), __RISCV_ISA_EXT_DATA_VALIDATE(svadu, RISCV_ISA_EXT_SVADU, riscv_ext_svadu_validate), @@ -897,7 +947,7 @@ static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap) * marchid. */ if (acpi_disabled && boot_vendorid == THEAD_VENDOR_ID && boot_archid == 0x0) - clear_bit(RISCV_ISA_EXT_v, source_isa); + clear_bit(RISCV_ISA_EXT_V, source_isa); riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap); @@ -1105,13 +1155,13 @@ void __init riscv_fill_hwcap(void) unsigned long isa2hwcap[RISCV_ISA_EXT_BASE] = {0}; int i, j; - isa2hwcap[RISCV_ISA_EXT_i] = COMPAT_HWCAP_ISA_I; - isa2hwcap[RISCV_ISA_EXT_m] = COMPAT_HWCAP_ISA_M; - isa2hwcap[RISCV_ISA_EXT_a] = COMPAT_HWCAP_ISA_A; - isa2hwcap[RISCV_ISA_EXT_f] = COMPAT_HWCAP_ISA_F; - isa2hwcap[RISCV_ISA_EXT_d] = COMPAT_HWCAP_ISA_D; - isa2hwcap[RISCV_ISA_EXT_c] = COMPAT_HWCAP_ISA_C; - isa2hwcap[RISCV_ISA_EXT_v] = COMPAT_HWCAP_ISA_V; + isa2hwcap[RISCV_ISA_EXT_I] = COMPAT_HWCAP_ISA_I; + isa2hwcap[RISCV_ISA_EXT_M] = COMPAT_HWCAP_ISA_M; + isa2hwcap[RISCV_ISA_EXT_A] = COMPAT_HWCAP_ISA_A; + isa2hwcap[RISCV_ISA_EXT_F] = COMPAT_HWCAP_ISA_F; + isa2hwcap[RISCV_ISA_EXT_D] = COMPAT_HWCAP_ISA_D; + isa2hwcap[RISCV_ISA_EXT_C] = COMPAT_HWCAP_ISA_C; + isa2hwcap[RISCV_ISA_EXT_V] = COMPAT_HWCAP_ISA_V; if (!acpi_disabled) { riscv_fill_hwcap_from_isa_string(isa2hwcap); diff --git a/arch/riscv/kernel/efi.c b/arch/riscv/kernel/efi.c index b64bf1624a05..2d3cc57b4535 100644 --- a/arch/riscv/kernel/efi.c +++ b/arch/riscv/kernel/efi.c @@ -95,3 +95,8 @@ int __init efi_set_mapping_permissions(struct mm_struct *mm, md->num_pages << EFI_PAGE_SHIFT, set_permissions, md); } + +bool efi_poweroff_required(void) +{ + return efi_enabled(EFI_RUNTIME_SERVICES); +} diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c index b430edfb83f4..be8b68514417 100644 --- a/arch/riscv/kernel/ftrace.c +++ b/arch/riscv/kernel/ftrace.c @@ -12,6 +12,7 @@ #include <linux/stop_machine.h> #include <asm/cacheflush.h> #include <asm/text-patching.h> +#include <asm/insn.h> #ifdef CONFIG_DYNAMIC_FTRACE void ftrace_arch_code_modify_prepare(void) @@ -63,7 +64,9 @@ static int __ftrace_modify_call(unsigned long source, unsigned long target, bool if (copy_from_kernel_nofault(replaced, (void *)source, 2 * MCOUNT_INSN_SIZE)) return -EFAULT; - if (replaced[0] != call[0]) { + /* Bypass the check if the auipc insn is a kprobe breakpoint */ + if (replaced[0] != call[0] && + !(riscv_insn_is_ebreak(replaced[0]) || riscv_insn_is_c_ebreak(replaced[0]))) { pr_err("%p: expected (%08x) but got (%08x)\n", (void *)source, call[0], replaced[0]); return -EINVAL; diff --git a/arch/riscv/kernel/kernel_mode_vector.c b/arch/riscv/kernel/kernel_mode_vector.c index 99972a48e86b..77e98b504485 100644 --- a/arch/riscv/kernel/kernel_mode_vector.c +++ b/arch/riscv/kernel/kernel_mode_vector.c @@ -10,18 +10,34 @@ #include <linux/percpu.h> #include <linux/preempt.h> #include <linux/types.h> +#include <linux/kvm_types.h> #include <asm/vector.h> +#include <asm/kvm_vcpu_vector.h> #include <asm/switch_to.h> #include <asm/simd.h> #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE #include <asm/asm-prototypes.h> #endif -static inline void riscv_v_flags_set(u32 flags) +static void (* __rcu kvm_flush_vector_ctx_callback)(void); + +void kvm_riscv_register_vctx_callback(void (*func)(void)) +{ + if (WARN_ON_ONCE(rcu_access_pointer(kvm_flush_vector_ctx_callback))) + return; + + rcu_assign_pointer(kvm_flush_vector_ctx_callback, func); +} +EXPORT_SYMBOL_GPL(kvm_riscv_register_vctx_callback); + +void kvm_riscv_unregister_vctx_callback(void) { - WRITE_ONCE(current->thread.riscv_v_flags, flags); + rcu_assign_pointer(kvm_flush_vector_ctx_callback, NULL); + synchronize_rcu(); } +EXPORT_SYMBOL_GPL(kvm_riscv_unregister_vctx_callback); + static inline void riscv_v_start(u32 flags) { @@ -55,13 +71,16 @@ void get_cpu_vector_context(void) * disable softirqs so it is impossible for softirqs to nest * get_cpu_vector_context() when kernel is actively using Vector. */ - if (!IS_ENABLED(CONFIG_PREEMPT_RT)) - local_bh_disable(); - else + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) { + if (!irqs_disabled()) + local_bh_disable(); + } else { preempt_disable(); + } riscv_v_start(RISCV_KERNEL_MODE_V); } +EXPORT_SYMBOL_FOR_KVM(get_cpu_vector_context); /* * Release the CPU vector context. @@ -74,10 +93,29 @@ void put_cpu_vector_context(void) { riscv_v_stop(RISCV_KERNEL_MODE_V); - if (!IS_ENABLED(CONFIG_PREEMPT_RT)) - local_bh_enable(); - else + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) { + if (!irqs_disabled()) + local_bh_enable(); + } else { preempt_enable(); + } +} +EXPORT_SYMBOL_FOR_KVM(put_cpu_vector_context); + +static void __riscv_flush_vector_context(void) +{ + void (*vcpu_flush_v_callback)(void); + + if (riscv_v_flags() & RISCV_V_VCPU_CTX) { + rcu_read_lock(); + vcpu_flush_v_callback = rcu_dereference(kvm_flush_vector_ctx_callback); + vcpu_flush_v_callback(); + rcu_read_unlock(); + return; + } + + riscv_v_vstate_save(¤t->thread.vstate, task_pt_regs(current)); + riscv_v_vstate_set_restore(current, task_pt_regs(current)); } #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE @@ -121,9 +159,9 @@ static int riscv_v_stop_kernel_context(void) return 0; } -static int riscv_v_start_kernel_context(bool *is_nested) +static int riscv_v_start_kernel_context(void) { - struct __riscv_v_ext_state *kvstate, *uvstate; + struct __riscv_v_ext_state *kvstate; kvstate = ¤t->thread.kernel_vstate; if (!kvstate->datap) @@ -131,7 +169,6 @@ static int riscv_v_start_kernel_context(bool *is_nested) if (riscv_preempt_v_started(current)) { WARN_ON(riscv_v_ctx_get_depth() == 0); - *is_nested = true; get_cpu_vector_context(); if (riscv_preempt_v_dirty(current)) { __riscv_v_vstate_save(kvstate, kvstate->datap); @@ -142,12 +179,18 @@ static int riscv_v_start_kernel_context(bool *is_nested) } /* Transfer the ownership of V from user to kernel, then save */ - riscv_v_start(RISCV_PREEMPT_V | RISCV_PREEMPT_V_DIRTY); - if (__riscv_v_vstate_check(task_pt_regs(current)->status, DIRTY)) { - uvstate = ¤t->thread.vstate; - __riscv_v_vstate_save(uvstate, uvstate->datap); - } - riscv_preempt_v_clear_dirty(current); + get_cpu_vector_context(); + __riscv_flush_vector_context(); + put_cpu_vector_context(); + /* + * A voluntary context switch caused by put_cpu_vector_context() can + * raise the NEED_RESTORE flag if preempt_v starts too early due to a + * failed risv_v_is_on() check. + * + * This causes the next context_nesting_end pollute the v-reg from + * the stale context memory in kernel-mode vector. + */ + riscv_v_start(RISCV_PREEMPT_V); return 0; } @@ -187,7 +230,7 @@ asmlinkage void riscv_v_context_nesting_end(struct pt_regs *regs) } } #else -#define riscv_v_start_kernel_context(nested) (-ENOENT) +#define riscv_v_start_kernel_context() (-ENOENT) #define riscv_v_stop_kernel_context() (-ENOENT) #endif /* CONFIG_RISCV_ISA_V_PREEMPTIVE */ @@ -206,21 +249,16 @@ asmlinkage void riscv_v_context_nesting_end(struct pt_regs *regs) */ void kernel_vector_begin(void) { - bool nested = false; - if (WARN_ON(!(has_vector() || has_xtheadvector()))) return; BUG_ON(!may_use_simd()); - if (riscv_v_start_kernel_context(&nested)) { + if (riscv_v_start_kernel_context()) { get_cpu_vector_context(); - riscv_v_vstate_save(¤t->thread.vstate, task_pt_regs(current)); + __riscv_flush_vector_context(); } - if (!nested) - riscv_v_vstate_set_restore(current, task_pt_regs(current)); - riscv_v_enable(); } EXPORT_SYMBOL_GPL(kernel_vector_begin); diff --git a/arch/riscv/kernel/machine_kexec_file.c b/arch/riscv/kernel/machine_kexec_file.c index 59d4bbc848a8..26cd2a8bd0cd 100644 --- a/arch/riscv/kernel/machine_kexec_file.c +++ b/arch/riscv/kernel/machine_kexec_file.c @@ -45,6 +45,15 @@ static int get_nr_ram_ranges_callback(struct resource *res, void *arg) return 0; } +unsigned int arch_get_system_nr_ranges(void) +{ + unsigned int nr_ranges = 2 + crashk_cma_cnt; /* For exclusion of crashkernel region */ + + walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback); + + return nr_ranges; +} + static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg) { struct crash_mem *cmem = arg; @@ -56,33 +65,9 @@ static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg) return 0; } -static int prepare_elf_headers(void **addr, unsigned long *sz) +int arch_crash_populate_cmem(struct crash_mem *cmem) { - struct crash_mem *cmem; - unsigned int nr_ranges; - int ret; - - nr_ranges = 1; /* For exclusion of crashkernel region */ - walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback); - - cmem = kmalloc_flex(*cmem, ranges, nr_ranges); - if (!cmem) - return -ENOMEM; - - cmem->max_nr_ranges = nr_ranges; - cmem->nr_ranges = 0; - ret = walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback); - if (ret) - goto out; - - /* Exclude crashkernel region */ - ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end); - if (!ret) - ret = crash_prepare_elf64_headers(cmem, true, addr, sz); - -out: - kfree(cmem); - return ret; + return walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback); } static char *setup_kdump_cmdline(struct kimage *image, char *cmdline, @@ -274,7 +259,7 @@ int load_extra_segments(struct kimage *image, unsigned long kernel_start, if (image->type == KEXEC_TYPE_CRASH) { void *headers; unsigned long headers_sz; - ret = prepare_elf_headers(&headers, &headers_sz); + ret = crash_prepare_headers(true, &headers, &headers_sz, NULL); if (ret) { pr_err("Preparing elf core header failed\n"); goto out; diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c index 16b243376f36..2239c28981bc 100644 --- a/arch/riscv/kernel/patch.c +++ b/arch/riscv/kernel/patch.c @@ -45,6 +45,8 @@ static __always_inline void *patch_map(void *addr, const unsigned int fixmap) phys_addr_t phys; if (core_kernel_text(uintaddr) || is_kernel_exittext(uintaddr)) { + if (!IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) + return addr; phys = __pa_symbol(addr); } else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) { struct page *page = vmalloc_to_page(addr); diff --git a/arch/riscv/kernel/probes/decode-insn.c b/arch/riscv/kernel/probes/decode-insn.c index 65d9590bfb9f..433d9035b888 100644 --- a/arch/riscv/kernel/probes/decode-insn.c +++ b/arch/riscv/kernel/probes/decode-insn.c @@ -29,12 +29,12 @@ riscv_probe_decode_insn(probe_opcode_t *addr, struct arch_probe_insn *api) * TODO: the REJECTED ones below need to be implemented */ #ifdef CONFIG_RISCV_ISA_C - RISCV_INSN_REJECTED(c_jal, insn); RISCV_INSN_REJECTED(c_ebreak, insn); RISCV_INSN_SET_SIMULATE(c_j, insn); RISCV_INSN_SET_SIMULATE(c_jr, insn); RISCV_INSN_SET_SIMULATE(c_jalr, insn); + RISCV_INSN_SET_SIMULATE(c_jal, insn); RISCV_INSN_SET_SIMULATE(c_beqz, insn); RISCV_INSN_SET_SIMULATE(c_bnez, insn); #endif diff --git a/arch/riscv/kernel/probes/simulate-insn.c b/arch/riscv/kernel/probes/simulate-insn.c index fa581590c1f8..f8a2f6857877 100644 --- a/arch/riscv/kernel/probes/simulate-insn.c +++ b/arch/riscv/kernel/probes/simulate-insn.c @@ -163,6 +163,13 @@ bool __kprobes simulate_c_j(u32 opcode, unsigned long addr, struct pt_regs *regs return true; } +bool __kprobes simulate_c_jal(u32 opcode, unsigned long addr, struct pt_regs *regs) +{ + regs->ra = addr + 2; + + return simulate_c_j(opcode, addr, regs); +} + static bool __kprobes simulate_c_jr_jalr(u32 opcode, unsigned long addr, struct pt_regs *regs, bool is_jalr) { diff --git a/arch/riscv/kernel/probes/simulate-insn.h b/arch/riscv/kernel/probes/simulate-insn.h index 44ebbc444db9..b89e1bb01842 100644 --- a/arch/riscv/kernel/probes/simulate-insn.h +++ b/arch/riscv/kernel/probes/simulate-insn.h @@ -25,6 +25,7 @@ bool simulate_branch(u32 opcode, unsigned long addr, struct pt_regs *regs); bool simulate_jal(u32 opcode, unsigned long addr, struct pt_regs *regs); bool simulate_jalr(u32 opcode, unsigned long addr, struct pt_regs *regs); bool simulate_c_j(u32 opcode, unsigned long addr, struct pt_regs *regs); +bool simulate_c_jal(u32 opcode, unsigned long addr, struct pt_regs *regs); bool simulate_c_jr(u32 opcode, unsigned long addr, struct pt_regs *regs); bool simulate_c_jalr(u32 opcode, unsigned long addr, struct pt_regs *regs); bool simulate_c_bnez(u32 opcode, unsigned long addr, struct pt_regs *regs); diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index b2df7f72241a..7cc5a6a5c020 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -349,10 +349,8 @@ long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg) if (arg & PR_TAGGED_ADDR_ENABLE && (tagged_addr_disabled || !pmlen)) return -EINVAL; - if (!(arg & PR_TAGGED_ADDR_ENABLE)) { + if (!(arg & PR_TAGGED_ADDR_ENABLE)) pmlen = PMLEN_0; - pmm = ENVCFG_PMM_PMLEN_0; - } if (mmap_write_lock_killable(mm)) return -EINTR; diff --git a/arch/riscv/kernel/qos.c b/arch/riscv/kernel/qos.c new file mode 100644 index 000000000000..99925c0e93dd --- /dev/null +++ b/arch/riscv/kernel/qos.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include <linux/cpu.h> +#include <linux/cpu_pm.h> +#include <linux/cpuhotplug.h> +#include <linux/notifier.h> +#include <linux/percpu-defs.h> +#include <linux/types.h> + +#include <asm/cpufeature-macros.h> +#include <asm/hwcap.h> +#include <asm/qos.h> + +/* + * Cached value of srmcfg csr for each cpu. Seeded to U32_MAX so the next + * __switch_to_srmcfg() unconditionally writes the CSR. The encoding + * MCID << 16 | RCID with both fields well under 16 bits can never + * produce this sentinel. This covers early-boot context switches that + * happen before riscv_srmcfg_init() runs as an arch_initcall. + */ +DEFINE_PER_CPU(u32, cpu_srmcfg) = U32_MAX; + +/* default srmcfg value for each cpu, set via resctrl cpu assignment */ +DEFINE_PER_CPU(u32, cpu_srmcfg_default); + +/* + * Invalidate the per-CPU srmcfg cache. Used as both the cpuhp startup + * and teardown callback. U32_MAX is not a valid srmcfg value + * (MCID << 16 | RCID, both fields under 16 bits), so the next + * __switch_to_srmcfg() always writes the CSR. + * + * Ssqosid leaves the CSR implementation-defined across hart stop/start, + * so the cached value cannot be trusted after online. The startup + * callback runs at CPUHP_AP_ONLINE_DYN, before CPUHP_AP_ACTIVE makes + * the CPU schedulable, so the cache is invalidated before any normal + * task runs and the CSR is written on that task's first switch. + * The teardown callback is not relied on. Idle and per-CPU kthreads keep + * switching as the CPU goes down and overwrite the sentinel with the CPU + * default, so it does not survive the offline period. + */ +static int riscv_srmcfg_reset_cache(unsigned int cpu) +{ + per_cpu(cpu_srmcfg, cpu) = U32_MAX; + return 0; +} + +/* + * CPU PM notifier: invalidate the cached srmcfg on resume from a deep + * idle / suspend. Ssqosid leaves CSR_SRMCFG state across low-power + * transitions implementation-defined, and the boot CPU never goes + * through the cpuhp online callback during system suspend, so without + * this hook __switch_to_srmcfg() would skip the CSR write when the + * outgoing task happens to share its srmcfg with the pre-suspend cache. + */ +static int riscv_srmcfg_pm_notify(struct notifier_block *nb, + unsigned long action, void *unused) +{ + switch (action) { + case CPU_PM_EXIT: + case CPU_PM_ENTER_FAILED: + /* + * The CSR is implementation-defined across the low-power + * transition. Invalidate the cache and eagerly rewrite the + * CSR for the current task so it does not run mis-tagged + * until the next context switch. + */ + __this_cpu_write(cpu_srmcfg, U32_MAX); + __switch_to_srmcfg(current); + break; + } + return NOTIFY_OK; +} + +static struct notifier_block riscv_srmcfg_pm_nb = { + .notifier_call = riscv_srmcfg_pm_notify, +}; + +static int __init riscv_srmcfg_init(void) +{ + int err; + + if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_SSQOSID)) + return 0; + + /* + * cpuhp_setup_state() invokes the startup callback locally on every + * already-online CPU, so no separate seed loop is needed here. + */ + err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "riscv/srmcfg:online", + riscv_srmcfg_reset_cache, riscv_srmcfg_reset_cache); + if (err < 0) + pr_warn("srmcfg: cpuhp setup failed (%d), cache not invalidated on CPU online\n", + err); + + /* + * Register the PM notifier even if the cpuhp setup failed. It is + * independent of the cpuhp state and guards suspend/resume. + */ + cpu_pm_register_notifier(&riscv_srmcfg_pm_nb); + return 0; +} +arch_initcall(riscv_srmcfg_init); diff --git a/arch/riscv/kernel/reset.c b/arch/riscv/kernel/reset.c index 912288572226..14eb08a6db85 100644 --- a/arch/riscv/kernel/reset.c +++ b/arch/riscv/kernel/reset.c @@ -3,10 +3,11 @@ * Copyright (C) 2012 Regents of the University of California */ +#include <linux/efi.h> #include <linux/reboot.h> #include <linux/pm.h> -static void default_power_off(void) +static void __noreturn default_power_off(void) { while (1) wait_for_interrupt(); @@ -17,6 +18,12 @@ EXPORT_SYMBOL(pm_power_off); void machine_restart(char *cmd) { + /* + * UpdateCapsule() depends on the system being reset via ResetSystem(). + */ + if (efi_enabled(EFI_RUNTIME_SERVICES)) + efi_reboot(reboot_mode, NULL); + do_kernel_restart(cmd); while (1); } diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index 52d1d2b8f338..a32344bb220d 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -321,6 +321,8 @@ void __init setup_arch(char **cmdline_p) efi_init(); paging_init(); + acpi_table_upgrade(); + /* Parse the ACPI tables for possible boot-time configuration */ acpi_boot_table_init(); diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c index caf6762427c8..7818e1d32622 100644 --- a/arch/riscv/kernel/sys_hwprobe.c +++ b/arch/riscv/kernel/sys_hwprobe.c @@ -5,9 +5,6 @@ * more details. */ #include <linux/syscalls.h> -#include <linux/completion.h> -#include <linux/atomic.h> -#include <linux/once.h> #include <asm/cacheflush.h> #include <asm/cpufeature.h> #include <asm/hwprobe.h> @@ -88,10 +85,10 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, if (has_fpu()) pair->value |= RISCV_HWPROBE_IMA_FD; - if (riscv_isa_extension_available(NULL, c)) + if (riscv_isa_extension_available(NULL, C)) pair->value |= RISCV_HWPROBE_IMA_C; - if (has_vector() && riscv_isa_extension_available(NULL, v)) + if (has_vector() && riscv_isa_extension_available(NULL, V)) pair->value |= RISCV_HWPROBE_IMA_V; /* @@ -205,6 +202,11 @@ static void hwprobe_isa_ext1(struct riscv_hwprobe *pair, * in the hart_isa bitmap, are made. */ EXT_KEY(isainfo->isa, ZICFISS, pair->value, missing); + EXT_KEY(isainfo->isa, ZICCLSM, pair->value, missing); + EXT_KEY(isainfo->isa, ZICCAMOA, pair->value, missing); + EXT_KEY(isainfo->isa, ZICCIF, pair->value, missing); + EXT_KEY(isainfo->isa, ZICCRSE, pair->value, missing); + EXT_KEY(isainfo->isa, ZA64RS, pair->value, missing); } /* Now turn off reporting features if any CPU is missing it. */ @@ -295,6 +297,8 @@ static u64 hwprobe_vec_misaligned(const struct cpumask *cpus) static void hwprobe_one_pair(struct riscv_hwprobe *pair, const struct cpumask *cpus) { + pair->value = 0; + switch (pair->key) { case RISCV_HWPROBE_KEY_MVENDORID: case RISCV_HWPROBE_KEY_MARCHID: @@ -329,17 +333,14 @@ static void hwprobe_one_pair(struct riscv_hwprobe *pair, break; case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE: - pair->value = 0; if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ)) pair->value = riscv_cboz_block_size; break; case RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE: - pair->value = 0; if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOM)) pair->value = riscv_cbom_block_size; break; case RISCV_HWPROBE_KEY_ZICBOP_BLOCK_SIZE: - pair->value = 0; if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOP)) pair->value = riscv_cbop_block_size; break; @@ -505,32 +506,28 @@ static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs, return 0; } -#ifdef CONFIG_MMU - -static DECLARE_COMPLETION(boot_probes_done); -static atomic_t pending_boot_probes = ATOMIC_INIT(1); - -void riscv_hwprobe_register_async_probe(void) +static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs, + size_t pair_count, size_t cpusetsize, + unsigned long __user *cpus_user, + unsigned int flags) { - atomic_inc(&pending_boot_probes); -} + if (flags & RISCV_HWPROBE_WHICH_CPUS) + return hwprobe_get_cpus(pairs, pair_count, cpusetsize, + cpus_user, flags); -void riscv_hwprobe_complete_async_probe(void) -{ - if (atomic_dec_and_test(&pending_boot_probes)) - complete(&boot_probes_done); + return hwprobe_get_values(pairs, pair_count, cpusetsize, + cpus_user, flags); } -static int complete_hwprobe_vdso_data(void) +#ifdef CONFIG_MMU + +static int __init init_hwprobe_vdso_data(void) { struct vdso_arch_data *avd = vdso_k_arch_data; u64 id_bitsmash = 0; struct riscv_hwprobe pair; int key; - if (unlikely(!atomic_dec_and_test(&pending_boot_probes))) - wait_for_completion(&boot_probes_done); - /* * Initialize vDSO data with the answers for the "all CPUs" case, to * save a syscall in the common case. @@ -558,52 +555,13 @@ static int complete_hwprobe_vdso_data(void) * vDSO should defer to the kernel for exotic cpu masks. */ avd->homogeneous_cpus = id_bitsmash != 0 && id_bitsmash != -1; - - /* - * Make sure all the VDSO values are visible before we look at them. - * This pairs with the implicit "no speculativly visible accesses" - * barrier in the VDSO hwprobe code. - */ - smp_wmb(); - avd->ready = true; - return 0; -} - -static int __init init_hwprobe_vdso_data(void) -{ - struct vdso_arch_data *avd = vdso_k_arch_data; - - /* - * Prevent the vDSO cached values from being used, as they're not ready - * yet. - */ - avd->ready = false; return 0; } arch_initcall_sync(init_hwprobe_vdso_data); -#else - -static int complete_hwprobe_vdso_data(void) { return 0; } - #endif /* CONFIG_MMU */ -static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs, - size_t pair_count, size_t cpusetsize, - unsigned long __user *cpus_user, - unsigned int flags) -{ - DO_ONCE_SLEEPABLE(complete_hwprobe_vdso_data); - - if (flags & RISCV_HWPROBE_WHICH_CPUS) - return hwprobe_get_cpus(pairs, pair_count, cpusetsize, - cpus_user, flags); - - return hwprobe_get_values(pairs, pair_count, cpusetsize, - cpus_user, flags); -} - SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs, size_t, pair_count, size_t, cpusetsize, unsigned long __user *, cpus, unsigned int, flags) diff --git a/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S b/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S index f16deee9e091..15e50934bbd7 100644 --- a/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S +++ b/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S @@ -179,6 +179,23 @@ test_kprobes_c_bnez_addr3: ret SYM_FUNC_END(test_kprobes_c_bnez) +#ifdef CONFIG_32BIT +SYM_FUNC_START(test_kprobes_c_jal) + li a0, 0 + mv a1, ra +test_kprobes_c_jal_addr1: + c.jal 2f + ret +1: li a0, KPROBE_TEST_MAGIC_UPPER + ret +test_kprobes_c_jal_addr2: +2: c.jal 1b + li a2, KPROBE_TEST_MAGIC_LOWER + add a0, a0, a2 + jr a1 +SYM_FUNC_END(test_kprobes_c_jal) +#endif + #endif /* CONFIG_RISCV_ISA_C */ .section .rodata @@ -209,6 +226,10 @@ SYM_DATA_START(test_kprobes_addresses) RISCV_PTR test_kprobes_c_bnez_addr1 RISCV_PTR test_kprobes_c_bnez_addr2 RISCV_PTR test_kprobes_c_bnez_addr3 +#ifdef CONFIG_32BIT + RISCV_PTR test_kprobes_c_jal_addr1 + RISCV_PTR test_kprobes_c_jal_addr2 +#endif #endif /* CONFIG_RISCV_ISA_C */ RISCV_PTR 0 SYM_DATA_END(test_kprobes_addresses) @@ -226,6 +247,9 @@ SYM_DATA_START(test_kprobes_functions) RISCV_PTR test_kprobes_c_jalr RISCV_PTR test_kprobes_c_beqz RISCV_PTR test_kprobes_c_bnez +#ifdef CONFIG_32BIT + RISCV_PTR test_kprobes_c_jal +#endif #endif /* CONFIG_RISCV_ISA_C */ RISCV_PTR 0 SYM_DATA_END(test_kprobes_functions) diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index 8c62c771a656..f8292adda099 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -7,7 +7,6 @@ #include <linux/kernel.h> #include <linux/init.h> #include <linux/irqflags.h> -#include <linux/randomize_kstack.h> #include <linux/sched.h> #include <linux/sched/debug.h> #include <linux/sched/signal.h> @@ -270,6 +269,7 @@ static bool probe_single_step_handler(struct pt_regs *regs) return user ? uprobe_single_step_handler(regs) : kprobe_single_step_handler(regs); } +NOKPROBE_SYMBOL(probe_single_step_handler); static bool probe_breakpoint_handler(struct pt_regs *regs) { @@ -277,6 +277,7 @@ static bool probe_breakpoint_handler(struct pt_regs *regs) return user ? uprobe_breakpoint_handler(regs) : kprobe_breakpoint_handler(regs); } +NOKPROBE_SYMBOL(probe_breakpoint_handler); void handle_break(struct pt_regs *regs) { @@ -301,6 +302,7 @@ void handle_break(struct pt_regs *regs) else die(regs, "Kernel BUG"); } +NOKPROBE_SYMBOL(handle_break); asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs) { @@ -333,15 +335,12 @@ void do_trap_ecall_u(struct pt_regs *regs) riscv_v_vstate_discard(regs); - syscall = syscall_enter_from_user_mode(regs, syscall); - - add_random_kstack_offset(); - - if (syscall >= 0 && syscall < NR_syscalls) { - syscall = array_index_nospec(syscall, NR_syscalls); - syscall_handler(regs, syscall); + if (likely(syscall_enter_from_user_mode_randomize_stack(regs, &syscall))) { + if (syscall >= 0 && syscall < NR_syscalls) { + syscall = array_index_nospec(syscall, NR_syscalls); + syscall_handler(regs, syscall); + } } - syscall_exit_to_user_mode(regs); } else { irqentry_state_t state = irqentry_nmi_enter(regs); diff --git a/arch/riscv/kernel/unaligned_access_speed.c b/arch/riscv/kernel/unaligned_access_speed.c index 5a5aa22124e7..67f1a0371659 100644 --- a/arch/riscv/kernel/unaligned_access_speed.c +++ b/arch/riscv/kernel/unaligned_access_speed.c @@ -6,7 +6,6 @@ #include <linux/cpu.h> #include <linux/cpumask.h> #include <linux/jump_label.h> -#include <linux/kthread.h> #include <linux/mm.h> #include <linux/smp.h> #include <linux/types.h> @@ -288,18 +287,9 @@ free: __free_pages(page, MISALIGNED_BUFFER_ORDER); } -/* Measure unaligned access speed on all CPUs present at boot in parallel. */ -static int vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) -{ - schedule_on_each_cpu(check_vector_unaligned_access); - riscv_hwprobe_complete_async_probe(); - - return 0; -} #else /* CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS */ -static int vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) +static void check_vector_unaligned_access(struct work_struct *work __always_unused) { - return 0; } #endif @@ -387,12 +377,7 @@ static int __init check_unaligned_access_all_cpus(void) per_cpu(vector_misaligned_access, cpu) = unaligned_vector_speed_param; } else if (!check_vector_unaligned_access_emulated_all_cpus() && IS_ENABLED(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS)) { - riscv_hwprobe_register_async_probe(); - if (IS_ERR(kthread_run(vec_check_unaligned_access_speed_all_cpus, - NULL, "vec_check_unaligned_access_speed_all_cpus"))) { - pr_warn("Failed to create vec_unalign_check kthread\n"); - riscv_hwprobe_complete_async_probe(); - } + schedule_on_each_cpu(check_vector_unaligned_access); } /* @@ -411,4 +396,10 @@ static int __init check_unaligned_access_all_cpus(void) return 0; } -late_initcall(check_unaligned_access_all_cpus); +/* + * Run after clocksource_done_booting() so measure_cycles() uses a stable + * clocksource, but before rootfs_initcall() enables usermode helpers. Those + * helpers can reach hwprobe and populate the vDSO cache, so async hwprobe + * probes must be registered first. + */ +fs_initcall_sync(check_unaligned_access_all_cpus); diff --git a/arch/riscv/kernel/usercfi.c b/arch/riscv/kernel/usercfi.c index 5a7113d69bad..dec0ba5eff5e 100644 --- a/arch/riscv/kernel/usercfi.c +++ b/arch/riscv/kernel/usercfi.c @@ -110,7 +110,7 @@ void set_indir_lp_lock(struct task_struct *task, bool lock) } /* * The shadow stack only stores the return address and not any variables - * this should be more than sufficient for most applications. + * 512M should be more than sufficient for most applications. * Else PAGE_ALIGN it and return back */ static unsigned long calc_shstk_size(unsigned long size) @@ -118,7 +118,7 @@ static unsigned long calc_shstk_size(unsigned long size) if (size) return PAGE_ALIGN(size); - return PAGE_ALIGN(min(rlimit(RLIMIT_STACK) / 2, SZ_2G)); + return PAGE_ALIGN(min(rlimit(RLIMIT_STACK) / 8, SZ_512M)); } /* @@ -525,9 +525,8 @@ static int __init setup_global_riscv_enable(char *str) if (riscv_nousercfi) pr_info("RISC-V user CFI disabled via cmdline - shadow stack status : %s, landing pad status : %s\n", - (riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_BCFI) ? "disabled" : - "enabled", (riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_FCFI) ? - "disabled" : "enabled"); + str_disabled_enabled(riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_BCFI), + str_disabled_enabled(riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_FCFI)); return 1; } diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index 43ee881f6c6f..8dbf2532a573 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -41,6 +41,8 @@ ccflags-y += $(CFI_FULL) asflags-y += $(KBUILD_BASE_ISA)$(CFI_MARCH) asflags-y += $(CFI_FULL) +ccflags-remove-y += $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_LTO) + ifneq ($(c-gettimeofday-y),) CFLAGS_vgettimeofday.o += -fPIC -include $(c-gettimeofday-y) endif @@ -68,11 +70,6 @@ ifneq ($(filter vgettimeofday, $(vdso-syms)),) CPPFLAGS_$(vdso_lds) += -DHAS_VGETTIMEOFDAY endif -# Disable -pg to prevent insert call site -CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_LTO) -CFLAGS_REMOVE_getrandom.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_LTO) -CFLAGS_REMOVE_hwprobe.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_LTO) - # Force dependency $(obj)/$(vdso_o): $(obj)/$(vdso_so) diff --git a/arch/riscv/kernel/vdso/hwprobe.c b/arch/riscv/kernel/vdso/hwprobe.c index 8f45500d0a6e..2ddeba6c68dd 100644 --- a/arch/riscv/kernel/vdso/hwprobe.c +++ b/arch/riscv/kernel/vdso/hwprobe.c @@ -27,7 +27,7 @@ static int riscv_vdso_get_values(struct riscv_hwprobe *pairs, size_t pair_count, * homogeneous, then this function can handle requests for arbitrary * masks. */ - if (flags != 0 || (!all_cpus && !avd->homogeneous_cpus) || unlikely(!avd->ready)) + if ((flags != 0) || (!all_cpus && !avd->homogeneous_cpus)) return riscv_hwprobe(pairs, pair_count, cpusetsize, cpus, flags); /* This is something we can handle, fill out the pairs. */ diff --git a/arch/riscv/kernel/vdso/vdso.lds.S b/arch/riscv/kernel/vdso/vdso.lds.S index c29ef12a63bb..5e9424df932b 100644 --- a/arch/riscv/kernel/vdso/vdso.lds.S +++ b/arch/riscv/kernel/vdso/vdso.lds.S @@ -45,9 +45,11 @@ SECTIONS .text : { *(.text .text.*) } :text . = ALIGN(4); + __vdso_alternatives_start = .; .alternative : { *(.alternative) } + __vdso_alternatives_end = .; } /* diff --git a/arch/riscv/kernel/vmlinux-xip.lds.S b/arch/riscv/kernel/vmlinux-xip.lds.S deleted file mode 100644 index a7611789bad5..000000000000 --- a/arch/riscv/kernel/vmlinux-xip.lds.S +++ /dev/null @@ -1,143 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 Regents of the University of California - * Copyright (C) 2017 SiFive - * Copyright (C) 2020 Vitaly Wool, Konsulko AB - */ - -#include <asm/pgtable.h> -#define LOAD_OFFSET KERNEL_LINK_ADDR -/* No __ro_after_init data in the .rodata section - which will always be ro */ -#define RO_AFTER_INIT_DATA - -#include <asm/vmlinux.lds.h> -#include <asm/page.h> -#include <asm/cache.h> -#include <asm/thread_info.h> -#include <asm/set_memory.h> - -OUTPUT_ARCH(riscv) -ENTRY(_start) - -jiffies = jiffies_64; - -SECTIONS -{ - /* Beginning of code and text segment */ - . = LOAD_OFFSET; - _xiprom = .; - _start = .; - HEAD_TEXT_SECTION - INIT_TEXT_SECTION(PAGE_SIZE) - /* we have to discard exit text and such at runtime, not link time */ - __exittext_begin = .; - .exit.text : - { - EXIT_TEXT - } - __exittext_end = .; - - .text : { - _text = .; - _stext = .; - TEXT_TEXT - SCHED_TEXT - LOCK_TEXT - KPROBES_TEXT - ENTRY_TEXT - IRQENTRY_TEXT - SOFTIRQENTRY_TEXT - _etext = .; - } - RO_DATA(L1_CACHE_BYTES) - .srodata : { - *(.srodata*) - } - .init.rodata : { - INIT_SETUP(16) - INIT_CALLS - CON_INITCALL - INIT_RAM_FS - } - _exiprom = .; /* End of XIP ROM area */ - - -/* - * From this point, stuff is considered writable and will be copied to RAM - */ - __data_loc = ALIGN(PAGE_SIZE); /* location in file */ - . = ALIGN(SECTION_ALIGN); /* location in memory */ - -#undef LOAD_OFFSET -#define LOAD_OFFSET (KERNEL_LINK_ADDR + _sdata - __data_loc) - - _sdata = .; /* Start of data section */ - _data = .; - RW_DATA(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE) - _edata = .; - __start_ro_after_init = .; - .data.ro_after_init : AT(ADDR(.data.ro_after_init) - LOAD_OFFSET) { - *(.data..ro_after_init) - } - __end_ro_after_init = .; - - . = ALIGN(PAGE_SIZE); - __init_begin = .; - .init.data : { - INIT_DATA - } - .exit.data : { - EXIT_DATA - } - . = ALIGN(8); - __soc_early_init_table : { - __soc_early_init_table_start = .; - KEEP(*(__soc_early_init_table)) - __soc_early_init_table_end = .; - } - __soc_builtin_dtb_table : { - __soc_builtin_dtb_table_start = .; - KEEP(*(__soc_builtin_dtb_table)) - __soc_builtin_dtb_table_end = .; - } - - __init_end = .; - - . = ALIGN(16); - .xip.traps : { - __xip_traps_start = .; - *(.xip.traps) - __xip_traps_end = .; - } - - . = ALIGN(PAGE_SIZE); - .sdata : { - __global_pointer$ = . + 0x800; - *(.sdata*) - *(.sbss*) - } - - BSS_SECTION(PAGE_SIZE, PAGE_SIZE, 0) - - PERCPU_SECTION(L1_CACHE_BYTES) - - .rel.dyn : AT(ADDR(.rel.dyn) - LOAD_OFFSET) { - *(.rel.dyn*) - } - - /* - * End of copied data. We need a dummy section to get its LMA. - * Also located before final ALIGN() as trailing padding is not stored - * in the resulting binary file and useless to copy. - */ - .data.endmark : AT(ADDR(.data.endmark) - LOAD_OFFSET) { } - _edata_loc = LOADADDR(.data.endmark); - - . = ALIGN(PAGE_SIZE); - _end = .; - - STABS_DEBUG - DWARF_DEBUG - - DISCARDS -} |