diff options
Diffstat (limited to 'arch/riscv/include/asm')
63 files changed, 1006 insertions, 426 deletions
diff --git a/arch/riscv/include/asm/Kbuild b/arch/riscv/include/asm/Kbuild index bd5fc9403295..7721b63642f4 100644 --- a/arch/riscv/include/asm/Kbuild +++ b/arch/riscv/include/asm/Kbuild @@ -14,5 +14,6 @@ generic-y += ticket_spinlock.h generic-y += qrwlock.h generic-y += qrwlock_types.h generic-y += qspinlock.h +generic-y += ring_buffer.h generic-y += user.h generic-y += vmlinux.lds.h diff --git a/arch/riscv/include/asm/acpi.h b/arch/riscv/include/asm/acpi.h index 6e13695120bc..cdefa1f2d67d 100644 --- a/arch/riscv/include/asm/acpi.h +++ b/arch/riscv/include/asm/acpi.h @@ -12,6 +12,8 @@ #ifndef _ASM_ACPI_H #define _ASM_ACPI_H +#include <linux/cpuidle.h> + /* Basic configuration for ACPI */ #ifdef CONFIG_ACPI @@ -61,16 +63,30 @@ static inline void arch_fix_phys_package_id(int num, u32 slot) { } void acpi_init_rintc_map(void); struct acpi_madt_rintc *acpi_cpu_get_madt_rintc(int cpu); -static inline u32 get_acpi_id_for_cpu(int cpu) -{ - return acpi_cpu_get_madt_rintc(cpu)->uid; -} int acpi_get_riscv_isa(struct acpi_table_header *table, unsigned int cpu, const char **isa); void acpi_get_cbo_block_size(struct acpi_table_header *table, u32 *cbom_size, u32 *cboz_size, u32 *cbop_size); +acpi_handle acpi_get_riscv_gsi_handle(u32 gsi); + +/* + * RISC-V Functional Fixed Hardware Specification Version v1.0.1, + * Chapter 3.1.2, Table 4: Arch. Context Lost Flags + */ +#define RISCV_LPI_HART_TIMER_CTXT_LOST BIT(0) + +static inline unsigned int arch_get_idle_state_flags(u32 arch_flags) +{ + if (arch_flags & RISCV_LPI_HART_TIMER_CTXT_LOST) + return CPUIDLE_FLAG_TIMER_STOP; + + return 0; +} + +#define arch_get_idle_state_flags arch_get_idle_state_flags + #else static inline void acpi_init_rintc_map(void) { } static inline struct acpi_madt_rintc *acpi_cpu_get_madt_rintc(int cpu) @@ -96,4 +112,6 @@ void acpi_map_cpus_to_nodes(void); static inline void acpi_map_cpus_to_nodes(void) { } #endif /* CONFIG_ACPI_NUMA */ +#define ACPI_TABLE_UPGRADE_MAX_PHYS MEMBLOCK_ALLOC_ACCESSIBLE + #endif /*_ASM_ACPI_H*/ diff --git a/arch/riscv/include/asm/asm-prototypes.h b/arch/riscv/include/asm/asm-prototypes.h index a9988bf21ec8..a0ca9efff267 100644 --- a/arch/riscv/include/asm/asm-prototypes.h +++ b/arch/riscv/include/asm/asm-prototypes.h @@ -5,6 +5,10 @@ #include <linux/ftrace.h> #include <asm-generic/asm-prototypes.h> +long long __lshrdi3(long long a, int b); +long long __ashrdi3(long long a, int b); +long long __ashldi3(long long a, int b); + long long __lshrti3(long long a, int b); long long __ashrti3(long long a, int b); long long __ashlti3(long long a, int b); @@ -40,6 +44,7 @@ asmlinkage void riscv_v_context_nesting_end(struct pt_regs *regs); #define DECLARE_DO_ERROR_INFO(name) asmlinkage void name(struct pt_regs *regs) DECLARE_DO_ERROR_INFO(do_trap_unknown); +DECLARE_DO_ERROR_INFO(do_trap_hardware_error); DECLARE_DO_ERROR_INFO(do_trap_insn_misaligned); DECLARE_DO_ERROR_INFO(do_trap_insn_fault); DECLARE_DO_ERROR_INFO(do_trap_insn_illegal); @@ -51,6 +56,7 @@ DECLARE_DO_ERROR_INFO(do_trap_ecall_u); DECLARE_DO_ERROR_INFO(do_trap_ecall_s); DECLARE_DO_ERROR_INFO(do_trap_ecall_m); DECLARE_DO_ERROR_INFO(do_trap_break); +DECLARE_DO_ERROR_INFO(do_trap_software_check); asmlinkage void ret_from_fork_kernel(void *fn_arg, int (*fn)(void *), struct pt_regs *regs); asmlinkage void ret_from_fork_user(struct pt_regs *regs); diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h index e9e8ba83e632..b8bf842d4c13 100644 --- a/arch/riscv/include/asm/asm.h +++ b/arch/riscv/include/asm/asm.h @@ -34,6 +34,7 @@ #define SZREG __REG_SEL(8, 4) #define LGREG __REG_SEL(3, 2) #define SRLI __REG_SEL(srliw, srli) +#define SLLI __REG_SEL(slliw, slli) #if __SIZEOF_POINTER__ == 8 #ifdef __ASSEMBLER__ diff --git a/arch/riscv/include/asm/assembler.h b/arch/riscv/include/asm/assembler.h index 16931712beab..a8df1999118b 100644 --- a/arch/riscv/include/asm/assembler.h +++ b/arch/riscv/include/asm/assembler.h @@ -80,3 +80,47 @@ .endm #endif /* __ASM_ASSEMBLER_H */ + +#if defined(VDSO_CFI) && (__riscv_xlen == 64) +.macro vdso_lpad, label = 0 +lpad \label +.endm +#else +.macro vdso_lpad, label = 0 +.endm +#endif + +/* + * This macro emits a program property note section identifying + * architecture features which require special handling, mainly for + * use in assembly files included in the VDSO. + */ +#define NT_GNU_PROPERTY_TYPE_0 5 +#define GNU_PROPERTY_RISCV_FEATURE_1_AND 0xc0000000 + +#define GNU_PROPERTY_RISCV_FEATURE_1_ZICFILP BIT(0) +#define GNU_PROPERTY_RISCV_FEATURE_1_ZICFISS BIT(1) + +#if defined(VDSO_CFI) && (__riscv_xlen == 64) +#define GNU_PROPERTY_RISCV_FEATURE_1_DEFAULT \ + (GNU_PROPERTY_RISCV_FEATURE_1_ZICFILP | GNU_PROPERTY_RISCV_FEATURE_1_ZICFISS) +#endif + +#ifdef GNU_PROPERTY_RISCV_FEATURE_1_DEFAULT +.macro emit_riscv_feature_1_and, feat = GNU_PROPERTY_RISCV_FEATURE_1_DEFAULT + .pushsection .note.gnu.property, "a" + .p2align 3 + .word 4 + .word 16 + .word NT_GNU_PROPERTY_TYPE_0 + .asciz "GNU" + .word GNU_PROPERTY_RISCV_FEATURE_1_AND + .word 4 + .word \feat + .word 0 + .popsection +.endm +#else +.macro emit_riscv_feature_1_and, feat = 0 +.endm +#endif diff --git a/arch/riscv/include/asm/atomic.h b/arch/riscv/include/asm/atomic.h index 3f33dc54f94b..616b8b332ac5 100644 --- a/arch/riscv/include/asm/atomic.h +++ b/arch/riscv/include/asm/atomic.h @@ -46,7 +46,7 @@ static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i) #endif /* - * First, the atomic ops that have no ordering constraints and therefor don't + * First, the atomic ops that have no ordering constraints and therefore don't * have the AQ or RL bits set. These don't return anything, so there's only * one version to worry about. */ @@ -81,7 +81,7 @@ ATOMIC_OPS(xor, xor, i) /* * Atomic ops that have ordered, relaxed, acquire, and release variants. - * There's two flavors of these: the arithmatic ops have both fetch and return + * There's two flavors of these: the arithmetic ops have both fetch and return * versions, while the logical ops only have fetch versions. */ #define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix) \ diff --git a/arch/riscv/include/asm/bitrev.h b/arch/riscv/include/asm/bitrev.h new file mode 100644 index 000000000000..4b9b8d34cc3b --- /dev/null +++ b/arch/riscv/include/asm/bitrev.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_BITREV_H +#define __ASM_BITREV_H + +#include <linux/types.h> +#include <asm/cpufeature-macros.h> +#include <asm/hwcap.h> +#include <asm-generic/bitops/__bitrev.h> + +static __always_inline __attribute_const__ u32 __arch_bitrev32(u32 x) +{ + unsigned long result; + + if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBKB)) + return generic___bitrev32(x); + + asm volatile( + ".option push\n" + ".option arch,+zbkb\n" + "rev8 %0, %1\n" + "brev8 %0, %0\n" + ".option pop" + : "=r" (result) : "r" ((long)x) + ); + + return result >> (__riscv_xlen - 32); +} + +static __always_inline __attribute_const__ u16 __arch_bitrev16(u16 x) +{ + return __arch_bitrev32(x) >> 16; +} + +static __always_inline __attribute_const__ u8 __arch_bitrev8(u8 x) +{ + unsigned long result; + + if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBKB)) + return generic___bitrev8(x); + + asm volatile( + ".option push\n" + ".option arch,+zbkb\n" + "brev8 %0, %1\n" + ".option pop" + : "=r" (result) : "r" ((long)x) + ); + + return result; +} +#endif diff --git a/arch/riscv/include/asm/bug.h b/arch/riscv/include/asm/bug.h index 6f581b84d8fc..699c0cf3e4ef 100644 --- a/arch/riscv/include/asm/bug.h +++ b/arch/riscv/include/asm/bug.h @@ -29,13 +29,8 @@ typedef u32 bug_insn_t; -#ifdef CONFIG_GENERIC_BUG_RELATIVE_POINTERS #define __BUG_ENTRY_ADDR RISCV_INT " 1b - ." #define __BUG_ENTRY_FILE(file) RISCV_INT " " file " - ." -#else -#define __BUG_ENTRY_ADDR RISCV_PTR " 1b" -#define __BUG_ENTRY_FILE(file) RISCV_PTR " " file -#endif #ifdef CONFIG_DEBUG_BUGVERBOSE #define __BUG_ENTRY(file, line, flags) \ diff --git a/arch/riscv/include/asm/cacheflush.h b/arch/riscv/include/asm/cacheflush.h index 0092513c3376..c2b0a2928f06 100644 --- a/arch/riscv/include/asm/cacheflush.h +++ b/arch/riscv/include/asm/cacheflush.h @@ -40,23 +40,25 @@ do { \ flush_icache_mm(vma->vm_mm, 0); \ } while (0) -#ifdef CONFIG_64BIT -extern u64 new_vmalloc[NR_CPUS / sizeof(u64) + 1]; +#if defined(CONFIG_64BIT) && defined(CONFIG_MMU) +/* This is accessed in assembly code. cpumask_var_t would be too complex. */ +extern DECLARE_BITMAP(new_valid_map_cpus, NR_CPUS); extern char _end[]; +static inline void mark_new_valid_map(void) +{ + /* + * We don't care if concurrently a cpu resets this value since + * the only place this can happen is in handle_exception() where + * an sfence.vma is emitted. + */ + bitmap_fill(new_valid_map_cpus, NR_CPUS); +} #define flush_cache_vmap flush_cache_vmap static inline void flush_cache_vmap(unsigned long start, unsigned long end) { - if (is_vmalloc_or_module_addr((void *)start)) { - int i; - - /* - * We don't care if concurrently a cpu resets this value since - * the only place this can happen is in handle_exception() where - * an sfence.vma is emitted. - */ - for (i = 0; i < ARRAY_SIZE(new_vmalloc); ++i) - new_vmalloc[i] = -1ULL; - } + if (is_vmalloc_or_module_addr((void *)start) || + (start >= VMEMMAP_START && end <= VMEMMAP_END)) + mark_new_valid_map(); } #define flush_cache_vmap_early(start, end) local_flush_tlb_kernel_range(start, end) #endif diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h index 8712cf9c69dc..662e160b0522 100644 --- a/arch/riscv/include/asm/cmpxchg.h +++ b/arch/riscv/include/asm/cmpxchg.h @@ -23,7 +23,10 @@ riscv_has_extension_unlikely(RISCV_ISA_EXT_ZABHA)) { \ __asm__ __volatile__ ( \ prepend \ + " .option push\n" \ + " .option arch, +zabha\n" \ " amoswap" swap_sfx " %0, %z2, %1\n" \ + " .option pop\n" \ swap_append \ : "=&r" (r), "+A" (*(p)) \ : "rJ" (n) \ @@ -141,7 +144,10 @@ \ __asm__ __volatile__ ( \ cas_prepend \ + " .option push\n" \ + " .option arch, +zacas, +zabha\n" \ " amocas" cas_sfx " %0, %z2, %1\n" \ + " .option pop\n" \ cas_append \ : "+&r" (r), "+A" (*(p)) \ : "rJ" (n) \ @@ -188,7 +194,10 @@ \ __asm__ __volatile__ ( \ cas_prepend \ + " .option push\n" \ + " .option arch, +zacas\n" \ " amocas" cas_sfx " %0, %z2, %1\n" \ + " .option pop\n" \ cas_append \ : "+&r" (r), "+A" (*(p)) \ : "rJ" (n) \ @@ -340,7 +349,10 @@ union __u128_halves { register unsigned long t4 asm ("t4") = __ho.high; \ \ __asm__ __volatile__ ( \ - " amocas.q" cas_sfx " %0, %z3, %2" \ + " .option push\n" \ + " .option arch, +zacas\n" \ + " amocas.q" cas_sfx " %0, %z3, %2\n" \ + " .option pop\n" \ : "+&r" (t3), "+&r" (t4), "+A" (*(p)) \ : "rJ" (t1), "rJ" (t2) \ : "memory"); \ diff --git a/arch/riscv/include/asm/compat.h b/arch/riscv/include/asm/compat.h index 6081327e55f5..28e115eed218 100644 --- a/arch/riscv/include/asm/compat.h +++ b/arch/riscv/include/asm/compat.h @@ -2,7 +2,7 @@ #ifndef __ASM_COMPAT_H #define __ASM_COMPAT_H -#define COMPAT_UTS_MACHINE "riscv\0\0" +#define COMPAT_UTS_MACHINE "riscv32\0\0" /* * Architecture specific compatibility types diff --git a/arch/riscv/include/asm/cpu_ops.h b/arch/riscv/include/asm/cpu_ops.h index 176b570ef982..065811fca594 100644 --- a/arch/riscv/include/asm/cpu_ops.h +++ b/arch/riscv/include/asm/cpu_ops.h @@ -24,7 +24,7 @@ struct cpu_operations { struct task_struct *tidle); #ifdef CONFIG_HOTPLUG_CPU void (*cpu_stop)(void); - int (*cpu_is_stopped)(unsigned int cpu); + bool (*cpu_is_stopped)(unsigned int cpu); #endif }; diff --git a/arch/riscv/include/asm/cpufeature.h b/arch/riscv/include/asm/cpufeature.h index 62837fa981e8..739fcc84bf7b 100644 --- a/arch/riscv/include/asm/cpufeature.h +++ b/arch/riscv/include/asm/cpufeature.h @@ -152,4 +152,16 @@ static __always_inline bool riscv_cpu_has_extension_unlikely(int cpu, const unsi return __riscv_isa_extension_available(hart_isa[cpu].isa, ext); } +static inline bool cpu_supports_shadow_stack(void) +{ + return (IS_ENABLED(CONFIG_RISCV_USER_CFI) && + riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICFISS)); +} + +static inline bool cpu_supports_indirect_br_lp_instr(void) +{ + return (IS_ENABLED(CONFIG_RISCV_USER_CFI) && + riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICFILP)); +} + #endif diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h index 4a37a98398ad..6c823361be86 100644 --- a/arch/riscv/include/asm/csr.h +++ b/arch/riscv/include/asm/csr.h @@ -18,6 +18,15 @@ #define SR_MPP _AC(0x00001800, UL) /* Previously Machine */ #define SR_SUM _AC(0x00040000, UL) /* Supervisor User Memory Access */ +/* zicfilp landing pad status bit */ +#define SR_SPELP _AC(0x00800000, UL) +#define SR_MPELP _AC(0x020000000000, UL) +#ifdef CONFIG_RISCV_M_MODE +#define SR_ELP SR_MPELP +#else +#define SR_ELP SR_SPELP +#endif + #define SR_FS _AC(0x00006000, UL) /* Floating-point Status */ #define SR_FS_OFF _AC(0x00000000, UL) #define SR_FS_INITIAL _AC(0x00002000, UL) @@ -75,6 +84,10 @@ #define SATP_ASID_MASK _AC(0xFFFF, UL) #endif +/* SRMCFG fields */ +#define SRMCFG_RCID_MASK GENMASK(11, 0) +#define SRMCFG_MCID_MASK GENMASK(27, 16) + /* Exception cause high bit - is an interrupt if set */ #define CAUSE_IRQ_FLAG (_AC(1, UL) << (__riscv_xlen - 1)) @@ -108,6 +121,7 @@ #define EXC_INST_PAGE_FAULT 12 #define EXC_LOAD_PAGE_FAULT 13 #define EXC_STORE_PAGE_FAULT 15 +#define EXC_SOFTWARE_CHECK 18 #define EXC_INST_GUEST_PAGE_FAULT 20 #define EXC_LOAD_GUEST_PAGE_FAULT 21 #define EXC_VIRTUAL_INST_FAULT 22 @@ -212,6 +226,8 @@ #define ENVCFG_PMM_PMLEN_16 (_AC(0x3, ULL) << 32) #define ENVCFG_CBZE (_AC(1, UL) << 7) #define ENVCFG_CBCFE (_AC(1, UL) << 6) +#define ENVCFG_LPE (_AC(1, UL) << 2) +#define ENVCFG_SSE (_AC(1, UL) << 3) #define ENVCFG_CBIE_SHIFT 4 #define ENVCFG_CBIE (_AC(0x3, UL) << ENVCFG_CBIE_SHIFT) #define ENVCFG_CBIE_ILL _AC(0x0, UL) @@ -230,6 +246,23 @@ #define SMSTATEEN0_HSENVCFG (_ULL(1) << SMSTATEEN0_HSENVCFG_SHIFT) #define SMSTATEEN0_SSTATEEN0_SHIFT 63 #define SMSTATEEN0_SSTATEEN0 (_ULL(1) << SMSTATEEN0_SSTATEEN0_SHIFT) +/* HPMEVENT bits. These are accessible in S-mode via Smcdeleg/Ssccfg */ +#define HPMEVENT_OF (BIT_ULL(63)) +#define HPMEVENT_MINH (BIT_ULL(62)) +#define HPMEVENT_SINH (BIT_ULL(61)) +#define HPMEVENT_UINH (BIT_ULL(60)) +#define HPMEVENT_VSINH (BIT_ULL(59)) +#define HPMEVENT_VUINH (BIT_ULL(58)) +#ifndef CONFIG_64BIT +#define HPMEVENTH_OF (BIT(31)) +#define HPMEVENTH_MINH (BIT(30)) +#define HPMEVENTH_SINH (BIT(29)) +#define HPMEVENTH_UINH (BIT(28)) +#define HPMEVENTH_VSINH (BIT(27)) +#define HPMEVENTH_VUINH (BIT(26)) +#endif + +#define SISELECT_SSCCFG_BASE 0x40 /* mseccfg bits */ #define MSECCFG_PMM ENVCFG_PMM @@ -311,16 +344,21 @@ #define CSR_SCOUNTEREN 0x106 #define CSR_SENVCFG 0x10a #define CSR_SSTATEEN0 0x10c +#define CSR_SCOUNTINHIBIT 0x120 #define CSR_SSCRATCH 0x140 #define CSR_SEPC 0x141 #define CSR_SCAUSE 0x142 #define CSR_STVAL 0x143 #define CSR_SIP 0x144 #define CSR_SATP 0x180 +#define CSR_SRMCFG 0x181 #define CSR_STIMECMP 0x14D #define CSR_STIMECMPH 0x15D +/* zicfiss user mode csr. CSR_SSP holds current shadow stack pointer */ +#define CSR_SSP 0x011 + /* xtheadvector symbolic CSR names */ #define CSR_VXSAT 0x9 #define CSR_VXRM 0xa @@ -333,6 +371,12 @@ /* Supervisor-Level Window to Indirectly Accessed Registers (AIA) */ #define CSR_SISELECT 0x150 #define CSR_SIREG 0x151 +/* Supervisor-Level Window to Indirectly Accessed Registers (Sscsrind) */ +#define CSR_SIREG2 0x152 +#define CSR_SIREG3 0x153 +#define CSR_SIREG4 0x155 +#define CSR_SIREG5 0x156 +#define CSR_SIREG6 0x157 /* Supervisor-Level Interrupts (AIA) */ #define CSR_STOPEI 0x15c @@ -380,6 +424,14 @@ /* VS-Level Window to Indirectly Accessed Registers (H-extension with AIA) */ #define CSR_VSISELECT 0x250 #define CSR_VSIREG 0x251 +/* + * VS-Level Window to Indirectly Accessed Registers (H-extension with Sscsrind) + */ +#define CSR_VSIREG2 0x252 +#define CSR_VSIREG3 0x253 +#define CSR_VSIREG4 0x255 +#define CSR_VSIREG5 0x256 +#define CSR_VSIREG6 0x257 /* VS-Level Interrupts (H-extension with AIA) */ #define CSR_VSTOPEI 0x25c @@ -422,6 +474,12 @@ /* Machine-Level Window to Indirectly Accessed Registers (AIA) */ #define CSR_MISELECT 0x350 #define CSR_MIREG 0x351 +/* Machine-Level Window to Indirectly Accessed Registers (Smcsrind) */ +#define CSR_MIREG2 0x352 +#define CSR_MIREG3 0x353 +#define CSR_MIREG4 0x355 +#define CSR_MIREG5 0x356 +#define CSR_MIREG6 0x357 /* Machine-Level Interrupts (AIA) */ #define CSR_MTOPEI 0x35c @@ -444,6 +502,23 @@ #define CSR_VTYPE 0xc21 #define CSR_VLENB 0xc22 +#define VTYPE_VLMUL _AC(7, UL) +#define VTYPE_VLMUL_FRAC _AC(4, UL) +#define VTYPE_VSEW_SHIFT 3 +#define VTYPE_VSEW (_AC(7, UL) << VTYPE_VSEW_SHIFT) +#define VTYPE_VTA_SHIFT 6 +#define VTYPE_VTA (_AC(1, UL) << VTYPE_VTA_SHIFT) +#define VTYPE_VMA_SHIFT 7 +#define VTYPE_VMA (_AC(1, UL) << VTYPE_VMA_SHIFT) +#define VTYPE_VILL_SHIFT (__riscv_xlen - 1) +#define VTYPE_VILL (_AC(1, UL) << VTYPE_VILL_SHIFT) + +#define VTYPE_VLMUL_THEAD _AC(3, UL) +#define VTYPE_VSEW_THEAD_SHIFT 2 +#define VTYPE_VSEW_THEAD (_AC(7, UL) << VTYPE_VSEW_THEAD_SHIFT) +#define VTYPE_VEDIV_THEAD_SHIFT 5 +#define VTYPE_VEDIV_THEAD (_AC(3, UL) << VTYPE_VEDIV_THEAD_SHIFT) + /* Scalar Crypto Extension - Entropy */ #define CSR_SEED 0x015 #define SEED_OPST_MASK _AC(0xC0000000, UL) @@ -467,6 +542,11 @@ # define CSR_IEH CSR_MIEH # define CSR_ISELECT CSR_MISELECT # define CSR_IREG CSR_MIREG +# define CSR_IREG2 CSR_MIREG2 +# define CSR_IREG3 CSR_MIREG3 +# define CSR_IREG4 CSR_MIREG4 +# define CSR_IREG5 CSR_MIREG5 +# define CSR_IREG6 CSR_MIREG6 # define CSR_IPH CSR_MIPH # define CSR_TOPEI CSR_MTOPEI # define CSR_TOPI CSR_MTOPI @@ -492,6 +572,11 @@ # define CSR_IEH CSR_SIEH # define CSR_ISELECT CSR_SISELECT # define CSR_IREG CSR_SIREG +# define CSR_IREG2 CSR_SIREG2 +# define CSR_IREG3 CSR_SIREG3 +# define CSR_IREG4 CSR_SIREG4 +# define CSR_IREG5 CSR_SIREG5 +# define CSR_IREG6 CSR_SIREG6 # define CSR_IPH CSR_SIPH # define CSR_TOPEI CSR_STOPEI # define CSR_TOPI CSR_STOPI diff --git a/arch/riscv/include/asm/csr_indirect.h b/arch/riscv/include/asm/csr_indirect.h new file mode 100644 index 000000000000..0f558fac8f5f --- /dev/null +++ b/arch/riscv/include/asm/csr_indirect.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _ASM_RISCV_CSR_INDIRECT_H +#define _ASM_RISCV_CSR_INDIRECT_H + +#include <linux/irqflags.h> + +#include <asm/csr.h> + +/* + * These have to be macros rather than functions: csr_read()/csr_write() + * stringify their CSR argument into the inline asm template via __ASM_STR(), + * so the CSR number must be a literal token at preprocessing time. Passing it + * as a function parameter emits "csrr %0, iregcsr", which no assembler can + * resolve. RISC-V has no register-indirect form of csrr/csrw - the CSR is a + * 12-bit immediate - so the sireg CSR selecting the indirect window cannot + * itself be a variable. + */ +#define csr_indirect_read(iregcsr, iselbase, iseloff) ({ \ + unsigned long __value = 0; \ + unsigned long __flags; \ + local_irq_save(__flags); \ + csr_write(CSR_ISELECT, (iselbase) + (iseloff)); \ + __value = csr_read(iregcsr); \ + local_irq_restore(__flags); \ + __value; \ +}) + +#define csr_indirect_write(iregcsr, iselbase, iseloff, value) ({ \ + unsigned long __flags; \ + local_irq_save(__flags); \ + csr_write(CSR_ISELECT, (iselbase) + (iseloff)); \ + csr_write(iregcsr, (value)); \ + local_irq_restore(__flags); \ +}) + +#define csr_indirect_warl(iregcsr, iselbase, iseloff, warl_val) ({ \ + unsigned long __old_val = 0, __value = 0; \ + unsigned long __flags; \ + local_irq_save(__flags); \ + csr_write(CSR_ISELECT, (iselbase) + (iseloff)); \ + __old_val = csr_read(iregcsr); \ + csr_write(iregcsr, (warl_val)); \ + __value = csr_read(iregcsr); \ + csr_write(iregcsr, __old_val); \ + local_irq_restore(__flags); \ + __value; \ +}) + +#endif diff --git a/arch/riscv/include/asm/elf.h b/arch/riscv/include/asm/elf.h index c7aea7886d22..aa4961b0e208 100644 --- a/arch/riscv/include/asm/elf.h +++ b/arch/riscv/include/asm/elf.h @@ -59,8 +59,8 @@ extern bool compat_elf_check_arch(Elf32_Ehdr *hdr); #endif /* - * Provides information on the availiable set of ISA extensions to userspace, - * via a bitmap that coorespends to each single-letter ISA extension. This is + * Provides information on the available set of ISA extensions to userspace, + * via a bitmap that corresponds to each single-letter ISA extension. This is * essentially defunct, but will remain for compatibility with userspace. */ #define ELF_HWCAP riscv_get_elf_hwcap() diff --git a/arch/riscv/include/asm/entry-common.h b/arch/riscv/include/asm/entry-common.h index b28ccc6cdeea..34ed149af5d1 100644 --- a/arch/riscv/include/asm/entry-common.h +++ b/arch/riscv/include/asm/entry-common.h @@ -40,4 +40,6 @@ static inline int handle_misaligned_store(struct pt_regs *regs) } #endif +bool handle_user_cfi_violation(struct pt_regs *regs); + #endif /* _ASM_RISCV_ENTRY_COMMON_H */ diff --git a/arch/riscv/include/asm/fixmap.h b/arch/riscv/include/asm/fixmap.h index 0a55099bb734..a81823c5ee4a 100644 --- a/arch/riscv/include/asm/fixmap.h +++ b/arch/riscv/include/asm/fixmap.h @@ -55,9 +55,6 @@ enum fixed_addresses { #define __early_set_fixmap __set_fixmap -#define __late_set_fixmap __set_fixmap -#define __late_clear_fixmap(idx) __set_fixmap((idx), 0, FIXMAP_PAGE_CLEAR) - extern void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot); diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 4369a2338541..f8db798b2654 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -10,15 +10,15 @@ #include <uapi/asm/hwcap.h> -#define RISCV_ISA_EXT_a ('a' - 'a') -#define RISCV_ISA_EXT_c ('c' - 'a') -#define RISCV_ISA_EXT_d ('d' - 'a') -#define RISCV_ISA_EXT_f ('f' - 'a') -#define RISCV_ISA_EXT_h ('h' - 'a') -#define RISCV_ISA_EXT_i ('i' - 'a') -#define RISCV_ISA_EXT_m ('m' - 'a') -#define RISCV_ISA_EXT_q ('q' - 'a') -#define RISCV_ISA_EXT_v ('v' - 'a') +#define RISCV_ISA_EXT_A ('a' - 'a') +#define RISCV_ISA_EXT_C ('c' - 'a') +#define RISCV_ISA_EXT_D ('d' - 'a') +#define RISCV_ISA_EXT_F ('f' - 'a') +#define RISCV_ISA_EXT_H ('h' - 'a') +#define RISCV_ISA_EXT_I ('i' - 'a') +#define RISCV_ISA_EXT_M ('m' - 'a') +#define RISCV_ISA_EXT_Q ('q' - 'a') +#define RISCV_ISA_EXT_V ('v' - 'a') /* * These macros represent the logical IDs of each multi-letter RISC-V ISA @@ -110,6 +110,18 @@ #define RISCV_ISA_EXT_ZALASR 101 #define RISCV_ISA_EXT_ZILSD 102 #define RISCV_ISA_EXT_ZCLSD 103 +#define RISCV_ISA_EXT_ZICFILP 104 +#define RISCV_ISA_EXT_ZICFISS 105 +#define RISCV_ISA_EXT_SSCSRIND 106 +#define RISCV_ISA_EXT_SMCSRIND 107 +#define RISCV_ISA_EXT_SMCNTRPMF 108 +#define RISCV_ISA_EXT_SSCCFG 109 +#define RISCV_ISA_EXT_SMCDELEG 110 +#define RISCV_ISA_EXT_SSQOSID 111 +#define RISCV_ISA_EXT_ZICCLSM 112 +#define RISCV_ISA_EXT_ZICCAMOA 113 +#define RISCV_ISA_EXT_ZICCIF 114 +#define RISCV_ISA_EXT_ZA64RS 115 #define RISCV_ISA_EXT_XLINUXENVCFG 127 @@ -119,9 +131,11 @@ #ifdef CONFIG_RISCV_M_MODE #define RISCV_ISA_EXT_SxAIA RISCV_ISA_EXT_SMAIA #define RISCV_ISA_EXT_SUPM RISCV_ISA_EXT_SMNPM +#define RISCV_ISA_EXT_SxCSRIND RISCV_ISA_EXT_SMCSRIND #else #define RISCV_ISA_EXT_SxAIA RISCV_ISA_EXT_SSAIA #define RISCV_ISA_EXT_SUPM RISCV_ISA_EXT_SSNPM +#define RISCV_ISA_EXT_SxCSRIND RISCV_ISA_EXT_SSCSRIND #endif #endif /* _ASM_RISCV_HWCAP_H */ diff --git a/arch/riscv/include/asm/hwprobe.h b/arch/riscv/include/asm/hwprobe.h index 8c572a464719..9b04377c0f98 100644 --- a/arch/riscv/include/asm/hwprobe.h +++ b/arch/riscv/include/asm/hwprobe.h @@ -8,7 +8,7 @@ #include <uapi/asm/hwprobe.h> -#define RISCV_HWPROBE_MAX_KEY 15 +#define RISCV_HWPROBE_MAX_KEY 16 static inline bool riscv_hwprobe_key_is_valid(__s64 key) { @@ -20,6 +20,7 @@ static inline bool hwprobe_key_is_bitmask(__s64 key) switch (key) { case RISCV_HWPROBE_KEY_BASE_BEHAVIOR: case RISCV_HWPROBE_KEY_IMA_EXT_0: + case RISCV_HWPROBE_KEY_IMA_EXT_1: case RISCV_HWPROBE_KEY_CPUPERF_0: case RISCV_HWPROBE_KEY_VENDOR_EXT_THEAD_0: case RISCV_HWPROBE_KEY_VENDOR_EXT_MIPS_0: @@ -42,11 +43,4 @@ static inline bool riscv_hwprobe_pair_cmp(struct riscv_hwprobe *pair, return pair->value == other_pair->value; } -#ifdef CONFIG_MMU -void riscv_hwprobe_register_async_probe(void); -void riscv_hwprobe_complete_async_probe(void); -#else -static inline void riscv_hwprobe_register_async_probe(void) {} -static inline void riscv_hwprobe_complete_async_probe(void) {} -#endif #endif diff --git a/arch/riscv/include/asm/io.h b/arch/riscv/include/asm/io.h index 09bb5f57a9d3..92d5f831f349 100644 --- a/arch/riscv/include/asm/io.h +++ b/arch/riscv/include/asm/io.h @@ -102,12 +102,14 @@ __io_reads_ins(reads, u32, l, __io_br(), __io_ar(addr)) #define readsw(addr, buffer, count) __readsw(addr, buffer, count) #define readsl(addr, buffer, count) __readsl(addr, buffer, count) +#ifdef CONFIG_HAS_IOPORT __io_reads_ins(ins, u8, b, __io_pbr(), __io_par(addr)) __io_reads_ins(ins, u16, w, __io_pbr(), __io_par(addr)) __io_reads_ins(ins, u32, l, __io_pbr(), __io_par(addr)) #define insb(addr, buffer, count) __insb(PCI_IOBASE + (addr), buffer, count) #define insw(addr, buffer, count) __insw(PCI_IOBASE + (addr), buffer, count) #define insl(addr, buffer, count) __insl(PCI_IOBASE + (addr), buffer, count) +#endif __io_writes_outs(writes, u8, b, __io_bw(), __io_aw()) __io_writes_outs(writes, u16, w, __io_bw(), __io_aw()) @@ -116,26 +118,32 @@ __io_writes_outs(writes, u32, l, __io_bw(), __io_aw()) #define writesw(addr, buffer, count) __writesw(addr, buffer, count) #define writesl(addr, buffer, count) __writesl(addr, buffer, count) +#ifdef CONFIG_HAS_IOPORT __io_writes_outs(outs, u8, b, __io_pbw(), __io_paw()) __io_writes_outs(outs, u16, w, __io_pbw(), __io_paw()) __io_writes_outs(outs, u32, l, __io_pbw(), __io_paw()) #define outsb(addr, buffer, count) __outsb(PCI_IOBASE + (addr), buffer, count) #define outsw(addr, buffer, count) __outsw(PCI_IOBASE + (addr), buffer, count) #define outsl(addr, buffer, count) __outsl(PCI_IOBASE + (addr), buffer, count) +#endif #ifdef CONFIG_64BIT __io_reads_ins(reads, u64, q, __io_br(), __io_ar(addr)) #define readsq(addr, buffer, count) __readsq(addr, buffer, count) +#ifdef CONFIG_HAS_IOPORT __io_reads_ins(ins, u64, q, __io_pbr(), __io_par(addr)) #define insq(addr, buffer, count) __insq(PCI_IOBASE + (addr), buffer, count) +#endif __io_writes_outs(writes, u64, q, __io_bw(), __io_aw()) #define writesq(addr, buffer, count) __writesq(addr, buffer, count) +#ifdef CONFIG_HAS_IOPORT __io_writes_outs(outs, u64, q, __io_pbr(), __io_paw()) #define outsq(addr, buffer, count) __outsq(PCI_IOBASE + (addr), buffer, count) #endif +#endif #include <asm-generic/io.h> diff --git a/arch/riscv/include/asm/kfence.h b/arch/riscv/include/asm/kfence.h index d08bf7fb3aee..29cb3a6ee113 100644 --- a/arch/riscv/include/asm/kfence.h +++ b/arch/riscv/include/asm/kfence.h @@ -6,6 +6,7 @@ #include <linux/kfence.h> #include <linux/pfn.h> #include <asm-generic/pgalloc.h> +#include <asm/cacheflush.h> #include <asm/pgtable.h> static inline bool arch_kfence_init_pool(void) @@ -17,10 +18,12 @@ static inline bool kfence_protect_page(unsigned long addr, bool protect) { pte_t *pte = virt_to_kpte(addr); - if (protect) + if (protect) { set_pte(pte, __pte(pte_val(ptep_get(pte)) & ~_PAGE_PRESENT)); - else + } else { set_pte(pte, __pte(pte_val(ptep_get(pte)) | _PAGE_PRESENT)); + mark_new_valid_map(); + } preempt_disable(); local_flush_tlb_kernel_range(addr, addr + PAGE_SIZE); diff --git a/arch/riscv/include/asm/kvm_aia.h b/arch/riscv/include/asm/kvm_aia.h index b04ecdd1a860..ab03f853e507 100644 --- a/arch/riscv/include/asm/kvm_aia.h +++ b/arch/riscv/include/asm/kvm_aia.h @@ -79,7 +79,7 @@ struct kvm_vcpu_aia { #define irqchip_in_kernel(k) ((k)->arch.aia.in_kernel) -extern unsigned int kvm_riscv_aia_nr_hgei; +extern atomic_t kvm_riscv_aia_nr_hgei; extern unsigned int kvm_riscv_aia_max_ids; DECLARE_STATIC_KEY_FALSE(kvm_riscv_aia_available); #define kvm_riscv_aia_available() \ @@ -165,6 +165,9 @@ int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner, void __iomem **hgei_va, phys_addr_t *hgei_pa); void kvm_riscv_aia_free_hgei(int cpu, int hgei); +void kvm_riscv_aia_pm_exit(void); +void kvm_riscv_aia_pm_enter(void); + void kvm_riscv_aia_enable(void); void kvm_riscv_aia_disable(void); int kvm_riscv_aia_init(void); diff --git a/arch/riscv/include/asm/kvm_gstage.h b/arch/riscv/include/asm/kvm_gstage.h index 595e2183173e..aaf080ba1b77 100644 --- a/arch/riscv/include/asm/kvm_gstage.h +++ b/arch/riscv/include/asm/kvm_gstage.h @@ -15,6 +15,7 @@ struct kvm_gstage { #define KVM_GSTAGE_FLAGS_LOCAL BIT(0) unsigned long vmid; pgd_t *pgd; + unsigned long pgd_levels; }; struct kvm_gstage_mapping { @@ -29,16 +30,22 @@ struct kvm_gstage_mapping { #define kvm_riscv_gstage_index_bits 10 #endif -extern unsigned long kvm_riscv_gstage_mode; -extern unsigned long kvm_riscv_gstage_pgd_levels; +extern unsigned long kvm_riscv_gstage_max_pgd_levels; #define kvm_riscv_gstage_pgd_xbits 2 #define kvm_riscv_gstage_pgd_size (1UL << (HGATP_PAGE_SHIFT + kvm_riscv_gstage_pgd_xbits)) -#define kvm_riscv_gstage_gpa_bits (HGATP_PAGE_SHIFT + \ - (kvm_riscv_gstage_pgd_levels * \ - kvm_riscv_gstage_index_bits) + \ - kvm_riscv_gstage_pgd_xbits) -#define kvm_riscv_gstage_gpa_size ((gpa_t)(1ULL << kvm_riscv_gstage_gpa_bits)) + +static inline unsigned long kvm_riscv_gstage_gpa_bits(unsigned long pgd_levels) +{ + return (HGATP_PAGE_SHIFT + + pgd_levels * kvm_riscv_gstage_index_bits + + kvm_riscv_gstage_pgd_xbits); +} + +static inline gpa_t kvm_riscv_gstage_gpa_size(unsigned long pgd_levels) +{ + return BIT_ULL(kvm_riscv_gstage_gpa_bits(pgd_levels)); +} bool kvm_riscv_gstage_get_leaf(struct kvm_gstage *gstage, gpa_t addr, pte_t **ptepp, u32 *ptep_level); @@ -47,26 +54,63 @@ int kvm_riscv_gstage_set_pte(struct kvm_gstage *gstage, struct kvm_mmu_memory_cache *pcache, const struct kvm_gstage_mapping *map); +bool kvm_riscv_gstage_try_update_pte(struct kvm_gstage *gstage, u32 level, + gpa_t addr, pte_t *ptep, + pte_t old_pte, pte_t new_pte); + int kvm_riscv_gstage_map_page(struct kvm_gstage *gstage, struct kvm_mmu_memory_cache *pcache, gpa_t gpa, phys_addr_t hpa, unsigned long page_size, bool page_rdonly, bool page_exec, struct kvm_gstage_mapping *out_map); +bool kvm_riscv_gstage_split_huge(struct kvm_gstage *gstage, + struct kvm_mmu_memory_cache *pcache, + gpa_t addr, u32 target_level, bool flush); + enum kvm_riscv_gstage_op { GSTAGE_OP_NOP = 0, /* Nothing */ GSTAGE_OP_CLEAR, /* Clear/Unmap */ GSTAGE_OP_WP, /* Write-protect */ }; -void kvm_riscv_gstage_op_pte(struct kvm_gstage *gstage, gpa_t addr, +bool kvm_riscv_gstage_op_pte(struct kvm_gstage *gstage, gpa_t addr, pte_t *ptep, u32 ptep_level, enum kvm_riscv_gstage_op op); -void kvm_riscv_gstage_unmap_range(struct kvm_gstage *gstage, +bool kvm_riscv_gstage_unmap_range(struct kvm_gstage *gstage, gpa_t start, gpa_t size, bool may_block); -void kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end); +bool kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end); + +bool kvm_riscv_gstage_wp_pt_masked(struct kvm_gstage *gstage, gfn_t base_gfn, + unsigned long mask); void kvm_riscv_gstage_mode_detect(void); +static inline unsigned long kvm_riscv_gstage_mode(unsigned long pgd_levels) +{ + switch (pgd_levels) { + case 2: + return HGATP_MODE_SV32X4; + case 3: + return HGATP_MODE_SV39X4; + case 4: + return HGATP_MODE_SV48X4; + case 5: + return HGATP_MODE_SV57X4; + default: + WARN_ON_ONCE(1); + return HGATP_MODE_OFF; + } +} + +static inline void kvm_riscv_gstage_init(struct kvm_gstage *gstage, struct kvm *kvm) +{ + gstage->kvm = kvm; + gstage->flags = 0; + gstage->vmid = READ_ONCE(kvm->arch.vmid.vmid); + gstage->pgd = kvm->arch.pgd; + gstage->pgd_levels = kvm->arch.pgd_levels; +} + #endif diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index 24585304c02b..a30600579231 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -18,6 +18,7 @@ #include <asm/ptrace.h> #include <asm/kvm_tlb.h> #include <asm/kvm_vmid.h> +#include <asm/kvm_vcpu_config.h> #include <asm/kvm_vcpu_fp.h> #include <asm/kvm_vcpu_insn.h> #include <asm/kvm_vcpu_sbi.h> @@ -47,17 +48,7 @@ #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE -#define KVM_HEDELEG_DEFAULT (BIT(EXC_INST_MISALIGNED) | \ - BIT(EXC_INST_ILLEGAL) | \ - BIT(EXC_BREAKPOINT) | \ - BIT(EXC_SYSCALL) | \ - BIT(EXC_INST_PAGE_FAULT) | \ - BIT(EXC_LOAD_PAGE_FAULT) | \ - BIT(EXC_STORE_PAGE_FAULT)) - -#define KVM_HIDELEG_DEFAULT (BIT(IRQ_VS_SOFT) | \ - BIT(IRQ_VS_TIMER) | \ - BIT(IRQ_VS_EXT)) +#define KVM_HAVE_MMU_RWLOCK #define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ KVM_DIRTY_LOG_INITIALLY_SET) @@ -94,6 +85,8 @@ struct kvm_arch { /* G-stage page table */ pgd_t *pgd; phys_addr_t pgd_phys; + unsigned long pgd_levels; + struct kvm_mmu_memory_cache pgd_split_page_cache; /* Guest Timer */ struct kvm_guest_timer timer; @@ -167,16 +160,14 @@ struct kvm_vcpu_csr { unsigned long senvcfg; }; -struct kvm_vcpu_config { - u64 henvcfg; - u64 hstateen0; - unsigned long hedeleg; -}; - struct kvm_vcpu_smstateen_csr { unsigned long sstateen0; }; +struct kvm_vcpu_zicfiss_csr { + unsigned long ssp; +}; + struct kvm_vcpu_reset_state { spinlock_t lock; unsigned long pc; @@ -217,19 +208,22 @@ struct kvm_vcpu_arch { /* CPU Smstateen CSR context of Guest VCPU */ struct kvm_vcpu_smstateen_csr smstateen_csr; + /* CPU Zicfiss CSR context of Guest VCPU */ + struct kvm_vcpu_zicfiss_csr zicfiss_csr; + /* CPU reset state of Guest VCPU */ struct kvm_vcpu_reset_state reset_state; /* * VCPU interrupts * - * We have a lockless approach for tracking pending VCPU interrupts - * implemented using atomic bitops. The irqs_pending bitmap represent - * pending interrupts whereas irqs_pending_mask represent bits changed - * in irqs_pending. Our approach is modeled around multiple producer - * and single consumer problem where the consumer is the VCPU itself. + * The irqs_pending bitmap represents pending interrupts whereas + * irqs_pending_mask represents bits changed in irqs_pending. Updates + * to these bitmaps are serialized so vcpu interrupt sync/flush cannot + * drop a newly injected interrupt while syncing guest-visible HVIP. */ #define KVM_RISCV_VCPU_NR_IRQS 64 + raw_spinlock_t irqs_pending_lock; DECLARE_BITMAP(irqs_pending, KVM_RISCV_VCPU_NR_IRQS); DECLARE_BITMAP(irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS); @@ -273,6 +267,9 @@ struct kvm_vcpu_arch { /* 'static' configurations which are set only once */ struct kvm_vcpu_config cfg; + /* Indicates modified guest CSRs */ + bool csr_dirty; + /* SBI steal-time accounting */ struct { gpa_t shmem; @@ -293,6 +290,8 @@ static inline bool kvm_arch_pmi_in_guest(struct kvm_vcpu *vcpu) static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} +void kvm_riscv_clear_former_vcpu(void); + int kvm_riscv_setup_default_irq_routing(struct kvm *kvm, u32 lines); void __kvm_riscv_unpriv_trap(void); diff --git a/arch/riscv/include/asm/kvm_isa.h b/arch/riscv/include/asm/kvm_isa.h new file mode 100644 index 000000000000..bc4b956d5f17 --- /dev/null +++ b/arch/riscv/include/asm/kvm_isa.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2026 Qualcomm Technologies, Inc. + */ + +#ifndef __KVM_RISCV_ISA_H +#define __KVM_RISCV_ISA_H + +#include <linux/types.h> + +unsigned long kvm_riscv_base2isa_ext(unsigned long base_ext); + +int __kvm_riscv_isa_check_host(unsigned long ext, unsigned long *base_ext); +#define kvm_riscv_isa_check_host(ext) \ + __kvm_riscv_isa_check_host(KVM_RISCV_ISA_EXT_##ext, NULL) + +bool kvm_riscv_isa_enable_allowed(unsigned long ext); +bool kvm_riscv_isa_disable_allowed(unsigned long ext); + +#endif diff --git a/arch/riscv/include/asm/kvm_nacl.h b/arch/riscv/include/asm/kvm_nacl.h index 4124d5e06a0f..f45407bcaa26 100644 --- a/arch/riscv/include/asm/kvm_nacl.h +++ b/arch/riscv/include/asm/kvm_nacl.h @@ -60,9 +60,11 @@ int kvm_riscv_nacl_init(void); #ifdef CONFIG_32BIT #define lelong_to_cpu(__x) le32_to_cpu(__x) #define cpu_to_lelong(__x) cpu_to_le32(__x) +#define __lelong __le32 #else #define lelong_to_cpu(__x) le64_to_cpu(__x) #define cpu_to_lelong(__x) cpu_to_le64(__x) +#define __lelong __le64 #endif #define nacl_shmem() \ @@ -70,7 +72,7 @@ int kvm_riscv_nacl_init(void); #define nacl_scratch_read_long(__shmem, __offset) \ ({ \ - unsigned long *__p = (__shmem) + \ + __lelong *__p = (__shmem) + \ SBI_NACL_SHMEM_SCRATCH_OFFSET + \ (__offset); \ lelong_to_cpu(*__p); \ @@ -78,7 +80,7 @@ int kvm_riscv_nacl_init(void); #define nacl_scratch_write_long(__shmem, __offset, __val) \ do { \ - unsigned long *__p = (__shmem) + \ + __lelong *__p = (__shmem) + \ SBI_NACL_SHMEM_SCRATCH_OFFSET + \ (__offset); \ *__p = cpu_to_lelong(__val); \ @@ -87,7 +89,7 @@ do { \ #define nacl_scratch_write_longs(__shmem, __offset, __array, __count) \ do { \ unsigned int __i; \ - unsigned long *__p = (__shmem) + \ + __lelong *__p = (__shmem) + \ SBI_NACL_SHMEM_SCRATCH_OFFSET + \ (__offset); \ for (__i = 0; __i < (__count); __i++) \ @@ -168,7 +170,7 @@ __kvm_riscv_nacl_hfence(__shmem, \ #define nacl_csr_read(__shmem, __csr) \ ({ \ - unsigned long *__a = (__shmem) + SBI_NACL_SHMEM_CSR_OFFSET; \ + __lelong *__a = (__shmem) + SBI_NACL_SHMEM_CSR_OFFSET; \ lelong_to_cpu(__a[SBI_NACL_SHMEM_CSR_INDEX(__csr)]); \ }) @@ -176,7 +178,7 @@ __kvm_riscv_nacl_hfence(__shmem, \ do { \ void *__s = (__shmem); \ unsigned int __i = SBI_NACL_SHMEM_CSR_INDEX(__csr); \ - unsigned long *__a = (__s) + SBI_NACL_SHMEM_CSR_OFFSET; \ + __lelong *__a = (__s) + SBI_NACL_SHMEM_CSR_OFFSET; \ u8 *__b = (__s) + SBI_NACL_SHMEM_DBITMAP_OFFSET; \ __a[__i] = cpu_to_lelong(__val); \ __b[__i >> 3] |= 1U << (__i & 0x7); \ @@ -186,7 +188,7 @@ do { \ ({ \ void *__s = (__shmem); \ unsigned int __i = SBI_NACL_SHMEM_CSR_INDEX(__csr); \ - unsigned long *__a = (__s) + SBI_NACL_SHMEM_CSR_OFFSET; \ + __lelong *__a = (__s) + SBI_NACL_SHMEM_CSR_OFFSET; \ u8 *__b = (__s) + SBI_NACL_SHMEM_DBITMAP_OFFSET; \ unsigned long __r = lelong_to_cpu(__a[__i]); \ __a[__i] = cpu_to_lelong(__val); \ diff --git a/arch/riscv/include/asm/kvm_vcpu_config.h b/arch/riscv/include/asm/kvm_vcpu_config.h new file mode 100644 index 000000000000..fcc15a0296b3 --- /dev/null +++ b/arch/riscv/include/asm/kvm_vcpu_config.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2026 Qualcomm Technologies, Inc. + */ + +#ifndef __KVM_VCPU_RISCV_CONFIG_H +#define __KVM_VCPU_RISCV_CONFIG_H + +#include <linux/types.h> + +struct kvm_vcpu; + +struct kvm_vcpu_config { + u64 henvcfg; + u64 hstateen0; + unsigned long hedeleg; + unsigned long hideleg; +}; + +void kvm_riscv_vcpu_config_init(struct kvm_vcpu *vcpu); +void kvm_riscv_vcpu_config_guest_debug(struct kvm_vcpu *vcpu); +void kvm_riscv_vcpu_config_ran_once(struct kvm_vcpu *vcpu); +void kvm_riscv_vcpu_config_load(struct kvm_vcpu *vcpu); + +#endif diff --git a/arch/riscv/include/asm/kvm_vcpu_insn.h b/arch/riscv/include/asm/kvm_vcpu_insn.h index 350011c83581..a5c50dd4a884 100644 --- a/arch/riscv/include/asm/kvm_vcpu_insn.h +++ b/arch/riscv/include/asm/kvm_vcpu_insn.h @@ -38,10 +38,10 @@ int kvm_riscv_vcpu_virtual_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, struct kvm_cpu_trap *trap); int kvm_riscv_vcpu_mmio_load(struct kvm_vcpu *vcpu, struct kvm_run *run, - unsigned long fault_addr, + gpa_t fault_addr, unsigned long htinst); int kvm_riscv_vcpu_mmio_store(struct kvm_vcpu *vcpu, struct kvm_run *run, - unsigned long fault_addr, + gpa_t fault_addr, unsigned long htinst); int kvm_riscv_vcpu_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run); diff --git a/arch/riscv/include/asm/kvm_vcpu_sbi.h b/arch/riscv/include/asm/kvm_vcpu_sbi.h index c1a7e3b40d9c..f01a2860c751 100644 --- a/arch/riscv/include/asm/kvm_vcpu_sbi.h +++ b/arch/riscv/include/asm/kvm_vcpu_sbi.h @@ -60,6 +60,9 @@ struct kvm_vcpu_sbi_extension { void (*reset)(struct kvm_vcpu *vcpu); + /* Allow the extension to correct its parameters before the first run */ + void (*validate)(struct kvm_vcpu *vcpu); + unsigned long state_reg_subtype; unsigned long (*get_state_reg_count)(struct kvm_vcpu *vcpu); int (*get_state_reg_id)(struct kvm_vcpu *vcpu, int index, u64 *reg_id); @@ -93,6 +96,7 @@ int kvm_riscv_vcpu_sbi_ecall(struct kvm_vcpu *vcpu, struct kvm_run *run); void kvm_riscv_vcpu_sbi_init(struct kvm_vcpu *vcpu); void kvm_riscv_vcpu_sbi_deinit(struct kvm_vcpu *vcpu); void kvm_riscv_vcpu_sbi_reset(struct kvm_vcpu *vcpu); +void kvm_riscv_vcpu_sbi_validate(struct kvm_vcpu *vcpu); #ifdef CONFIG_RISCV_SBI_V01 extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_v01; diff --git a/arch/riscv/include/asm/kvm_vcpu_vector.h b/arch/riscv/include/asm/kvm_vcpu_vector.h index 57a798a4cb0d..6371d5ea5392 100644 --- a/arch/riscv/include/asm/kvm_vcpu_vector.h +++ b/arch/riscv/include/asm/kvm_vcpu_vector.h @@ -35,6 +35,22 @@ void kvm_riscv_vcpu_host_vector_save(struct kvm_cpu_context *cntx); void kvm_riscv_vcpu_host_vector_restore(struct kvm_cpu_context *cntx); int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu); void kvm_riscv_vcpu_free_vector_context(struct kvm_vcpu *vcpu); +void kvm_riscv_register_vctx_callback(void (*func)(void)); +void kvm_riscv_unregister_vctx_callback(void); +void kvm_riscv_vcpu_flush_vector(void); + +static inline void kvm_riscv_v_init(void) +{ + if (has_vector()) + kvm_riscv_register_vctx_callback(&kvm_riscv_vcpu_flush_vector); +} + +static inline void kvm_riscv_v_exit(void) +{ + if (has_vector()) + kvm_riscv_unregister_vctx_callback(); +} + #else struct kvm_cpu_context; @@ -69,6 +85,14 @@ static inline int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu) static inline void kvm_riscv_vcpu_free_vector_context(struct kvm_vcpu *vcpu) { } + +static inline void kvm_riscv_v_init(void) +{ +} + +static inline void kvm_riscv_v_exit(void) +{ +} #endif int kvm_riscv_vcpu_get_reg_vector(struct kvm_vcpu *vcpu, diff --git a/arch/riscv/include/asm/linkage.h b/arch/riscv/include/asm/linkage.h index 9e88ba23cd2b..7e0210ef4eb4 100644 --- a/arch/riscv/include/asm/linkage.h +++ b/arch/riscv/include/asm/linkage.h @@ -9,4 +9,6 @@ #define __ALIGN .balign 4 #define __ALIGN_STR ".balign 4" +#define _THIS_IP_ ({ unsigned long __ip; asm volatile("auipc %0, 0" : "=r" (__ip)); __ip; }) + #endif /* _ASM_RISCV_LINKAGE_H */ diff --git a/arch/riscv/include/asm/mman.h b/arch/riscv/include/asm/mman.h new file mode 100644 index 000000000000..0ad1d19832eb --- /dev/null +++ b/arch/riscv/include/asm/mman.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_MMAN_H__ +#define __ASM_MMAN_H__ + +#include <linux/compiler.h> +#include <linux/types.h> +#include <linux/mm.h> +#include <uapi/asm/mman.h> + +static inline unsigned long arch_calc_vm_prot_bits(unsigned long prot, + unsigned long pkey __always_unused) +{ + unsigned long ret = 0; + + /* + * If PROT_WRITE was specified, force it to VM_READ | VM_WRITE. + * Only VM_WRITE means shadow stack. + */ + if (prot & PROT_WRITE) + ret = (VM_READ | VM_WRITE); + return ret; +} + +#define arch_calc_vm_prot_bits(prot, pkey) arch_calc_vm_prot_bits(prot, pkey) + +#endif /* ! __ASM_MMAN_H__ */ diff --git a/arch/riscv/include/asm/mmu_context.h b/arch/riscv/include/asm/mmu_context.h index 8c4bc49a3a0f..dbf27a78df6c 100644 --- a/arch/riscv/include/asm/mmu_context.h +++ b/arch/riscv/include/asm/mmu_context.h @@ -48,6 +48,13 @@ static inline unsigned long mm_untag_mask(struct mm_struct *mm) } #endif +#define deactivate_mm deactivate_mm +static inline void deactivate_mm(struct task_struct *tsk, + struct mm_struct *mm) +{ + shstk_release(tsk); +} + #include <asm-generic/mmu_context.h> #endif /* _ASM_RISCV_MMU_CONTEXT_H */ diff --git a/arch/riscv/include/asm/module.lds.h b/arch/riscv/include/asm/module.lds.h index 1075beae1ac6..9ced27c8ccb6 100644 --- a/arch/riscv/include/asm/module.lds.h +++ b/arch/riscv/include/asm/module.lds.h @@ -2,8 +2,8 @@ /* Copyright (C) 2017 Andes Technology Corporation */ #ifdef CONFIG_MODULE_SECTIONS SECTIONS { - .plt : { BYTE(0) } - .got : { BYTE(0) } - .got.plt : { BYTE(0) } + .plt 0 : { BYTE(0) } + .got 0 : { BYTE(0) } + .got.plt 0 : { BYTE(0) } } #endif diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h index ffe213ad65a4..709a36fb4323 100644 --- a/arch/riscv/include/asm/page.h +++ b/arch/riscv/include/asm/page.h @@ -29,11 +29,7 @@ #define PAGE_OFFSET_L5 _AC(0xff60000000000000, UL) #define PAGE_OFFSET_L4 _AC(0xffffaf8000000000, UL) #define PAGE_OFFSET_L3 _AC(0xffffffd600000000, UL) -#ifdef CONFIG_XIP_KERNEL -#define PAGE_OFFSET PAGE_OFFSET_L3 -#else #define PAGE_OFFSET kernel_map.page_offset -#endif /* CONFIG_XIP_KERNEL */ #else #define PAGE_OFFSET _AC(0xc0000000, UL) #endif /* CONFIG_64BIT */ @@ -50,9 +46,7 @@ void clear_page(void *page); #endif #define copy_page(to, from) memcpy((to), (from), PAGE_SIZE) -#define clear_user_page(pgaddr, vaddr, page) clear_page(pgaddr) -#define copy_user_page(vto, vfrom, vaddr, topg) \ - memcpy((vto), (vfrom), PAGE_SIZE) +#define copy_user_page(vto, vfrom, vaddr, topg) copy_page(vto, vfrom) /* * Use struct definitions to apply C type checking @@ -106,15 +100,8 @@ struct kernel_mapping { /* Offset between linear mapping virtual address and kernel load address */ unsigned long va_pa_offset; /* Offset between kernel mapping virtual address and kernel load address */ -#ifdef CONFIG_XIP_KERNEL - unsigned long va_kernel_xip_text_pa_offset; - unsigned long va_kernel_xip_data_pa_offset; - uintptr_t xiprom; - uintptr_t xiprom_sz; -#else unsigned long page_offset; unsigned long va_kernel_pa_offset; -#endif }; extern struct kernel_mapping kernel_map; @@ -133,16 +120,7 @@ extern unsigned long vmemmap_start_pfn; void *linear_mapping_pa_to_va(unsigned long x); #endif -#ifdef CONFIG_XIP_KERNEL -#define kernel_mapping_pa_to_va(y) ({ \ - unsigned long _y = (unsigned long)(y); \ - (_y < phys_ram_base) ? \ - (void *)(_y + kernel_map.va_kernel_xip_text_pa_offset) : \ - (void *)(_y + kernel_map.va_kernel_xip_data_pa_offset); \ - }) -#else #define kernel_mapping_pa_to_va(y) ((void *)((unsigned long)(y) + kernel_map.va_kernel_pa_offset)) -#endif #define __pa_to_va_nodebug(x) linear_mapping_pa_to_va(x) @@ -152,16 +130,7 @@ void *linear_mapping_pa_to_va(unsigned long x); phys_addr_t linear_mapping_va_to_pa(unsigned long x); #endif -#ifdef CONFIG_XIP_KERNEL -#define kernel_mapping_va_to_pa(y) ({ \ - unsigned long _y = (unsigned long)(y); \ - (_y < kernel_map.virt_addr + kernel_map.xiprom_sz) ? \ - (_y - kernel_map.va_kernel_xip_text_pa_offset) : \ - (_y - kernel_map.va_kernel_xip_data_pa_offset); \ - }) -#else #define kernel_mapping_va_to_pa(y) ((unsigned long)(y) - kernel_map.va_kernel_pa_offset) -#endif #define __va_to_pa_nodebug(x) ({ \ unsigned long _x = x; \ @@ -192,7 +161,10 @@ extern phys_addr_t __phys_addr_symbol(unsigned long x); #define sym_to_pfn(x) __phys_to_pfn(__pa_symbol(x)) -unsigned long kaslr_offset(void); +static inline unsigned long kaslr_offset(void) +{ + return kernel_map.virt_offset; +} static __always_inline void *pfn_to_kaddr(unsigned long pfn) { @@ -206,7 +178,7 @@ static __always_inline void *pfn_to_kaddr(unsigned long pfn) (unsigned long)(_addr) >= PAGE_OFFSET && pfn_valid(virt_to_pfn(_addr)); \ }) -#define VM_DATA_DEFAULT_FLAGS VM_DATA_FLAGS_NON_EXEC +#define VMA_DATA_DEFAULT_FLAGS VMA_DATA_FLAGS_NON_EXEC #include <asm-generic/memory_model.h> #include <asm-generic/getorder.h> diff --git a/arch/riscv/include/asm/paravirt.h b/arch/riscv/include/asm/paravirt.h index c0abde70fc2c..c49c55b266f3 100644 --- a/arch/riscv/include/asm/paravirt.h +++ b/arch/riscv/include/asm/paravirt.h @@ -3,20 +3,6 @@ #define _ASM_RISCV_PARAVIRT_H #ifdef CONFIG_PARAVIRT -#include <linux/static_call_types.h> - -struct static_key; -extern struct static_key paravirt_steal_enabled; -extern struct static_key paravirt_steal_rq_enabled; - -u64 dummy_steal_clock(int cpu); - -DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); - -static inline u64 paravirt_steal_clock(int cpu) -{ - return static_call(pv_steal_clock)(cpu); -} int __init pv_time_init(void); diff --git a/arch/riscv/include/asm/paravirt_api_clock.h b/arch/riscv/include/asm/paravirt_api_clock.h deleted file mode 100644 index 65ac7cee0dad..000000000000 --- a/arch/riscv/include/asm/paravirt_api_clock.h +++ /dev/null @@ -1 +0,0 @@ -#include <asm/paravirt.h> diff --git a/arch/riscv/include/asm/pgtable-bits.h b/arch/riscv/include/asm/pgtable-bits.h index b422d9691e60..d5a86b4df3ce 100644 --- a/arch/riscv/include/asm/pgtable-bits.h +++ b/arch/riscv/include/asm/pgtable-bits.h @@ -40,20 +40,20 @@ #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -/* ext_svrsw60t59b: Bit(60) for uffd-wp tracking */ -#define _PAGE_UFFD_WP \ +/* ext_svrsw60t59b: Bit(60) for userfaultfd tracking */ +#define _PAGE_UFFD \ ((riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B)) ? \ (1UL << 60) : 0) /* * Bit 4 is not involved into swap entry computation, so we - * can borrow it for swap page uffd-wp tracking. + * can borrow it for swap page userfaultfd tracking. */ -#define _PAGE_SWP_UFFD_WP \ +#define _PAGE_SWP_UFFD \ ((riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B)) ? \ _PAGE_USER : 0) #else -#define _PAGE_UFFD_WP 0 -#define _PAGE_SWP_UFFD_WP 0 +#define _PAGE_UFFD 0 +#define _PAGE_SWP_UFFD 0 #endif #define _PAGE_TABLE _PAGE_PRESENT diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 9acd58a67123..40b1ed4f3ea8 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -93,6 +93,16 @@ */ #define vmemmap ((struct page *)VMEMMAP_START - vmemmap_start_pfn) +/* Needed to limit get_free_mem_region() */ +#if defined(CONFIG_FLATMEM) +#define DIRECT_MAP_PHYSMEM_END (phys_ram_base + KERN_VIRT_SIZE - 1) +#elif defined(CONFIG_SPARSEMEM_VMEMMAP) +#define DIRECT_MAP_PHYSMEM_END \ + ((vmemmap_start_pfn + VMEMMAP_SIZE / sizeof(struct page)) * PAGE_SIZE - 1) +#elif defined(CONFIG_SPARSEMEM) +/* DIRECT_MAP_PHYSMEM_END is not limited by VA space assignment in this case */ +#endif + #define PCI_IO_SIZE SZ_16M #define PCI_IO_END VMEMMAP_START #define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE) @@ -134,21 +144,6 @@ #include <linux/page_table_check.h> -#ifdef CONFIG_XIP_KERNEL -#define XIP_FIXUP(addr) ({ \ - extern char _sdata[], _start[], _end[]; \ - uintptr_t __rom_start_data = CONFIG_XIP_PHYS_ADDR \ - + (uintptr_t)&_sdata - (uintptr_t)&_start; \ - uintptr_t __rom_end_data = CONFIG_XIP_PHYS_ADDR \ - + (uintptr_t)&_end - (uintptr_t)&_start; \ - uintptr_t __a = (uintptr_t)(addr); \ - (__a >= __rom_start_data && __a < __rom_end_data) ? \ - __a - __rom_start_data + CONFIG_PHYS_RAM_BASE : __a; \ - }) -#else -#define XIP_FIXUP(addr) (addr) -#endif /* CONFIG_XIP_KERNEL */ - struct pt_alloc_ops { pte_t *(*get_pte_virt)(phys_addr_t pa); phys_addr_t (*alloc_pte)(uintptr_t va); @@ -178,6 +173,7 @@ extern struct pt_alloc_ops pt_ops __meminitdata; #define PAGE_READ_EXEC __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_EXEC) #define PAGE_WRITE_EXEC __pgprot(_PAGE_BASE | _PAGE_READ | \ _PAGE_EXEC | _PAGE_WRITE) +#define PAGE_SHADOWSTACK __pgprot(_PAGE_BASE | _PAGE_WRITE) #define PAGE_COPY PAGE_READ #define PAGE_COPY_EXEC PAGE_READ_EXEC @@ -410,51 +406,60 @@ static inline int pte_special(pte_t pte) static inline pte_t pte_wrprotect(pte_t pte) { - return __pte(pte_val(pte) & ~(_PAGE_WRITE)); + return __pte((pte_val(pte) & ~(_PAGE_WRITE)) | (_PAGE_READ)); } #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -#define pgtable_supports_uffd_wp() \ +#define pgtable_supports_uffd() \ riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B) -static inline bool pte_uffd_wp(pte_t pte) +static inline bool pte_uffd(pte_t pte) { - return !!(pte_val(pte) & _PAGE_UFFD_WP); + return !!(pte_val(pte) & _PAGE_UFFD); } -static inline pte_t pte_mkuffd_wp(pte_t pte) +static inline pte_t pte_mkuffd(pte_t pte) { - return pte_wrprotect(__pte(pte_val(pte) | _PAGE_UFFD_WP)); + return pte_wrprotect(__pte(pte_val(pte) | _PAGE_UFFD)); } -static inline pte_t pte_clear_uffd_wp(pte_t pte) +static inline pte_t pte_clear_uffd(pte_t pte) { - return __pte(pte_val(pte) & ~(_PAGE_UFFD_WP)); + return __pte(pte_val(pte) & ~(_PAGE_UFFD)); } -static inline bool pte_swp_uffd_wp(pte_t pte) +static inline bool pte_swp_uffd(pte_t pte) { - return !!(pte_val(pte) & _PAGE_SWP_UFFD_WP); + return !!(pte_val(pte) & _PAGE_SWP_UFFD); } -static inline pte_t pte_swp_mkuffd_wp(pte_t pte) +static inline pte_t pte_swp_mkuffd(pte_t pte) { - return __pte(pte_val(pte) | _PAGE_SWP_UFFD_WP); + return __pte(pte_val(pte) | _PAGE_SWP_UFFD); } -static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) +static inline pte_t pte_swp_clear_uffd(pte_t pte) { - return __pte(pte_val(pte) & ~(_PAGE_SWP_UFFD_WP)); + return __pte(pte_val(pte) & ~(_PAGE_SWP_UFFD)); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ /* static inline pte_t pte_mkread(pte_t pte) */ +struct vm_area_struct; +pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma); +#define pte_mkwrite pte_mkwrite + static inline pte_t pte_mkwrite_novma(pte_t pte) { return __pte(pte_val(pte) | _PAGE_WRITE); } +static inline pte_t pte_mkwrite_shstk(pte_t pte) +{ + return __pte((pte_val(pte) & ~(_PAGE_LEAF)) | _PAGE_WRITE); +} + /* static inline pte_t pte_mkexec(pte_t pte) */ static inline pte_t pte_mkdirty(pte_t pte) @@ -529,10 +534,7 @@ static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) PAGE_SIZE) #endif -#ifdef CONFIG_NUMA_BALANCING -/* - * See the comment in include/asm-generic/pgtable.h - */ +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pte_protnone(pte_t pte) { return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE)) == _PAGE_PROT_NONE; @@ -542,7 +544,7 @@ static inline int pmd_protnone(pmd_t pmd) { return pte_protnone(pmd_pte(pmd)); } -#endif +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ /* Modify page protection bits */ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) @@ -563,6 +565,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { + unsigned long asid = get_mm_asid(vma->vm_mm); + /* * Svvptc guarantees that the new valid pte will be visible within * a bounded timeframe, so when the uarch does not cache invalid @@ -571,6 +575,14 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC)) return; + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)) { + local_sfence_w_inval(); + while (nr--) + local_sinval_vma(address + nr * PAGE_SIZE, asid); + local_sfence_inval_ir(); + return; + } + /* * The kernel assumes that TLBs don't cache invalid entries, but * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a @@ -579,7 +591,7 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, * the extra traps reduce performance. So, eagerly SFENCE.VMA. */ while (nr--) - local_flush_tlb_page(address + nr * PAGE_SIZE); + local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid); } #define update_mmu_cache(vma, addr, ptep) \ @@ -627,7 +639,7 @@ static inline void __set_pte_at(struct mm_struct *mm, pte_t *ptep, pte_t pteval) static inline void set_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pteval, unsigned int nr) { - page_table_check_ptes_set(mm, ptep, pteval, nr); + page_table_check_ptes_set(mm, addr, ptep, pteval, nr); for (;;) { __set_pte_at(mm, ptep, pteval); @@ -649,8 +661,8 @@ static inline void pte_clear(struct mm_struct *mm, extern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, pte_t *ptep, pte_t entry, int dirty); #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG /* defined in mm/pgtable.c */ -extern int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, - pte_t *ptep); +bool ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep); #define __HAVE_ARCH_PTEP_GET_AND_CLEAR static inline pte_t ptep_get_and_clear(struct mm_struct *mm, @@ -664,7 +676,7 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, set_pte(ptep, __pte(0)); #endif - page_table_check_pte_clear(mm, pte); + page_table_check_pte_clear(mm, address, pte); return pte; } @@ -673,12 +685,20 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep) { - atomic_long_and(~(unsigned long)_PAGE_WRITE, (atomic_long_t *)ptep); + pte_t read_pte = READ_ONCE(*ptep); + /* + * ptep_set_wrprotect can be called for shadow stack ranges too. + * shadow stack memory is XWR = 010 and thus clearing _PAGE_WRITE will lead to + * encoding 000b which is wrong encoding with V = 1. This should lead to page fault + * but we dont want this wrong configuration to be set in page tables. + */ + atomic_long_set((atomic_long_t *)ptep, + ((pte_val(read_pte) & ~(unsigned long)_PAGE_WRITE) | _PAGE_READ)); } #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH -static inline int ptep_clear_flush_young(struct vm_area_struct *vma, - unsigned long address, pte_t *ptep) +static inline bool ptep_clear_flush_young(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep) { /* * This comment is borrowed from x86, but applies equally to RISC-V: @@ -833,11 +853,19 @@ static inline pmd_t pmd_mkyoung(pmd_t pmd) return pte_pmd(pte_mkyoung(pmd_pte(pmd))); } +pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma); +#define pmd_mkwrite pmd_mkwrite + static inline pmd_t pmd_mkwrite_novma(pmd_t pmd) { return pte_pmd(pte_mkwrite_novma(pmd_pte(pmd))); } +static inline pmd_t pmd_mkwrite_shstk(pmd_t pte) +{ + return __pmd((pmd_val(pte) & ~(_PAGE_LEAF)) | _PAGE_WRITE); +} + static inline pmd_t pmd_wrprotect(pmd_t pmd) { return pte_pmd(pte_wrprotect(pmd_pte(pmd))); @@ -878,34 +906,34 @@ static inline pud_t pud_mkspecial(pud_t pud) #endif #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static inline bool pmd_uffd_wp(pmd_t pmd) +static inline bool pmd_uffd(pmd_t pmd) { - return pte_uffd_wp(pmd_pte(pmd)); + return pte_uffd(pmd_pte(pmd)); } -static inline pmd_t pmd_mkuffd_wp(pmd_t pmd) +static inline pmd_t pmd_mkuffd(pmd_t pmd) { - return pte_pmd(pte_mkuffd_wp(pmd_pte(pmd))); + return pte_pmd(pte_mkuffd(pmd_pte(pmd))); } -static inline pmd_t pmd_clear_uffd_wp(pmd_t pmd) +static inline pmd_t pmd_clear_uffd(pmd_t pmd) { - return pte_pmd(pte_clear_uffd_wp(pmd_pte(pmd))); + return pte_pmd(pte_clear_uffd(pmd_pte(pmd))); } -static inline bool pmd_swp_uffd_wp(pmd_t pmd) +static inline bool pmd_swp_uffd(pmd_t pmd) { - return pte_swp_uffd_wp(pmd_pte(pmd)); + return pte_swp_uffd(pmd_pte(pmd)); } -static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_mkuffd(pmd_t pmd) { - return pte_pmd(pte_swp_mkuffd_wp(pmd_pte(pmd))); + return pte_pmd(pte_swp_mkuffd(pmd_pte(pmd))); } -static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_clear_uffd(pmd_t pmd) { - return pte_pmd(pte_swp_clear_uffd_wp(pmd_pte(pmd))); + return pte_pmd(pte_swp_clear_uffd(pmd_pte(pmd))); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ @@ -925,7 +953,7 @@ static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) return pte_pmd(pte_clear_soft_dirty(pmd_pte(pmd))); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static inline bool pmd_swp_soft_dirty(pmd_t pmd) { return pte_swp_soft_dirty(pmd_pte(pmd)); @@ -940,35 +968,35 @@ static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) { return pte_pmd(pte_swp_clear_soft_dirty(pmd_pte(pmd))); } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd) { - page_table_check_pmd_set(mm, pmdp, pmd); + page_table_check_pmd_set(mm, addr, pmdp, pmd); return __set_pte_at(mm, (pte_t *)pmdp, pmd_pte(pmd)); } static inline void set_pud_at(struct mm_struct *mm, unsigned long addr, pud_t *pudp, pud_t pud) { - page_table_check_pud_set(mm, pudp, pud); + page_table_check_pud_set(mm, addr, pudp, pud); return __set_pte_at(mm, (pte_t *)pudp, pud_pte(pud)); } #ifdef CONFIG_PAGE_TABLE_CHECK -static inline bool pte_user_accessible_page(pte_t pte) +static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte) { return pte_present(pte) && pte_user(pte); } -static inline bool pmd_user_accessible_page(pmd_t pmd) +static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd) { return pmd_leaf(pmd) && pmd_user(pmd); } -static inline bool pud_user_accessible_page(pud_t pud) +static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud) { return pud_leaf(pud) && pud_user(pud); } @@ -989,8 +1017,8 @@ static inline int pmdp_set_access_flags(struct vm_area_struct *vma, } #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG -static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, - unsigned long address, pmd_t *pmdp) +static inline bool pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp) { return ptep_test_and_clear_young(vma, address, (pte_t *)pmdp); } @@ -1007,7 +1035,7 @@ static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, pmd_clear(pmdp); #endif - page_table_check_pmd_clear(mm, pmd); + page_table_check_pmd_clear(mm, address, pmd); return pmd; } @@ -1023,7 +1051,7 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm, static inline pmd_t pmdp_establish(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp, pmd_t pmd) { - page_table_check_pmd_set(vma->vm_mm, pmdp, pmd); + page_table_check_pmd_set(vma->vm_mm, address, pmdp, pmd); return __pmd(atomic_long_xchg((atomic_long_t *)pmdp, pmd_val(pmd))); } @@ -1083,8 +1111,8 @@ static inline int pudp_set_access_flags(struct vm_area_struct *vma, return ptep_set_access_flags(vma, address, (pte_t *)pudp, pud_pte(entry), dirty); } -static inline int pudp_test_and_clear_young(struct vm_area_struct *vma, - unsigned long address, pud_t *pudp) +static inline bool pudp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, pud_t *pudp) { return ptep_test_and_clear_young(vma, address, (pte_t *)pudp); } @@ -1101,7 +1129,7 @@ static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm, pud_clear(pudp); #endif - page_table_check_pud_clear(mm, pud); + page_table_check_pud_clear(mm, address, pud); return pud; } @@ -1122,7 +1150,7 @@ static inline void update_mmu_cache_pud(struct vm_area_struct *vma, static inline pud_t pudp_establish(struct vm_area_struct *vma, unsigned long address, pud_t *pudp, pud_t pud) { - page_table_check_pud_set(vma->vm_mm, pudp, pud); + page_table_check_pud_set(vma->vm_mm, address, pudp, pud); return __pud(atomic_long_xchg((atomic_long_t *)pudp, pud_val(pud))); } @@ -1149,7 +1177,7 @@ static inline pud_t pud_modify(pud_t pud, pgprot_t newprot) * bit 0: _PAGE_PRESENT (zero) * bit 1 to 2: (zero) * bit 3: _PAGE_SWP_SOFT_DIRTY - * bit 4: _PAGE_SWP_UFFD_WP + * bit 4: _PAGE_SWP_UFFD * bit 5: _PAGE_PROT_NONE (zero) * bit 6: exclusive marker * bits 7 to 11: swap type @@ -1187,10 +1215,10 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) return __pte(pte_val(pte) & ~_PAGE_SWP_EXCLUSIVE); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES #define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) }) #define __swp_entry_to_pmd(swp) __pmd((swp).val) -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ /* * In the RV64 Linux scheme, we give the user half of the virtual-address space @@ -1246,26 +1274,14 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) extern char _start[]; extern void *_dtb_early_va; extern uintptr_t _dtb_early_pa; -#if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_MMU) -#define dtb_early_va (*(void **)XIP_FIXUP(&_dtb_early_va)) -#define dtb_early_pa (*(uintptr_t *)XIP_FIXUP(&_dtb_early_pa)) -#else #define dtb_early_va _dtb_early_va #define dtb_early_pa _dtb_early_pa -#endif /* CONFIG_XIP_KERNEL */ extern u64 satp_mode; void paging_init(void); void misc_mem_init(void); /* - * ZERO_PAGE is a global shared page that is always zero, - * used for zero-mapped memory areas, etc. - */ -extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; -#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) - -/* * Use set_p*_safe(), and elide TLB flushing, when confident that *no* * TLB flush will be required as a result of the "set". For example, use * in scenarios where it is known ahead of time that the routine is diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h index da5426122d28..815715c67f94 100644 --- a/arch/riscv/include/asm/processor.h +++ b/arch/riscv/include/asm/processor.h @@ -16,6 +16,7 @@ #include <asm/insn-def.h> #include <asm/alternative-macros.h> #include <asm/hwcap.h> +#include <asm/usercfi.h> #define arch_get_mmap_end(addr, len, flags) \ ({ \ @@ -66,6 +67,12 @@ struct pt_regs; * - bit 0: indicates whether the in-kernel Vector context is active. The * activation of this state disables the preemption. On a non-RT kernel, it * also disable bh. + * - bit 1: tells kvm that the vcpu process has guest context saved in vcpu's + * context memory and need to be restore upon returing back to the guest. + * - bit 2: represents that the vector context has now loaded and belongs to + * the guest kernel. Any non-scheduler context saving routing needs to save + * the register file to vcpu's context memory. The bit is set upon returing + * back to the guest and cleared after loading the host's vector context. * - bits 8: is used for tracking preemptible kernel-mode Vector, when * RISCV_ISA_V_PREEMPTIVE is enabled. Calling kernel_vector_begin() does not * disable the preemption if the thread's kernel_vstate.datap is allocated. @@ -85,7 +92,7 @@ struct pt_regs; * preempt_v. All preempt_v context should be dropped in such case because * V-regs are caller-saved. Only sstatus.VS=ON is persisted across a * schedule() call. - * - bit 30: The in-kernel preempt_v context is saved, and requries to be + * - bit 30: The in-kernel preempt_v context is saved, and is required to be * restored when returning to the context that owns the preempt_v. * - bit 31: The in-kernel preempt_v context is dirty, as signaled by the * trap entry code. Any context switches out-of current task need to save @@ -96,6 +103,8 @@ struct pt_regs; #define RISCV_V_CTX_UNIT_DEPTH 0x00010000 #define RISCV_KERNEL_MODE_V 0x00000001 +#define RISCV_V_VCPU_NEED_RESTORE 0x00000002 +#define RISCV_V_VCPU_CTX 0x00000004 #define RISCV_PREEMPT_V 0x00000100 #define RISCV_PREEMPT_V_DIRTY 0x80000000 #define RISCV_PREEMPT_V_NEED_RESTORE 0x40000000 @@ -122,6 +131,9 @@ struct thread_struct { /* A forced icache flush is not needed if migrating to the previous cpu. */ unsigned int prev_cpu; #endif +#ifdef CONFIG_RISCV_ISA_SSQOSID + u32 srmcfg; +#endif }; /* Whitelist the fstate from the task_struct for hardened usercopy */ diff --git a/arch/riscv/include/asm/purgatory.h b/arch/riscv/include/asm/purgatory.h new file mode 100644 index 000000000000..5a827e6752b7 --- /dev/null +++ b/arch/riscv/include/asm/purgatory.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_RISCV_PURGATORY_H +#define _ASM_RISCV_PURGATORY_H + +#ifndef __ASSEMBLER__ +#include <linux/purgatory.h> + +void purgatory(void); +#endif /* __ASSEMBLER__ */ + +#endif /* _ASM_RISCV_PURGATORY_H */ diff --git a/arch/riscv/include/asm/qos.h b/arch/riscv/include/asm/qos.h new file mode 100644 index 000000000000..cf19e8438bb9 --- /dev/null +++ b/arch/riscv/include/asm/qos.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_RISCV_QOS_H +#define _ASM_RISCV_QOS_H + +#include <linux/percpu-defs.h> + +#ifdef CONFIG_RISCV_ISA_SSQOSID + +#include <linux/bitfield.h> +#include <linux/cpufeature.h> +#include <linux/sched.h> + +#include <asm/csr.h> +#include <asm/hwcap.h> + +/* cached value of srmcfg csr for each cpu */ +DECLARE_PER_CPU(u32, cpu_srmcfg); + +/* default srmcfg value for each cpu, set via resctrl cpu assignment */ +DECLARE_PER_CPU(u32, cpu_srmcfg_default); + +static inline void __switch_to_srmcfg(struct task_struct *next) +{ + u32 thread_srmcfg, default_srmcfg; + + thread_srmcfg = READ_ONCE(next->thread.srmcfg); + default_srmcfg = __this_cpu_read(cpu_srmcfg_default); + + /* + * RCID and MCID inherit from cpu_srmcfg_default independently. + * RESCTRL_RESERVED_CLOSID and RESCTRL_RESERVED_RMID are both 0, so a + * zero field means "unassigned" and takes the CPU default. + */ + if (thread_srmcfg == 0) { + thread_srmcfg = default_srmcfg; + } else { + u32 rcid = FIELD_GET(SRMCFG_RCID_MASK, thread_srmcfg); + u32 mcid = FIELD_GET(SRMCFG_MCID_MASK, thread_srmcfg); + + if (rcid == 0 || mcid == 0) { + if (rcid == 0) + rcid = FIELD_GET(SRMCFG_RCID_MASK, default_srmcfg); + if (mcid == 0) + mcid = FIELD_GET(SRMCFG_MCID_MASK, default_srmcfg); + thread_srmcfg = FIELD_PREP(SRMCFG_RCID_MASK, rcid) | + FIELD_PREP(SRMCFG_MCID_MASK, mcid); + } + } + + if (thread_srmcfg != __this_cpu_read(cpu_srmcfg)) { + /* + * No fence around the csrw. Ssqosid is silent on srmcfg + * ordering versus memory accesses, so a few accesses at the + * switch boundary may carry the previous RCID/MCID. The + * tagging inaccuracy is bounded and acceptable for QoS. + */ + __this_cpu_write(cpu_srmcfg, thread_srmcfg); + csr_write(CSR_SRMCFG, thread_srmcfg); + } +} + +static __always_inline bool has_srmcfg(void) +{ + return riscv_has_extension_unlikely(RISCV_ISA_EXT_SSQOSID); +} + +#else /* ! CONFIG_RISCV_ISA_SSQOSID */ + +struct task_struct; +static __always_inline bool has_srmcfg(void) { return false; } +static inline void __switch_to_srmcfg(struct task_struct *next) { } + +#endif /* CONFIG_RISCV_ISA_SSQOSID */ +#endif /* _ASM_RISCV_QOS_H */ diff --git a/arch/riscv/include/asm/runtime-const.h b/arch/riscv/include/asm/runtime-const.h index d766e2b9e6df..ce2008bb5227 100644 --- a/arch/riscv/include/asm/runtime-const.h +++ b/arch/riscv/include/asm/runtime-const.h @@ -2,6 +2,10 @@ #ifndef _ASM_RISCV_RUNTIME_CONST_H #define _ASM_RISCV_RUNTIME_CONST_H +#ifdef MODULE + #error "Cannot use runtime-const infrastructure from modules" +#endif + #include <asm/asm.h> #include <asm/alternative.h> #include <asm/cacheflush.h> @@ -11,21 +15,24 @@ #include <linux/uaccess.h> +#define RUNTIME_MAGIC __ASM_STR(0x89ABCDEF) + #ifdef CONFIG_32BIT -#define runtime_const_ptr(sym) \ -({ \ - typeof(sym) __ret; \ - asm_inline(".option push\n\t" \ - ".option norvc\n\t" \ - "1:\t" \ - "lui %[__ret],0x89abd\n\t" \ - "addi %[__ret],%[__ret],-0x211\n\t" \ - ".option pop\n\t" \ - ".pushsection runtime_ptr_" #sym ",\"a\"\n\t" \ - ".long 1b - .\n\t" \ - ".popsection" \ - : [__ret] "=r" (__ret)); \ - __ret; \ +#define runtime_const_ptr(sym) \ +({ \ + typeof(sym) __ret; \ + asm_inline(".option push\n\t" \ + ".option norvc\n\t" \ + ".option norelax\n\t" \ + "1:\t" \ + "lui %[__ret], %%hi(" RUNTIME_MAGIC ")\n\t" \ + "addi %[__ret],%[__ret], %%lo(" RUNTIME_MAGIC ")\n\t" \ + ".option pop\n\t" \ + ".pushsection runtime_ptr_" #sym ",\"a\"\n\t" \ + ".long 1b - .\n\t" \ + ".popsection" \ + : [__ret] "=r" (__ret)); \ + __ret; \ }) #else /* @@ -41,11 +48,12 @@ #define RISCV_RUNTIME_CONST_64_PREAMBLE \ ".option push\n\t" \ ".option norvc\n\t" \ + ".option norelax\n\t" \ "1:\t" \ - "lui %[__ret],0x89abd\n\t" \ - "lui %[__tmp],0x1234\n\t" \ - "addiw %[__ret],%[__ret],-0x211\n\t" \ - "addiw %[__tmp],%[__tmp],0x567\n\t" \ + "lui %[__ret], %%hi(" RUNTIME_MAGIC ")\n\t" \ + "lui %[__tmp], %%hi(" RUNTIME_MAGIC ")\n\t" \ + "addiw %[__ret],%[__ret], %%lo(" RUNTIME_MAGIC ")\n\t" \ + "addiw %[__tmp],%[__tmp], %%lo(" RUNTIME_MAGIC ")\n\t" \ #define RISCV_RUNTIME_CONST_64_BASE \ "slli %[__tmp],%[__tmp],32\n\t" \ @@ -153,6 +161,23 @@ __ret; \ }) +#define runtime_const_mask_32(val, sym) \ +({ \ + u32 __ret; \ + asm_inline(".option push\n\t" \ + ".option norvc\n\t" \ + "1:\t" \ + SLLI " %[__ret],%[__val],12\n\t" \ + SRLI " %[__ret],%[__ret],12\n\t" \ + ".option pop\n\t" \ + ".pushsection runtime_mask_" #sym ",\"a\"\n\t" \ + ".long 1b - .\n\t" \ + ".popsection" \ + : [__ret] "=r" (__ret) \ + : [__val] "r" (val)); \ + __ret; \ +}) + #define runtime_const_init(type, sym) do { \ extern s32 __start_runtime_##type##_##sym[]; \ extern s32 __stop_runtime_##type##_##sym[]; \ @@ -256,6 +281,33 @@ static inline void __runtime_fixup_shift(void *where, unsigned long val) mutex_unlock(&text_mutex); } +static inline void __runtime_fixup_mask(void *where, unsigned long val) +{ + unsigned int width = (val) ? __fls(val) + 1 : 0; + + /* + * XXX: Current implementation only supports patching masks of + * form GENMASK(width, 0) (width >= 0) using a SRLI + SLLI + * sequence instead of LUI + ADDI + AND sequence to improve + * performance, density, and covers all the current use-cases. + * + * When the need arises to support any generic mask, and this + * BUG_ON() is tripped, consider using a: + * + * lui %[__ret], #imm16 + * addi %[__ret], #imm16 + * + * sequence to load the 32bit const mask, and perform a logical + * and outside the asm block before returning the result. Fixup + * can simply reuse the existing __runtime_fixup_32() to patch + * the LUI + ADDI sequence. + */ + BUG_ON(!val || width > 31 || (GENMASK(width - 1, 0) != val)); + + __runtime_fixup_shift(where, 32 - width); + __runtime_fixup_shift(where + 4, 32 - width); +} + static inline void runtime_const_fixup(void (*fn)(void *, unsigned long), unsigned long val, s32 *start, s32 *end) { diff --git a/arch/riscv/include/asm/scs.h b/arch/riscv/include/asm/scs.h index ab7714aa93bd..023a412fe38d 100644 --- a/arch/riscv/include/asm/scs.h +++ b/arch/riscv/include/asm/scs.h @@ -10,7 +10,6 @@ /* Load init_shadow_call_stack to gp. */ .macro scs_load_init_stack la gp, init_shadow_call_stack - XIP_FIXUP_OFFSET gp .endm /* Load the per-CPU IRQ shadow call stack to gp. */ diff --git a/arch/riscv/include/asm/set_memory.h b/arch/riscv/include/asm/set_memory.h index 87389e93325a..ef59e1716a2c 100644 --- a/arch/riscv/include/asm/set_memory.h +++ b/arch/riscv/include/asm/set_memory.h @@ -47,7 +47,7 @@ bool kernel_page_present(struct page *page); #endif /* __ASSEMBLER__ */ -#if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_XIP_KERNEL) +#if defined(CONFIG_STRICT_KERNEL_RWX) #ifdef CONFIG_64BIT #define SECTION_ALIGN (1 << 21) #else diff --git a/arch/riscv/include/asm/simd.h b/arch/riscv/include/asm/simd.h index adb50f3ec205..f176a8072c62 100644 --- a/arch/riscv/include/asm/simd.h +++ b/arch/riscv/include/asm/simd.h @@ -36,20 +36,10 @@ static __must_check inline bool may_use_simd(void) /* * Nesting is achieved in preempt_v by spreading the control for * preemptible and non-preemptible kernel-mode Vector into two fields. - * Always try to match with preempt_v if kernel V-context exists. Then, - * fallback to check non preempt_v if nesting happens, or if the config - * is not set. + * Only non-preempt_v can nest on top of preempt_v, if non-preempt_v is + * unavailable, then preempt_v is not allowed. */ - if (IS_ENABLED(CONFIG_RISCV_ISA_V_PREEMPTIVE) && current->thread.kernel_vstate.datap) { - if (!riscv_preempt_v_started(current)) - return true; - } - /* - * Non-preemptible kernel-mode Vector temporarily disables bh. So we - * must not return true on irq_disabled(). Otherwise we would fail the - * lockdep check calling local_bh_enable() - */ - return !irqs_disabled() && !(riscv_v_flags() & RISCV_KERNEL_MODE_V); + return !(riscv_v_flags() & RISCV_KERNEL_MODE_V); } #else /* ! CONFIG_RISCV_ISA_V */ diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h index 7ac80e9f2288..0ecc67641b09 100644 --- a/arch/riscv/include/asm/smp.h +++ b/arch/riscv/include/asm/smp.h @@ -105,7 +105,7 @@ static inline void riscv_ipi_set_virq_range(int virq, int nr) #endif /* CONFIG_SMP */ -#if defined(CONFIG_HOTPLUG_CPU) && (CONFIG_SMP) +#if defined(CONFIG_HOTPLUG_CPU) bool cpu_has_hotplug(unsigned int cpu); #else static inline bool cpu_has_hotplug(unsigned int cpu) diff --git a/arch/riscv/include/asm/string.h b/arch/riscv/include/asm/string.h index 5ba77f60bf0b..764ffe8f6479 100644 --- a/arch/riscv/include/asm/string.h +++ b/arch/riscv/include/asm/string.h @@ -28,6 +28,15 @@ extern asmlinkage __kernel_size_t strlen(const char *); #define __HAVE_ARCH_STRNCMP extern asmlinkage int strncmp(const char *cs, const char *ct, size_t count); + +#define __HAVE_ARCH_STRNLEN +extern asmlinkage __kernel_size_t strnlen(const char *, size_t); + +#define __HAVE_ARCH_STRCHR +extern asmlinkage char *strchr(const char *, int); + +#define __HAVE_ARCH_STRRCHR +extern asmlinkage char *strrchr(const char *, int); #endif /* For those files which don't want to check by kasan. */ diff --git a/arch/riscv/include/asm/switch_to.h b/arch/riscv/include/asm/switch_to.h index 0e71eb82f920..123c89b694e8 100644 --- a/arch/riscv/include/asm/switch_to.h +++ b/arch/riscv/include/asm/switch_to.h @@ -14,6 +14,7 @@ #include <asm/processor.h> #include <asm/ptrace.h> #include <asm/csr.h> +#include <asm/qos.h> #ifdef CONFIG_FPU extern void __fstate_save(struct task_struct *save_to); @@ -60,8 +61,8 @@ static inline void __switch_to_fpu(struct task_struct *prev, static __always_inline bool has_fpu(void) { - return riscv_has_extension_likely(RISCV_ISA_EXT_f) || - riscv_has_extension_likely(RISCV_ISA_EXT_d); + /* D extension depends on F, so checking D alone is sufficient. */ + return riscv_has_extension_likely(RISCV_ISA_EXT_D); } #else static __always_inline bool has_fpu(void) { return false; } @@ -119,6 +120,8 @@ do { \ __switch_to_fpu(__prev, __next); \ if (has_vector() || has_xtheadvector()) \ __switch_to_vector(__prev, __next); \ + if (has_srmcfg()) \ + __switch_to_srmcfg(__next); \ if (switch_to_should_flush_icache(__next)) \ local_flush_icache_all(); \ __switch_to_envcfg(__next); \ diff --git a/arch/riscv/include/asm/syscall.h b/arch/riscv/include/asm/syscall.h index 34313387f977..987c9a78806f 100644 --- a/arch/riscv/include/asm/syscall.h +++ b/arch/riscv/include/asm/syscall.h @@ -20,7 +20,7 @@ extern void * const sys_call_table[]; extern void * const compat_sys_call_table[]; /* - * Only the low 32 bits of orig_r0 are meaningful, so we return int. + * Only the low 32 bits of orig_a0 are meaningful, so we return int. * This importantly ignores the high bits on 64-bit, so comparisons * sign-extend the low 32 bits. */ @@ -112,11 +112,6 @@ static inline void syscall_handler(struct pt_regs *regs, ulong syscall) regs->a0 = fn(regs); } -static inline bool arch_syscall_is_vdso_sigreturn(struct pt_regs *regs) -{ - return false; -} - asmlinkage long sys_riscv_flush_icache(uintptr_t, uintptr_t, uintptr_t); asmlinkage long sys_riscv_hwprobe(struct riscv_hwprobe *, size_t, size_t, diff --git a/arch/riscv/include/asm/syscall_wrapper.h b/arch/riscv/include/asm/syscall_wrapper.h index ac80216549ff..226289c3b5c8 100644 --- a/arch/riscv/include/asm/syscall_wrapper.h +++ b/arch/riscv/include/asm/syscall_wrapper.h @@ -32,6 +32,10 @@ asmlinkage long __riscv_sys_ni_syscall(const struct pt_regs *); __diag_push(); \ __diag_ignore(GCC, 8, "-Wattribute-alias", \ "Type aliasing is used to sanitize syscall arguments"); \ + __diag_ignore(clang, 23, "-Wunknown-warning-option", \ + "Avoid breaking versions without -Wattribute-alias"); \ + __diag_ignore(clang, 23, "-Wattribute-alias", \ + "Type aliasing is used to sanitize syscall arguments"); \ static long __se_##prefix##name(ulong, ulong, ulong, ulong, ulong, ulong, \ ulong) \ __attribute__((alias(__stringify(___se_##prefix##name)))); \ diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index 836d80dd2921..55019fdfa9ec 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -73,6 +73,9 @@ struct thread_info { */ unsigned long a0, a1, a2; #endif +#ifdef CONFIG_RISCV_USER_CFI + struct cfi_state user_cfi_state; +#endif }; #ifdef CONFIG_SHADOW_CALL_STACK @@ -117,7 +120,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src); #include <asm-generic/thread_info_tif.h> #define TIF_32BIT 16 /* compat-mode 32bit process */ -#define TIF_RISCV_V_DEFER_RESTORE 17 /* restore Vector before returing to user */ +#define TIF_RISCV_V_DEFER_RESTORE 17 /* restore Vector before returning to user */ #define _TIF_RISCV_V_DEFER_RESTORE BIT(TIF_RISCV_V_DEFER_RESTORE) diff --git a/arch/riscv/include/asm/timex.h b/arch/riscv/include/asm/timex.h index a06697846e69..f660afa30530 100644 --- a/arch/riscv/include/asm/timex.h +++ b/arch/riscv/include/asm/timex.h @@ -22,13 +22,13 @@ static inline cycles_t get_cycles(void) #else /* !CONFIG_64BIT */ static inline u32 get_cycles(void) { - return readl_relaxed(((u32 *)clint_time_val)); + return readl_relaxed(((u32 __iomem *)clint_time_val)); } #define get_cycles get_cycles static inline u32 get_cycles_hi(void) { - return readl_relaxed(((u32 *)clint_time_val) + 1); + return readl_relaxed(((u32 __iomem *)clint_time_val) + 1); } #define get_cycles_hi get_cycles_hi #endif /* CONFIG_64BIT */ @@ -80,12 +80,4 @@ static inline u64 get_cycles64(void) return ((u64)hi << 32) | lo; } #endif /* CONFIG_64BIT */ - -#define ARCH_HAS_READ_CURRENT_TIMER -static inline int read_current_timer(unsigned long *timer_val) -{ - *timer_val = get_cycles(); - return 0; -} - #endif /* _ASM_RISCV_TIMEX_H */ diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h index eed0abc40514..9636d07fe9ee 100644 --- a/arch/riscv/include/asm/tlbflush.h +++ b/arch/riscv/include/asm/tlbflush.h @@ -15,6 +15,29 @@ #define FLUSH_TLB_NO_ASID ((unsigned long)-1) #ifdef CONFIG_MMU +static inline unsigned long get_mm_asid(struct mm_struct *mm) +{ + return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID; +} + +static inline void local_sfence_inval_ir(void) +{ + asm volatile(SFENCE_INVAL_IR() ::: "memory"); +} + +static inline void local_sfence_w_inval(void) +{ + asm volatile(SFENCE_W_INVAL() ::: "memory"); +} + +static inline void local_sinval_vma(unsigned long vma, unsigned long asid) +{ + if (asid != FLUSH_TLB_NO_ASID) + asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory"); + else + asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory"); +} + static inline void local_flush_tlb_all(void) { __asm__ __volatile__ ("sfence.vma" : : : "memory"); diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index 11c9886c3b70..5d4ec15584cf 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -112,7 +112,7 @@ do { \ _ASM_EXTABLE_UACCESS_ERR(1b, %l2, %0) \ : "=&r" (__tmp) \ : "m" (*(ptr)) : : label); \ - (x) = (__typeof__(x))(unsigned long)__tmp; \ + (x) = (__force __typeof__(x))(unsigned long)__tmp; \ } while (0) #else /* !CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ #define __get_user_asm(insn, x, ptr, label) \ diff --git a/arch/riscv/include/asm/usercfi.h b/arch/riscv/include/asm/usercfi.h new file mode 100644 index 000000000000..61ee02cee297 --- /dev/null +++ b/arch/riscv/include/asm/usercfi.h @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: GPL-2.0 + * Copyright (C) 2024 Rivos, Inc. + * Deepak Gupta <debug@rivosinc.com> + */ +#ifndef _ASM_RISCV_USERCFI_H +#define _ASM_RISCV_USERCFI_H + +#define CMDLINE_DISABLE_RISCV_USERCFI_FCFI 1 +#define CMDLINE_DISABLE_RISCV_USERCFI_BCFI 2 +#define CMDLINE_DISABLE_RISCV_USERCFI 3 + +#ifndef __ASSEMBLER__ +#include <linux/types.h> +#include <linux/prctl.h> +#include <linux/errno.h> + +struct task_struct; +struct kernel_clone_args; + +extern unsigned long riscv_nousercfi; + +#ifdef CONFIG_RISCV_USER_CFI +struct cfi_state { + unsigned long ubcfi_en : 1; /* Enable for backward cfi. */ + unsigned long ubcfi_locked : 1; + unsigned long ufcfi_en : 1; /* Enable for forward cfi. Note that ELP goes in sstatus */ + unsigned long ufcfi_locked : 1; + unsigned long user_shdw_stk; /* Current user shadow stack pointer */ + unsigned long shdw_stk_base; /* Base address of shadow stack */ + unsigned long shdw_stk_size; /* size of shadow stack */ +}; + +unsigned long shstk_alloc_thread_stack(struct task_struct *tsk, + const struct kernel_clone_args *args); +void shstk_release(struct task_struct *tsk); +void set_shstk_base(struct task_struct *task, unsigned long shstk_addr, unsigned long size); +unsigned long get_shstk_base(struct task_struct *task, unsigned long *size); +void set_active_shstk(struct task_struct *task, unsigned long shstk_addr); +bool is_shstk_enabled(struct task_struct *task); +bool is_shstk_locked(struct task_struct *task); +bool is_shstk_allocated(struct task_struct *task); +void set_shstk_lock(struct task_struct *task, bool lock); +void set_shstk_status(struct task_struct *task, bool enable); +unsigned long get_active_shstk(struct task_struct *task); +int restore_user_shstk(struct task_struct *tsk, unsigned long shstk_ptr); +int save_user_shstk(struct task_struct *tsk, unsigned long *saved_shstk_ptr); +bool is_indir_lp_enabled(struct task_struct *task); +bool is_indir_lp_locked(struct task_struct *task); +void set_indir_lp_status(struct task_struct *task, bool enable); +void set_indir_lp_lock(struct task_struct *task, bool lock); + +#define PR_SHADOW_STACK_SUPPORTED_STATUS_MASK (PR_SHADOW_STACK_ENABLE) +#define PR_CFI_SUPPORTED_STATUS_MASK (PR_CFI_ENABLE | PR_CFI_DISABLE | PR_CFI_LOCK) + +#else + +#define shstk_alloc_thread_stack(tsk, args) 0 + +#define shstk_release(tsk) + +#define get_shstk_base(task, size) 0UL + +#define set_shstk_base(task, shstk_addr, size) do {} while (0) + +#define set_active_shstk(task, shstk_addr) do {} while (0) + +#define is_shstk_enabled(task) false + +#define is_shstk_locked(task) false + +#define is_shstk_allocated(task) false + +#define set_shstk_lock(task, lock) do {} while (0) + +#define set_shstk_status(task, enable) do {} while (0) + +#define is_indir_lp_enabled(task) false + +#define is_indir_lp_locked(task) false + +#define set_indir_lp_status(task, enable) do {} while (0) + +#define set_indir_lp_lock(task, lock) do {} while (0) + +#define restore_user_shstk(tsk, shstk_ptr) -EINVAL + +#define save_user_shstk(tsk, saved_shstk_ptr) -EINVAL + +#define get_active_shstk(task) 0UL + +#endif /* CONFIG_RISCV_USER_CFI */ + +bool is_user_shstk_enabled(void); +bool is_user_lpad_enabled(void); + +#endif /* __ASSEMBLER__ */ + +#endif /* _ASM_RISCV_USERCFI_H */ diff --git a/arch/riscv/include/asm/vdso.h b/arch/riscv/include/asm/vdso.h index f80357fe24d1..0259bbc98e93 100644 --- a/arch/riscv/include/asm/vdso.h +++ b/arch/riscv/include/asm/vdso.h @@ -12,15 +12,32 @@ * All systems with an MMU have a VDSO, but systems without an MMU don't * support shared libraries and therefore don't have one. */ -#ifdef CONFIG_MMU #define __VDSO_PAGES 4 #ifndef __ASSEMBLER__ + +#ifdef CONFIG_MMU #include <generated/vdso-offsets.h> +#else +extern size_t __vdso_alternatives_start_offset, __vdso_alternatives_end_offset; +#endif +#ifdef CONFIG_RISCV_USER_CFI +#include <generated/vdso-cfi-offsets.h> +#else +extern size_t __vdso_alternatives_start_cfi_offset, __vdso_alternatives_end_cfi_offset; +#endif + +#ifdef CONFIG_RISCV_USER_CFI +#define VDSO_SYMBOL(base, name) \ + (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZIMOP) ? \ + (void __user *)((unsigned long)(base) + __vdso_##name##_cfi_offset) : \ + (void __user *)((unsigned long)(base) + __vdso_##name##_offset)) +#else #define VDSO_SYMBOL(base, name) \ - (void __user *)((unsigned long)(base) + __vdso_##name##_offset) + ((void __user *)((unsigned long)(base) + __vdso_##name##_offset)) +#endif #ifdef CONFIG_COMPAT #include <generated/compat_vdso-offsets.h> @@ -28,14 +45,14 @@ #define COMPAT_VDSO_SYMBOL(base, name) \ (void __user *)((unsigned long)(base) + compat__vdso_##name##_offset) -extern char compat_vdso_start[], compat_vdso_end[]; - +#else +extern size_t compat__vdso_alternatives_start_offset, compat__vdso_alternatives_end_offset; #endif /* CONFIG_COMPAT */ extern char vdso_start[], vdso_end[]; +extern char vdso_cfi_start[], vdso_cfi_end[]; +extern char compat_vdso_start[], compat_vdso_end[]; #endif /* !__ASSEMBLER__ */ -#endif /* CONFIG_MMU */ - #endif /* _ASM_RISCV_VDSO_H */ diff --git a/arch/riscv/include/asm/vdso/arch_data.h b/arch/riscv/include/asm/vdso/arch_data.h index 88b37af55175..da57a3786f7a 100644 --- a/arch/riscv/include/asm/vdso/arch_data.h +++ b/arch/riscv/include/asm/vdso/arch_data.h @@ -12,12 +12,6 @@ struct vdso_arch_data { /* Boolean indicating all CPUs have the same static hwprobe values. */ __u8 homogeneous_cpus; - - /* - * A gate to check and see if the hwprobe data is actually ready, as - * probing is deferred to avoid boot slowdowns. - */ - __u8 ready; }; #endif /* __RISCV_ASM_VDSO_ARCH_DATA_H */ diff --git a/arch/riscv/include/asm/vdso/gettimeofday.h b/arch/riscv/include/asm/vdso/gettimeofday.h index 9ec08fa04d35..61cb3cbab143 100644 --- a/arch/riscv/include/asm/vdso/gettimeofday.h +++ b/arch/riscv/include/asm/vdso/gettimeofday.h @@ -9,12 +9,6 @@ #include <asm/csr.h> #include <uapi/linux/time.h> -/* - * 32-bit land is lacking generic time vsyscalls as well as the legacy 32-bit - * time syscalls like gettimeofday. Skip these definitions since on 32-bit. - */ -#ifdef CONFIG_GENERIC_TIME_VSYSCALL - #define VDSO_HAS_CLOCK_GETRES 1 static __always_inline @@ -66,8 +60,6 @@ int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) return ret; } -#endif /* CONFIG_GENERIC_TIME_VSYSCALL */ - static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, const struct vdso_time_data *vd) { diff --git a/arch/riscv/include/asm/vector.h b/arch/riscv/include/asm/vector.h index 00cb9c0982b1..fffe72a77208 100644 --- a/arch/riscv/include/asm/vector.h +++ b/arch/riscv/include/asm/vector.h @@ -58,6 +58,11 @@ static inline u32 riscv_v_flags(void) return READ_ONCE(current->thread.riscv_v_flags); } +static inline void riscv_v_flags_set(u32 flags) +{ + WRITE_ONCE(current->thread.riscv_v_flags, flags); +} + static __always_inline bool has_vector(void) { return riscv_has_extension_unlikely(RISCV_ISA_EXT_ZVE32X); diff --git a/arch/riscv/include/asm/xip_fixup.h b/arch/riscv/include/asm/xip_fixup.h deleted file mode 100644 index f3d56299bc22..000000000000 --- a/arch/riscv/include/asm/xip_fixup.h +++ /dev/null @@ -1,49 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * XIP fixup macros, only useful in assembly. - */ -#ifndef _ASM_RISCV_XIP_FIXUP_H -#define _ASM_RISCV_XIP_FIXUP_H - -#include <linux/pgtable.h> - -#ifdef CONFIG_XIP_KERNEL -.macro XIP_FIXUP_OFFSET reg - /* Fix-up address in Flash into address in RAM early during boot before - * MMU is up. Because generated code "thinks" data is in Flash, but it - * is actually in RAM (actually data is also in Flash, but Flash is - * read-only, thus we need to use the data residing in RAM). - * - * The start of data in Flash is _sdata and the start of data in RAM is - * CONFIG_PHYS_RAM_BASE. So this fix-up essentially does this: - * reg += CONFIG_PHYS_RAM_BASE - _start - */ - li t0, CONFIG_PHYS_RAM_BASE - add \reg, \reg, t0 - la t0, _sdata - sub \reg, \reg, t0 -.endm -.macro XIP_FIXUP_FLASH_OFFSET reg - /* In linker script, at the transition from read-only section to - * writable section, the VMA is increased while LMA remains the same. - * (See in linker script how _sdata, __data_loc and LOAD_OFFSET is - * changed) - * - * Consequently, early during boot before MMU is up, the generated code - * reads the "writable" section at wrong addresses, because VMA is used - * by compiler to generate code, but the data is located in Flash using - * LMA. - */ - la t0, _sdata - sub \reg, \reg, t0 - la t0, __data_loc - add \reg, \reg, t0 -.endm -#else -.macro XIP_FIXUP_OFFSET reg -.endm -.macro XIP_FIXUP_FLASH_OFFSET reg -.endm -#endif /* CONFIG_XIP_KERNEL */ - -#endif diff --git a/arch/riscv/include/asm/xor.h b/arch/riscv/include/asm/xor.h deleted file mode 100644 index 96011861e46b..000000000000 --- a/arch/riscv/include/asm/xor.h +++ /dev/null @@ -1,68 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) 2021 SiFive - */ - -#include <linux/hardirq.h> -#include <asm-generic/xor.h> -#ifdef CONFIG_RISCV_ISA_V -#include <asm/vector.h> -#include <asm/switch_to.h> -#include <asm/asm-prototypes.h> - -static void xor_vector_2(unsigned long bytes, unsigned long *__restrict p1, - const unsigned long *__restrict p2) -{ - kernel_vector_begin(); - xor_regs_2_(bytes, p1, p2); - kernel_vector_end(); -} - -static void xor_vector_3(unsigned long bytes, unsigned long *__restrict p1, - const unsigned long *__restrict p2, - const unsigned long *__restrict p3) -{ - kernel_vector_begin(); - xor_regs_3_(bytes, p1, p2, p3); - kernel_vector_end(); -} - -static void xor_vector_4(unsigned long bytes, unsigned long *__restrict p1, - const unsigned long *__restrict p2, - const unsigned long *__restrict p3, - const unsigned long *__restrict p4) -{ - kernel_vector_begin(); - xor_regs_4_(bytes, p1, p2, p3, p4); - kernel_vector_end(); -} - -static void xor_vector_5(unsigned long bytes, unsigned long *__restrict p1, - const unsigned long *__restrict p2, - const unsigned long *__restrict p3, - const unsigned long *__restrict p4, - const unsigned long *__restrict p5) -{ - kernel_vector_begin(); - xor_regs_5_(bytes, p1, p2, p3, p4, p5); - kernel_vector_end(); -} - -static struct xor_block_template xor_block_rvv = { - .name = "rvv", - .do_2 = xor_vector_2, - .do_3 = xor_vector_3, - .do_4 = xor_vector_4, - .do_5 = xor_vector_5 -}; - -#undef XOR_TRY_TEMPLATES -#define XOR_TRY_TEMPLATES \ - do { \ - xor_speed(&xor_block_8regs); \ - xor_speed(&xor_block_32regs); \ - if (has_vector()) { \ - xor_speed(&xor_block_rvv);\ - } \ - } while (0) -#endif |