diff options
Diffstat (limited to 'arch/riscv/kernel')
| -rw-r--r-- | arch/riscv/kernel/acpi.c | 12 | ||||
| -rw-r--r-- | arch/riscv/kernel/cpu-hotplug.c | 2 | ||||
| -rw-r--r-- | arch/riscv/kernel/entry.S | 1 | ||||
| -rw-r--r-- | arch/riscv/kernel/ftrace.c | 2 | ||||
| -rw-r--r-- | arch/riscv/kernel/head.S | 43 | ||||
| -rw-r--r-- | arch/riscv/kernel/head.h | 3 | ||||
| -rw-r--r-- | arch/riscv/kernel/mcount-dyn.S | 2 | ||||
| -rw-r--r-- | arch/riscv/kernel/module-sections.c | 2 | ||||
| -rw-r--r-- | arch/riscv/kernel/probes/kprobes.c | 2 | ||||
| -rw-r--r-- | arch/riscv/kernel/probes/uprobes.c | 2 | ||||
| -rw-r--r-- | arch/riscv/kernel/setup.c | 6 | ||||
| -rw-r--r-- | arch/riscv/kernel/smpboot.c | 8 | ||||
| -rw-r--r-- | arch/riscv/kernel/soc.c | 2 | ||||
| -rw-r--r-- | arch/riscv/kernel/suspend.c | 2 | ||||
| -rw-r--r-- | arch/riscv/kernel/suspend_entry.S | 2 | ||||
| -rw-r--r-- | arch/riscv/kernel/traps.c | 6 | ||||
| -rw-r--r-- | arch/riscv/kernel/unaligned_access_speed.c | 221 | ||||
| -rw-r--r-- | arch/riscv/kernel/vdso_cfi/.gitignore | 8 | ||||
| -rw-r--r-- | arch/riscv/kernel/vdso_cfi/Makefile | 3 | ||||
| -rw-r--r-- | arch/riscv/kernel/vmcore_info.c | 14 | ||||
| -rw-r--r-- | arch/riscv/kernel/vmlinux.lds.S | 5 |
21 files changed, 123 insertions, 225 deletions
diff --git a/arch/riscv/kernel/acpi.c b/arch/riscv/kernel/acpi.c index 322ea92aa39f..068e0b404b6f 100644 --- a/arch/riscv/kernel/acpi.c +++ b/arch/riscv/kernel/acpi.c @@ -69,7 +69,7 @@ static int __init acpi_fadt_sanity_check(void) /* * FADT is required on riscv; retrieve it to check its presence - * and carry out revision and ACPI HW reduced compliancy tests + * and carry out revision and ACPI HW reduced compliance tests */ status = acpi_get_table(ACPI_SIG_FADT, 0, &table); if (ACPI_FAILURE(status)) { @@ -85,12 +85,12 @@ static int __init acpi_fadt_sanity_check(void) * The revision in the table header is the FADT's Major revision. The * FADT also has a minor revision, which is stored in the FADT itself. * - * TODO: Currently, we check for 6.5 as the minimum version to check - * for HW_REDUCED flag. However, once RISC-V updates are released in - * the ACPI spec, we need to update this check for exact minor revision + * ACPI 6.6 is required for RISC-V as it introduces RISC-V specific + * tables such as RHCT (RISC-V Hart Capabilities Table) and RIMT + * (RISC-V I/O Mapping Table). */ - if (table->revision < 6 || (table->revision == 6 && fadt->minor_revision < 5)) - pr_err(FW_BUG "Unsupported FADT revision %d.%d, should be 6.5+\n", + if (table->revision < 6 || (table->revision == 6 && fadt->minor_revision < 6)) + pr_err(FW_BUG "Unsupported FADT revision %d.%d, should be 6.6+\n", table->revision, fadt->minor_revision); if (!(fadt->flags & ACPI_FADT_HW_REDUCED)) { diff --git a/arch/riscv/kernel/cpu-hotplug.c b/arch/riscv/kernel/cpu-hotplug.c index 3f50d3dd76c6..a0ee426f6d93 100644 --- a/arch/riscv/kernel/cpu-hotplug.c +++ b/arch/riscv/kernel/cpu-hotplug.c @@ -43,7 +43,6 @@ int __cpu_disable(void) return 0; } -#ifdef CONFIG_HOTPLUG_CPU /* * Called on the thread which is asking for a CPU to be shutdown, if the * CPU reported dead to the hotplug core. @@ -75,4 +74,3 @@ void __noreturn arch_cpu_idle_dead(void) /* It should never reach here */ BUG(); } -#endif diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index 60eb221296a6..d011fb51c59a 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -498,6 +498,7 @@ SYM_DATA_START_LOCAL(excp_vect_table) RISCV_PTR do_trap_unknown /* cause=16 */ RISCV_PTR do_trap_unknown /* cause=17 */ RISCV_PTR do_trap_software_check /* cause=18 is sw check exception */ + RISCV_PTR do_trap_hardware_error /* hardware error (19) */ SYM_DATA_END_LABEL(excp_vect_table, SYM_L_LOCAL, excp_vect_table_end) #ifndef CONFIG_MMU diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c index 8d18d6727f0f..b430edfb83f4 100644 --- a/arch/riscv/kernel/ftrace.c +++ b/arch/riscv/kernel/ftrace.c @@ -148,7 +148,7 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long ad /* * This is called early on, and isn't wrapped by - * ftrace_arch_code_modify_{prepare,post_process}() and therefor doesn't hold + * ftrace_arch_code_modify_{prepare,post_process}() and therefore doesn't hold * text_mutex, which triggers a lockdep failure. SMP isn't running so we could * just directly poke the text, but it's simpler to just take the lock * ourselves. diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index 9c99c5ad6fe8..f6a8ca49e627 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -14,7 +14,6 @@ #include <asm/hwcap.h> #include <asm/image.h> #include <asm/scs.h> -#include <asm/xip_fixup.h> #include <asm/usercfi.h> #include "efi-header.S" @@ -75,13 +74,12 @@ pe_head_start: relocate_enable_mmu: /* Relocate return address */ la a1, kernel_map - XIP_FIXUP_OFFSET a1 REG_L a1, KERNEL_MAP_VIRT_ADDR(a1) la a2, _start sub a1, a1, a2 add ra, ra, a1 - /* Point stvec to virtual address of intruction after satp write */ + /* Point stvec to virtual address of instruction after satp write */ la a2, 1f add a2, a2, a1 csrw CSR_TVEC, a2 @@ -89,7 +87,6 @@ relocate_enable_mmu: /* Compute satp for kernel page tables, but don't load it yet */ srl a2, a0, PAGE_SHIFT la a1, satp_mode - XIP_FIXUP_OFFSET a1 REG_L a1, 0(a1) or a2, a2, a1 @@ -100,7 +97,6 @@ relocate_enable_mmu: * to ensure the new translations are in use. */ la a0, trampoline_pg_dir - XIP_FIXUP_OFFSET a0 srl a0, a0, PAGE_SHIFT or a0, a0, a1 sfence.vma @@ -154,11 +150,9 @@ secondary_start_sbi: /* a0 contains the hartid & a1 contains boot data */ li a2, SBI_HART_BOOT_TASK_PTR_OFFSET - XIP_FIXUP_OFFSET a2 add a2, a2, a1 REG_L tp, (a2) li a3, SBI_HART_BOOT_STACK_PTR_OFFSET - XIP_FIXUP_OFFSET a3 add a3, a3, a1 REG_L sp, (a3) @@ -167,7 +161,6 @@ secondary_start_sbi: #ifdef CONFIG_MMU /* Enable virtual memory and relocate to virtual address */ la a0, swapper_pg_dir - XIP_FIXUP_OFFSET a0 call relocate_enable_mmu #endif call .Lsetup_trap_vector @@ -269,40 +262,13 @@ SYM_CODE_START(_start_kernel) .Lgood_cores: /* The lottery system is only required for spinwait booting method */ -#ifndef CONFIG_XIP_KERNEL /* Pick one hart to run the main boot sequence */ la a3, hart_lottery li a2, 1 amoadd.w a3, a2, (a3) bnez a3, .Lsecondary_start - -#else - /* hart_lottery in flash contains a magic number */ - la a3, hart_lottery - mv a2, a3 - XIP_FIXUP_OFFSET a2 - XIP_FIXUP_FLASH_OFFSET a3 - lw t1, (a3) - amoswap.w t0, t1, (a2) - /* first time here if hart_lottery in RAM is not set */ - beq t0, t1, .Lsecondary_start - -#endif /* CONFIG_XIP */ #endif /* CONFIG_RISCV_BOOT_SPINWAIT */ -#ifdef CONFIG_XIP_KERNEL - la sp, _end + THREAD_SIZE - XIP_FIXUP_OFFSET sp - mv s0, a0 - mv s1, a1 - call __copy_data - - /* Restore a0 & a1 copy */ - mv a0, s0 - mv a1, s1 -#endif - -#ifndef CONFIG_XIP_KERNEL /* Clear BSS for flat non-ELF images */ la a3, __bss_start la a4, __bss_stop @@ -312,20 +278,16 @@ SYM_CODE_START(_start_kernel) add a3, a3, RISCV_SZPTR blt a3, a4, .Lclear_bss .Lclear_bss_done: -#endif la a2, boot_cpu_hartid - XIP_FIXUP_OFFSET a2 REG_S a0, (a2) /* Initialize page tables and relocate to virtual addresses */ la tp, init_task la sp, init_thread_union + THREAD_SIZE - XIP_FIXUP_OFFSET sp addi sp, sp, -PT_SIZE_ON_STACK scs_load_init_stack #ifdef CONFIG_BUILTIN_DTB la a0, __dtb_start - XIP_FIXUP_OFFSET a0 #else mv a0, a1 #endif /* CONFIG_BUILTIN_DTB */ @@ -335,7 +297,6 @@ SYM_CODE_START(_start_kernel) call setup_vm #ifdef CONFIG_MMU la a0, early_pg_dir - XIP_FIXUP_OFFSET a0 call relocate_enable_mmu #endif /* CONFIG_MMU */ @@ -374,9 +335,7 @@ SYM_CODE_START(_start_kernel) slli a3, a0, LGREG la a1, __cpu_spinwait_stack_pointer - XIP_FIXUP_OFFSET a1 la a2, __cpu_spinwait_task_pointer - XIP_FIXUP_OFFSET a2 add a1, a3, a1 add a2, a3, a2 diff --git a/arch/riscv/kernel/head.h b/arch/riscv/kernel/head.h index a556fdaafed9..05a04bef442b 100644 --- a/arch/riscv/kernel/head.h +++ b/arch/riscv/kernel/head.h @@ -11,9 +11,6 @@ extern atomic_t hart_lottery; asmlinkage void __init setup_vm(uintptr_t dtb_pa); -#ifdef CONFIG_XIP_KERNEL -asmlinkage void __init __copy_data(void); -#endif #ifdef CONFIG_RISCV_BOOT_SPINWAIT extern void *__cpu_spinwait_stack_pointer[]; diff --git a/arch/riscv/kernel/mcount-dyn.S b/arch/riscv/kernel/mcount-dyn.S index 48f6c4f7dca0..082fe0b0e3c0 100644 --- a/arch/riscv/kernel/mcount-dyn.S +++ b/arch/riscv/kernel/mcount-dyn.S @@ -66,7 +66,7 @@ * 8(sp) stores the function return address (i.e. parent IP) that * can be accessed by &(fregs)->ra in tracing function. * -* The other regs are saved at the respective localtion and accessed +* The other regs are saved at the respective location and accessed * by the respective ftrace_regs member. * * Here is the layout of stack for your reference. diff --git a/arch/riscv/kernel/module-sections.c b/arch/riscv/kernel/module-sections.c index 1675cbad8619..98eaac6f6606 100644 --- a/arch/riscv/kernel/module-sections.c +++ b/arch/riscv/kernel/module-sections.c @@ -148,7 +148,7 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, return -ENOEXEC; } - /* Calculate the maxinum number of entries */ + /* Calculate the maximum number of entries */ for (i = 0; i < ehdr->e_shnum; i++) { size_t num_relas = sechdrs[i].sh_size / sizeof(Elf_Rela); Elf_Rela *relas = (void *)ehdr + sechdrs[i].sh_offset; diff --git a/arch/riscv/kernel/probes/kprobes.c b/arch/riscv/kernel/probes/kprobes.c index 8723390c7cad..9e2afabf94e8 100644 --- a/arch/riscv/kernel/probes/kprobes.c +++ b/arch/riscv/kernel/probes/kprobes.c @@ -149,7 +149,7 @@ static void __kprobes set_current_kprobe(struct kprobe *p) /* * Interrupts need to be disabled before single-step mode is set, and not - * reenabled until after single-step mode ends. + * re-enabled until after single-step mode ends. * Without disabling interrupt on local CPU, there is a chance of * interrupt occurrence in the period of exception return and start of * out-of-line single-step, that result in wrongly single stepping diff --git a/arch/riscv/kernel/probes/uprobes.c b/arch/riscv/kernel/probes/uprobes.c index f0d0691a8688..eb177d0ce8ab 100644 --- a/arch/riscv/kernel/probes/uprobes.c +++ b/arch/riscv/kernel/probes/uprobes.c @@ -111,7 +111,7 @@ void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) current->thread.bad_cause = utask->autask.saved_cause; /* - * Task has received a fatal signal, so reset back to probbed + * Task has received a fatal signal, so reset back to probed * address. */ instruction_pointer_set(regs, utask->vaddr); diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index b5bc5fc65cea..c89cc272440b 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -46,11 +46,7 @@ * This is used before the kernel initializes the BSS so it can't be in the * BSS. */ -atomic_t hart_lottery __section(".sdata") -#ifdef CONFIG_XIP_KERNEL -= ATOMIC_INIT(0xC001BEEF) -#endif -; +atomic_t hart_lottery __section(".sdata"); unsigned long boot_cpu_hartid; EXPORT_SYMBOL_GPL(boot_cpu_hartid); diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index d85916a3660c..8b628580fe11 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -251,18 +251,14 @@ asmlinkage __visible void smp_callin(void) set_cpu_online(curr_cpuid, true); /* - * Remote cache and TLB flushes are ignored while the CPU is offline, - * so flush them both right now just in case. + * Remote instruction cache and TLB flushes are ignored while the CPU + * is offline, so flush them both right now just in case. */ local_flush_icache_all(); local_flush_tlb_all(); #ifndef CONFIG_HOTPLUG_PARALLEL complete(&cpu_running); #endif - /* - * Disable preemption before enabling interrupts, so we don't try to - * schedule a CPU that hasn't actually started yet. - */ local_irq_enable(); cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); } diff --git a/arch/riscv/kernel/soc.c b/arch/riscv/kernel/soc.c index a0516172a33c..7d69e3b6bab4 100644 --- a/arch/riscv/kernel/soc.c +++ b/arch/riscv/kernel/soc.c @@ -8,7 +8,7 @@ #include <asm/soc.h> /* - * This is called extremly early, before parse_dtb(), to allow initializing + * This is called extremely early, before parse_dtb(), to allow initializing * SoC hardware before memory or any device driver initialization. */ void __init soc_early_init(void) diff --git a/arch/riscv/kernel/suspend.c b/arch/riscv/kernel/suspend.c index aff93090c4ef..3efbf7874f3b 100644 --- a/arch/riscv/kernel/suspend.c +++ b/arch/riscv/kernel/suspend.c @@ -78,7 +78,7 @@ int cpu_suspend(unsigned long arg, suspend_save_csrs(&context); /* - * Function graph tracer state gets incosistent when the kernel + * Function graph tracer state gets inconsistent when the kernel * calls functions that never return (aka finishers) hence disable * graph tracing during their execution. */ diff --git a/arch/riscv/kernel/suspend_entry.S b/arch/riscv/kernel/suspend_entry.S index 2d54f309c140..d71b55fd6259 100644 --- a/arch/riscv/kernel/suspend_entry.S +++ b/arch/riscv/kernel/suspend_entry.S @@ -10,7 +10,6 @@ #include <asm/asm-offsets.h> #include <asm/assembler.h> #include <asm/csr.h> -#include <asm/xip_fixup.h> .text .altmacro @@ -70,7 +69,6 @@ SYM_TYPED_FUNC_START(__cpu_resume_enter) /* Enable MMU */ la a0, swapper_pg_dir - XIP_FIXUP_OFFSET a0 call relocate_enable_mmu /* Restore A0 and A1 */ diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index 461279a7bd86..8c62c771a656 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -142,11 +142,7 @@ static void do_trap_error(struct pt_regs *regs, int signo, int code, } } -#if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_RISCV_ALTERNATIVE) -#define __trap_section __noinstr_section(".xip.traps") -#else #define __trap_section noinstr -#endif #define DO_ERROR_INFO(name, signo, code, str) \ asmlinkage __visible __trap_section void name(struct pt_regs *regs) \ { \ @@ -165,6 +161,8 @@ asmlinkage __visible __trap_section void name(struct pt_regs *regs) \ DO_ERROR_INFO(do_trap_unknown, SIGILL, ILL_ILLTRP, "unknown exception"); +DO_ERROR_INFO(do_trap_hardware_error, + SIGBUS, BUS_MCEERR_AR, "hardware error"); DO_ERROR_INFO(do_trap_insn_misaligned, SIGBUS, BUS_ADRALN, "instruction address misaligned"); DO_ERROR_INFO(do_trap_insn_fault, diff --git a/arch/riscv/kernel/unaligned_access_speed.c b/arch/riscv/kernel/unaligned_access_speed.c index b36a6a56f404..485ab1d105d3 100644 --- a/arch/riscv/kernel/unaligned_access_speed.c +++ b/arch/riscv/kernel/unaligned_access_speed.c @@ -16,7 +16,7 @@ #include "copy-unaligned.h" -#define MISALIGNED_ACCESS_JIFFIES_LG2 1 +#define MISALIGNED_ACCESS_NS 8000000 #define MISALIGNED_BUFFER_SIZE 0x4000 #define MISALIGNED_BUFFER_ORDER get_order(MISALIGNED_BUFFER_SIZE) #define MISALIGNED_COPY_SIZE ((MISALIGNED_BUFFER_SIZE / 2) - 0x80) @@ -29,109 +29,125 @@ static long unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNO static cpumask_t fast_misaligned_access; -#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS -static int check_unaligned_access(void *param) +static u64 __maybe_unused +measure_cycles(void (*func)(void *dst, const void *src, size_t len), + void *dst, void *src, size_t len) { - int cpu = smp_processor_id(); - u64 start_cycles, end_cycles; - u64 word_cycles; - u64 byte_cycles; - int ratio; - unsigned long start_jiffies, now; - struct page *page = param; - void *dst; - void *src; - long speed = RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW; - - if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) - return 0; + u64 start_cycles, end_cycles, cycles = -1ULL; + u64 start_ns; - /* Make an unaligned destination buffer. */ - dst = (void *)((unsigned long)page_address(page) | 0x1); - /* Unalign src as well, but differently (off by 1 + 2 = 3). */ - src = dst + (MISALIGNED_BUFFER_SIZE / 2); - src += 2; - word_cycles = -1ULL; /* Do a warmup. */ - __riscv_copy_words_unaligned(dst, src, MISALIGNED_COPY_SIZE); + func(dst, src, len); + preempt_disable(); - start_jiffies = jiffies; - while ((now = jiffies) == start_jiffies) - cpu_relax(); /* * For a fixed amount of time, repeatedly try the function, and take * the best time in cycles as the measurement. */ - while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) { + start_ns = ktime_get_mono_fast_ns(); + while (ktime_get_mono_fast_ns() < start_ns + MISALIGNED_ACCESS_NS) { start_cycles = get_cycles64(); /* Ensure the CSR read can't reorder WRT to the copy. */ mb(); - __riscv_copy_words_unaligned(dst, src, MISALIGNED_COPY_SIZE); + func(dst, src, len); /* Ensure the copy ends before the end time is snapped. */ mb(); end_cycles = get_cycles64(); - if ((end_cycles - start_cycles) < word_cycles) - word_cycles = end_cycles - start_cycles; + if ((end_cycles - start_cycles) < cycles) + cycles = end_cycles - start_cycles; } - byte_cycles = -1ULL; - __riscv_copy_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE); - start_jiffies = jiffies; - while ((now = jiffies) == start_jiffies) - cpu_relax(); + preempt_enable(); - while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) { - start_cycles = get_cycles64(); - mb(); - __riscv_copy_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE); - mb(); - end_cycles = get_cycles64(); - if ((end_cycles - start_cycles) < byte_cycles) - byte_cycles = end_cycles - start_cycles; - } + return cycles; +} - preempt_enable(); +/* + * Return: + * 1 if unaligned accesses are fast + * 0 if unaligned accesses are slow + * -1 if check cannot be done + */ +static int __maybe_unused +compare_unaligned_access(void (*word_copy)(void *dst, const void *src, size_t len), + void (*byte_copy)(void *dst, const void *src, size_t len), + void *buf, const char *type) +{ + int cpu = smp_processor_id(); + u64 word_cycles; + u64 byte_cycles; + void *dst, *src; + bool fast; + int ratio; + + /* Make an unaligned destination buffer. */ + dst = (void *)((unsigned long)buf | 0x1); + /* Unalign src as well, but differently (off by 1 + 2 = 3). */ + src = dst + (MISALIGNED_BUFFER_SIZE / 2); + src += 2; + + word_cycles = measure_cycles(word_copy, dst, src, MISALIGNED_COPY_SIZE); + byte_cycles = measure_cycles(byte_copy, dst, src, MISALIGNED_COPY_SIZE); /* Don't divide by zero. */ if (!word_cycles || !byte_cycles) { - pr_warn("cpu%d: rdtime lacks granularity needed to measure unaligned access speed\n", - cpu); + pr_warn("cpu%d: rdtime lacks granularity needed to measure %s unaligned access speed\n", + cpu, type); - return 0; + return -1; } - if (word_cycles < byte_cycles) - speed = RISCV_HWPROBE_MISALIGNED_SCALAR_FAST; + fast = word_cycles < byte_cycles; ratio = div_u64((byte_cycles * 100), word_cycles); - pr_info("cpu%d: Ratio of byte access time to unaligned word access is %d.%02d, unaligned accesses are %s\n", + pr_info("cpu%d: %s unaligned word access speed is %d.%02dx byte access speed (%s)\n", cpu, + type, ratio / 100, ratio % 100, - (speed == RISCV_HWPROBE_MISALIGNED_SCALAR_FAST) ? "fast" : "slow"); + fast ? "fast" : "slow"); - per_cpu(misaligned_access_speed, cpu) = speed; + return fast; +} + +#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS +static int check_unaligned_access(struct page *page) +{ + void *buf = page_address(page); + int cpu = smp_processor_id(); + int ret; + + if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) + return 0; + + ret = compare_unaligned_access(__riscv_copy_words_unaligned, + __riscv_copy_bytes_unaligned, + buf, "scalar"); + if (ret < 0) + return 0; /* * Set the value of fast_misaligned_access of a CPU. These operations * are atomic to avoid race conditions. */ - if (speed == RISCV_HWPROBE_MISALIGNED_SCALAR_FAST) + if (ret) { + per_cpu(misaligned_access_speed, cpu) = RISCV_HWPROBE_MISALIGNED_SCALAR_FAST; cpumask_set_cpu(cpu, &fast_misaligned_access); - else + } else { + per_cpu(misaligned_access_speed, cpu) = RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW; cpumask_clear_cpu(cpu, &fast_misaligned_access); + } return 0; } -static void __init check_unaligned_access_nonboot_cpu(void *param) +static void __init _check_unaligned_access(void *param) { unsigned int cpu = smp_processor_id(); struct page **pages = param; - if (smp_processor_id() != 0) - check_unaligned_access(pages[cpu]); + check_unaligned_access(pages[cpu]); } /* Measure unaligned access speed on all CPUs present at boot in parallel. */ @@ -158,11 +174,7 @@ static void __init check_unaligned_access_speed_all_cpus(void) } } - /* Check everybody except 0, who stays behind to tend jiffies. */ - on_each_cpu(check_unaligned_access_nonboot_cpu, bufs, 1); - - /* Check core 0. */ - smp_call_on_cpu(0, check_unaligned_access, bufs[0], true); + on_each_cpu(_check_unaligned_access, bufs, 1); out: for_each_cpu(cpu, cpu_online_mask) { @@ -281,15 +293,8 @@ static int riscv_offline_cpu(unsigned int cpu) static void check_vector_unaligned_access(struct work_struct *work __always_unused) { int cpu = smp_processor_id(); - u64 start_cycles, end_cycles; - u64 word_cycles; - u64 byte_cycles; - int ratio; - unsigned long start_jiffies, now; struct page *page; - void *dst; - void *src; - long speed = RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW; + int ret; if (per_cpu(vector_misaligned_access, cpu) != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) return; @@ -300,76 +305,20 @@ static void check_vector_unaligned_access(struct work_struct *work __always_unus return; } - /* Make an unaligned destination buffer. */ - dst = (void *)((unsigned long)page_address(page) | 0x1); - /* Unalign src as well, but differently (off by 1 + 2 = 3). */ - src = dst + (MISALIGNED_BUFFER_SIZE / 2); - src += 2; - word_cycles = -1ULL; - - /* Do a warmup. */ kernel_vector_begin(); - __riscv_copy_vec_words_unaligned(dst, src, MISALIGNED_COPY_SIZE); - - start_jiffies = jiffies; - while ((now = jiffies) == start_jiffies) - cpu_relax(); - - /* - * For a fixed amount of time, repeatedly try the function, and take - * the best time in cycles as the measurement. - */ - while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) { - start_cycles = get_cycles64(); - /* Ensure the CSR read can't reorder WRT to the copy. */ - mb(); - __riscv_copy_vec_words_unaligned(dst, src, MISALIGNED_COPY_SIZE); - /* Ensure the copy ends before the end time is snapped. */ - mb(); - end_cycles = get_cycles64(); - if ((end_cycles - start_cycles) < word_cycles) - word_cycles = end_cycles - start_cycles; - } - - byte_cycles = -1ULL; - __riscv_copy_vec_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE); - start_jiffies = jiffies; - while ((now = jiffies) == start_jiffies) - cpu_relax(); - - while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) { - start_cycles = get_cycles64(); - /* Ensure the CSR read can't reorder WRT to the copy. */ - mb(); - __riscv_copy_vec_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE); - /* Ensure the copy ends before the end time is snapped. */ - mb(); - end_cycles = get_cycles64(); - if ((end_cycles - start_cycles) < byte_cycles) - byte_cycles = end_cycles - start_cycles; - } + ret = compare_unaligned_access(__riscv_copy_vec_words_unaligned, + __riscv_copy_vec_bytes_unaligned, + page_address(page), "vector"); kernel_vector_end(); - /* Don't divide by zero. */ - if (!word_cycles || !byte_cycles) { - pr_warn("cpu%d: rdtime lacks granularity needed to measure unaligned vector access speed\n", - cpu); - + if (ret < 0) goto free; - } - if (word_cycles < byte_cycles) - speed = RISCV_HWPROBE_MISALIGNED_VECTOR_FAST; - - ratio = div_u64((byte_cycles * 100), word_cycles); - pr_info("cpu%d: Ratio of vector byte access time to vector unaligned word access is %d.%02d, unaligned accesses are %s\n", - cpu, - ratio / 100, - ratio % 100, - (speed == RISCV_HWPROBE_MISALIGNED_VECTOR_FAST) ? "fast" : "slow"); - - per_cpu(vector_misaligned_access, cpu) = speed; + if (ret) + per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_FAST; + else + per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW; free: __free_pages(page, MISALIGNED_BUFFER_ORDER); @@ -494,4 +443,4 @@ static int __init check_unaligned_access_all_cpus(void) return 0; } -arch_initcall(check_unaligned_access_all_cpus); +late_initcall(check_unaligned_access_all_cpus); diff --git a/arch/riscv/kernel/vdso_cfi/.gitignore b/arch/riscv/kernel/vdso_cfi/.gitignore new file mode 100644 index 000000000000..220b6ebece4f --- /dev/null +++ b/arch/riscv/kernel/vdso_cfi/.gitignore @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copied source files from the main vdso directory +*.c +*.S +!vdso-cfi.S +vdso.lds +*.tmp +vdso-syms.S diff --git a/arch/riscv/kernel/vdso_cfi/Makefile b/arch/riscv/kernel/vdso_cfi/Makefile index 8ebd190782b0..10292498b765 100644 --- a/arch/riscv/kernel/vdso_cfi/Makefile +++ b/arch/riscv/kernel/vdso_cfi/Makefile @@ -23,3 +23,6 @@ $(vdso_c_objects): $(obj)/%.c: $(src)/%.c # Include the main VDSO Makefile which contains all the build rules and sources # The VDSO_CFI_BUILD variable will be passed to it to enable CFI compilation include $(src)/Makefile + +# Clean rules - remove the copied source files +clean-files += $(notdir $(vdso_c_sources)) $(notdir $(vdso_S_sources)) diff --git a/arch/riscv/kernel/vmcore_info.c b/arch/riscv/kernel/vmcore_info.c index d5e448aa90e7..c27efceec3cc 100644 --- a/arch/riscv/kernel/vmcore_info.c +++ b/arch/riscv/kernel/vmcore_info.c @@ -3,6 +3,11 @@ #include <linux/vmcore_info.h> #include <linux/pagemap.h> +static inline u64 get_satp_value(void) +{ + return csr_read(CSR_SATP); +} + void arch_crash_save_vmcoreinfo(void) { VMCOREINFO_NUMBER(phys_ram_base); @@ -19,13 +24,8 @@ void arch_crash_save_vmcoreinfo(void) #endif #endif vmcoreinfo_append_str("NUMBER(KERNEL_LINK_ADDR)=0x%lx\n", KERNEL_LINK_ADDR); -#ifdef CONFIG_XIP_KERNEL - /* TODO: Communicate with crash-utility developers on the information to - * export. The XIP case is more complicated, because the virtual-physical - * address offset depends on whether the address is in ROM or in RAM. - */ -#else vmcoreinfo_append_str("NUMBER(va_kernel_pa_offset)=0x%lx\n", kernel_map.va_kernel_pa_offset); -#endif + vmcoreinfo_append_str("KERNELOFFSET=%lx\n", kaslr_offset()); + vmcoreinfo_append_str("NUMBER(satp)=0x%llx\n", get_satp_value()); } diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 997f9eb3b22b..1f4f8496941a 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -7,10 +7,6 @@ #define RO_EXCEPTION_TABLE_ALIGN 4 #define RUNTIME_DISCARD_EXIT -#ifdef CONFIG_XIP_KERNEL -#include "vmlinux-xip.lds.S" -#else - #include <asm/pgtable.h> #define LOAD_OFFSET KERNEL_LINK_ADDR @@ -176,4 +172,3 @@ SECTIONS DISCARDS } -#endif /* CONFIG_XIP_KERNEL */ |