diff options
Diffstat (limited to 'arch/riscv')
288 files changed, 15703 insertions, 4385 deletions
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 6b39f37f769a..d6c2dbf8455c 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -21,14 +21,16 @@ config RISCV select ARCH_DMA_DEFAULT_COHERENT select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM_VMEMMAP - select ARCH_ENABLE_MEMORY_HOTREMOVE if MEMORY_HOTPLUG select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE + select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_BINFMT_FLAT + select ARCH_HAS_CC_CAN_LINK select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DEBUG_VIRTUAL if MMU select ARCH_HAS_DEBUG_VM_PGTABLE select ARCH_HAS_DEBUG_WX + select ARCH_HAS_DELAY_TIMER select ARCH_HAS_ELF_CORE_EFLAGS if BINFMT_ELF && ELF_CORE select ARCH_HAS_FAST_MULTIPLIER select ARCH_HAS_FORTIFY_SOURCE @@ -48,22 +50,21 @@ config RISCV select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_SET_DIRECT_MAP if MMU select ARCH_HAS_SET_MEMORY if MMU - select ARCH_HAS_STRICT_KERNEL_RWX if MMU && !XIP_KERNEL - select ARCH_HAS_STRICT_MODULE_RWX if MMU && !XIP_KERNEL + select ARCH_HAS_STRICT_KERNEL_RWX if MMU + select ARCH_HAS_STRICT_MODULE_RWX if MMU select ARCH_HAS_SYNC_CORE_BEFORE_USERMODE select ARCH_HAS_SYSCALL_WRAPPER select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAS_UBSAN - select ARCH_HAS_VDSO_ARCH_DATA if HAVE_GENERIC_VDSO + select ARCH_HAS_VDSO_ARCH_DATA select ARCH_HAVE_NMI_SAFE_CMPXCHG - select ARCH_KEEP_MEMBLOCK if ACPI + select ARCH_KEEP_MEMBLOCK if ACPI || KEXEC select ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE if 64BIT && MMU select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT select ARCH_STACKWALK select ARCH_SUPPORTS_ATOMIC_RMW - # clang >= 17: https://github.com/llvm/llvm-project/commit/62fa708ceb027713b386c7e0efda994f8bdc27e2 - select ARCH_SUPPORTS_CFI if (!CC_IS_CLANG || CLANG_VERSION >= 170000) + select ARCH_SUPPORTS_CFI select ARCH_SUPPORTS_DEBUG_PAGEALLOC if MMU select ARCH_SUPPORTS_HUGE_PFNMAP if TRANSPARENT_HUGEPAGE select ARCH_SUPPORTS_HUGETLBFS if MMU @@ -72,6 +73,7 @@ config RISCV select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS if 64BIT && MMU select ARCH_SUPPORTS_PAGE_TABLE_CHECK if MMU select ARCH_SUPPORTS_PER_VMA_LOCK if MMU + select ARCH_HAS_PTE_PROTNONE if MMU select ARCH_SUPPORTS_RT select ARCH_SUPPORTS_SHADOW_CALL_STACK if HAVE_SHADOW_CALL_STACK select ARCH_SUPPORTS_SCHED_MC if SMP @@ -85,7 +87,7 @@ config RISCV select ARCH_WANT_FRAME_POINTERS select ARCH_WANT_GENERAL_HUGETLB if !RISCV_ISA_SVNAPOT select ARCH_WANT_HUGE_PMD_SHARE if 64BIT - select ARCH_WANT_LD_ORPHAN_WARN if !XIP_KERNEL + select ARCH_WANT_LD_ORPHAN_WARN select ARCH_WANT_OPTIMIZE_DAX_VMEMMAP select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP select ARCH_WANTS_NO_INSTR @@ -105,14 +107,16 @@ config RISCV select FUNCTION_ALIGNMENT_8B if DYNAMIC_FTRACE_WITH_CALL_OPS select GENERIC_ARCH_TOPOLOGY select GENERIC_ATOMIC64 if !64BIT + select GENERIC_BITREVERSE if HAVE_ARCH_BITREVERSE select GENERIC_CLOCKEVENTS_BROADCAST if SMP select GENERIC_CPU_DEVICES select GENERIC_CPU_VULNERABILITIES select GENERIC_EARLY_IOREMAP select GENERIC_ENTRY - select GENERIC_GETTIMEOFDAY if HAVE_GENERIC_VDSO && 64BIT + select GENERIC_GETTIMEOFDAY if MMU && 64BIT select GENERIC_IDLE_POLL_SETUP select GENERIC_IOREMAP if MMU + select HAVE_IOREMAP_PROT if MMU select GENERIC_IRQ_IPI if SMP select GENERIC_IRQ_IPI_MUX if SMP select GENERIC_IRQ_MULTI_HANDLER @@ -123,20 +127,20 @@ config RISCV select GENERIC_PCI_IOMAP select GENERIC_SCHED_CLOCK select GENERIC_SMP_IDLE_THREAD - select GENERIC_TIME_VSYSCALL if GENERIC_GETTIMEOFDAY select HARDIRQS_SW_RESEND select HAS_IOPORT if MMU select HAVE_ALIGNED_STRUCT_PAGE select HAVE_ARCH_AUDITSYSCALL + select HAVE_ARCH_BITREVERSE if RISCV_ISA_ZBKB && BITREVERSE select HAVE_ARCH_HUGE_VMALLOC if HAVE_ARCH_HUGE_VMAP select HAVE_ARCH_HUGE_VMAP if MMU && 64BIT - select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL - select HAVE_ARCH_JUMP_LABEL_RELATIVE if !XIP_KERNEL + select HAVE_ARCH_JUMP_LABEL + select HAVE_ARCH_JUMP_LABEL_RELATIVE select HAVE_ARCH_KASAN if MMU && 64BIT select HAVE_ARCH_KASAN_VMALLOC if MMU && 64BIT select HAVE_ARCH_KFENCE if MMU && 64BIT select HAVE_ARCH_KSTACK_ERASE - select HAVE_ARCH_KGDB if !XIP_KERNEL + select HAVE_ARCH_KGDB select HAVE_ARCH_KGDB_QXFER_PKT select HAVE_ARCH_MMAP_RND_BITS if MMU select HAVE_ARCH_MMAP_RND_COMPAT_BITS if COMPAT @@ -151,36 +155,36 @@ config RISCV select HAVE_ARCH_USERFAULTFD_WP if 64BIT && MMU && USERFAULTFD && RISCV_ISA_SVRSW60T59B select HAVE_ARCH_VMAP_STACK if MMU && 64BIT select HAVE_ASM_MODVERSIONS + select HAVE_BUILDTIME_MCOUNT_SORT select HAVE_CONTEXT_TRACKING_USER select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS if MMU - select HAVE_DYNAMIC_FTRACE if !XIP_KERNEL && MMU && (CLANG_SUPPORTS_DYNAMIC_FTRACE || GCC_SUPPORTS_DYNAMIC_FTRACE) - select FUNCTION_ALIGNMENT_4B if HAVE_DYNAMIC_FTRACE && RISCV_ISA_C + select HAVE_DYNAMIC_FTRACE if MMU && (CLANG_SUPPORTS_DYNAMIC_FTRACE || GCC_SUPPORTS_DYNAMIC_FTRACE) + select FUNCTION_ALIGNMENT_4B if DYNAMIC_FTRACE && RISCV_ISA_C select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS if HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS if (DYNAMIC_FTRACE_WITH_ARGS && !CFI) select HAVE_DYNAMIC_FTRACE_WITH_ARGS if HAVE_DYNAMIC_FTRACE select HAVE_FTRACE_GRAPH_FUNC select HAVE_FUNCTION_GRAPH_TRACER if HAVE_DYNAMIC_FTRACE_WITH_ARGS select HAVE_FUNCTION_GRAPH_FREGS - select HAVE_FUNCTION_TRACER if !XIP_KERNEL && HAVE_DYNAMIC_FTRACE + select HAVE_FUNCTION_TRACER if HAVE_DYNAMIC_FTRACE select HAVE_EBPF_JIT if MMU select HAVE_GENERIC_TIF_BITS select HAVE_GUP_FAST if MMU select HAVE_FUNCTION_ARG_ACCESS_API select HAVE_FUNCTION_ERROR_INJECTION select HAVE_GCC_PLUGINS - select HAVE_GENERIC_VDSO if MMU select HAVE_IRQ_TIME_ACCOUNTING - select HAVE_KERNEL_BZIP2 if !XIP_KERNEL && !EFI_ZBOOT - select HAVE_KERNEL_GZIP if !XIP_KERNEL && !EFI_ZBOOT - select HAVE_KERNEL_LZ4 if !XIP_KERNEL && !EFI_ZBOOT - select HAVE_KERNEL_LZMA if !XIP_KERNEL && !EFI_ZBOOT - select HAVE_KERNEL_LZO if !XIP_KERNEL && !EFI_ZBOOT - select HAVE_KERNEL_UNCOMPRESSED if !XIP_KERNEL && !EFI_ZBOOT - select HAVE_KERNEL_ZSTD if !XIP_KERNEL && !EFI_ZBOOT - select HAVE_KERNEL_XZ if !XIP_KERNEL && !EFI_ZBOOT - select HAVE_KPROBES if !XIP_KERNEL - select HAVE_KRETPROBES if !XIP_KERNEL + select HAVE_KERNEL_BZIP2 if !EFI_ZBOOT + select HAVE_KERNEL_GZIP if !EFI_ZBOOT + select HAVE_KERNEL_LZ4 if !EFI_ZBOOT + select HAVE_KERNEL_LZMA if !EFI_ZBOOT + select HAVE_KERNEL_LZO if !EFI_ZBOOT + select HAVE_KERNEL_UNCOMPRESSED if !EFI_ZBOOT + select HAVE_KERNEL_ZSTD if !EFI_ZBOOT + select HAVE_KERNEL_XZ if !EFI_ZBOOT + select HAVE_KPROBES + select HAVE_KRETPROBES # https://github.com/ClangBuiltLinux/linux/issues/1881 select HAVE_LD_DEAD_CODE_DATA_ELIMINATION if !LD_IS_LLD select HAVE_MOVE_PMD @@ -191,9 +195,9 @@ config RISCV select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP select HAVE_POSIX_CPU_TIMERS_TASK_WORK - select HAVE_PREEMPT_DYNAMIC_KEY if !XIP_KERNEL + select HAVE_PREEMPT_DYNAMIC_KEY select HAVE_REGS_AND_STACK_ACCESS_API - select HAVE_RETHOOK if !XIP_KERNEL + select HAVE_RETHOOK select HAVE_RSEQ select HAVE_RUST if RUSTC_SUPPORTS_RISCV && CC_IS_CLANG select HAVE_SAMPLE_FTRACE_DIRECT @@ -214,7 +218,7 @@ config RISCV select PCI_ECAM if (ACPI && PCI) select PCI_MSI if PCI select RELOCATABLE if !MMU && !PHYS_RAM_BASE_FIXED - select RISCV_ALTERNATIVE if !XIP_KERNEL + select RISCV_ALTERNATIVE select RISCV_APLIC select RISCV_IMSIC select RISCV_INTC @@ -225,16 +229,14 @@ config RISCV select THREAD_INFO_IN_TASK select TRACE_IRQFLAGS_SUPPORT select UACCESS_MEMCPY if !MMU - select VDSO_GETRANDOM if HAVE_GENERIC_VDSO && 64BIT + select VDSO_DATASTORE if MMU + select VDSO_GETRANDOM if MMU && 64BIT select USER_STACKTRACE_SUPPORT select ZONE_DMA32 if 64BIT config RUSTC_SUPPORTS_RISCV def_bool y depends on 64BIT - # Shadow call stack requires rustc version 1.82+ due to use of the - # -Zsanitizer=shadow-call-stack flag. - depends on !SHADOW_CALL_STACK || RUSTC_VERSION >= 108200 config CLANG_SUPPORTS_DYNAMIC_FTRACE def_bool CC_IS_CLANG @@ -324,7 +326,7 @@ config STACKTRACE_SUPPORT config GENERIC_BUG def_bool y depends on BUG - select GENERIC_BUG_RELATIVE_POINTERS if 64BIT + select GENERIC_BUG_RELATIVE_POINTERS config GENERIC_BUG_RELATIVE_POINTERS bool @@ -407,9 +409,6 @@ config ARCH_RV32I bool "RV32I" depends on NONPORTABLE select 32BIT - select GENERIC_LIB_ASHLDI3 - select GENERIC_LIB_ASHRDI3 - select GENERIC_LIB_LSHRDI3 select GENERIC_LIB_UCMPDI2 config ARCH_RV64I @@ -458,7 +457,8 @@ config NR_CPUS range 2 32 if RISCV_SBI_V01 && 32BIT range 2 64 if RISCV_SBI_V01 && 64BIT default "32" if 32BIT - default "64" if 64BIT + default "64" if RISCV_SBI_V01 && 64BIT + default "256" if !RISCV_SBI_V01 && 64BIT config HOTPLUG_CPU bool "Support for hot-pluggable CPUs" @@ -541,7 +541,6 @@ endchoice config RISCV_ALTERNATIVE bool - depends on !XIP_KERNEL help This Kconfig allows the kernel to automatically patch the erratum or cpufeature required by the execution platform at run @@ -595,6 +594,24 @@ config RISCV_ISA_SVNAPOT If you don't know what to do here, say Y. +config RISCV_ISA_SSQOSID + bool "Ssqosid extension support for supervisor mode Quality of Service ID" + depends on 64BIT + default n + help + Adds support for the Ssqosid ISA extension (Supervisor-mode + Quality of Service ID). + + Ssqosid defines the srmcfg CSR which allows the system to tag the + running process with an RCID (Resource Control ID) and MCID + (Monitoring Counter ID). The RCID is used to determine resource + allocation. The MCID is used to track resource usage in event + counters. + + For example, a cache controller may use the RCID to apply a + cache partitioning scheme and use the MCID to track how much + cache a process, or a group of processes, is using. + config RISCV_ISA_SVPBMT bool "Svpbmt extension support for supervisor mode page-based memory types" depends on 64BIT && MMU @@ -878,19 +895,18 @@ config TOOLCHAIN_NEEDS_EXPLICIT_ZICSR_ZIFENCEI and Zifencei are supported in binutils from version 2.36 onwards. To make life easier, and avoid forcing toolchains that default to a newer ISA spec to version 2.2, relax the check to binutils >= 2.36. - For clang < 17 or GCC < 11.3.0, for which this is not possible or need - special treatment, this is dealt with in TOOLCHAIN_NEEDS_OLD_ISA_SPEC. + For GCC < 11.3.0, for which this is not possible or need special + treatment, this is dealt with in TOOLCHAIN_NEEDS_OLD_ISA_SPEC. config TOOLCHAIN_NEEDS_OLD_ISA_SPEC def_bool y depends on TOOLCHAIN_NEEDS_EXPLICIT_ZICSR_ZIFENCEI - # https://github.com/llvm/llvm-project/commit/22e199e6afb1263c943c0c0d4498694e15bf8a16 # https://gcc.gnu.org/git/?p=gcc.git;a=commit;h=d29f5d6ab513c52fd872f532c492e35ae9fd6671 - depends on (CC_IS_CLANG && CLANG_VERSION < 170000) || (CC_IS_GCC && GCC_VERSION < 110300) + depends on CC_IS_GCC && GCC_VERSION < 110300 help - Certain versions of clang and GCC do not support zicsr and zifencei via - -march. This option causes an older ISA spec compatible with these older - versions of clang and GCC to be passed to GAS, which has the same result + Certain versions of GCC do not support zicsr and zifencei via -march. + This option causes an older ISA spec compatible with these older + versions of GCC to be passed to GAS, which has the same result as passing zicsr and zifencei to -march. config FPU @@ -941,6 +957,28 @@ config RISCV_VECTOR_MISALIGNED help Enable detecting support for vector misaligned loads and stores. +config RISCV_SBI_FWFT_DELEGATE_MISALIGNED + bool "Request firmware delegation of unaligned access exceptions" + depends on RISCV_SBI + depends on NONPORTABLE + help + Use SBI FWFT to request delegation of load address misaligned and + store address misaligned exceptions, if possible, and prefer Linux + kernel emulation of these accesses to firmware emulation. + + Unfortunately, Linux's emulation is still incomplete. Namely, it + currently does not handle vector instructions and KVM guest accesses. + On platforms where these accesses would have been handled by firmware, + enabling this causes unexpected kernel oopses, userspaces crashes and + KVM guest crashes. If you are sure that these are not a problem for + your platform, you can say Y here, which may improve performance. + + Saying N here will not worsen emulation support for unaligned accesses + even in the case where the firmware also has incomplete support. It + simply keeps the firmware's emulation enabled. + + If you don't know what to do here, say N. + choice prompt "Unaligned Accesses Support" default RISCV_PROBE_UNALIGNED_ACCESS @@ -1005,7 +1043,6 @@ choice config RISCV_PROBE_VECTOR_UNALIGNED_ACCESS bool "Probe speed of vector unaligned accesses" select RISCV_VECTOR_MISALIGNED - depends on RISCV_ISA_V help During boot, the kernel will run a series of tests to determine the speed of vector unaligned accesses if they are supported. This probing @@ -1111,6 +1148,7 @@ config COMPAT config PARAVIRT bool "Enable paravirtualization code" depends on RISCV_SBI + select HAVE_PV_STEAL_CLOCK_GEN help This changes the kernel so it can modify itself when it is run under a hypervisor, potentially improving performance significantly @@ -1129,7 +1167,6 @@ config PARAVIRT_TIME_ACCOUNTING config RELOCATABLE bool "Build a relocatable kernel" - depends on !XIP_KERNEL select MODULE_SECTIONS if MODULES select ARCH_VMLINUX_NEEDS_RELOCS help @@ -1145,8 +1182,8 @@ config RELOCATABLE config RANDOMIZE_BASE bool "Randomize the address of the kernel image" - select RELOCATABLE - depends on MMU && 64BIT && !XIP_KERNEL + depends on RELOCATABLE + depends on MMU && 64BIT help Randomizes the virtual address at which the kernel image is loaded, as a security feature that deters exploit attempts @@ -1162,6 +1199,28 @@ config RANDOMIZE_BASE If unsure, say N. +config RISCV_USER_CFI + def_bool y + bool "riscv userspace control flow integrity" + depends on 64BIT && MMU && \ + $(cc-option,-mabi=lp64 -march=rv64ima_zicfiss_zicfilp -fcf-protection=full) + depends on RISCV_ALTERNATIVE + select RISCV_SBI + select ARCH_HAS_USER_SHADOW_STACK + select ARCH_USES_HIGH_VMA_FLAGS + select DYNAMIC_SIGFRAME + help + Provides CPU-assisted control flow integrity to userspace tasks. + Control flow integrity is provided by implementing shadow stack for + backward edge and indirect branch tracking for forward edge. + Shadow stack protection is a hardware feature that detects function + return address corruption. This helps mitigate ROP attacks. + Indirect branch tracking enforces that all indirect branches must land + on a landing pad instruction else CPU will fault. This mitigates against + JOP / COP attacks. Applications must be enabled to use it, and old userspace + does not get protection "for free". + default y. + endmenu # "Kernel features" menu "Boot options" @@ -1214,7 +1273,7 @@ config EFI_STUB config EFI bool "UEFI runtime support" - depends on OF && !XIP_KERNEL + depends on OF depends on MMU default y select ARCH_SUPPORTS_ACPI if 64BIT @@ -1265,44 +1324,6 @@ config PHYS_RAM_BASE explicitly specified to run early relocations of read-write data from flash to RAM. -config XIP_KERNEL - bool "Kernel Execute-In-Place from ROM" - depends on MMU && SPARSEMEM && NONPORTABLE - # This prevents XIP from being enabled by all{yes,mod}config, which - # fail to build since XIP doesn't support large kernels. - depends on !COMPILE_TEST - select PHYS_RAM_BASE_FIXED - help - Execute-In-Place allows the kernel to run from non-volatile storage - directly addressable by the CPU, such as NOR flash. This saves RAM - space since the text section of the kernel is not loaded from flash - to RAM. Read-write sections, such as the data section and stack, - are still copied to RAM. The XIP kernel is not compressed since - it has to run directly from flash, so it will take more space to - store it. The flash address used to link the kernel object files, - and for storing it, is configuration dependent. Therefore, if you - say Y here, you must know the proper physical address where to - store the kernel image depending on your own flash memory usage. - - Also note that the make target becomes "make xipImage" rather than - "make zImage" or "make Image". The final kernel binary to put in - ROM memory will be arch/riscv/boot/xipImage. - - SPARSEMEM is required because the kernel text and rodata that are - flash resident are not backed by memmap, then any attempt to get - a struct page on those regions will trigger a fault. - - If unsure, say N. - -config XIP_PHYS_ADDR - hex "XIP Kernel Physical Location" - depends on XIP_KERNEL - default "0x21000000" - help - This is the physical address in your flash memory the kernel will - be linked for and stored to. This address is dependent on your - own flash usage. - config RISCV_ISA_FALLBACK bool "Permit falling back to parsing riscv,isa for extension support by default" default y @@ -1351,6 +1372,20 @@ config PORTABLE config ARCH_PROC_KCORE_TEXT def_bool y +config ARCH_CC_CAN_LINK + bool + default $(cc_can_link_user,-march=rv64g -mabi=lp64d) if 64BIT && FPU + default $(cc_can_link_user,-march=rv64g -mabi=lp64) if 64BIT + default $(cc_can_link_user,-march=rv32g -mabi=ilp32d) if FPU + default $(cc_can_link_user,-march=rv32g -mabi=ilp32) + +config ARCH_USERFLAGS + string + default "-march=rv64g -mabi=lp64d" if 64BIT && FPU + default "-march=rv64g -mabi=lp64" if 64BIT + default "-march=rv32g -mabi=ilp32d" if FPU + default "-march=rv32g -mabi=ilp32" + menu "Power management options" source "kernel/power/Kconfig" diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs index d621b85dd63b..429e07589306 100644 --- a/arch/riscv/Kconfig.socs +++ b/arch/riscv/Kconfig.socs @@ -2,7 +2,7 @@ menu "SoC selection" config ARCH_ANDES bool "Andes SoCs" - depends on MMU && !XIP_KERNEL + depends on MMU select ERRATA_ANDES help This enables support for Andes SoC platform hardware. @@ -33,7 +33,7 @@ config ARCH_RENESAS config ARCH_SIFIVE bool "SiFive SoCs" - select ERRATA_SIFIVE if !XIP_KERNEL + select ERRATA_SIFIVE help This enables support for SiFive SoC platform hardware. @@ -61,7 +61,7 @@ config SOC_STARFIVE config ARCH_SUNXI bool "Allwinner sun20i SoCs" - depends on MMU && !XIP_KERNEL + depends on MMU select ERRATA_THEAD select SUN4I_TIMER help @@ -78,12 +78,18 @@ config ARCH_TENSTORRENT config ARCH_THEAD bool "T-HEAD RISC-V SoCs" - depends on MMU && !XIP_KERNEL + depends on MMU select ERRATA_THEAD select PM_GENERIC_DOMAINS if PM help This enables support for the RISC-V based T-HEAD SoCs. +config ARCH_ULTRARISC + bool "UltraRISC RISC-V SoCs" + help + This enables support for UltraRISC SoC platform hardware, + including boards based on the UR-DP1000. + config ARCH_VIRT bool "QEMU Virt Machine" select POWER_RESET diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile index 4c6de57f65ef..44f5a439741a 100644 --- a/arch/riscv/Makefile +++ b/arch/riscv/Makefile @@ -28,7 +28,6 @@ endif export BITS ifeq ($(CONFIG_ARCH_RV64I),y) BITS := 64 - UTS_MACHINE := riscv64 KBUILD_CFLAGS += -mabi=lp64 KBUILD_AFLAGS += -mabi=lp64 @@ -39,13 +38,14 @@ ifeq ($(CONFIG_ARCH_RV64I),y) -Cno-redzone else BITS := 32 - UTS_MACHINE := riscv32 KBUILD_CFLAGS += -mabi=ilp32 KBUILD_AFLAGS += -mabi=ilp32 KBUILD_LDFLAGS += -melf32lriscv endif +UTS_MACHINE := riscv$(BITS) + # LLVM has an issue with target-features and LTO: https://github.com/llvm/llvm-project/issues/59350 # Ensure it is aware of linker relaxation with LTO, otherwise relocations may # be incorrect: https://github.com/llvm/llvm-project/issues/65090 @@ -75,15 +75,12 @@ else riscv-march-$(CONFIG_TOOLCHAIN_NEEDS_EXPLICIT_ZICSR_ZIFENCEI) := $(riscv-march-y)_zicsr_zifencei endif -# Check if the toolchain supports Zacas -riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZACAS) := $(riscv-march-y)_zacas - -# Check if the toolchain supports Zabha -riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZABHA) := $(riscv-march-y)_zabha +KBUILD_BASE_ISA = -march=$(shell echo $(riscv-march-y) | sed -E 's/(rv32ima|rv64ima)fd([^v_]*)v?/\1\2/') +export KBUILD_BASE_ISA # Remove F,D,V from isa string for all. Keep extensions between "fd" and "v" by # matching non-v and non-multi-letter extensions out with the filter ([^v_]*) -KBUILD_CFLAGS += -march=$(shell echo $(riscv-march-y) | sed -E 's/(rv32ima|rv64ima)fd([^v_]*)v?/\1\2/') +KBUILD_CFLAGS += $(KBUILD_BASE_ISA) KBUILD_AFLAGS += -march=$(riscv-march-y) @@ -147,7 +144,6 @@ ifdef CONFIG_RISCV_M_MODE boot-image-$(CONFIG_SOC_CANAAN_K210) := loader.bin endif boot-image-$(CONFIG_EFI_ZBOOT) := vmlinuz.efi -boot-image-$(CONFIG_XIP_KERNEL) := xipImage KBUILD_IMAGE := $(boot)/$(boot-image-y) libs-y += arch/riscv/lib/ @@ -158,21 +154,24 @@ ifeq ($(CONFIG_MMU),y) prepare: vdso_prepare vdso_prepare: prepare0 $(Q)$(MAKE) $(build)=arch/riscv/kernel/vdso include/generated/vdso-offsets.h + $(if $(CONFIG_RISCV_USER_CFI),$(Q)$(MAKE) \ + $(build)=arch/riscv/kernel/vdso_cfi include/generated/vdso-cfi-offsets.h) $(if $(CONFIG_COMPAT),$(Q)$(MAKE) \ $(build)=arch/riscv/kernel/compat_vdso include/generated/compat_vdso-offsets.h) endif endif -vdso-install-y += arch/riscv/kernel/vdso/vdso.so.dbg +vdso-install-$(CONFIG_MMU) += arch/riscv/kernel/vdso/vdso.so.dbg +vdso-install-$(CONFIG_RISCV_USER_CFI) += arch/riscv/kernel/vdso_cfi/vdso-cfi.so.dbg vdso-install-$(CONFIG_COMPAT) += arch/riscv/kernel/compat_vdso/compat_vdso.so.dbg -BOOT_TARGETS := Image Image.gz Image.bz2 Image.lz4 Image.lzma Image.lzo Image.zst Image.xz loader loader.bin xipImage vmlinuz.efi +BOOT_TARGETS := Image Image.gz Image.bz2 Image.lz4 Image.lzma Image.lzo Image.zst Image.xz loader loader.bin vmlinuz.efi all: $(notdir $(KBUILD_IMAGE)) loader.bin: loader -Image.gz Image.bz2 Image.lz4 Image.lzma Image.lzo Image.zst Image.xz loader xipImage vmlinuz.efi: Image +Image.gz Image.bz2 Image.lz4 Image.lzma Image.lzo Image.zst Image.xz loader vmlinuz.efi: Image $(BOOT_TARGETS): vmlinux $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ @@ -212,8 +211,6 @@ define archhelp echo ' Image.xz - Compressed kernel image (arch/riscv/boot/Image.xz)' echo ' vmlinuz.efi - Compressed EFI kernel image (arch/riscv/boot/vmlinuz.efi)' echo ' Default when CONFIG_EFI_ZBOOT=y' - echo ' xipImage - Execute-in-place kernel image (arch/riscv/boot/xipImage)' - echo ' Default when CONFIG_XIP_KERNEL=y' echo ' install - Install kernel using (your) ~/bin/$(INSTALLKERNEL) or' echo ' (distribution) /sbin/$(INSTALLKERNEL) or install to ' echo ' $$(INSTALL_PATH)' diff --git a/arch/riscv/boot/.gitignore b/arch/riscv/boot/.gitignore index e1bc507e8cb2..5d123ceaa012 100644 --- a/arch/riscv/boot/.gitignore +++ b/arch/riscv/boot/.gitignore @@ -5,4 +5,3 @@ loader loader.lds loader.bin vmlinuz* -xipImage diff --git a/arch/riscv/boot/Makefile b/arch/riscv/boot/Makefile index 5301adf5f3f5..4a2d75b7e42d 100644 --- a/arch/riscv/boot/Makefile +++ b/arch/riscv/boot/Makefile @@ -16,20 +16,8 @@ OBJCOPYFLAGS_Image :=-O binary -R .note -R .note.gnu.build-id -R .comment -S OBJCOPYFLAGS_loader.bin :=-O binary -OBJCOPYFLAGS_xipImage :=-O binary -R .note -R .note.gnu.build-id -R .comment -S -targets := Image Image.* loader loader.o loader.lds loader.bin xipImage - -ifeq ($(CONFIG_XIP_KERNEL),y) - -quiet_cmd_mkxip = $(quiet_cmd_objcopy) -cmd_mkxip = $(cmd_objcopy) - -$(obj)/xipImage: vmlinux FORCE - $(call if_changed,mkxip) - @$(kecho) ' Physical Address of xipImage: $(CONFIG_XIP_PHYS_ADDR)' - -endif +targets := Image Image.* loader loader.o loader.lds loader.bin $(obj)/Image: vmlinux FORCE $(call if_changed,objcopy) diff --git a/arch/riscv/boot/dts/allwinner/sun20i-d1-lichee-rv-dock.dts b/arch/riscv/boot/dts/allwinner/sun20i-d1-lichee-rv-dock.dts index 08cf716328a0..feaa75d5aead 100644 --- a/arch/riscv/boot/dts/allwinner/sun20i-d1-lichee-rv-dock.dts +++ b/arch/riscv/boot/dts/allwinner/sun20i-d1-lichee-rv-dock.dts @@ -59,6 +59,18 @@ status = "okay"; }; +&ledc { + pinctrl-0 = <&ledc_pc0_pin>; + pinctrl-names = "default"; + status = "okay"; + + multi-led@0 { + reg = <0x0>; + color = <LED_COLOR_ID_RGB>; + function = LED_FUNCTION_STATUS; + }; +}; + &mmc1 { bus-width = <4>; mmc-pwrseq = <&wifi_pwrseq>; diff --git a/arch/riscv/boot/dts/allwinner/sun20i-d1-nezha.dts b/arch/riscv/boot/dts/allwinner/sun20i-d1-nezha.dts index 8dbe717c79ce..73840ea300f0 100644 --- a/arch/riscv/boot/dts/allwinner/sun20i-d1-nezha.dts +++ b/arch/riscv/boot/dts/allwinner/sun20i-d1-nezha.dts @@ -22,6 +22,7 @@ #include <dt-bindings/gpio/gpio.h> #include <dt-bindings/input/input.h> +#include <dt-bindings/leds/common.h> /dts-v1/; @@ -121,6 +122,18 @@ }; }; +&ledc { + pinctrl-0 = <&ledc_pc0_pin>; + pinctrl-names = "default"; + status = "okay"; + + multi-led@0 { + reg = <0x0>; + color = <LED_COLOR_ID_RGB>; + function = LED_FUNCTION_STATUS; + }; +}; + &mdio { ext_rgmii_phy: ethernet-phy@1 { compatible = "ethernet-phy-ieee802.3-c22"; diff --git a/arch/riscv/boot/dts/allwinner/sun20i-d1.dtsi b/arch/riscv/boot/dts/allwinner/sun20i-d1.dtsi index b18f368e06e0..b0fb0ea377bc 100644 --- a/arch/riscv/boot/dts/allwinner/sun20i-d1.dtsi +++ b/arch/riscv/boot/dts/allwinner/sun20i-d1.dtsi @@ -59,6 +59,12 @@ }; /omit-if-no-ref/ + ledc_pc0_pin: ledc-pc0-pin { + pins = "PC0"; + function = "ledc"; + }; + + /omit-if-no-ref/ uart0_pb8_pins: uart0-pb8-pins { pins = "PB8", "PB9"; function = "uart0"; diff --git a/arch/riscv/boot/dts/allwinner/sun20i-d1s.dtsi b/arch/riscv/boot/dts/allwinner/sun20i-d1s.dtsi index a7442a508433..3f4ee820ef56 100644 --- a/arch/riscv/boot/dts/allwinner/sun20i-d1s.dtsi +++ b/arch/riscv/boot/dts/allwinner/sun20i-d1s.dtsi @@ -1,6 +1,8 @@ // SPDX-License-Identifier: (GPL-2.0+ OR MIT) // Copyright (C) 2021-2022 Samuel Holland <samuel@sholland.org> +#include <dt-bindings/thermal/thermal.h> + #define SOC_PERIPHERAL_IRQ(nr) (nr + 16) #include "sunxi-d1s-t113.dtsi" @@ -115,4 +117,33 @@ <0x00000000 0x0000000e 0xffffffff 0xffffffff 0x00010000>, <0x00000000 0x0000000f 0xffffffff 0xffffffff 0x00020000>; }; + + thermal-zones { + cpu-thermal { + polling-delay-passive = <0>; + polling-delay = <0>; + thermal-sensors = <&ths>; + + cooling-maps { + map0 { + trip = <&cpu_alert>; + cooling-device = <&cpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + + trips { + cpu_alert: cpu-alert { + temperature = <85000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu-crit { + temperature = <100000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + }; }; diff --git a/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi b/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi index 63e252b44973..13b1e5716604 100644 --- a/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi +++ b/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi @@ -191,6 +191,12 @@ pins = "PB6", "PB7"; function = "uart3"; }; + + /omit-if-no-ref/ + uart4_pb_pins: uart4-pb-pins { + pins = "PB2", "PB3"; + function = "uart4"; + }; }; ccu: clock-controller@2001000 { @@ -204,6 +210,21 @@ #reset-cells = <1>; }; + ledc: led-controller@2008000 { + compatible = "allwinner,sun20i-d1-ledc", + "allwinner,sun50i-a100-ledc"; + reg = <0x2008000 0x400>; + interrupts = <SOC_PERIPHERAL_IRQ(20) IRQ_TYPE_LEVEL_HIGH>; + clocks = <&ccu CLK_BUS_LEDC>, <&ccu CLK_LEDC>; + clock-names = "bus", "mod"; + resets = <&ccu RST_BUS_LEDC>; + dmas = <&dma 42>; + dma-names = "tx"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + gpadc: adc@2009000 { compatible = "allwinner,sun20i-d1-gpadc"; reg = <0x2009000 0x400>; @@ -214,6 +235,18 @@ #io-channel-cells = <1>; }; + ths: thermal-sensor@2009400 { + compatible = "allwinner,sun20i-d1-ths"; + reg = <0x2009400 0x400>; + interrupts = <SOC_PERIPHERAL_IRQ(58) IRQ_TYPE_LEVEL_HIGH>; + clocks = <&ccu CLK_BUS_THS>; + clock-names = "bus"; + resets = <&ccu RST_BUS_THS>; + nvmem-cells = <&ths_calibration>; + nvmem-cell-names = "calibration"; + #thermal-sensor-cells = <0>; + }; + dmic: dmic@2031000 { compatible = "allwinner,sun20i-d1-dmic", "allwinner,sun50i-h6-dmic"; @@ -268,6 +301,15 @@ clocks = <&dcxo>; }; + hstimer@3008000 { + compatible = "allwinner,sun20i-d1-hstimer"; + reg = <0x03008000 0x1000>; + interrupts = <SOC_PERIPHERAL_IRQ(55) IRQ_TYPE_LEVEL_HIGH>, + <SOC_PERIPHERAL_IRQ(56) IRQ_TYPE_LEVEL_HIGH>; + clocks = <&ccu CLK_BUS_HSTIMER>; + resets = <&ccu RST_BUS_HSTIMER>; + }; + wdt: watchdog@20500a0 { compatible = "allwinner,sun20i-d1-wdt-reset", "allwinner,sun20i-d1-wdt"; @@ -474,6 +516,10 @@ reg = <0x3006000 0x1000>; #address-cells = <1>; #size-cells = <1>; + + ths_calibration: thermal-sensor-calibration@14 { + reg = <0x14 0x8>; + }; }; crypto: crypto@3040000 { diff --git a/arch/riscv/boot/dts/anlogic/dr1v90.dtsi b/arch/riscv/boot/dts/anlogic/dr1v90.dtsi index a5d0765ade32..9fe183f5f5c8 100644 --- a/arch/riscv/boot/dts/anlogic/dr1v90.dtsi +++ b/arch/riscv/boot/dts/anlogic/dr1v90.dtsi @@ -27,8 +27,9 @@ mmu-type = "riscv,sv39"; reg = <0>; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "zba", "zbb", "zbc", - "zbkc", "zbs", "zicntr", "zicsr", "zifencei", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", + "zba", "zbb", "zbc", "zbkc", "zbs", + "zicntr", "zicsr", "zifencei", "zihintpause", "zihpm"; cpu0_intc: interrupt-controller { diff --git a/arch/riscv/boot/dts/canaan/Makefile b/arch/riscv/boot/dts/canaan/Makefile index 987d1f0c41f0..7d54ea5c6f3d 100644 --- a/arch/riscv/boot/dts/canaan/Makefile +++ b/arch/riscv/boot/dts/canaan/Makefile @@ -1,6 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 dtb-$(CONFIG_ARCH_CANAAN) += canaan_kd233.dtb dtb-$(CONFIG_ARCH_CANAAN) += k210_generic.dtb +dtb-$(CONFIG_ARCH_CANAAN) += k230-canmv.dtb +dtb-$(CONFIG_ARCH_CANAAN) += k230-evb.dtb dtb-$(CONFIG_ARCH_CANAAN) += sipeed_maix_bit.dtb dtb-$(CONFIG_ARCH_CANAAN) += sipeed_maix_dock.dtb dtb-$(CONFIG_ARCH_CANAAN) += sipeed_maix_go.dtb diff --git a/arch/riscv/boot/dts/canaan/k230-canmv.dts b/arch/riscv/boot/dts/canaan/k230-canmv.dts new file mode 100644 index 000000000000..2ba555f6de13 --- /dev/null +++ b/arch/riscv/boot/dts/canaan/k230-canmv.dts @@ -0,0 +1,343 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2024 Yangyu Chen <cyy@cyyself.name> + */ + +#include "k230.dtsi" + +/ { + model = "Canaan CanMV-K230"; + compatible = "canaan,canmv-k230", "canaan,kendryte-k230"; + + aliases { + serial0 = &uart0; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + ddr: memory@0 { + device_type = "memory"; + reg = <0x0 0x0 0x0 0x20000000>; + }; + + timerx_pulse_in: clock-50000000 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <50000000>; + }; +}; + +&uart0 { + status = "okay"; +}; + +&pinctrl { + jtag_pins: jtag-pins { + jtag-tck-cfg { + pinmux = <K230_PINMUX(2, 1)>; + slew-rate = <0>; + drive-strength = <4>; + power-source = <K230_MSC_1V8>; + input-enable; + bias-pull-down; + input-schmitt-enable; + }; + + jtag-tdi-cfg { + pinmux = <K230_PINMUX(3, 1)>; + slew-rate = <0>; + drive-strength = <4>; + power-source = <K230_MSC_1V8>; + input-enable; + bias-disable; + }; + + jtag-tdo-cfg { + pinmux = <K230_PINMUX(4, 1)>; + slew-rate = <0>; + drive-strength = <4>; + power-source = <K230_MSC_1V8>; + output-enable; + bias-disable; + }; + + jtag-tms-cfg { + pinmux = <K230_PINMUX(5, 1)>; + slew-rate = <0>; + drive-strength = <4>; + power-source = <K230_MSC_1V8>; + input-enable; + bias-pull-up; + }; + }; + + uart2_pins: uart2-pins { + uart2-pins-cfg { + pinmux = <K230_PINMUX(5, 3)>, /* uart2 txd */ + <K230_PINMUX(6, 3)>; /* uart2 rxd */ + slew-rate = <0>; + drive-strength = <4>; + power-source = <K230_MSC_1V8>; + input-enable; + output-enable; + bias-disable; + }; + }; + + pwm2_pins: pwm2-pins { + pwm2-pin-cfg { + pinmux = <K230_PINMUX(7, 1)>; + slew-rate = <0>; + drive-strength = <7>; + power-source = <K230_MSC_1V8>; + input-enable; + output-enable; + bias-disable; + input-schmitt-enable; + }; + }; + + pwm3_pins: pwm3-pins { + pwm3-pin-cfg { + pinmux = <K230_PINMUX(8, 1)>; + slew-rate = <0>; + drive-strength = <7>; + power-source = <K230_MSC_1V8>; + input-enable; + output-enable; + bias-disable; + input-schmitt-enable; + }; + }; + + pwm4_pins: pwm4-pins { + pwm4-pin-cfg { + pinmux = <K230_PINMUX(9, 1)>; + slew-rate = <0>; + drive-strength = <7>; + power-source = <K230_MSC_1V8>; + input-enable; + output-enable; + bias-disable; + input-schmitt-enable; + }; + }; + + iis_pins: iis-pins { + iis-clk-cfg { + pinmux = <K230_PINMUX(32, 2)>; + slew-rate = <0>; + drive-strength = <4>; + power-source = <K230_MSC_1V8>; + output-enable; + bias-disable; + }; + + iis-ws-cfg { + pinmux = <K230_PINMUX(33, 2)>; + slew-rate = <0>; + drive-strength = <4>; + power-source = <K230_MSC_1V8>; + output-enable; + bias-disable; + }; + + iis-din0-cfg { + pinmux = <K230_PINMUX(34, 2)>; + slew-rate = <0>; + drive-strength = <4>; + power-source = <K230_MSC_1V8>; + input-enable; + bias-disable; + }; + + iis-dout0-cfg { + pinmux = <K230_PINMUX(35, 2)>; + slew-rate = <0>; + drive-strength = <4>; + power-source = <K230_MSC_1V8>; + output-enable; + bias-disable; + }; + }; + + uart4_pins: uart4-pins { + uart4-txd-cfg { + pinmux = <K230_PINMUX(36, 4)>; + slew-rate = <0>; + drive-strength = <7>; + power-source = <K230_MSC_1V8>; + output-enable; + bias-disable; + input-schmitt-enable; + }; + + uart4-rxd-cfg { + pinmux = <K230_PINMUX(37, 4)>; + slew-rate = <0>; + drive-strength = <7>; + power-source = <K230_MSC_1V8>; + input-enable; + bias-disable; + input-schmitt-enable; + }; + }; + + uart0_pins: uart0-pins { + uart0-txd-cfg { + pinmux = <K230_PINMUX(38, 1)>; + slew-rate = <0>; + drive-strength = <7>; + power-source = <K230_MSC_1V8>; + output-enable; + bias-disable; + input-schmitt-enable; + }; + + uart0-rxd-cfg { + pinmux = <K230_PINMUX(39, 1)>; + slew-rate = <0>; + drive-strength = <7>; + power-source = <K230_MSC_1V8>; + input-enable; + bias-disable; + input-schmitt-enable; + }; + }; + + iic1_pins: iic1-pins { + iic1-pins-cfg { + pinmux = <K230_PINMUX(40, 2)>, /* iic1 scl */ + <K230_PINMUX(41, 2)>; /* iic1 sda */ + slew-rate = <0>; + drive-strength = <7>; + power-source = <K230_MSC_1V8>; + input-enable; + output-enable; + bias-pull-up; + input-schmitt-enable; + }; + }; + + iic3_pins: iic3-pins { + iic3-pins-cfg { + pinmux = <K230_PINMUX(44, 2)>, /* iic3 scl */ + <K230_PINMUX(45, 2)>; /* iic3 sda */ + slew-rate = <0>; + drive-strength = <7>; + power-source = <K230_MSC_1V8>; + input-enable; + output-enable; + bias-pull-up; + input-schmitt-enable; + }; + }; + + iic4_pins: iic4-pins { + iic4-pins-cfg { + pinmux = <K230_PINMUX(46, 3)>, /* iic4 scl */ + <K230_PINMUX(47, 3)>; /* iic4 sda */ + slew-rate = <0>; + drive-strength = <7>; + power-source = <K230_MSC_1V8>; + input-enable; + output-enable; + bias-pull-up; + input-schmitt-enable; + }; + }; + + iic0_pins: iic0-pins { + iic0-pins-cfg { + pinmux = <K230_PINMUX(48, 3)>, /* iic0 scl */ + <K230_PINMUX(49, 3)>; /* iic0 sda */ + slew-rate = <0>; + drive-strength = <7>; + power-source = <K230_MSC_1V8>; + input-enable; + output-enable; + bias-pull-up; + input-schmitt-enable; + }; + }; + + uart3_pins: uart3-pins { + uart3-txd-cfg { + pinmux = <K230_PINMUX(50, 1)>; + slew-rate = <0>; + drive-strength = <7>; + power-source = <K230_MSC_3V3>; + output-enable; + bias-disable; + input-schmitt-enable; + }; + + uart3-rxd-cfg { + pinmux = <K230_PINMUX(51, 1)>; + slew-rate = <0>; + drive-strength = <7>; + power-source = <K230_MSC_3V3>; + input-enable; + bias-disable; + input-schmitt-enable; + }; + }; + + key_pins: key-pins { + key-pins-cfg { + pinmux = <K230_PINMUX(52, 0)>, /* key0 */ + <K230_PINMUX(53, 0)>; /* key1 */ + slew-rate = <0>; + drive-strength = <7>; + power-source = <K230_MSC_3V3>; + input-enable; + output-enable; + bias-disable; + input-schmitt-enable; + }; + }; + + mmc1_pins: mmc1-pins { + mmc1-cmd-cfg { + pinmux = <K230_PINMUX(54, 2)>; + slew-rate = <0>; + drive-strength = <7>; + power-source = <K230_MSC_3V3>; + input-enable; + output-enable; + bias-pull-up; + input-schmitt-enable; + }; + + mmc1-clk-cfg { + pinmux = <K230_PINMUX(55, 2)>; + slew-rate = <0>; + drive-strength = <7>; + power-source = <K230_MSC_3V3>; + output-enable; + bias-disable; + input-schmitt-enable; + }; + + mmc1-data-cfg { + pinmux = <K230_PINMUX(56, 2)>, /* mmc1 data0 */ + <K230_PINMUX(57, 2)>, /* mmc1 data1 */ + <K230_PINMUX(58, 2)>, /* mmc1 data2 */ + <K230_PINMUX(59, 2)>; /* mmc1 data3 */ + slew-rate = <0>; + drive-strength = <7>; + power-source = <K230_MSC_3V3>; + input-enable; + output-enable; + bias-pull-up; + input-schmitt-enable; + }; + }; +}; + +&sysclk { + clocks = <&osc24m>, <&timerx_pulse_in>; + clock-names = "osc24m", "timer-pulse-in"; +}; diff --git a/arch/riscv/boot/dts/canaan/k230-evb.dts b/arch/riscv/boot/dts/canaan/k230-evb.dts new file mode 100644 index 000000000000..4246e4bba6cc --- /dev/null +++ b/arch/riscv/boot/dts/canaan/k230-evb.dts @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2024 Yangyu Chen <cyy@cyyself.name> + */ + +#include "k230.dtsi" + +/ { + model = "Kendryte K230 EVB"; + compatible = "canaan,k230-usip-lp3-evb", "canaan,kendryte-k230"; + + aliases { + serial0 = &uart0; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + ddr: memory@0 { + device_type = "memory"; + reg = <0x0 0x0 0x0 0x20000000>; + }; + + timerx_pulse_in: clock-50000000 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <50000000>; + }; +}; + +&uart0 { + status = "okay"; +}; + +&sysclk { + clocks = <&osc24m>, <&timerx_pulse_in>; + clock-names = "osc24m", "timer-pulse-in"; +}; diff --git a/arch/riscv/boot/dts/canaan/k230-pinctrl.h b/arch/riscv/boot/dts/canaan/k230-pinctrl.h new file mode 100644 index 000000000000..63dd999ca55b --- /dev/null +++ b/arch/riscv/boot/dts/canaan/k230-pinctrl.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */ +/* + * Copyright (C) 2024 Canaan Bright Sight Co. Ltd + * Copyright (C) 2024 Ze Huang <18771902331@163.com> + */ + +#ifndef _K230_PINCTRL_H +#define _K230_PINCTRL_H + +#define K230_MSC_3V3 0 +#define K230_MSC_1V8 1 + +#define BANK_VOLTAGE_DEFAULT K230_MSC_1V8 +#define BANK_VOLTAGE_IO50_IO61 K230_MSC_3V3 + +#define K230_PINMUX(pin, mode) (((pin) << 8) | (mode)) + +#endif /* _K230_PINCTRL_H */ diff --git a/arch/riscv/boot/dts/canaan/k230.dtsi b/arch/riscv/boot/dts/canaan/k230.dtsi new file mode 100644 index 000000000000..6dcb628a960b --- /dev/null +++ b/arch/riscv/boot/dts/canaan/k230.dtsi @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2024 Yangyu Chen <cyy@cyyself.name> + */ + +#include <dt-bindings/clock/canaan,k230-clk.h> +#include <dt-bindings/interrupt-controller/irq.h> +#include <dt-bindings/reset/canaan,k230-rst.h> +#include "k230-pinctrl.h" + +/dts-v1/; +/ { + #address-cells = <2>; + #size-cells = <2>; + compatible = "canaan,kendryte-k230"; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + timebase-frequency = <27000000>; + + cpu@0 { + compatible = "thead,c908", "riscv"; + device_type = "cpu"; + reg = <0>; + riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicsr_zifencei_zihpm_zfh_zba_zbb_zbc_zbs_zvfh_svpbmt"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", + "zicbop", "zicboz", "zicntr", "zicsr", "zifencei", + "zihpm", "zfh", "zba", "zbb", "zbc", "zbs", "zvfh", + "svpbmt"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + d-cache-block-size = <64>; + d-cache-sets = <128>; + d-cache-size = <32768>; + i-cache-block-size = <64>; + i-cache-sets = <128>; + i-cache-size = <32768>; + next-level-cache = <&l2_cache>; + mmu-type = "riscv,sv39"; + + cpu0_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + l2_cache: l2-cache { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-size = <262144>; + cache-sets = <256>; + cache-unified; + }; + }; + + osc24m: clock-24000000 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <24000000>; + clock-output-names = "osc24m"; + }; + + soc { + compatible = "simple-bus"; + interrupt-parent = <&plic>; + #address-cells = <2>; + #size-cells = <2>; + dma-noncoherent; + ranges; + + plic: interrupt-controller@f00000000 { + compatible = "canaan,k230-plic", "thead,c900-plic"; + reg = <0xf 0x00000000 0x0 0x04000000>; + interrupts-extended = <&cpu0_intc 11>, <&cpu0_intc 9>; + interrupt-controller; + #address-cells = <0>; + #interrupt-cells = <2>; + riscv,ndev = <208>; + }; + + clint: timer@f04000000 { + compatible = "canaan,k230-clint", "thead,c900-clint"; + reg = <0xf 0x04000000 0x0 0x00010000>; + interrupts-extended = <&cpu0_intc 3>, <&cpu0_intc 7>; + }; + + rst: reset-controller@91101000 { + compatible = "canaan,k230-rst"; + reg = <0x0 0x91101000 0x0 0x1000>; + #reset-cells = <1>; + }; + + pinctrl: pinctrl@91105000 { + compatible = "canaan,k230-pinctrl"; + reg = <0x0 0x91105000 0x0 0x100>; + }; + + sysclk: clock-controller@91102000 { + compatible = "canaan,k230-clk"; + reg = <0x0 0x91102000 0x0 0x40>, + <0x0 0x91100000 0x0 0x108>; + clocks = <&osc24m>; + clock-names = "osc24m"; + #clock-cells = <1>; + }; + + uart0: serial@91400000 { + compatible = "snps,dw-apb-uart"; + reg = <0x0 0x91400000 0x0 0x1000>; + clocks = <&sysclk K230_LS_UART0_RATE>; + interrupts = <16 IRQ_TYPE_LEVEL_HIGH>; + reg-io-width = <4>; + reg-shift = <2>; + resets = <&rst RST_UART0>; + status = "disabled"; + }; + + uart1: serial@91401000 { + compatible = "snps,dw-apb-uart"; + reg = <0x0 0x91401000 0x0 0x1000>; + clocks = <&sysclk K230_LS_UART1_RATE>; + interrupts = <17 IRQ_TYPE_LEVEL_HIGH>; + reg-io-width = <4>; + reg-shift = <2>; + resets = <&rst RST_UART1>; + status = "disabled"; + }; + + uart2: serial@91402000 { + compatible = "snps,dw-apb-uart"; + reg = <0x0 0x91402000 0x0 0x1000>; + clocks = <&sysclk K230_LS_UART2_RATE>; + interrupts = <18 IRQ_TYPE_LEVEL_HIGH>; + reg-io-width = <4>; + reg-shift = <2>; + resets = <&rst RST_UART2>; + status = "disabled"; + }; + + uart3: serial@91403000 { + compatible = "snps,dw-apb-uart"; + reg = <0x0 0x91403000 0x0 0x1000>; + clocks = <&sysclk K230_LS_UART3_RATE>; + interrupts = <19 IRQ_TYPE_LEVEL_HIGH>; + reg-io-width = <4>; + reg-shift = <2>; + resets = <&rst RST_UART3>; + status = "disabled"; + }; + + uart4: serial@91404000 { + compatible = "snps,dw-apb-uart"; + reg = <0x0 0x91404000 0x0 0x1000>; + clocks = <&sysclk K230_LS_UART4_RATE>; + interrupts = <20 IRQ_TYPE_LEVEL_HIGH>; + reg-io-width = <4>; + reg-shift = <2>; + resets = <&rst RST_UART4>; + status = "disabled"; + }; + }; +}; diff --git a/arch/riscv/boot/dts/microchip/Makefile b/arch/riscv/boot/dts/microchip/Makefile index 345ed7a48cc1..43f91cbf338b 100644 --- a/arch/riscv/boot/dts/microchip/Makefile +++ b/arch/riscv/boot/dts/microchip/Makefile @@ -1,9 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 -dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-beaglev-fire.dtb -dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-disco-kit.dtb -dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-icicle-kit.dtb -dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-icicle-kit-prod.dtb -dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-m100pfsevp.dtb -dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-polarberry.dtb -dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-sev-kit.dtb -dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-tysom-m.dtb +dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-beaglev-fire.dtb +dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-disco-kit.dtb +dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-icicle-kit.dtb +dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-icicle-kit-prod.dtb +dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-m100pfsevp.dtb +dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-polarberry.dtb +dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-sev-kit.dtb +dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-tysom-m.dtb +dtb-$(CONFIG_ARCH_MICROCHIP) += pic64gx-curiosity-kit.dtb diff --git a/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts b/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts index f44ad8e6f4e4..6d7037c9528d 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts +++ b/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts @@ -4,6 +4,7 @@ /dts-v1/; #include <dt-bindings/gpio/gpio.h> +#include <dt-bindings/interrupt-controller/irq.h> #include "mpfs.dtsi" #include "mpfs-beaglev-fire-fabric.dtsi" @@ -63,20 +64,29 @@ regulator-max-microvolt = <2800000>; }; - imx219_vdig: fixedregulator-1 { + imx219_vdig: regulator-1v8 { compatible = "regulator-fixed"; regulator-name = "imx219_vdig"; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; }; - imx219_vddl: fixedregulator-2 { + imx219_vddl: regulator-1v2 { compatible = "regulator-fixed"; regulator-name = "imx219_vddl"; regulator-min-microvolt = <1200000>; regulator-max-microvolt = <1200000>; }; + regulator-1v8-syzygy { + compatible = "regulator-fixed"; + gpios = <&gpio2 30 GPIO_ACTIVE_HIGH>; + regulator-name = "syzygy_1v8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + enable-active-high; + }; }; &gpio0 { @@ -85,6 +95,8 @@ <21>, <22>, <23>, <24>, <25>, <26>; ngpios = <14>; + gpio-line-names = "", "", "", "", "", "", "", + "", "", "", "", "", "SD_CARD_CS", "USER_BUTTON"; status = "okay"; }; @@ -96,6 +108,9 @@ <43>, <44>, <45>, <46>, <47>, <48>, <49>, <50>; ngpios = <24>; + gpio-line-names = "", "", "", "", "", "", "", "", "", "", + "", "", "", "", "", "", "", "", "", "", + "ADC_IRQn", "", "", "USB_OCn"; status = "okay"; }; @@ -118,20 +133,6 @@ "P8_PIN27", "P8_PIN28", "P8_PIN29", "P8_PIN30", "M2_W_DISABLE1", "M2_W_DISABLE2", "VIO_ENABLE", "SD_DET"; status = "okay"; - - vio-enable-hog { - gpio-hog; - gpios = <30 30>; - output-high; - line-name = "VIO_ENABLE"; - }; - - sd-det-hog { - gpio-hog; - gpios = <31 31>; - input; - line-name = "SD_DET"; - }; }; &i2c0 { @@ -164,6 +165,35 @@ }; }; +&irqmux { + interrupt-map = <0 &plic 13>, <1 &plic 14>, <2 &plic 15>, + <3 &plic 16>, <4 &plic 17>, <5 &plic 18>, + <6 &plic 19>, <7 &plic 20>, <8 &plic 21>, + <9 &plic 22>, <10 &plic 23>, <11 &plic 24>, + <12 &plic 25>, <13 &plic 26>, + + <32 &plic 27>, <33 &plic 28>, <34 &plic 29>, + <35 &plic 30>, <36 &plic 31>, <37 &plic 32>, + <38 &plic 33>, <39 &plic 34>, <40 &plic 35>, + <41 &plic 36>, <42 &plic 37>, <43 &plic 38>, + <44 &plic 39>, <45 &plic 40>, <46 &plic 41>, + <47 &plic 42>, <48 &plic 43>, <49 &plic 44>, + <50 &plic 45>, <51 &plic 46>, <52 &plic 47>, + <53 &plic 48>, <54 &plic 49>, <55 &plic 50>, + + <64 &plic 53>, <65 &plic 53>, <66 &plic 53>, + <67 &plic 53>, <68 &plic 53>, <69 &plic 53>, + <70 &plic 53>, <71 &plic 53>, <72 &plic 53>, + <73 &plic 53>, <74 &plic 53>, <75 &plic 53>, + <76 &plic 53>, <77 &plic 53>, <78 &plic 53>, + <79 &plic 53>, <80 &plic 53>, <81 &plic 53>, + <82 &plic 53>, <83 &plic 53>, <84 &plic 53>, + <85 &plic 53>, <86 &plic 53>, <87 &plic 53>, + <88 &plic 53>, <89 &plic 53>, <90 &plic 53>, + <91 &plic 53>, <92 &plic 53>, <93 &plic 53>, + <94 &plic 53>, <95 &plic 53>; +}; + &mac0 { status = "okay"; phy-mode = "sgmii"; @@ -231,6 +261,7 @@ spi-cpha; spi-max-frequency = <5000000>; microchip,hw-device-address = <1>; + interrupts-extended = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>; #address-cells = <1>; #size-cells = <0>; status = "okay"; @@ -287,7 +318,7 @@ mmc@1 { compatible = "mmc-spi-slot"; reg = <1>; - gpios = <&gpio2 31 1>; + cd-gpios = <&gpio2 31 GPIO_ACTIVE_LOW>; voltage-ranges = <3300 3300>; spi-max-frequency = <5000000>; disable-wp; @@ -297,7 +328,6 @@ &syscontroller { microchip,bitstream-flash = <&sys_ctrl_flash>; - status = "okay"; }; &syscontroller_qspi { diff --git a/arch/riscv/boot/dts/microchip/mpfs-disco-kit.dts b/arch/riscv/boot/dts/microchip/mpfs-disco-kit.dts index c068b9bb5bfd..da2fcde263e3 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-disco-kit.dts +++ b/arch/riscv/boot/dts/microchip/mpfs-disco-kit.dts @@ -97,24 +97,10 @@ }; &gpio1 { - interrupts = <27>, <28>, <29>, <30>, - <31>, <32>, <33>, <47>, - <35>, <36>, <37>, <38>, - <39>, <40>, <41>, <42>, - <43>, <44>, <45>, <46>, - <47>, <48>, <49>, <50>; status = "okay"; }; &gpio2 { - interrupts = <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>; status = "okay"; }; @@ -130,6 +116,35 @@ status = "okay"; }; +&irqmux { + interrupt-map = <0 &plic 13>, <1 &plic 14>, <2 &plic 15>, + <3 &plic 16>, <4 &plic 17>, <5 &plic 18>, + <6 &plic 19>, <7 &plic 20>, <8 &plic 21>, + <9 &plic 22>, <10 &plic 23>, <11 &plic 24>, + <12 &plic 25>, <13 &plic 26>, + + <32 &plic 27>, <33 &plic 28>, <34 &plic 29>, + <35 &plic 30>, <36 &plic 31>, <37 &plic 32>, + <38 &plic 33>, <39 &plic 34>, <40 &plic 35>, + <41 &plic 36>, <42 &plic 37>, <43 &plic 38>, + <44 &plic 39>, <45 &plic 40>, <46 &plic 41>, + <47 &plic 42>, <48 &plic 43>, <49 &plic 44>, + <50 &plic 45>, <51 &plic 46>, <52 &plic 47>, + <53 &plic 48>, <54 &plic 49>, <55 &plic 50>, + + <64 &plic 53>, <65 &plic 53>, <66 &plic 53>, + <67 &plic 53>, <68 &plic 53>, <69 &plic 53>, + <70 &plic 53>, <71 &plic 53>, <72 &plic 53>, + <73 &plic 53>, <74 &plic 53>, <75 &plic 53>, + <76 &plic 53>, <77 &plic 53>, <78 &plic 53>, + <79 &plic 53>, <80 &plic 53>, <81 &plic 53>, + <82 &plic 53>, <83 &plic 53>, <84 &plic 53>, + <85 &plic 53>, <86 &plic 53>, <87 &plic 53>, + <88 &plic 53>, <89 &plic 53>, <90 &plic 53>, + <91 &plic 53>, <92 &plic 53>, <93 &plic 53>, + <94 &plic 53>, <95 &plic 53>; +}; + &mac0 { phy-mode = "sgmii"; phy-handle = <&phy0>; @@ -184,7 +199,3 @@ &spi1 { status = "okay"; }; - -&syscontroller { - status = "okay"; -}; diff --git a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-common.dtsi b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-common.dtsi index e01a216e6c3a..a11e3103b8f2 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-common.dtsi +++ b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-common.dtsi @@ -3,7 +3,6 @@ /dts-v1/; -#include "mpfs.dtsi" #include "mpfs-icicle-kit-fabric.dtsi" #include <dt-bindings/gpio/gpio.h> #include <dt-bindings/leds/common.h> @@ -77,14 +76,6 @@ }; &gpio2 { - interrupts = <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>; status = "okay"; }; @@ -136,6 +127,35 @@ status = "okay"; }; +&irqmux { + interrupt-map = <0 &plic 13>, <1 &plic 14>, <2 &plic 15>, + <3 &plic 16>, <4 &plic 17>, <5 &plic 18>, + <6 &plic 19>, <7 &plic 20>, <8 &plic 21>, + <9 &plic 22>, <10 &plic 23>, <11 &plic 24>, + <12 &plic 25>, <13 &plic 26>, + + <32 &plic 27>, <33 &plic 28>, <34 &plic 29>, + <35 &plic 30>, <36 &plic 31>, <37 &plic 32>, + <38 &plic 33>, <39 &plic 34>, <40 &plic 35>, + <41 &plic 36>, <42 &plic 37>, <43 &plic 38>, + <44 &plic 39>, <45 &plic 40>, <46 &plic 41>, + <47 &plic 42>, <48 &plic 43>, <49 &plic 44>, + <50 &plic 45>, <51 &plic 46>, <52 &plic 47>, + <53 &plic 48>, <54 &plic 49>, <55 &plic 50>, + + <64 &plic 53>, <65 &plic 53>, <66 &plic 53>, + <67 &plic 53>, <68 &plic 53>, <69 &plic 53>, + <70 &plic 53>, <71 &plic 53>, <72 &plic 53>, + <73 &plic 53>, <74 &plic 53>, <75 &plic 53>, + <76 &plic 53>, <77 &plic 53>, <78 &plic 53>, + <79 &plic 53>, <80 &plic 53>, <81 &plic 53>, + <82 &plic 53>, <83 &plic 53>, <84 &plic 53>, + <85 &plic 53>, <86 &plic 53>, <87 &plic 53>, + <88 &plic 53>, <89 &plic 53>, <90 &plic 53>, + <91 &plic 53>, <92 &plic 53>, <93 &plic 53>, + <94 &plic 53>, <95 &plic 53>; +}; + &mac0 { phy-mode = "sgmii"; phy-handle = <&phy0>; @@ -218,10 +238,6 @@ status = "okay"; }; -&syscontroller { - status = "okay"; -}; - &syscontroller_qspi { /* * The flash *is* there, but Icicle kits that have engineering sample diff --git a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi index 71f724325578..9078e5b1e49c 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi +++ b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi @@ -1,6 +1,9 @@ // SPDX-License-Identifier: (GPL-2.0 OR MIT) /* Copyright (c) 2020-2021 Microchip Technology Inc */ +#include "mpfs.dtsi" +#include "mpfs-pinctrl.dtsi" + / { core_pwm0: pwm@40000000 { compatible = "microchip,corepwm-rtl-v4"; @@ -80,6 +83,16 @@ }; }; +&can0 { + pinctrl-names = "default"; + pinctrl-0 = <&can0_fabric>; +}; + +&can1 { + pinctrl-names = "default"; + pinctrl-0 = <&ikrd_can1_cfg>; +}; + &ccc_nw { clocks = <&refclk_ccc>, <&refclk_ccc>, <&refclk_ccc>, <&refclk_ccc>, <&refclk_ccc>, <&refclk_ccc>; @@ -87,3 +100,43 @@ "dll0_ref", "dll1_ref"; status = "okay"; }; + +&mmuart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_fabric>; +}; + +&mmuart2 { + pinctrl-names = "default"; + pinctrl-0 = <&uart2_fabric>; +}; + +&mmuart3 { + pinctrl-names = "default"; + pinctrl-0 = <&uart3_fabric>; +}; + +&mmuart4 { + pinctrl-names = "default"; + pinctrl-0 = <&uart4_fabric>; +}; + +&mssio { + pinctrl-names = "default"; + pinctrl-0 = <&spi1_mssio>, <&can1_mssio>, <&mdio0_mssio>, <&mdio1_mssio>; +}; + +&qspi { + pinctrl-names = "default"; + pinctrl-0 = <&qspi_fabric>; +}; + +&spi0 { + pinctrl-names = "default"; + pinctrl-0 = <&spi0_fabric>; +}; + +&spi1 { + pinctrl-names = "default"; + pinctrl-0 = <&ikrd_spi1_cfg>; +}; diff --git a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-prod.dts b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-prod.dts index 8afedece89d1..636493f6584d 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-prod.dts +++ b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-prod.dts @@ -14,6 +14,16 @@ "microchip,mpfs"; }; +&i2c0 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_fabric>; +}; + +&i2c1 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_mssio>; +}; + &syscontroller { microchip,bitstream-flash = <&sys_ctrl_flash>; }; diff --git a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts index 556aa9638282..6fadce815c9a 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts +++ b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts @@ -11,3 +11,22 @@ "microchip,mpfs-icicle-kit", "microchip,mpfs"; }; + +&i2c0 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_fabric>; +}; + +/* + * Due to silicon errata, routing via MSS IOs doesn't work on ES devices. + * Instead, i2c1, appearing on B1/C1, which are normally MSS IOs, is routed + * via the fabric and back to B1/C1 via "fabric-test" functionality. + * This is done silently by Libero, so the iomux0 setting for i2c1 has to + * be fabric IO, despite tooling etc saying that MSS IOs are used. + * + * See Section 3.3 of https://ww1.microchip.com/downloads/aemDocuments/documents/FPGA/ProductDocuments/Errata/polarfiresoc/microsemi_polarfire_soc_fpga_egineering_samples_errata_er0219_v1.pdf + */ +&i2c1 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_fabric>; +}; diff --git a/arch/riscv/boot/dts/microchip/mpfs-m100pfsevp.dts b/arch/riscv/boot/dts/microchip/mpfs-m100pfsevp.dts index a8d623ee9fa4..973e34b74be4 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-m100pfsevp.dts +++ b/arch/riscv/boot/dts/microchip/mpfs-m100pfsevp.dts @@ -52,11 +52,36 @@ status = "okay"; }; +&irqmux { + interrupt-map = <0 &plic 13>, <1 &plic 14>, <2 &plic 15>, + <3 &plic 16>, <4 &plic 17>, <5 &plic 18>, + <6 &plic 19>, <7 &plic 20>, <8 &plic 21>, + <9 &plic 22>, <10 &plic 23>, <11 &plic 24>, + <12 &plic 25>, <13 &plic 26>, + + <32 &plic 27>, <33 &plic 28>, <34 &plic 29>, + <35 &plic 30>, <36 &plic 31>, <37 &plic 32>, + <38 &plic 33>, <39 &plic 34>, <40 &plic 35>, + <41 &plic 36>, <42 &plic 37>, <43 &plic 38>, + <44 &plic 39>, <45 &plic 40>, <46 &plic 41>, + <47 &plic 42>, <48 &plic 43>, <49 &plic 44>, + <50 &plic 45>, <51 &plic 46>, <52 &plic 47>, + <53 &plic 48>, <54 &plic 49>, <55 &plic 50>, + + <64 &plic 53>, <65 &plic 53>, <66 &plic 53>, + <67 &plic 53>, <68 &plic 53>, <69 &plic 53>, + <70 &plic 53>, <71 &plic 53>, <72 &plic 53>, + <73 &plic 53>, <74 &plic 53>, <75 &plic 53>, + <76 &plic 53>, <77 &plic 53>, <78 &plic 53>, + <79 &plic 53>, <80 &plic 53>, <81 &plic 53>, + <82 &plic 53>, <83 &plic 53>, <84 &plic 53>, + <85 &plic 53>, <86 &plic 53>, <87 &plic 53>, + <88 &plic 53>, <89 &plic 53>, <90 &plic 53>, + <91 &plic 53>, <92 &plic 53>, <93 &plic 53>, + <94 &plic 53>, <95 &plic 53>; +}; + &gpio0 { - interrupts = <13>, <14>, <15>, <16>, - <17>, <18>, <19>, <20>, - <21>, <22>, <23>, <24>, - <25>, <26>; ngpios = <14>; status = "okay"; @@ -75,14 +100,6 @@ }; &gpio2 { - interrupts = <13>, <14>, <15>, <16>, - <17>, <18>, <19>, <20>, - <21>, <22>, <23>, <24>, - <25>, <26>, <27>, <28>, - <29>, <30>, <31>, <32>, - <33>, <34>, <35>, <36>, - <37>, <38>, <39>, <40>, - <41>, <42>, <43>, <44>; status = "okay"; }; @@ -162,10 +179,6 @@ status = "okay"; }; -&syscontroller { - status = "okay"; -}; - &usb { status = "okay"; dr_mode = "host"; diff --git a/arch/riscv/boot/dts/microchip/mpfs-pinctrl.dtsi b/arch/riscv/boot/dts/microchip/mpfs-pinctrl.dtsi new file mode 100644 index 000000000000..d37cabc6438a --- /dev/null +++ b/arch/riscv/boot/dts/microchip/mpfs-pinctrl.dtsi @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) + +&iomux0 { + spi0_fabric: mux-spi0-fabric { + function = "spi0"; + groups = "spi0_fabric"; + }; + + spi0_mssio: mux-spi0-mssio { + function = "spi0"; + groups = "spi0_mssio"; + }; + + spi1_fabric: mux-spi1-fabric { + function = "spi1"; + groups = "spi1_fabric"; + }; + + spi1_mssio: mux-spi1-mssio { + function = "spi1"; + groups = "spi1_mssio"; + }; + + i2c0_fabric: mux-i2c0-fabric { + function = "i2c0"; + groups = "i2c0_fabric"; + }; + + i2c0_mssio: mux-i2c0-mssio { + function = "i2c0"; + groups = "i2c0_mssio"; + }; + + i2c1_fabric: mux-i2c1-fabric { + function = "i2c1"; + groups = "i2c1_fabric"; + }; + + i2c1_mssio: mux-i2c1-mssio { + function = "i2c1"; + groups = "i2c1_mssio"; + }; + + can0_fabric: mux-can0-fabric { + function = "can0"; + groups = "can0_fabric"; + }; + + can0_mssio: mux-can0-mssio { + function = "can0"; + groups = "can0_mssio"; + }; + + can1_fabric: mux-can1-fabric { + function = "can1"; + groups = "can1_fabric"; + }; + + can1_mssio: mux-can1-mssio { + function = "can1"; + groups = "can1_mssio"; + }; + + qspi_fabric: mux-qspi-fabric { + function = "qspi"; + groups = "qspi_fabric"; + }; + + qspi_mssio: mux-qspi-mssio { + function = "qspi"; + groups = "qspi_mssio"; + }; + + uart0_fabric: mux-uart0-fabric { + function = "uart0"; + groups = "uart0_fabric"; + }; + + uart0_mssio: mux-uart0-mssio { + function = "uart0"; + groups = "uart0_mssio"; + }; + + uart1_fabric: mux-uart1-fabric { + function = "uart1"; + groups = "uart1_fabric"; + }; + + uart1_mssio: mux-uart1-mssio { + function = "uart1"; + groups = "uart1_mssio"; + }; + + uart2_fabric: mux-uart2-fabric { + function = "uart2"; + groups = "uart2_fabric"; + }; + + uart2_mssio: mux-uart2-mssio { + function = "uart2"; + groups = "uart2_mssio"; + }; + + uart3_fabric: mux-uart3-fabric { + function = "uart3"; + groups = "uart3_fabric"; + }; + + uart3_mssio: mux-uart3-mssio { + function = "uart3"; + groups = "uart3_mssio"; + }; + + uart4_fabric: mux-uart4-fabric { + function = "uart4"; + groups = "uart4_fabric"; + }; + + uart4_mssio: mux-uart4-mssio { + function = "uart4"; + groups = "uart4_mssio"; + }; + + mdio0_fabric: mux-mdio0-fabric { + function = "mdio0"; + groups = "mdio0_fabric"; + }; + + mdio0_mssio: mux-mdio0-mssio { + function = "mdio0"; + groups = "mdio0_mssio"; + }; + + mdio1_fabric: mux-mdio1-fabric { + function = "mdio1"; + groups = "mdio1_fabric"; + }; + + mdio1_mssio: mux-mdio1-mssio { + function = "mdio1"; + groups = "mdio1_mssio"; + }; +}; + +&mssio { + ikrd_can1_cfg: ikrd-can1-cfg { + can1-pins { + pins = <34>, <35>, <36>; + function = "can"; + bias-pull-up; + drive-strength = <8>; + power-source = <3300000>; + microchip,ibufmd = <0x1>; + }; + }; + + ikrd_spi1_cfg: ikrd-spi1-cfg { + spi1-pins { + pins = <30>, <31>, <32>, <33>; + function = "spi"; + bias-pull-up; + drive-strength = <8>; + power-source = <3300000>; + microchip,ibufmd = <0x1>; + }; + }; +}; diff --git a/arch/riscv/boot/dts/microchip/mpfs-polarberry.dts b/arch/riscv/boot/dts/microchip/mpfs-polarberry.dts index ea0808ab1042..cad0871a62a5 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-polarberry.dts +++ b/arch/riscv/boot/dts/microchip/mpfs-polarberry.dts @@ -30,6 +30,35 @@ }; }; +&irqmux { + interrupt-map = <0 &plic 13>, <1 &plic 14>, <2 &plic 15>, + <3 &plic 16>, <4 &plic 17>, <5 &plic 18>, + <6 &plic 19>, <7 &plic 20>, <8 &plic 21>, + <9 &plic 22>, <10 &plic 23>, <11 &plic 24>, + <12 &plic 25>, <13 &plic 26>, + + <32 &plic 27>, <33 &plic 28>, <34 &plic 29>, + <35 &plic 30>, <36 &plic 31>, <37 &plic 32>, + <38 &plic 33>, <39 &plic 34>, <40 &plic 35>, + <41 &plic 36>, <42 &plic 37>, <43 &plic 38>, + <44 &plic 39>, <45 &plic 40>, <46 &plic 41>, + <47 &plic 42>, <48 &plic 43>, <49 &plic 44>, + <50 &plic 45>, <51 &plic 46>, <52 &plic 47>, + <53 &plic 48>, <54 &plic 49>, <55 &plic 50>, + + <64 &plic 53>, <65 &plic 53>, <66 &plic 53>, + <67 &plic 53>, <68 &plic 53>, <69 &plic 53>, + <70 &plic 53>, <71 &plic 53>, <72 &plic 53>, + <73 &plic 53>, <74 &plic 53>, <75 &plic 53>, + <76 &plic 53>, <77 &plic 53>, <78 &plic 53>, + <79 &plic 53>, <80 &plic 53>, <81 &plic 53>, + <82 &plic 53>, <83 &plic 53>, <84 &plic 53>, + <85 &plic 53>, <86 &plic 53>, <87 &plic 53>, + <88 &plic 53>, <89 &plic 53>, <90 &plic 53>, + <91 &plic 53>, <92 &plic 53>, <93 &plic 53>, + <94 &plic 53>, <95 &plic 53>; +}; + /* * phy0 is connected to mac0, but the port itself is on the (optional) carrier * board. @@ -83,7 +112,3 @@ &rtc { status = "okay"; }; - -&syscontroller { - status = "okay"; -}; diff --git a/arch/riscv/boot/dts/microchip/mpfs-sev-kit.dts b/arch/riscv/boot/dts/microchip/mpfs-sev-kit.dts index f9a890579438..46ded8a4fc36 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-sev-kit.dts +++ b/arch/riscv/boot/dts/microchip/mpfs-sev-kit.dts @@ -56,15 +56,36 @@ status = "okay"; }; +&irqmux { + interrupt-map = <0 &plic 13>, <1 &plic 14>, <2 &plic 15>, + <3 &plic 16>, <4 &plic 17>, <5 &plic 18>, + <6 &plic 19>, <7 &plic 20>, <8 &plic 21>, + <9 &plic 22>, <10 &plic 23>, <11 &plic 24>, + <12 &plic 25>, <13 &plic 26>, + + <32 &plic 27>, <33 &plic 28>, <34 &plic 29>, + <35 &plic 30>, <36 &plic 31>, <37 &plic 32>, + <38 &plic 33>, <39 &plic 34>, <40 &plic 35>, + <41 &plic 36>, <42 &plic 37>, <43 &plic 38>, + <44 &plic 39>, <45 &plic 40>, <46 &plic 41>, + <47 &plic 42>, <48 &plic 43>, <49 &plic 44>, + <50 &plic 45>, <51 &plic 46>, <52 &plic 47>, + <53 &plic 48>, <54 &plic 49>, <55 &plic 50>, + + <64 &plic 53>, <65 &plic 53>, <66 &plic 53>, + <67 &plic 53>, <68 &plic 53>, <69 &plic 53>, + <70 &plic 53>, <71 &plic 53>, <72 &plic 53>, + <73 &plic 53>, <74 &plic 53>, <75 &plic 53>, + <76 &plic 53>, <77 &plic 53>, <78 &plic 53>, + <79 &plic 53>, <80 &plic 53>, <81 &plic 53>, + <82 &plic 53>, <83 &plic 53>, <84 &plic 53>, + <85 &plic 53>, <86 &plic 53>, <87 &plic 53>, + <88 &plic 53>, <89 &plic 53>, <90 &plic 53>, + <91 &plic 53>, <92 &plic 53>, <93 &plic 53>, + <94 &plic 53>, <95 &plic 53>; +}; + &gpio2 { - interrupts = <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>; status = "okay"; }; @@ -128,10 +149,6 @@ status = "okay"; }; -&syscontroller { - status = "okay"; -}; - &usb { status = "okay"; dr_mode = "otg"; diff --git a/arch/riscv/boot/dts/microchip/mpfs-tysom-m.dts b/arch/riscv/boot/dts/microchip/mpfs-tysom-m.dts index d1120f5f2c01..c6950410e918 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-tysom-m.dts +++ b/arch/riscv/boot/dts/microchip/mpfs-tysom-m.dts @@ -69,13 +69,36 @@ }; }; +&irqmux { + interrupt-map = <0 &plic 13>, <1 &plic 14>, <2 &plic 15>, + <3 &plic 16>, <4 &plic 17>, <5 &plic 18>, + <6 &plic 19>, <7 &plic 20>, <8 &plic 21>, + <9 &plic 22>, <10 &plic 23>, <11 &plic 24>, + <12 &plic 25>, <13 &plic 26>, + + <32 &plic 27>, <33 &plic 28>, <34 &plic 29>, + <35 &plic 30>, <36 &plic 31>, <37 &plic 32>, + <38 &plic 33>, <39 &plic 34>, <40 &plic 35>, + <41 &plic 36>, <42 &plic 37>, <43 &plic 38>, + <44 &plic 39>, <45 &plic 40>, <46 &plic 41>, + <47 &plic 42>, <48 &plic 43>, <49 &plic 44>, + <50 &plic 45>, <51 &plic 46>, <52 &plic 47>, + <53 &plic 48>, <54 &plic 49>, <55 &plic 50>, + + <64 &plic 53>, <65 &plic 53>, <66 &plic 53>, + <67 &plic 53>, <68 &plic 53>, <69 &plic 53>, + <70 &plic 53>, <71 &plic 53>, <72 &plic 53>, + <73 &plic 53>, <74 &plic 53>, <75 &plic 53>, + <76 &plic 53>, <77 &plic 53>, <78 &plic 53>, + <79 &plic 53>, <80 &plic 53>, <81 &plic 53>, + <82 &plic 53>, <83 &plic 53>, <84 &plic 53>, + <85 &plic 53>, <86 &plic 53>, <87 &plic 53>, + <88 &plic 53>, <89 &plic 53>, <90 &plic 53>, + <91 &plic 53>, <92 &plic 53>, <93 &plic 53>, + <94 &plic 53>, <95 &plic 53>; +}; + &gpio1 { - interrupts = <27>, <28>, <29>, <30>, - <31>, <32>, <33>, <47>, - <35>, <36>, <37>, <38>, - <39>, <40>, <41>, <42>, - <43>, <44>, <45>, <46>, - <47>, <48>, <49>, <50>; status = "okay"; }; @@ -148,10 +171,6 @@ }; }; -&syscontroller { - status = "okay"; -}; - &usb { status = "okay"; dr_mode = "host"; diff --git a/arch/riscv/boot/dts/microchip/mpfs.dtsi b/arch/riscv/boot/dts/microchip/mpfs.dtsi index 9883ca3554c5..85d8df6437f2 100644 --- a/arch/riscv/boot/dts/microchip/mpfs.dtsi +++ b/arch/riscv/boot/dts/microchip/mpfs.dtsi @@ -251,12 +251,39 @@ #dma-cells = <1>; }; - clkcfg: clkcfg@20002000 { - compatible = "microchip,mpfs-clkcfg"; - reg = <0x0 0x20002000 0x0 0x1000>, <0x0 0x3E001000 0x0 0x1000>; - clocks = <&refclk>; - #clock-cells = <1>; + mss_top_sysreg: syscon@20002000 { + compatible = "microchip,mpfs-mss-top-sysreg", "syscon", "simple-mfd"; + reg = <0x0 0x20002000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <1>; #reset-cells = <1>; + + irqmux: interrupt-controller@54 { + compatible = "microchip,mpfs-irqmux"; + reg = <0x54 0x4>; + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-map-mask = <0x7f>; + }; + + iomux0: pinctrl@200 { + compatible = "microchip,mpfs-pinctrl-iomux0"; + reg = <0x200 0x4>; + pinctrl-use-default; + + }; + + mssio: pinctrl@204 { + compatible = "microchip,mpfs-pinctrl-mssio"; + reg = <0x204 0x7c>; + /* on icicle ref design at least */ + pinctrl-use-default; + }; + }; + + sysreg_scb: syscon@20003000 { + compatible = "microchip,mpfs-sysreg-scb", "syscon"; + reg = <0x0 0x20003000 0x0 0x1000>; }; ccc_se: clock-controller@38010000 { @@ -425,6 +452,7 @@ clocks = <&clkcfg CLK_CAN0>, <&clkcfg CLK_MSSPLL3>; interrupt-parent = <&plic>; interrupts = <56>; + resets = <&mss_top_sysreg CLK_CAN0>; status = "disabled"; }; @@ -434,6 +462,7 @@ clocks = <&clkcfg CLK_CAN1>, <&clkcfg CLK_MSSPLL3>; interrupt-parent = <&plic>; interrupts = <57>; + resets = <&mss_top_sysreg CLK_CAN1>; status = "disabled"; }; @@ -445,9 +474,9 @@ interrupt-parent = <&plic>; interrupts = <64>, <65>, <66>, <67>, <68>, <69>; local-mac-address = [00 00 00 00 00 00]; - clocks = <&clkcfg CLK_MAC0>, <&clkcfg CLK_AHB>; - clock-names = "pclk", "hclk"; - resets = <&clkcfg CLK_MAC0>; + clocks = <&clkcfg CLK_MAC0>, <&clkcfg CLK_AHB>, <&refclk>; + clock-names = "pclk", "hclk", "tsu_clk"; + resets = <&mss_top_sysreg CLK_MAC0>; status = "disabled"; }; @@ -459,45 +488,66 @@ interrupt-parent = <&plic>; interrupts = <70>, <71>, <72>, <73>, <74>, <75>; local-mac-address = [00 00 00 00 00 00]; - clocks = <&clkcfg CLK_MAC1>, <&clkcfg CLK_AHB>; - clock-names = "pclk", "hclk"; - resets = <&clkcfg CLK_MAC1>; + clocks = <&clkcfg CLK_MAC1>, <&clkcfg CLK_AHB>, <&refclk>; + clock-names = "pclk", "hclk", "tsu_clk"; + resets = <&mss_top_sysreg CLK_MAC1>; status = "disabled"; }; gpio0: gpio@20120000 { compatible = "microchip,mpfs-gpio"; reg = <0x0 0x20120000 0x0 0x1000>; - interrupt-parent = <&plic>; + interrupt-parent = <&irqmux>; interrupt-controller; - #interrupt-cells = <1>; + #interrupt-cells = <2>; + interrupts = <0>, <1>, <2>, <3>, + <4>, <5>, <6>, <7>, + <8>, <9>, <10>, <11>, + <12>, <13>; clocks = <&clkcfg CLK_GPIO0>; gpio-controller; #gpio-cells = <2>; + ngpios = <14>; status = "disabled"; }; gpio1: gpio@20121000 { compatible = "microchip,mpfs-gpio"; reg = <0x0 0x20121000 0x0 0x1000>; - interrupt-parent = <&plic>; + interrupt-parent = <&irqmux>; interrupt-controller; - #interrupt-cells = <1>; + #interrupt-cells = <2>; + interrupts = <32>, <33>, <34>, <35>, + <36>, <37>, <38>, <39>, + <40>, <41>, <42>, <43>, + <44>, <45>, <46>, <47>, + <48>, <49>, <50>, <51>, + <52>, <53>, <54>, <55>; clocks = <&clkcfg CLK_GPIO1>; gpio-controller; #gpio-cells = <2>; + ngpios = <24>; status = "disabled"; }; gpio2: gpio@20122000 { compatible = "microchip,mpfs-gpio"; reg = <0x0 0x20122000 0x0 0x1000>; - interrupt-parent = <&plic>; + interrupt-parent = <&irqmux>; interrupt-controller; - #interrupt-cells = <1>; + #interrupt-cells = <2>; + interrupts = <64>, <65>, <66>, <67>, + <68>, <69>, <70>, <71>, + <72>, <73>, <74>, <75>, + <76>, <77>, <78>, <79>, + <80>, <81>, <82>, <83>, + <84>, <85>, <86>, <87>, + <88>, <89>, <90>, <91>, + <92>, <93>, <94>, <95>; clocks = <&clkcfg CLK_GPIO2>; gpio-controller; #gpio-cells = <2>; + ngpios = <32>; status = "disabled"; }; @@ -521,10 +571,14 @@ status = "disabled"; }; - mbox: mailbox@37020000 { + control_scb: syscon@37020000 { + compatible = "microchip,mpfs-control-scb", "syscon"; + reg = <0x0 0x37020000 0x0 0x100>; + }; + + mbox: mailbox@37020800 { compatible = "microchip,mpfs-mailbox"; - reg = <0x0 0x37020000 0x0 0x58>, <0x0 0x2000318C 0x0 0x40>, - <0x0 0x37020800 0x0 0x100>; + reg = <0x0 0x37020800 0x0 0x1000>; interrupt-parent = <&plic>; interrupts = <96>; #mbox-cells = <1>; @@ -541,5 +595,12 @@ clocks = <&scbclk>; status = "disabled"; }; + + clkcfg: clkcfg@3e001000 { + compatible = "microchip,mpfs-clkcfg"; + reg = <0x0 0x3e001000 0x0 0x1000>; + clocks = <&refclk>; + #clock-cells = <1>; + }; }; }; diff --git a/arch/riscv/boot/dts/microchip/pic64gx-curiosity-kit.dts b/arch/riscv/boot/dts/microchip/pic64gx-curiosity-kit.dts new file mode 100644 index 000000000000..180fd8891903 --- /dev/null +++ b/arch/riscv/boot/dts/microchip/pic64gx-curiosity-kit.dts @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Device Tree Source for the PIC64GX Curiosity Kit + * + * Copyright (C) 2024 Microchip Technology Inc. and its subsidiaries + * + * Author: Pierre-Henry Moussay <pierre-henry.moussay@microchip.com> + * + * The Curiosity-GX10000 (PIC64GX SoC Curiosity Kit) is a compact SoC + * prototyping board featuring a Microchip PIC64GX SoC + * PIC64GX-1000. Features include: + * - 1 GB DDR4 SDRAM + * - Gigabit Ethernet + * - microSD-card slot + * + * https://www.microchip.com/en-us/development-tool/curiosity-pic64gx1000-kit-es + */ + +/dts-v1/; + +#include "pic64gx.dtsi" +#include "pic64gx-pinctrl.dtsi" + +/* Clock frequency (in Hz) of the rtcclk */ +#define RTCCLK_FREQ 1000000 + +/ { + #address-cells = <2>; + #size-cells = <2>; + model = "Microchip PIC64GX Curiosity Kit"; + compatible = "microchip,pic64gx-curiosity-kit", "microchip,pic64gx"; + + aliases { + ethernet0 = &mac0; + serial1 = &mmuart1; + serial2 = &mmuart2; + }; + + chosen { + stdout-path = "serial1:115200n8"; + }; + + cpus { + timebase-frequency = <RTCCLK_FREQ>; + }; + + memory@80000000 { + device_type = "memory"; + reg = <0x0 0x80000000 0x0 0x40000000>; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + hss: hss-buffer@bfc00000 { + compatible = "shared-dma-pool"; + reg = <0x0 0xbfc00000 0x0 0x400000>; + no-map; + }; + }; +}; + +&gpio0 { + status ="okay"; + gpio-line-names = + "", "", "", "", "", "", "", "", + "", "", "", "", "MIPI_CAM_RESET", "MIPI_CAM_STANDBY"; +}; + +&gpio1 { + status ="okay"; + gpio-line-names = + "", "", "LED1", "LED2", "LED3", "LED4", "LED5", "LED6", + "LED7", "LED8", "", "", "", "", "", "", + "", "", "", "", "HDMI_HPD", "", "", "GPIO_1_23"; +}; + +&gpio2 { + pinctrl-names = "default"; + pinctrl-0 = <&mdio1_gpio>, <&spi0_gpio>, <&can0_gpio>, <&pcie_gpio>, + <&qspi_gpio>, <&uart3_gpio>, <&uart4_gpio>, <&can1_gpio>; + status ="okay"; + gpio-line-names = + "", "", "", "", "", "", "SWITCH2", "USR_IO12", + "DIP1", "DIP2", "", "DIP3", "USR_IO1", "USR_IO2", "USR_IO7", "USR_IO8", + "USR_IO3", "USR_IO4", "USR_IO5", "USR_IO6", "", "", "USR_IO9", "USR_IO10", + "DIP4", "USR_IO11", "", "", "SWITCH1", "", "", ""; +}; + +&i2c0 { + status = "okay"; +}; + +&i2c1 { + status = "okay"; +}; + +&irqmux { + interrupt-map = <0 &plic 13>, <1 &plic 14>, <2 &plic 15>, + <3 &plic 16>, <4 &plic 17>, <5 &plic 18>, + <6 &plic 19>, <7 &plic 20>, <8 &plic 21>, + <9 &plic 22>, <10 &plic 23>, <11 &plic 24>, + <12 &plic 25>, <13 &plic 26>, + + <32 &plic 27>, <33 &plic 28>, <34 &plic 29>, + <35 &plic 30>, <36 &plic 31>, <37 &plic 32>, + <38 &plic 33>, <39 &plic 34>, <40 &plic 35>, + <41 &plic 36>, <42 &plic 37>, <43 &plic 38>, + <44 &plic 39>, <45 &plic 40>, <46 &plic 41>, + <47 &plic 42>, <48 &plic 43>, <49 &plic 44>, + <50 &plic 45>, <51 &plic 46>, <52 &plic 47>, + <53 &plic 48>, <54 &plic 49>, <55 &plic 50>, + + <64 &plic 53>, <65 &plic 53>, <66 &plic 53>, + <67 &plic 53>, <68 &plic 53>, <69 &plic 53>, + <70 &plic 53>, <71 &plic 53>, <72 &plic 53>, + <73 &plic 53>, <74 &plic 53>, <75 &plic 53>, + <76 &plic 53>, <77 &plic 53>, <78 &plic 53>, + <79 &plic 53>, <80 &plic 53>, <81 &plic 53>, + <82 &plic 53>, <83 &plic 53>, <84 &plic 53>, + <85 &plic 53>, <86 &plic 53>, <87 &plic 53>, + <88 &plic 53>, <89 &plic 53>, <90 &plic 53>, + <91 &plic 53>, <92 &plic 53>, <93 &plic 53>, + <94 &plic 53>, <95 &plic 53>; +}; + +&mac0 { + status = "okay"; + phy-mode = "sgmii"; + phy-handle = <&phy0>; + pinctrl-names = "default"; + pinctrl-0 = <&mdio0_default>; + + phy0: ethernet-phy@b { + reg = <0xb>; + }; +}; + +&mbox { + status = "okay"; +}; + +&mmc { + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-mmc-highspeed; + sdhci-caps-mask = <0x00000007 0x00000000>; + status = "okay"; +}; + +&mmuart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_fio>; + status = "okay"; +}; + +&mmuart2 { + pinctrl-names = "default"; + pinctrl-0 = <&uart2_default>; + status = "okay"; +}; + +&refclk { + clock-frequency = <125000000>; +}; + +&rtc { + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/microchip/pic64gx-pinctrl.dtsi b/arch/riscv/boot/dts/microchip/pic64gx-pinctrl.dtsi new file mode 100644 index 000000000000..cf73b0556633 --- /dev/null +++ b/arch/riscv/boot/dts/microchip/pic64gx-pinctrl.dtsi @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) + +&iomux0 { + spi0_fio: mux-spi0-fabric { + function = "spi0"; + groups = "spi0_fabric"; + }; + + spi1_mssio: mux-spi1-mssio { + function = "spi1"; + groups = "spi1_mssio"; + }; + + i2c0_mssio: mux-i2c0-mssio { + function = "i2c0"; + groups = "i2c0_mssio"; + }; + + i2c1_mssio: mux-i2c1-mssio { + function = "i2c1"; + groups = "i2c1_mssio"; + }; + + can0_fio: mux-can0-fabric { + function = "can0"; + groups = "can0_fabric"; + }; + + can1_fio: mux-can1-fabric { + function = "can1"; + groups = "can1_fabric"; + }; + + qspi_fio: mux-qspi-fabric { + function = "qspi"; + groups = "qspi_fabric"; + }; + + uart0_mssio: mux-uart0-mssio { + function = "uart0"; + groups = "uart0_mssio"; + }; + + uart1_fio: mux-uart1-fabric { + function = "uart1"; + groups = "uart1_fabric"; + }; + + uart2_fio: mux-uart2-fabric { + function = "uart2"; + groups = "uart2_fabric"; + }; + + uart3_fio: mux-uart3-fabric { + function = "uart3"; + groups = "uart3_fabric"; + }; + + uart4_fio: mux-uart4-fabric { + function = "uart4"; + groups = "uart4_fabric"; + }; + + mdio0_fio: mux-mdio0-fabric { + function = "mdio0"; + groups = "mdio0_fabric"; + }; + + mdio1_fio: mux-mdio1-fabric { + function = "mdio1"; + groups = "mdio1_fabric"; + }; +}; + +&gpio2_pinctrl { + //TODO rethink the labels, since a bunch of these are not defaults or + //just outright remove the non-default groups + mdio0_default: mux-mac0 { + function = "mdio0"; + groups = "mdio0"; + }; + + mdio0_gpio: mux-mac0-gpio2 { + function = "gpio"; + groups = "gpio_mdio0"; + }; + + mdio1_default: mux-mac1 { + function = "mdio1"; + groups = "mdio1"; + }; + + mdio1_gpio: mux-mac1-gpio2 { + function = "gpio"; + groups = "gpio_mdio1"; + }; + + spi0_default: mux-spi0 { + function = "spi0"; + groups = "spi0"; + }; + + spi0_gpio: mux-spi0-gpio2 { + function = "gpio"; + groups = "gpio_spi0"; + }; + + can0_default: mux-can0 { + function = "can0"; + groups = "can0"; + }; + + can0_gpio: mux-can0-gpio2 { + function = "gpio"; + groups = "gpio_can0"; + }; + + pcie_default: mux-pcie { + function = "pcie"; + groups = "pcie"; + }; + + pcie_gpio: mux-pcie-gpio2 { + function = "gpio"; + groups = "gpio_pcie"; + }; + + qspi_default: mux-qspi { + function = "qspi"; + groups = "qspi"; + }; + + qspi_gpio: mux-qspi-gpio2 { + function = "gpio"; + groups = "gpio_qspi"; + }; + + uart3_default: mux-uart3 { + function = "uart3"; + groups = "uart3"; + }; + + uart3_gpio: mux-uart3-gpio2 { + function = "gpio"; + groups = "gpio_uart3"; + }; + + uart4_default: mux-uart4 { + function = "uart4"; + groups = "uart4"; + }; + + uart4_gpio: mux-uart4-gpio2 { + function = "gpio"; + groups = "gpio_uart4"; + }; + + can1_default: mux-can1 { + function = "can1"; + groups = "can1"; + }; + + can1_gpio: mux-can1-gpio2 { + function = "gpio"; + groups = "gpio_can1"; + }; + + uart2_default: mux-uart2 { + function = "uart2"; + groups = "uart2"; + }; + + uart2_gpio: mux-uart2-gpio2 { + function = "gpio"; + groups = "gpio_uart2"; + }; +}; diff --git a/arch/riscv/boot/dts/microchip/pic64gx.dtsi b/arch/riscv/boot/dts/microchip/pic64gx.dtsi new file mode 100644 index 000000000000..5addfd435711 --- /dev/null +++ b/arch/riscv/boot/dts/microchip/pic64gx.dtsi @@ -0,0 +1,656 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Device Tree Source for the PIC64GX SoCs + * + * Copyright (C) 2024 Microchip Technology Inc. and its subsidiaries + * + * Author: Pierre-Henry Moussay <pierre-henry.moussay@microchip.com> + * + * PIC64GX is a series RISC-V multicore SoCs: + * https://www.microchip.com/en-us/products/microprocessors/64-bit-mpus/pic64gx + */ + +/dts-v1/; +#include "dt-bindings/clock/microchip,mpfs-clock.h" + +/ { + #address-cells = <2>; + #size-cells = <2>; + model = "Microchip PIC64GX SoC"; + compatible = "microchip,pic64gx"; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu0: cpu@0 { + compatible = "sifive,e51", "sifive,rocket0", "riscv"; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <128>; + i-cache-size = <16384>; + reg = <0>; + riscv,isa = "rv64imac"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "c", "zicntr", "zicsr", + "zifencei", "zihpm"; + clocks = <&clkcfg CLK_CPU>; + status = "disabled"; + + cpu0_intc: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + + cpu1: cpu@1 { + compatible = "sifive,u54-mc", "sifive,rocket0", "riscv"; + d-cache-block-size = <64>; + d-cache-sets = <64>; + d-cache-size = <32768>; + d-tlb-sets = <1>; + d-tlb-size = <32>; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <64>; + i-cache-size = <32768>; + i-tlb-sets = <1>; + i-tlb-size = <32>; + mmu-type = "riscv,sv39"; + reg = <1>; + riscv,isa = "rv64imafdc"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "zicntr", + "zicsr", "zifencei", "zihpm"; + clocks = <&clkcfg CLK_CPU>; + tlb-split; + next-level-cache = <&cctrllr>; + status = "okay"; + + cpu1_intc: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + + cpu2: cpu@2 { + compatible = "sifive,u54-mc", "sifive,rocket0", "riscv"; + d-cache-block-size = <64>; + d-cache-sets = <64>; + d-cache-size = <32768>; + d-tlb-sets = <1>; + d-tlb-size = <32>; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <64>; + i-cache-size = <32768>; + i-tlb-sets = <1>; + i-tlb-size = <32>; + mmu-type = "riscv,sv39"; + reg = <2>; + riscv,isa = "rv64imafdc"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "zicntr", + "zicsr", "zifencei", "zihpm"; + clocks = <&clkcfg CLK_CPU>; + tlb-split; + next-level-cache = <&cctrllr>; + status = "okay"; + + cpu2_intc: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + + cpu3: cpu@3 { + compatible = "sifive,u54-mc", "sifive,rocket0", "riscv"; + d-cache-block-size = <64>; + d-cache-sets = <64>; + d-cache-size = <32768>; + d-tlb-sets = <1>; + d-tlb-size = <32>; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <64>; + i-cache-size = <32768>; + i-tlb-sets = <1>; + i-tlb-size = <32>; + mmu-type = "riscv,sv39"; + reg = <3>; + riscv,isa = "rv64imafdc"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "zicntr", + "zicsr", "zifencei", "zihpm"; + clocks = <&clkcfg CLK_CPU>; + tlb-split; + next-level-cache = <&cctrllr>; + status = "okay"; + + cpu3_intc: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + + cpu4: cpu@4 { + compatible = "sifive,u54-mc", "sifive,rocket0", "riscv"; + d-cache-block-size = <64>; + d-cache-sets = <64>; + d-cache-size = <32768>; + d-tlb-sets = <1>; + d-tlb-size = <32>; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <64>; + i-cache-size = <32768>; + i-tlb-sets = <1>; + i-tlb-size = <32>; + mmu-type = "riscv,sv39"; + reg = <4>; + riscv,isa = "rv64imafdc"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "zicntr", + "zicsr", "zifencei", "zihpm"; + clocks = <&clkcfg CLK_CPU>; + tlb-split; + next-level-cache = <&cctrllr>; + status = "okay"; + cpu4_intc: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + + cpu-map { + cluster0 { + core0 { + cpu = <&cpu0>; + }; + + core1 { + cpu = <&cpu1>; + }; + + core2 { + cpu = <&cpu2>; + }; + + core3 { + cpu = <&cpu3>; + }; + + core4 { + cpu = <&cpu4>; + }; + }; + }; + }; + + scbclk: clock-80000000 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <80000000>; + }; + + refclk: mssrefclk { + compatible = "fixed-clock"; + #clock-cells = <0>; + }; + + axiclk: axi-aclk0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <125000000>; + }; + + videoclk: video-aclk0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <125000000>; + }; + + syscontroller: syscontroller { + compatible = "microchip,pic64gx-sys-controller"; + mboxes = <&mbox 0>; + }; + + soc { + #address-cells = <2>; + #size-cells = <2>; + compatible = "simple-bus"; + ranges; + + clint: clint@2000000 { + compatible = "microchip,pic64gx-clint", "sifive,clint0"; + reg = <0x0 0x2000000 0x0 0xC000>; + interrupts-extended = <&cpu0_intc 0xffffffff>, <&cpu0_intc 0xffffffff>, + <&cpu1_intc 3>, <&cpu1_intc 7>, + <&cpu2_intc 3>, <&cpu2_intc 7>, + <&cpu3_intc 3>, <&cpu3_intc 7>, + <&cpu4_intc 3>, <&cpu4_intc 7>; + }; + + cctrllr: cache-controller@2010000 { + compatible = "microchip,pic64gx-ccache", "microchip,mpfs-ccache", + "sifive,fu540-c000-ccache", "cache"; + reg = <0x0 0x2010000 0x0 0x1000>; + cache-block-size = <64>; + cache-level = <2>; + cache-sets = <1024>; + cache-size = <2097152>; + cache-unified; + interrupt-parent = <&plic>; + interrupts = <1>, <3>, <4>, <2>; + }; + + pdma: dma-controller@3000000 { + compatible = "microchip,pic64gx-pdma", "microchip,mpfs-pdma", + "sifive,pdma0"; + reg = <0x0 0x3000000 0x0 0x8000>; + interrupt-parent = <&plic>; + interrupts = <5 6>, <7 8>, <9 10>, <11 12>; + #dma-cells = <1>; + }; + + plic: interrupt-controller@c000000 { + compatible = "microchip,pic64gx-plic", "sifive,plic-1.0.0"; + reg = <0x0 0xc000000 0x0 0x4000000>; + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-controller; + interrupts-extended = <&cpu0_intc 0xffffffff>, + <&cpu1_intc 0xffffffff>, <&cpu1_intc 9>, + <&cpu2_intc 0xffffffff>, <&cpu2_intc 9>, + <&cpu3_intc 0xffffffff>, <&cpu3_intc 9>, + <&cpu4_intc 0xffffffff>, <&cpu4_intc 9>; + riscv,ndev = <186>; + }; + + mmuart0: serial@20000000 { + compatible = "ns16550a"; + reg = <0x0 0x20000000 0x0 0x400>; + reg-io-width = <4>; + reg-shift = <2>; + interrupt-parent = <&plic>; + interrupts = <90>; + current-speed = <115200>; + clocks = <&clkcfg CLK_MMUART0>; + pinctrl-names = "default"; + pinctrl-0 = <&uart0_mssio>; + status = "disabled"; /* Reserved for the HSS */ + }; + + mss_top_sysreg: syscon@20002000 { + compatible = "microchip,pic64gx-mss-top-sysreg", + "microchip,mpfs-mss-top-sysreg", + "syscon", "simple-mfd"; + reg = <0x0 0x20002000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <1>; + #reset-cells = <1>; + + irqmux: interrupt-controller@54 { + compatible = "microchip,pic64gx-irqmux", "microchip,mpfs-irqmux"; + reg = <0x54 0x4>; + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-map-mask = <0x7f>; + }; + + iomux0: pinctrl@200 { + compatible = "microchip,pic64gx-pinctrl-iomux0", + "microchip,mpfs-pinctrl-iomux0"; + reg = <0x200 0x4>; + pinctrl-use-default; + }; + + mssio: pinctrl@204 { + compatible = "microchip,pic64gx-pinctrl-mssio", + "microchip,mpfs-pinctrl-mssio"; + reg = <0x204 0x7c>; + /* on icicle ref design at least */ + pinctrl-use-default; + }; + }; + + sysreg_scb: syscon@20003000 { + compatible = "microchip,pic64gx-sysreg-scb", + "microchip,mpfs-sysreg-scb", + "syscon"; + reg = <0x0 0x20003000 0x0 0x1000>; + }; + + /* Common node entry for emmc/sd */ + mmc: mmc@20008000 { + compatible = "microchip,pic64gx-sd4hc", "cdns,sd4hc"; + reg = <0x0 0x20008000 0x0 0x1000>; + interrupt-parent = <&plic>; + interrupts = <88>; + clocks = <&clkcfg CLK_MMC>; + max-frequency = <200000000>; + status = "disabled"; + }; + + mmuart1: serial@20100000 { + compatible = "ns16550a"; + reg = <0x0 0x20100000 0x0 0x400>; + reg-io-width = <4>; + reg-shift = <2>; + interrupt-parent = <&plic>; + interrupts = <91>; + current-speed = <115200>; + clocks = <&clkcfg CLK_MMUART1>; + status = "disabled"; + }; + + mmuart2: serial@20102000 { + compatible = "ns16550a"; + reg = <0x0 0x20102000 0x0 0x400>; + reg-io-width = <4>; + reg-shift = <2>; + interrupt-parent = <&plic>; + interrupts = <92>; + current-speed = <115200>; + clocks = <&clkcfg CLK_MMUART2>; + status = "disabled"; + }; + + mmuart3: serial@20104000 { + compatible = "ns16550a"; + reg = <0x0 0x20104000 0x0 0x400>; + reg-io-width = <4>; + reg-shift = <2>; + interrupt-parent = <&plic>; + interrupts = <93>; + current-speed = <115200>; + clocks = <&clkcfg CLK_MMUART3>; + status = "disabled"; + }; + + mmuart4: serial@20106000 { + compatible = "ns16550a"; + reg = <0x0 0x20106000 0x0 0x400>; + reg-io-width = <4>; + reg-shift = <2>; + interrupt-parent = <&plic>; + interrupts = <94>; + clocks = <&clkcfg CLK_MMUART4>; + current-speed = <115200>; + status = "disabled"; + }; + + spi0: spi@20108000 { + compatible = "microchip,pic64gx-spi", "microchip,mpfs-spi"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x20108000 0x0 0x1000>; + interrupt-parent = <&plic>; + interrupts = <54>; + clocks = <&clkcfg CLK_SPI0>; + status = "disabled"; + }; + + spi1: spi@20109000 { + compatible = "microchip,pic64gx-spi", "microchip,mpfs-spi"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x20109000 0x0 0x1000>; + interrupt-parent = <&plic>; + interrupts = <55>; + clocks = <&clkcfg CLK_SPI1>; + pinctrl-names = "default"; + pinctrl-0 = <&spi1_mssio>; + status = "disabled"; + }; + + i2c0: i2c@2010a000 { + compatible = "microchip,pic64gx-i2c", "microchip,corei2c-rtl-v7"; + reg = <0x0 0x2010a000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <0>; + interrupt-parent = <&plic>; + interrupts = <58>; + clocks = <&clkcfg CLK_I2C0>; + clock-frequency = <100000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_mssio>; + status = "disabled"; + }; + + i2c1: i2c@2010b000 { + compatible = "microchip,pic64gx-i2c", "microchip,corei2c-rtl-v7"; + reg = <0x0 0x2010b000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <0>; + interrupt-parent = <&plic>; + interrupts = <61>; + clocks = <&clkcfg CLK_I2C1>; + clock-frequency = <100000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_mssio>; + status = "disabled"; + }; + + can0: can@2010c000 { + compatible = "microchip,pic64gx-can", "microchip,mpfs-can"; + reg = <0x0 0x2010c000 0x0 0x1000>; + clocks = <&clkcfg CLK_CAN0>, <&clkcfg CLK_MSSPLL3>; + interrupt-parent = <&plic>; + interrupts = <56>; + resets = <&mss_top_sysreg CLK_CAN0>; + status = "disabled"; + }; + + can1: can@2010d000 { + compatible = "microchip,pic64gx-can", "microchip,mpfs-can"; + reg = <0x0 0x2010d000 0x0 0x1000>; + clocks = <&clkcfg CLK_CAN1>, <&clkcfg CLK_MSSPLL3>; + interrupt-parent = <&plic>; + interrupts = <57>; + resets = <&mss_top_sysreg CLK_CAN1>; + status = "disabled"; + }; + + mac0: ethernet@20110000 { + compatible = "microchip,pic64gx-macb", "microchip,mpfs-macb", + "cdns,macb"; + reg = <0x0 0x20110000 0x0 0x2000>; + #address-cells = <1>; + #size-cells = <0>; + interrupt-parent = <&plic>; + interrupts = <64>, <65>, <66>, <67>, <68>, <69>; + /* Filled in by a bootloader */ + local-mac-address = [00 00 00 00 00 00]; + clocks = <&clkcfg CLK_MAC0>, <&clkcfg CLK_AHB>, <&refclk>; + clock-names = "pclk", "hclk", "tsu_clk"; + resets = <&mss_top_sysreg CLK_MAC0>; + status = "disabled"; + }; + + mac1: ethernet@20112000 { + compatible = "microchip,pic64gx-macb", "microchip,mpfs-macb", + "cdns,macb"; + reg = <0x0 0x20112000 0x0 0x2000>; + #address-cells = <1>; + #size-cells = <0>; + interrupt-parent = <&plic>; + interrupts = <70>, <71>, <72>, <73>, <74>, <75>; + /* Filled in by a bootloader */ + local-mac-address = [00 00 00 00 00 00]; + clocks = <&clkcfg CLK_MAC0>, <&clkcfg CLK_AHB>, <&refclk>; + clock-names = "pclk", "hclk", "tsu_clk"; + resets = <&mss_top_sysreg CLK_MAC1>; + status = "disabled"; + }; + + gpio0: gpio@20120000 { + compatible = "microchip,pic64gx-gpio", "microchip,mpfs-gpio"; + reg = <0x0 0x20120000 0x0 0x1000>; + interrupt-parent = <&irqmux>; + interrupt-controller; + #interrupt-cells = <2>; + interrupts = <0>, <1>, <2>, <3>, + <4>, <5>, <6>, <7>, + <8>, <9>, <10>, <11>, + <12>, <13>; + clocks = <&clkcfg CLK_GPIO0>; + gpio-controller; + #gpio-cells = <2>; + ngpios = <14>; + status = "disabled"; + }; + + gpio1: gpio@20121000 { + compatible = "microchip,pic64gx-gpio", "microchip,mpfs-gpio"; + reg = <0x0 0x20121000 0x0 0x1000>; + interrupt-parent = <&irqmux>; + interrupt-controller; + #interrupt-cells = <2>; + interrupts = <32>, <33>, <34>, <35>, + <36>, <37>, <38>, <39>, + <40>, <41>, <42>, <43>, + <44>, <45>, <46>, <47>, + <48>, <49>, <50>, <51>, + <52>, <53>, <54>, <55>; + clocks = <&clkcfg CLK_GPIO1>; + gpio-controller; + #gpio-cells = <2>; + ngpios = <24>; + status = "disabled"; + }; + + gpio2: gpio@20122000 { + compatible = "microchip,pic64gx-gpio", "microchip,mpfs-gpio"; + reg = <0x0 0x20122000 0x0 0x1000>; + interrupt-parent = <&irqmux>; + interrupt-controller; + #interrupt-cells = <2>; + interrupts = <64>, <65>, <66>, <67>, + <68>, <69>, <70>, <71>, + <72>, <73>, <74>, <75>, + <76>, <77>, <78>, <79>, + <80>, <81>, <82>, <83>, + <84>, <85>, <86>, <87>, + <88>, <89>, <90>, <91>, + <92>, <93>, <94>, <95>; + clocks = <&clkcfg CLK_GPIO2>; + gpio-controller; + #gpio-cells = <2>; + ngpios = <32>; + status = "disabled"; + }; + + rtc: rtc@20124000 { + compatible = "microchip,pic64gx-rtc", "microchip,mpfs-rtc"; + reg = <0x0 0x20124000 0x0 0x1000>; + interrupt-parent = <&plic>; + interrupts = <80>, <81>; + clocks = <&clkcfg CLK_RTC>, <&clkcfg CLK_RTCREF>; + clock-names = "rtc", "rtcref"; + status = "disabled"; + }; + + usb: usb@20201000 { + compatible = "microchip,pic64gx-musb", "microchip,mpfs-musb"; + reg = <0x0 0x20201000 0x0 0x1000>; + interrupt-parent = <&plic>; + interrupts = <86>, <87>; + clocks = <&clkcfg CLK_USB>; + interrupt-names = "dma", "mc"; + status = "disabled"; + }; + + qspi: spi@21000000 { + compatible = "microchip,pic64gx-qspi", "microchip,coreqspi-rtl-v2"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x21000000 0x0 0x1000>; + interrupt-parent = <&plic>; + interrupts = <85>; + clocks = <&clkcfg CLK_QSPI>; + status = "disabled"; + }; + + control_scb: syscon@37020000 { + compatible = "microchip,pic64gx-control-scb", + "microchip,mpfs-control-scb", + "syscon"; + reg = <0x0 0x37020000 0x0 0x100>; + }; + + syscontroller_qspi: spi@37020100 { + compatible = "microchip,pic64gx-qspi", "microchip,coreqspi-rtl-v2"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x37020100 0x0 0x100>; + interrupt-parent = <&plic>; + interrupts = <110>; + clocks = <&scbclk>; + status = "disabled"; + }; + + mbox: mailbox@37020800 { + compatible = "microchip,pic64gx-mailbox", "microchip,mpfs-mailbox"; + reg = <0x0 0x37020800 0x0 0x100>; + interrupt-parent = <&plic>; + interrupts = <96>; + #mbox-cells = <1>; + status = "disabled"; + }; + + ccc_se: clock-controller@38010000 { + compatible = "microchip,pic64gx-ccc", "microchip,mpfs-ccc"; + reg = <0x0 0x38010000 0x0 0x1000>, <0x0 0x38020000 0x0 0x1000>, + <0x0 0x39010000 0x0 0x1000>, <0x0 0x39020000 0x0 0x1000>; + #clock-cells = <1>; + status = "disabled"; + }; + + ccc_ne: clock-controller@38040000 { + compatible = "microchip,pic64gx-ccc", "microchip,mpfs-ccc"; + reg = <0x0 0x38040000 0x0 0x1000>, <0x0 0x38080000 0x0 0x1000>, + <0x0 0x39040000 0x0 0x1000>, <0x0 0x39080000 0x0 0x1000>; + #clock-cells = <1>; + status = "disabled"; + }; + + ccc_nw: clock-controller@38100000 { + compatible = "microchip,pic64gx-ccc", "microchip,mpfs-ccc"; + reg = <0x0 0x38100000 0x0 0x1000>, <0x0 0x38200000 0x0 0x1000>, + <0x0 0x39100000 0x0 0x1000>, <0x0 0x39200000 0x0 0x1000>; + #clock-cells = <1>; + status = "disabled"; + }; + + ccc_sw: clock-controller@38400000 { + compatible = "microchip,pic64gx-ccc", "microchip,mpfs-ccc"; + reg = <0x0 0x38400000 0x0 0x1000>, <0x0 0x38800000 0x0 0x1000>, + <0x0 0x39400000 0x0 0x1000>, <0x0 0x39800000 0x0 0x1000>; + #clock-cells = <1>; + status = "disabled"; + }; + + clkcfg: clkcfg@3e001000 { + compatible = "microchip,pic64gx-clkcfg", "microchip,mpfs-clkcfg"; + reg = <0x0 0x3e001000 0x0 0x1000>; + clocks = <&refclk>; + #clock-cells = <1>; + }; + + gpio2_pinctrl: pinctrl@41000000 { + compatible = "microchip,pic64gx-pinctrl-gpio2"; + reg = <0x0 0x41000000 0x0 0x4>; + pinctrl-use-default; + pinctrl-names = "default"; + pinctrl-0 = <&mdio0_fio>, <&mdio1_fio>, <&spi0_fio>, <&qspi_fio>, + <&uart3_fio>, <&uart4_fio>, <&can1_fio>, <&can0_fio>, + <&uart2_fio>; + }; + }; +}; diff --git a/arch/riscv/boot/dts/renesas/r9a07g043f.dtsi b/arch/riscv/boot/dts/renesas/r9a07g043f.dtsi index a8bcb26f4270..571de3cafa82 100644 --- a/arch/riscv/boot/dts/renesas/r9a07g043f.dtsi +++ b/arch/riscv/boot/dts/renesas/r9a07g043f.dtsi @@ -12,6 +12,8 @@ #include <arm64/renesas/r9a07g043.dtsi> / { + interrupt-parent = <&plic>; + cpus { #address-cells = <1>; #size-cells = <0>; @@ -52,7 +54,6 @@ &soc { dma-noncoherent; - interrupt-parent = <&plic>; irqc: interrupt-controller@110a0000 { compatible = "renesas,r9a07g043f-irqc"; diff --git a/arch/riscv/boot/dts/sophgo/Makefile b/arch/riscv/boot/dts/sophgo/Makefile index 6f65526d4193..d5c80d4ff005 100644 --- a/arch/riscv/boot/dts/sophgo/Makefile +++ b/arch/riscv/boot/dts/sophgo/Makefile @@ -1,7 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 dtb-$(CONFIG_ARCH_SOPHGO) += cv1800b-milkv-duo.dtb dtb-$(CONFIG_ARCH_SOPHGO) += cv1812h-huashan-pi.dtb +dtb-$(CONFIG_ARCH_SOPHGO) += sg2000-milkv-duo-s.dtb dtb-$(CONFIG_ARCH_SOPHGO) += sg2002-licheerv-nano-b.dtb +dtb-$(CONFIG_ARCH_SOPHGO) += sg2002-milkv-duo256m.dtb dtb-$(CONFIG_ARCH_SOPHGO) += sg2042-milkv-pioneer.dtb dtb-$(CONFIG_ARCH_SOPHGO) += sg2042-evb-v1.dtb dtb-$(CONFIG_ARCH_SOPHGO) += sg2042-evb-v2.dtb diff --git a/arch/riscv/boot/dts/sophgo/cv180x.dtsi b/arch/riscv/boot/dts/sophgo/cv180x.dtsi index 1b2b1969a648..ec3bf305274a 100644 --- a/arch/riscv/boot/dts/sophgo/cv180x.dtsi +++ b/arch/riscv/boot/dts/sophgo/cv180x.dtsi @@ -417,7 +417,7 @@ }; dmac: dma-controller@4330000 { - compatible = "snps,axi-dma-1.01a"; + compatible = "sophgo,cv1800b-axi-dma", "snps,axi-dma-1.01a"; reg = <0x04330000 0x1000>; interrupts = <SOC_PERIPHERAL_IRQ(13) IRQ_TYPE_LEVEL_HIGH>; clocks = <&clk CLK_SDMA_AXI>, <&clk CLK_SDMA_AXI>; @@ -438,8 +438,8 @@ clocks = <&clk CLK_AXI4_USB>, <&clk CLK_APB_USB>; clock-names = "otg", "utmi"; g-np-tx-fifo-size = <32>; - g-rx-fifo-size = <536>; - g-tx-fifo-size = <768 512 512 384 128 128>; + g-rx-fifo-size = <1536>; + g-tx-fifo-size = <128 128 64 64 64 64 32 32>; interrupts = <SOC_PERIPHERAL_IRQ(14) IRQ_TYPE_LEVEL_HIGH>; phys = <&usbphy>; phy-names = "usb2-phy"; @@ -448,6 +448,24 @@ status = "disabled"; }; + gpio4: gpio@5021000 { + compatible = "snps,dw-apb-gpio"; + reg = <0x5021000 0x1000>; + #address-cells = <1>; + #size-cells = <0>; + + porte: gpio-controller@0 { + compatible = "snps,dw-apb-gpio-port"; + gpio-controller; + #gpio-cells = <2>; + ngpios = <32>; + reg = <0>; + interrupt-controller; + #interrupt-cells = <2>; + interrupts = <SOC_PERIPHERAL_IRQ(54) IRQ_TYPE_LEVEL_HIGH>; + }; + }; + rtc@5025000 { compatible = "sophgo,cv1800b-rtc", "syscon"; reg = <0x5025000 0x2000>; diff --git a/arch/riscv/boot/dts/sophgo/sg2000-milkv-duo-s.dts b/arch/riscv/boot/dts/sophgo/sg2000-milkv-duo-s.dts new file mode 100644 index 000000000000..78a257839f6f --- /dev/null +++ b/arch/riscv/boot/dts/sophgo/sg2000-milkv-duo-s.dts @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) + +/dts-v1/; + +#include <dt-bindings/pinctrl/pinctrl-sg2000.h> +#include "sg2000.dtsi" + +/ { + model = "Milk-V Duo S"; + compatible = "milkv,duo-s", "sophgo,sg2000"; + + aliases { + i2c4 = &i2c4; + mmc0 = &sdhci0; + mmc1 = &emmc; + mmc2 = &sdhci1; + serial0 = &uart0; + serial4 = &uart4; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + wifi_pwrseq: wifi-pwrseq { + compatible = "mmc-pwrseq-simple"; + reset-gpios = <&porta 15 GPIO_ACTIVE_LOW>; + power-off-delay-us = <50000>; + post-power-on-delay-ms = <200>; + }; +}; + +&osc { + clock-frequency = <25000000>; +}; + +&pinctrl { + emmc_cfg: emmc-cfg { + emmc-rstn-pins { + pinmux = <PINMUX(PIN_EMMC_RSTN, 0)>; + bias-pull-up; + drive-strength-microamp = <10800>; + power-source = <1800>; + }; + + emmc-clk-pins { + pinmux = <PINMUX(PIN_EMMC_CLK, 0)>; + bias-pull-up; + drive-strength-microamp = <10800>; + power-source = <1800>; + }; + + emmc-cmd-pins { + pinmux = <PINMUX(PIN_EMMC_CMD, 0)>; + bias-pull-up; + drive-strength-microamp = <10800>; + power-source = <1800>; + }; + + emmc-data-pins { + pinmux = <PINMUX(PIN_EMMC_DAT0, 0)>, + <PINMUX(PIN_EMMC_DAT1, 0)>, + <PINMUX(PIN_EMMC_DAT2, 0)>, + <PINMUX(PIN_EMMC_DAT3, 0)>; + bias-pull-up; + drive-strength-microamp = <10800>; + power-source = <1800>; + }; + }; + + i2c4_cfg: i2c4-cfg { + i2c4-pins { + pinmux = <PINMUX(PIN_VIVO_D1, 7)>, + <PINMUX(PIN_VIVO_D0, 7)>; + bias-pull-up; + drive-strength-microamp = <4000>; + power-source = <1800>; + }; + }; + + sdhci0_cfg: sdhci0-cfg { + sdhci0-cd-pins { + pinmux = <PINMUX(PIN_SD0_CD, 0)>; + bias-pull-up; + drive-strength-microamp = <10800>; + power-source = <3300>; + }; + + sdhci0-clk-pins { + pinmux = <PINMUX(PIN_SD0_CLK, 0)>; + bias-pull-up; + drive-strength-microamp = <16100>; + power-source = <3300>; + }; + + sdhci0-cmd-pins { + pinmux = <PINMUX(PIN_SD0_CMD, 0)>; + bias-pull-up; + drive-strength-microamp = <10800>; + power-source = <3300>; + }; + + sdhci0-data-pins { + pinmux = <PINMUX(PIN_SD0_D0, 0)>, + <PINMUX(PIN_SD0_D1, 0)>, + <PINMUX(PIN_SD0_D2, 0)>, + <PINMUX(PIN_SD0_D3, 0)>; + bias-pull-up; + drive-strength-microamp = <10800>; + power-source = <3300>; + }; + }; + + sdhci1_cfg: sdhci1-cfg { + sdhci1-clk-pins { + pinmux = <PINMUX(PIN_MIPI_TXM4, 1)>; + bias-pull-up; + drive-strength-microamp = <10800>; + power-source = <1800>; + }; + + sdhci1-cmd-pins { + pinmux = <PINMUX(PIN_MIPI_TXP4, 1)>; + bias-pull-up; + drive-strength-microamp = <10800>; + power-source = <1800>; + }; + + sdhci1-data-pins { + pinmux = <PINMUX(PIN_MIPI_TXM3, 1)>, + <PINMUX(PIN_MIPI_TXP3, 1)>, + <PINMUX(PIN_MIPI_TXM2, 7)>, + <PINMUX(PIN_MIPI_TXP2, 7)>; + bias-pull-up; + drive-strength-microamp = <10800>; + power-source = <1800>; + }; + }; + + uart0_cfg: uart0-cfg { + uart0-pins { + pinmux = <PINMUX(PIN_UART0_TX, 0)>, + <PINMUX(PIN_UART0_RX, 0)>; + bias-pull-up; + drive-strength-microamp = <10800>; + power-source = <3300>; + }; + }; + + uart4_cfg: uart4-cfg { + uart4-data-pins { + pinmux = <PINMUX(PIN_UART2_TX, 5)>, + <PINMUX(PIN_UART2_RX, 5)>; + drive-strength-microamp = <10800>; + power-source = <3300>; + }; + + uart4-control-pins { + pinmux = <PINMUX(PIN_UART2_CTS, 5)>, + <PINMUX(PIN_UART2_RTS, 5)>; + bias-pull-up; + drive-strength-microamp = <10800>; + power-source = <3300>; + }; + }; +}; + +&dmac { + status = "okay"; +}; + +&emmc { + bus-width = <4>; + no-1-8-v; + cap-mmc-hw-reset; + no-sd; + no-sdio; + non-removable; + pinctrl-0 = <&emmc_cfg>; + pinctrl-names = "default"; + status = "okay"; +}; + +&gmac0 { + status = "okay"; +}; + +&i2c4 { + pinctrl-0 = <&i2c4_cfg>; + pinctrl-names = "default"; + status = "okay"; +}; + +&mdio { + status = "okay"; +}; + +&saradc { + status = "okay"; +}; + +&sdhci0 { + bus-width = <4>; + disable-wp; + no-1-8-v; + pinctrl-0 = <&sdhci0_cfg>; + pinctrl-names = "default"; + status = "okay"; +}; + +&sdhci1 { + bus-width = <4>; + cap-sdio-irq; + cap-sd-highspeed; + keep-power-in-suspend; + no-mmc; + no-sd; + non-removable; + max-frequency = <150000000>; + mmc-pwrseq = <&wifi_pwrseq>; + pinctrl-0 = <&sdhci1_cfg>; + pinctrl-names = "default"; + sd-uhs-sdr50; + sd-uhs-sdr104; + status = "okay"; +}; + +&uart0 { + pinctrl-0 = <&uart0_cfg>; + pinctrl-names = "default"; + status = "okay"; +}; + +&uart4 { + pinctrl-0 = <&uart4_cfg>; + pinctrl-names = "default"; + uart-has-rtscts; + status = "okay"; +}; + +&usb { + dr_mode = "host"; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/sophgo/sg2000.dtsi b/arch/riscv/boot/dts/sophgo/sg2000.dtsi new file mode 100644 index 000000000000..38462769574f --- /dev/null +++ b/arch/riscv/boot/dts/sophgo/sg2000.dtsi @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) + +#define SOC_PERIPHERAL_IRQ(nr) ((nr) + 16) + +#include <dt-bindings/interrupt-controller/irq.h> +#include <dt-bindings/pinctrl/pinctrl-sg2000.h> +#include "cv180x-cpus.dtsi" +#include "cv180x.dtsi" +#include "cv181x.dtsi" + +/ { + compatible = "sophgo,sg2000"; + + memory@80000000 { + device_type = "memory"; + reg = <0x80000000 0x20000000>; + }; + + soc { + interrupt-parent = <&plic>; + dma-noncoherent; + + pinctrl: pinctrl@3001000 { + compatible = "sophgo,sg2000-pinctrl"; + reg = <0x03001000 0x1000>, + <0x05027000 0x1000>; + reg-names = "sys", "rtc"; + }; + + clk: clock-controller@3002000 { + compatible = "sophgo,sg2000-clk"; + reg = <0x03002000 0x1000>; + clocks = <&osc>; + #clock-cells = <1>; + }; + + plic: interrupt-controller@70000000 { + compatible = "sophgo,sg2000-plic", "thead,c900-plic"; + reg = <0x70000000 0x4000000>; + interrupts-extended = <&cpu0_intc 11>, <&cpu0_intc 9>; + interrupt-controller; + #address-cells = <0>; + #interrupt-cells = <2>; + riscv,ndev = <101>; + }; + + clint: timer@74000000 { + compatible = "sophgo,sg2000-clint", "thead,c900-clint"; + reg = <0x74000000 0x10000>; + interrupts-extended = <&cpu0_intc 3>, <&cpu0_intc 7>; + }; + }; +}; diff --git a/arch/riscv/boot/dts/sophgo/sg2002-milkv-duo256m.dts b/arch/riscv/boot/dts/sophgo/sg2002-milkv-duo256m.dts new file mode 100644 index 000000000000..a2797c0bd767 --- /dev/null +++ b/arch/riscv/boot/dts/sophgo/sg2002-milkv-duo256m.dts @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (C) 2023 Jisheng Zhang <jszhang@kernel.org> + * Copyright (C) 2026 Chen-Yu Yeh <chenyou910331@gmail.com> + */ + +/dts-v1/; + +#include "sg2002.dtsi" +#include <dt-bindings/gpio/gpio.h> +#include <dt-bindings/leds/common.h> + +/ { + model = "Milk-V Duo 256M"; + compatible = "milkv,duo256m", "sophgo,sg2002"; + + aliases { + serial0 = &uart0; + serial1 = &uart1; + serial2 = &uart2; + serial3 = &uart3; + serial4 = &uart4; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + leds { + compatible = "gpio-leds"; + + led-0 { + gpios = <&porte 2 GPIO_ACTIVE_HIGH>; + color = <LED_COLOR_ID_BLUE>; + function = LED_FUNCTION_STATUS; + linux,default-trigger = "heartbeat"; + }; + }; + + reserved-memory { + #address-cells = <1>; + #size-cells = <1>; + ranges; + + coprocessor_rtos: region@8fe00000 { + reg = <0x8fe00000 0x200000>; + no-map; + }; + }; +}; + +&osc { + clock-frequency = <25000000>; +}; + +&pinctrl { + uart0_cfg: uart0-cfg { + uart0-pins { + pinmux = <PINMUX(PIN_UART0_TX, 0)>, + <PINMUX(PIN_UART0_RX, 0)>; + bias-pull-up; + drive-strength-microamp = <10800>; + power-source = <3300>; + }; + }; + + sdhci0_cfg: sdhci0-cfg { + sdhci0-clk-pins { + pinmux = <PINMUX(PIN_SD0_CLK, 0)>; + bias-pull-up; + drive-strength-microamp = <16100>; + power-source = <3300>; + }; + + sdhci0-cmd-pins { + pinmux = <PINMUX(PIN_SD0_CMD, 0)>; + bias-pull-up; + drive-strength-microamp = <10800>; + power-source = <3300>; + }; + + sdhci0-data-pins { + pinmux = <PINMUX(PIN_SD0_D0, 0)>, + <PINMUX(PIN_SD0_D1, 0)>, + <PINMUX(PIN_SD0_D2, 0)>, + <PINMUX(PIN_SD0_D3, 0)>; + bias-pull-up; + drive-strength-microamp = <10800>; + power-source = <3300>; + }; + + sdhci0-cd-pins { + pinmux = <PINMUX(PIN_SD0_CD, 0)>; + bias-pull-up; + drive-strength-microamp = <10800>; + power-source = <3300>; + }; + }; +}; + +&sdhci0 { + pinctrl-0 = <&sdhci0_cfg>; + pinctrl-names = "default"; + status = "okay"; + bus-width = <4>; + no-1-8-v; + no-mmc; + no-sdio; + disable-wp; +}; + +&uart0 { + pinctrl-0 = <&uart0_cfg>; + pinctrl-names = "default"; + status = "okay"; +}; + +&usb { + dr_mode = "host"; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/sophgo/sg2042-cpus.dtsi b/arch/riscv/boot/dts/sophgo/sg2042-cpus.dtsi index 94a4b71acad3..fd8906b313d2 100644 --- a/arch/riscv/boot/dts/sophgo/sg2042-cpus.dtsi +++ b/arch/riscv/boot/dts/sophgo/sg2042-cpus.dtsi @@ -263,7 +263,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <0>; + reg = <0x0>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -291,7 +291,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <1>; + reg = <0x1>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -319,7 +319,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <2>; + reg = <0x2>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -347,7 +347,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <3>; + reg = <0x3>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -375,7 +375,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <4>; + reg = <0x4>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -403,7 +403,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <5>; + reg = <0x5>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -431,7 +431,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <6>; + reg = <0x6>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -459,7 +459,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <7>; + reg = <0x7>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -487,7 +487,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <8>; + reg = <0x8>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -515,7 +515,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <9>; + reg = <0x9>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -533,7 +533,7 @@ }; }; - cpu10: cpu@10 { + cpu10: cpu@a { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -543,7 +543,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <10>; + reg = <0xa>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -561,7 +561,7 @@ }; }; - cpu11: cpu@11 { + cpu11: cpu@b { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -571,7 +571,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <11>; + reg = <0xb>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -589,7 +589,7 @@ }; }; - cpu12: cpu@12 { + cpu12: cpu@c { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -599,7 +599,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <12>; + reg = <0xc>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -617,7 +617,7 @@ }; }; - cpu13: cpu@13 { + cpu13: cpu@d { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -627,7 +627,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <13>; + reg = <0xd>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -645,7 +645,7 @@ }; }; - cpu14: cpu@14 { + cpu14: cpu@e { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -655,7 +655,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <14>; + reg = <0xe>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -673,7 +673,7 @@ }; }; - cpu15: cpu@15 { + cpu15: cpu@f { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -683,7 +683,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <15>; + reg = <0xf>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -701,7 +701,7 @@ }; }; - cpu16: cpu@16 { + cpu16: cpu@10 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -711,7 +711,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <16>; + reg = <0x10>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -729,7 +729,7 @@ }; }; - cpu17: cpu@17 { + cpu17: cpu@11 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -739,7 +739,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <17>; + reg = <0x11>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -757,7 +757,7 @@ }; }; - cpu18: cpu@18 { + cpu18: cpu@12 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -767,7 +767,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <18>; + reg = <0x12>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -785,7 +785,7 @@ }; }; - cpu19: cpu@19 { + cpu19: cpu@13 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -795,7 +795,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <19>; + reg = <0x13>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -813,7 +813,7 @@ }; }; - cpu20: cpu@20 { + cpu20: cpu@14 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -823,7 +823,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <20>; + reg = <0x14>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -841,7 +841,7 @@ }; }; - cpu21: cpu@21 { + cpu21: cpu@15 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -851,7 +851,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <21>; + reg = <0x15>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -869,7 +869,7 @@ }; }; - cpu22: cpu@22 { + cpu22: cpu@16 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -879,7 +879,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <22>; + reg = <0x16>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -897,7 +897,7 @@ }; }; - cpu23: cpu@23 { + cpu23: cpu@17 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -907,7 +907,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <23>; + reg = <0x17>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -925,7 +925,7 @@ }; }; - cpu24: cpu@24 { + cpu24: cpu@18 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -935,7 +935,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <24>; + reg = <0x18>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -953,7 +953,7 @@ }; }; - cpu25: cpu@25 { + cpu25: cpu@19 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -963,7 +963,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <25>; + reg = <0x19>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -981,7 +981,7 @@ }; }; - cpu26: cpu@26 { + cpu26: cpu@1a { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -991,7 +991,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <26>; + reg = <0x1a>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1009,7 +1009,7 @@ }; }; - cpu27: cpu@27 { + cpu27: cpu@1b { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1019,7 +1019,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <27>; + reg = <0x1b>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1037,7 +1037,7 @@ }; }; - cpu28: cpu@28 { + cpu28: cpu@1c { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1047,7 +1047,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <28>; + reg = <0x1c>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1065,7 +1065,7 @@ }; }; - cpu29: cpu@29 { + cpu29: cpu@1d { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1075,7 +1075,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <29>; + reg = <0x1d>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1093,7 +1093,7 @@ }; }; - cpu30: cpu@30 { + cpu30: cpu@1e { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1103,7 +1103,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <30>; + reg = <0x1e>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1121,7 +1121,7 @@ }; }; - cpu31: cpu@31 { + cpu31: cpu@1f { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1131,7 +1131,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <31>; + reg = <0x1f>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1149,7 +1149,7 @@ }; }; - cpu32: cpu@32 { + cpu32: cpu@20 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1159,7 +1159,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <32>; + reg = <0x20>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1177,7 +1177,7 @@ }; }; - cpu33: cpu@33 { + cpu33: cpu@21 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1187,7 +1187,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <33>; + reg = <0x21>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1205,7 +1205,7 @@ }; }; - cpu34: cpu@34 { + cpu34: cpu@22 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1215,7 +1215,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <34>; + reg = <0x22>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1233,7 +1233,7 @@ }; }; - cpu35: cpu@35 { + cpu35: cpu@23 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1243,7 +1243,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <35>; + reg = <0x23>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1261,7 +1261,7 @@ }; }; - cpu36: cpu@36 { + cpu36: cpu@24 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1271,7 +1271,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <36>; + reg = <0x24>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1289,7 +1289,7 @@ }; }; - cpu37: cpu@37 { + cpu37: cpu@25 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1299,7 +1299,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <37>; + reg = <0x25>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1317,7 +1317,7 @@ }; }; - cpu38: cpu@38 { + cpu38: cpu@26 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1327,7 +1327,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <38>; + reg = <0x26>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1345,7 +1345,7 @@ }; }; - cpu39: cpu@39 { + cpu39: cpu@27 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1355,7 +1355,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <39>; + reg = <0x27>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1373,7 +1373,7 @@ }; }; - cpu40: cpu@40 { + cpu40: cpu@28 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1383,7 +1383,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <40>; + reg = <0x28>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1401,7 +1401,7 @@ }; }; - cpu41: cpu@41 { + cpu41: cpu@29 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1411,7 +1411,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <41>; + reg = <0x29>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1429,7 +1429,7 @@ }; }; - cpu42: cpu@42 { + cpu42: cpu@2a { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1439,7 +1439,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <42>; + reg = <0x2a>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1457,7 +1457,7 @@ }; }; - cpu43: cpu@43 { + cpu43: cpu@2b { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1467,7 +1467,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <43>; + reg = <0x2b>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1485,7 +1485,7 @@ }; }; - cpu44: cpu@44 { + cpu44: cpu@2c { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1495,7 +1495,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <44>; + reg = <0x2c>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1513,7 +1513,7 @@ }; }; - cpu45: cpu@45 { + cpu45: cpu@2d { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1523,7 +1523,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <45>; + reg = <0x2d>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1541,7 +1541,7 @@ }; }; - cpu46: cpu@46 { + cpu46: cpu@2e { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1551,7 +1551,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <46>; + reg = <0x2e>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1569,7 +1569,7 @@ }; }; - cpu47: cpu@47 { + cpu47: cpu@2f { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1579,7 +1579,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <47>; + reg = <0x2f>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1597,7 +1597,7 @@ }; }; - cpu48: cpu@48 { + cpu48: cpu@30 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1607,7 +1607,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <48>; + reg = <0x30>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1625,7 +1625,7 @@ }; }; - cpu49: cpu@49 { + cpu49: cpu@31 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1635,7 +1635,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <49>; + reg = <0x31>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1653,7 +1653,7 @@ }; }; - cpu50: cpu@50 { + cpu50: cpu@32 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1663,7 +1663,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <50>; + reg = <0x32>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1681,7 +1681,7 @@ }; }; - cpu51: cpu@51 { + cpu51: cpu@33 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1691,7 +1691,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <51>; + reg = <0x33>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1709,7 +1709,7 @@ }; }; - cpu52: cpu@52 { + cpu52: cpu@34 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1719,7 +1719,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <52>; + reg = <0x34>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1737,7 +1737,7 @@ }; }; - cpu53: cpu@53 { + cpu53: cpu@35 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1747,7 +1747,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <53>; + reg = <0x35>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1765,7 +1765,7 @@ }; }; - cpu54: cpu@54 { + cpu54: cpu@36 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1775,7 +1775,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <54>; + reg = <0x36>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1793,7 +1793,7 @@ }; }; - cpu55: cpu@55 { + cpu55: cpu@37 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1803,7 +1803,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <55>; + reg = <0x37>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1821,7 +1821,7 @@ }; }; - cpu56: cpu@56 { + cpu56: cpu@38 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1831,7 +1831,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <56>; + reg = <0x38>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1849,7 +1849,7 @@ }; }; - cpu57: cpu@57 { + cpu57: cpu@39 { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1859,7 +1859,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <57>; + reg = <0x39>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1877,7 +1877,7 @@ }; }; - cpu58: cpu@58 { + cpu58: cpu@3a { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1887,7 +1887,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <58>; + reg = <0x3a>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1905,7 +1905,7 @@ }; }; - cpu59: cpu@59 { + cpu59: cpu@3b { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1915,7 +1915,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <59>; + reg = <0x3b>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1933,7 +1933,7 @@ }; }; - cpu60: cpu@60 { + cpu60: cpu@3c { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1943,7 +1943,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <60>; + reg = <0x3c>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1961,7 +1961,7 @@ }; }; - cpu61: cpu@61 { + cpu61: cpu@3d { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1971,7 +1971,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <61>; + reg = <0x3d>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1989,7 +1989,7 @@ }; }; - cpu62: cpu@62 { + cpu62: cpu@3e { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -1999,7 +1999,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <62>; + reg = <0x3e>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -2017,7 +2017,7 @@ }; }; - cpu63: cpu@63 { + cpu63: cpu@3f { compatible = "thead,c920", "riscv"; device_type = "cpu"; riscv,isa = "rv64imafdc"; @@ -2027,7 +2027,7 @@ "zifencei", "zihpm", "zfh", "xtheadvector"; thead,vlenb = <16>; - reg = <63>; + reg = <0x3f>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -2189,4 +2189,309 @@ cache-unified; }; }; + + soc { + intc: interrupt-controller@7090000000 { + compatible = "sophgo,sg2042-plic", "thead,c900-plic"; + #address-cells = <0>; + #interrupt-cells = <2>; + reg = <0x00000070 0x90000000 0x00000000 0x04000000>; + interrupt-controller; + interrupts-extended = + <&cpu0_intc 11>, <&cpu0_intc 9>, + <&cpu1_intc 11>, <&cpu1_intc 9>, + <&cpu2_intc 11>, <&cpu2_intc 9>, + <&cpu3_intc 11>, <&cpu3_intc 9>, + <&cpu4_intc 11>, <&cpu4_intc 9>, + <&cpu5_intc 11>, <&cpu5_intc 9>, + <&cpu6_intc 11>, <&cpu6_intc 9>, + <&cpu7_intc 11>, <&cpu7_intc 9>, + <&cpu8_intc 11>, <&cpu8_intc 9>, + <&cpu9_intc 11>, <&cpu9_intc 9>, + <&cpu10_intc 11>, <&cpu10_intc 9>, + <&cpu11_intc 11>, <&cpu11_intc 9>, + <&cpu12_intc 11>, <&cpu12_intc 9>, + <&cpu13_intc 11>, <&cpu13_intc 9>, + <&cpu14_intc 11>, <&cpu14_intc 9>, + <&cpu15_intc 11>, <&cpu15_intc 9>, + <&cpu16_intc 11>, <&cpu16_intc 9>, + <&cpu17_intc 11>, <&cpu17_intc 9>, + <&cpu18_intc 11>, <&cpu18_intc 9>, + <&cpu19_intc 11>, <&cpu19_intc 9>, + <&cpu20_intc 11>, <&cpu20_intc 9>, + <&cpu21_intc 11>, <&cpu21_intc 9>, + <&cpu22_intc 11>, <&cpu22_intc 9>, + <&cpu23_intc 11>, <&cpu23_intc 9>, + <&cpu24_intc 11>, <&cpu24_intc 9>, + <&cpu25_intc 11>, <&cpu25_intc 9>, + <&cpu26_intc 11>, <&cpu26_intc 9>, + <&cpu27_intc 11>, <&cpu27_intc 9>, + <&cpu28_intc 11>, <&cpu28_intc 9>, + <&cpu29_intc 11>, <&cpu29_intc 9>, + <&cpu30_intc 11>, <&cpu30_intc 9>, + <&cpu31_intc 11>, <&cpu31_intc 9>, + <&cpu32_intc 11>, <&cpu32_intc 9>, + <&cpu33_intc 11>, <&cpu33_intc 9>, + <&cpu34_intc 11>, <&cpu34_intc 9>, + <&cpu35_intc 11>, <&cpu35_intc 9>, + <&cpu36_intc 11>, <&cpu36_intc 9>, + <&cpu37_intc 11>, <&cpu37_intc 9>, + <&cpu38_intc 11>, <&cpu38_intc 9>, + <&cpu39_intc 11>, <&cpu39_intc 9>, + <&cpu40_intc 11>, <&cpu40_intc 9>, + <&cpu41_intc 11>, <&cpu41_intc 9>, + <&cpu42_intc 11>, <&cpu42_intc 9>, + <&cpu43_intc 11>, <&cpu43_intc 9>, + <&cpu44_intc 11>, <&cpu44_intc 9>, + <&cpu45_intc 11>, <&cpu45_intc 9>, + <&cpu46_intc 11>, <&cpu46_intc 9>, + <&cpu47_intc 11>, <&cpu47_intc 9>, + <&cpu48_intc 11>, <&cpu48_intc 9>, + <&cpu49_intc 11>, <&cpu49_intc 9>, + <&cpu50_intc 11>, <&cpu50_intc 9>, + <&cpu51_intc 11>, <&cpu51_intc 9>, + <&cpu52_intc 11>, <&cpu52_intc 9>, + <&cpu53_intc 11>, <&cpu53_intc 9>, + <&cpu54_intc 11>, <&cpu54_intc 9>, + <&cpu55_intc 11>, <&cpu55_intc 9>, + <&cpu56_intc 11>, <&cpu56_intc 9>, + <&cpu57_intc 11>, <&cpu57_intc 9>, + <&cpu58_intc 11>, <&cpu58_intc 9>, + <&cpu59_intc 11>, <&cpu59_intc 9>, + <&cpu60_intc 11>, <&cpu60_intc 9>, + <&cpu61_intc 11>, <&cpu61_intc 9>, + <&cpu62_intc 11>, <&cpu62_intc 9>, + <&cpu63_intc 11>, <&cpu63_intc 9>; + riscv,ndev = <224>; + }; + + clint_mswi: interrupt-controller@7094000000 { + compatible = "sophgo,sg2042-aclint-mswi", "thead,c900-aclint-mswi"; + reg = <0x00000070 0x94000000 0x00000000 0x00004000>; + interrupts-extended = <&cpu0_intc 3>, + <&cpu1_intc 3>, + <&cpu2_intc 3>, + <&cpu3_intc 3>, + <&cpu4_intc 3>, + <&cpu5_intc 3>, + <&cpu6_intc 3>, + <&cpu7_intc 3>, + <&cpu8_intc 3>, + <&cpu9_intc 3>, + <&cpu10_intc 3>, + <&cpu11_intc 3>, + <&cpu12_intc 3>, + <&cpu13_intc 3>, + <&cpu14_intc 3>, + <&cpu15_intc 3>, + <&cpu16_intc 3>, + <&cpu17_intc 3>, + <&cpu18_intc 3>, + <&cpu19_intc 3>, + <&cpu20_intc 3>, + <&cpu21_intc 3>, + <&cpu22_intc 3>, + <&cpu23_intc 3>, + <&cpu24_intc 3>, + <&cpu25_intc 3>, + <&cpu26_intc 3>, + <&cpu27_intc 3>, + <&cpu28_intc 3>, + <&cpu29_intc 3>, + <&cpu30_intc 3>, + <&cpu31_intc 3>, + <&cpu32_intc 3>, + <&cpu33_intc 3>, + <&cpu34_intc 3>, + <&cpu35_intc 3>, + <&cpu36_intc 3>, + <&cpu37_intc 3>, + <&cpu38_intc 3>, + <&cpu39_intc 3>, + <&cpu40_intc 3>, + <&cpu41_intc 3>, + <&cpu42_intc 3>, + <&cpu43_intc 3>, + <&cpu44_intc 3>, + <&cpu45_intc 3>, + <&cpu46_intc 3>, + <&cpu47_intc 3>, + <&cpu48_intc 3>, + <&cpu49_intc 3>, + <&cpu50_intc 3>, + <&cpu51_intc 3>, + <&cpu52_intc 3>, + <&cpu53_intc 3>, + <&cpu54_intc 3>, + <&cpu55_intc 3>, + <&cpu56_intc 3>, + <&cpu57_intc 3>, + <&cpu58_intc 3>, + <&cpu59_intc 3>, + <&cpu60_intc 3>, + <&cpu61_intc 3>, + <&cpu62_intc 3>, + <&cpu63_intc 3>; + }; + + clint_mtimer0: timer@70ac004000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac004000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu0_intc 7>, + <&cpu1_intc 7>, + <&cpu2_intc 7>, + <&cpu3_intc 7>; + }; + + clint_mtimer1: timer@70ac014000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac014000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu4_intc 7>, + <&cpu5_intc 7>, + <&cpu6_intc 7>, + <&cpu7_intc 7>; + }; + + clint_mtimer2: timer@70ac024000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac024000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu8_intc 7>, + <&cpu9_intc 7>, + <&cpu10_intc 7>, + <&cpu11_intc 7>; + }; + + clint_mtimer3: timer@70ac034000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac034000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu12_intc 7>, + <&cpu13_intc 7>, + <&cpu14_intc 7>, + <&cpu15_intc 7>; + }; + + clint_mtimer4: timer@70ac044000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac044000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu16_intc 7>, + <&cpu17_intc 7>, + <&cpu18_intc 7>, + <&cpu19_intc 7>; + }; + + clint_mtimer5: timer@70ac054000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac054000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu20_intc 7>, + <&cpu21_intc 7>, + <&cpu22_intc 7>, + <&cpu23_intc 7>; + }; + + clint_mtimer6: timer@70ac064000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac064000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu24_intc 7>, + <&cpu25_intc 7>, + <&cpu26_intc 7>, + <&cpu27_intc 7>; + }; + + clint_mtimer7: timer@70ac074000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac074000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu28_intc 7>, + <&cpu29_intc 7>, + <&cpu30_intc 7>, + <&cpu31_intc 7>; + }; + + clint_mtimer8: timer@70ac084000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac084000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu32_intc 7>, + <&cpu33_intc 7>, + <&cpu34_intc 7>, + <&cpu35_intc 7>; + }; + + clint_mtimer9: timer@70ac094000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac094000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu36_intc 7>, + <&cpu37_intc 7>, + <&cpu38_intc 7>, + <&cpu39_intc 7>; + }; + + clint_mtimer10: timer@70ac0a4000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac0a4000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu40_intc 7>, + <&cpu41_intc 7>, + <&cpu42_intc 7>, + <&cpu43_intc 7>; + }; + + clint_mtimer11: timer@70ac0b4000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac0b4000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu44_intc 7>, + <&cpu45_intc 7>, + <&cpu46_intc 7>, + <&cpu47_intc 7>; + }; + + clint_mtimer12: timer@70ac0c4000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac0c4000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu48_intc 7>, + <&cpu49_intc 7>, + <&cpu50_intc 7>, + <&cpu51_intc 7>; + }; + + clint_mtimer13: timer@70ac0d4000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac0d4000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu52_intc 7>, + <&cpu53_intc 7>, + <&cpu54_intc 7>, + <&cpu55_intc 7>; + }; + + clint_mtimer14: timer@70ac0e4000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac0e4000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu56_intc 7>, + <&cpu57_intc 7>, + <&cpu58_intc 7>, + <&cpu59_intc 7>; + }; + + clint_mtimer15: timer@70ac0f4000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac0f4000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu60_intc 7>, + <&cpu61_intc 7>, + <&cpu62_intc 7>, + <&cpu63_intc 7>; + }; + }; }; diff --git a/arch/riscv/boot/dts/sophgo/sg2042-milkv-pioneer.dts b/arch/riscv/boot/dts/sophgo/sg2042-milkv-pioneer.dts index 54d8386bf9c0..ecf8c1e29079 100644 --- a/arch/riscv/boot/dts/sophgo/sg2042-milkv-pioneer.dts +++ b/arch/riscv/boot/dts/sophgo/sg2042-milkv-pioneer.dts @@ -52,6 +52,17 @@ status = "okay"; }; +&i2c0 { + pinctrl-0 = <&i2c0_cfg>; + pinctrl-names = "default"; + status = "okay"; + + rtc: rtc@68 { + compatible = "dallas,ds1307"; + reg = <0x68>; + }; +}; + &i2c1 { pinctrl-0 = <&i2c1_cfg>; pinctrl-names = "default"; @@ -89,6 +100,16 @@ }; }; + i2c0_cfg: i2c0-cfg { + i2c0-pins { + pinmux = <PINMUX(PIN_IIC0_SDA, 0)>, + <PINMUX(PIN_IIC0_SCL, 0)>; + bias-pull-up; + drive-strength-microamp = <26800>; + input-schmitt-enable; + }; + }; + i2c1_cfg: i2c1-cfg { i2c1-pins { pinmux = <PINMUX(PIN_IIC1_SDA, 0)>, diff --git a/arch/riscv/boot/dts/sophgo/sg2042.dtsi b/arch/riscv/boot/dts/sophgo/sg2042.dtsi index ec99da39150f..7eab0655f150 100644 --- a/arch/riscv/boot/dts/sophgo/sg2042.dtsi +++ b/arch/riscv/boot/dts/sophgo/sg2042.dtsi @@ -234,7 +234,7 @@ reg-names = "clr", "doorbell"; msi-controller; #msi-cells = <0>; - msi-ranges = <&intc 64 IRQ_TYPE_EDGE_RISING 32>; + msi-ranges = <&intc 64 IRQ_TYPE_EDGE_RISING 16>; }; rpgate: clock-controller@7030010368 { @@ -264,397 +264,6 @@ #clock-cells = <1>; }; - pcie_rc0: pcie@7060000000 { - compatible = "sophgo,sg2042-pcie-host"; - device_type = "pci"; - reg = <0x70 0x60000000 0x0 0x00800000>, - <0x40 0x00000000 0x0 0x00001000>; - reg-names = "reg", "cfg"; - linux,pci-domain = <0>; - #address-cells = <3>; - #size-cells = <2>; - ranges = <0x01000000 0x0 0x00000000 0x40 0xc0000000 0x0 0x00400000>, - <0x42000000 0x0 0xd0000000 0x40 0xd0000000 0x0 0x10000000>, - <0x02000000 0x0 0xe0000000 0x40 0xe0000000 0x0 0x20000000>, - <0x43000000 0x42 0x00000000 0x42 0x00000000 0x2 0x00000000>, - <0x03000000 0x41 0x00000000 0x41 0x00000000 0x1 0x00000000>; - bus-range = <0x0 0xff>; - vendor-id = <0x1f1c>; - device-id = <0x2042>; - cdns,no-bar-match-nbits = <48>; - msi-parent = <&msi>; - status = "disabled"; - }; - - pcie_rc1: pcie@7060800000 { - compatible = "sophgo,sg2042-pcie-host"; - device_type = "pci"; - reg = <0x70 0x60800000 0x0 0x00800000>, - <0x44 0x00000000 0x0 0x00001000>; - reg-names = "reg", "cfg"; - linux,pci-domain = <1>; - #address-cells = <3>; - #size-cells = <2>; - ranges = <0x01000000 0x0 0x00000000 0x44 0xc0400000 0x0 0x00400000>, - <0x42000000 0x0 0xd0000000 0x44 0xd0000000 0x0 0x10000000>, - <0x02000000 0x0 0xe0000000 0x44 0xe0000000 0x0 0x20000000>, - <0x43000000 0x46 0x00000000 0x46 0x00000000 0x2 0x00000000>, - <0x03000000 0x45 0x00000000 0x45 0x00000000 0x1 0x00000000>; - bus-range = <0x0 0xff>; - vendor-id = <0x1f1c>; - device-id = <0x2042>; - cdns,no-bar-match-nbits = <48>; - msi-parent = <&msi>; - status = "disabled"; - }; - - pcie_rc2: pcie@7062000000 { - compatible = "sophgo,sg2042-pcie-host"; - device_type = "pci"; - reg = <0x70 0x62000000 0x0 0x00800000>, - <0x48 0x00000000 0x0 0x00001000>; - reg-names = "reg", "cfg"; - linux,pci-domain = <2>; - #address-cells = <3>; - #size-cells = <2>; - ranges = <0x01000000 0x0 0x00000000 0x48 0xc0800000 0x0 0x00400000>, - <0x42000000 0x0 0xd0000000 0x48 0xd0000000 0x0 0x10000000>, - <0x02000000 0x0 0xe0000000 0x48 0xe0000000 0x0 0x20000000>, - <0x03000000 0x49 0x00000000 0x49 0x00000000 0x1 0x00000000>, - <0x43000000 0x4a 0x00000000 0x4a 0x00000000 0x2 0x00000000>; - bus-range = <0x0 0xff>; - vendor-id = <0x1f1c>; - device-id = <0x2042>; - cdns,no-bar-match-nbits = <48>; - msi-parent = <&msi>; - status = "disabled"; - }; - - pcie_rc3: pcie@7062800000 { - compatible = "sophgo,sg2042-pcie-host"; - device_type = "pci"; - reg = <0x70 0x62800000 0x0 0x00800000>, - <0x4c 0x00000000 0x0 0x00001000>; - reg-names = "reg", "cfg"; - linux,pci-domain = <3>; - #address-cells = <3>; - #size-cells = <2>; - ranges = <0x01000000 0x0 0x00000000 0x4c 0xc0c00000 0x0 0x00400000>, - <0x42000000 0x0 0xf8000000 0x4c 0xf8000000 0x0 0x04000000>, - <0x02000000 0x0 0xfc000000 0x4c 0xfc000000 0x0 0x04000000>, - <0x43000000 0x4e 0x00000000 0x4e 0x00000000 0x2 0x00000000>, - <0x03000000 0x4d 0x00000000 0x4d 0x00000000 0x1 0x00000000>; - bus-range = <0x0 0xff>; - vendor-id = <0x1f1c>; - device-id = <0x2042>; - cdns,no-bar-match-nbits = <48>; - msi-parent = <&msi>; - status = "disabled"; - }; - - clint_mswi: interrupt-controller@7094000000 { - compatible = "sophgo,sg2042-aclint-mswi", "thead,c900-aclint-mswi"; - reg = <0x00000070 0x94000000 0x00000000 0x00004000>; - interrupts-extended = <&cpu0_intc 3>, - <&cpu1_intc 3>, - <&cpu2_intc 3>, - <&cpu3_intc 3>, - <&cpu4_intc 3>, - <&cpu5_intc 3>, - <&cpu6_intc 3>, - <&cpu7_intc 3>, - <&cpu8_intc 3>, - <&cpu9_intc 3>, - <&cpu10_intc 3>, - <&cpu11_intc 3>, - <&cpu12_intc 3>, - <&cpu13_intc 3>, - <&cpu14_intc 3>, - <&cpu15_intc 3>, - <&cpu16_intc 3>, - <&cpu17_intc 3>, - <&cpu18_intc 3>, - <&cpu19_intc 3>, - <&cpu20_intc 3>, - <&cpu21_intc 3>, - <&cpu22_intc 3>, - <&cpu23_intc 3>, - <&cpu24_intc 3>, - <&cpu25_intc 3>, - <&cpu26_intc 3>, - <&cpu27_intc 3>, - <&cpu28_intc 3>, - <&cpu29_intc 3>, - <&cpu30_intc 3>, - <&cpu31_intc 3>, - <&cpu32_intc 3>, - <&cpu33_intc 3>, - <&cpu34_intc 3>, - <&cpu35_intc 3>, - <&cpu36_intc 3>, - <&cpu37_intc 3>, - <&cpu38_intc 3>, - <&cpu39_intc 3>, - <&cpu40_intc 3>, - <&cpu41_intc 3>, - <&cpu42_intc 3>, - <&cpu43_intc 3>, - <&cpu44_intc 3>, - <&cpu45_intc 3>, - <&cpu46_intc 3>, - <&cpu47_intc 3>, - <&cpu48_intc 3>, - <&cpu49_intc 3>, - <&cpu50_intc 3>, - <&cpu51_intc 3>, - <&cpu52_intc 3>, - <&cpu53_intc 3>, - <&cpu54_intc 3>, - <&cpu55_intc 3>, - <&cpu56_intc 3>, - <&cpu57_intc 3>, - <&cpu58_intc 3>, - <&cpu59_intc 3>, - <&cpu60_intc 3>, - <&cpu61_intc 3>, - <&cpu62_intc 3>, - <&cpu63_intc 3>; - }; - - clint_mtimer0: timer@70ac004000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac004000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu0_intc 7>, - <&cpu1_intc 7>, - <&cpu2_intc 7>, - <&cpu3_intc 7>; - }; - - clint_mtimer1: timer@70ac014000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac014000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu4_intc 7>, - <&cpu5_intc 7>, - <&cpu6_intc 7>, - <&cpu7_intc 7>; - }; - - clint_mtimer2: timer@70ac024000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac024000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu8_intc 7>, - <&cpu9_intc 7>, - <&cpu10_intc 7>, - <&cpu11_intc 7>; - }; - - clint_mtimer3: timer@70ac034000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac034000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu12_intc 7>, - <&cpu13_intc 7>, - <&cpu14_intc 7>, - <&cpu15_intc 7>; - }; - - clint_mtimer4: timer@70ac044000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac044000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu16_intc 7>, - <&cpu17_intc 7>, - <&cpu18_intc 7>, - <&cpu19_intc 7>; - }; - - clint_mtimer5: timer@70ac054000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac054000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu20_intc 7>, - <&cpu21_intc 7>, - <&cpu22_intc 7>, - <&cpu23_intc 7>; - }; - - clint_mtimer6: timer@70ac064000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac064000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu24_intc 7>, - <&cpu25_intc 7>, - <&cpu26_intc 7>, - <&cpu27_intc 7>; - }; - - clint_mtimer7: timer@70ac074000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac074000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu28_intc 7>, - <&cpu29_intc 7>, - <&cpu30_intc 7>, - <&cpu31_intc 7>; - }; - - clint_mtimer8: timer@70ac084000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac084000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu32_intc 7>, - <&cpu33_intc 7>, - <&cpu34_intc 7>, - <&cpu35_intc 7>; - }; - - clint_mtimer9: timer@70ac094000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac094000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu36_intc 7>, - <&cpu37_intc 7>, - <&cpu38_intc 7>, - <&cpu39_intc 7>; - }; - - clint_mtimer10: timer@70ac0a4000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac0a4000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu40_intc 7>, - <&cpu41_intc 7>, - <&cpu42_intc 7>, - <&cpu43_intc 7>; - }; - - clint_mtimer11: timer@70ac0b4000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac0b4000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu44_intc 7>, - <&cpu45_intc 7>, - <&cpu46_intc 7>, - <&cpu47_intc 7>; - }; - - clint_mtimer12: timer@70ac0c4000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac0c4000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu48_intc 7>, - <&cpu49_intc 7>, - <&cpu50_intc 7>, - <&cpu51_intc 7>; - }; - - clint_mtimer13: timer@70ac0d4000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac0d4000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu52_intc 7>, - <&cpu53_intc 7>, - <&cpu54_intc 7>, - <&cpu55_intc 7>; - }; - - clint_mtimer14: timer@70ac0e4000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac0e4000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu56_intc 7>, - <&cpu57_intc 7>, - <&cpu58_intc 7>, - <&cpu59_intc 7>; - }; - - clint_mtimer15: timer@70ac0f4000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac0f4000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu60_intc 7>, - <&cpu61_intc 7>, - <&cpu62_intc 7>, - <&cpu63_intc 7>; - }; - - intc: interrupt-controller@7090000000 { - compatible = "sophgo,sg2042-plic", "thead,c900-plic"; - #address-cells = <0>; - #interrupt-cells = <2>; - reg = <0x00000070 0x90000000 0x00000000 0x04000000>; - interrupt-controller; - interrupts-extended = - <&cpu0_intc 11>, <&cpu0_intc 9>, - <&cpu1_intc 11>, <&cpu1_intc 9>, - <&cpu2_intc 11>, <&cpu2_intc 9>, - <&cpu3_intc 11>, <&cpu3_intc 9>, - <&cpu4_intc 11>, <&cpu4_intc 9>, - <&cpu5_intc 11>, <&cpu5_intc 9>, - <&cpu6_intc 11>, <&cpu6_intc 9>, - <&cpu7_intc 11>, <&cpu7_intc 9>, - <&cpu8_intc 11>, <&cpu8_intc 9>, - <&cpu9_intc 11>, <&cpu9_intc 9>, - <&cpu10_intc 11>, <&cpu10_intc 9>, - <&cpu11_intc 11>, <&cpu11_intc 9>, - <&cpu12_intc 11>, <&cpu12_intc 9>, - <&cpu13_intc 11>, <&cpu13_intc 9>, - <&cpu14_intc 11>, <&cpu14_intc 9>, - <&cpu15_intc 11>, <&cpu15_intc 9>, - <&cpu16_intc 11>, <&cpu16_intc 9>, - <&cpu17_intc 11>, <&cpu17_intc 9>, - <&cpu18_intc 11>, <&cpu18_intc 9>, - <&cpu19_intc 11>, <&cpu19_intc 9>, - <&cpu20_intc 11>, <&cpu20_intc 9>, - <&cpu21_intc 11>, <&cpu21_intc 9>, - <&cpu22_intc 11>, <&cpu22_intc 9>, - <&cpu23_intc 11>, <&cpu23_intc 9>, - <&cpu24_intc 11>, <&cpu24_intc 9>, - <&cpu25_intc 11>, <&cpu25_intc 9>, - <&cpu26_intc 11>, <&cpu26_intc 9>, - <&cpu27_intc 11>, <&cpu27_intc 9>, - <&cpu28_intc 11>, <&cpu28_intc 9>, - <&cpu29_intc 11>, <&cpu29_intc 9>, - <&cpu30_intc 11>, <&cpu30_intc 9>, - <&cpu31_intc 11>, <&cpu31_intc 9>, - <&cpu32_intc 11>, <&cpu32_intc 9>, - <&cpu33_intc 11>, <&cpu33_intc 9>, - <&cpu34_intc 11>, <&cpu34_intc 9>, - <&cpu35_intc 11>, <&cpu35_intc 9>, - <&cpu36_intc 11>, <&cpu36_intc 9>, - <&cpu37_intc 11>, <&cpu37_intc 9>, - <&cpu38_intc 11>, <&cpu38_intc 9>, - <&cpu39_intc 11>, <&cpu39_intc 9>, - <&cpu40_intc 11>, <&cpu40_intc 9>, - <&cpu41_intc 11>, <&cpu41_intc 9>, - <&cpu42_intc 11>, <&cpu42_intc 9>, - <&cpu43_intc 11>, <&cpu43_intc 9>, - <&cpu44_intc 11>, <&cpu44_intc 9>, - <&cpu45_intc 11>, <&cpu45_intc 9>, - <&cpu46_intc 11>, <&cpu46_intc 9>, - <&cpu47_intc 11>, <&cpu47_intc 9>, - <&cpu48_intc 11>, <&cpu48_intc 9>, - <&cpu49_intc 11>, <&cpu49_intc 9>, - <&cpu50_intc 11>, <&cpu50_intc 9>, - <&cpu51_intc 11>, <&cpu51_intc 9>, - <&cpu52_intc 11>, <&cpu52_intc 9>, - <&cpu53_intc 11>, <&cpu53_intc 9>, - <&cpu54_intc 11>, <&cpu54_intc 9>, - <&cpu55_intc 11>, <&cpu55_intc 9>, - <&cpu56_intc 11>, <&cpu56_intc 9>, - <&cpu57_intc 11>, <&cpu57_intc 9>, - <&cpu58_intc 11>, <&cpu58_intc 9>, - <&cpu59_intc 11>, <&cpu59_intc 9>, - <&cpu60_intc 11>, <&cpu60_intc 9>, - <&cpu61_intc 11>, <&cpu61_intc 9>, - <&cpu62_intc 11>, <&cpu62_intc 9>, - <&cpu63_intc 11>, <&cpu63_intc 9>; - riscv,ndev = <224>; - }; - rstgen: reset-controller@7030013000 { compatible = "sophgo,sg2042-reset"; reg = <0x00000070 0x30013000 0x00000000 0x0000000c>; @@ -789,5 +398,97 @@ "timer"; status = "disabled"; }; + + pcie_rc0: pcie@7060000000 { + compatible = "sophgo,sg2042-pcie-host"; + device_type = "pci"; + reg = <0x70 0x60000000 0x0 0x00800000>, + <0x40 0x00000000 0x0 0x00001000>; + reg-names = "reg", "cfg"; + linux,pci-domain = <0>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x01000000 0x0 0x00000000 0x40 0xc0000000 0x0 0x00400000>, + <0x42000000 0x0 0xd0000000 0x40 0xd0000000 0x0 0x10000000>, + <0x02000000 0x0 0xe0000000 0x40 0xe0000000 0x0 0x20000000>, + <0x43000000 0x42 0x00000000 0x42 0x00000000 0x2 0x00000000>, + <0x03000000 0x41 0x00000000 0x41 0x00000000 0x1 0x00000000>; + bus-range = <0x0 0xff>; + vendor-id = <0x1f1c>; + device-id = <0x2042>; + cdns,no-bar-match-nbits = <48>; + dma-coherent; + msi-parent = <&msi>; + status = "disabled"; + }; + + pcie_rc1: pcie@7060800000 { + compatible = "sophgo,sg2042-pcie-host"; + device_type = "pci"; + reg = <0x70 0x60800000 0x0 0x00800000>, + <0x44 0x00000000 0x0 0x00001000>; + reg-names = "reg", "cfg"; + linux,pci-domain = <1>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x01000000 0x0 0x00000000 0x44 0xc0400000 0x0 0x00400000>, + <0x42000000 0x0 0xd0000000 0x44 0xd0000000 0x0 0x10000000>, + <0x02000000 0x0 0xe0000000 0x44 0xe0000000 0x0 0x20000000>, + <0x43000000 0x46 0x00000000 0x46 0x00000000 0x2 0x00000000>, + <0x03000000 0x45 0x00000000 0x45 0x00000000 0x1 0x00000000>; + bus-range = <0x0 0xff>; + vendor-id = <0x1f1c>; + device-id = <0x2042>; + cdns,no-bar-match-nbits = <48>; + dma-coherent; + msi-parent = <&msi>; + status = "disabled"; + }; + + pcie_rc2: pcie@7062000000 { + compatible = "sophgo,sg2042-pcie-host"; + device_type = "pci"; + reg = <0x70 0x62000000 0x0 0x00800000>, + <0x48 0x00000000 0x0 0x00001000>; + reg-names = "reg", "cfg"; + linux,pci-domain = <2>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x01000000 0x0 0x00000000 0x48 0xc0800000 0x0 0x00400000>, + <0x42000000 0x0 0xd0000000 0x48 0xd0000000 0x0 0x10000000>, + <0x02000000 0x0 0xe0000000 0x48 0xe0000000 0x0 0x20000000>, + <0x03000000 0x49 0x00000000 0x49 0x00000000 0x1 0x00000000>, + <0x43000000 0x4a 0x00000000 0x4a 0x00000000 0x2 0x00000000>; + bus-range = <0x0 0xff>; + vendor-id = <0x1f1c>; + device-id = <0x2042>; + cdns,no-bar-match-nbits = <48>; + dma-coherent; + msi-parent = <&msi>; + status = "disabled"; + }; + + pcie_rc3: pcie@7062800000 { + compatible = "sophgo,sg2042-pcie-host"; + device_type = "pci"; + reg = <0x70 0x62800000 0x0 0x00800000>, + <0x4c 0x00000000 0x0 0x00001000>; + reg-names = "reg", "cfg"; + linux,pci-domain = <3>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x01000000 0x0 0x00000000 0x4c 0xc0c00000 0x0 0x00400000>, + <0x42000000 0x0 0xf8000000 0x4c 0xf8000000 0x0 0x04000000>, + <0x02000000 0x0 0xfc000000 0x4c 0xfc000000 0x0 0x04000000>, + <0x43000000 0x4e 0x00000000 0x4e 0x00000000 0x2 0x00000000>, + <0x03000000 0x4d 0x00000000 0x4d 0x00000000 0x1 0x00000000>; + bus-range = <0x0 0xff>; + vendor-id = <0x1f1c>; + device-id = <0x2042>; + cdns,no-bar-match-nbits = <48>; + dma-coherent; + msi-parent = <&msi>; + status = "disabled"; + }; }; }; diff --git a/arch/riscv/boot/dts/sophgo/sg2044-cpus.dtsi b/arch/riscv/boot/dts/sophgo/sg2044-cpus.dtsi index 523799a1a8b8..f66a382c95bd 100644 --- a/arch/riscv/boot/dts/sophgo/sg2044-cpus.dtsi +++ b/arch/riscv/boot/dts/sophgo/sg2044-cpus.dtsi @@ -14,7 +14,7 @@ cpu0: cpu@0 { compatible = "thead,c920", "riscv"; - reg = <0>; + reg = <0x0>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -24,10 +24,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache0>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -50,7 +50,7 @@ cpu1: cpu@1 { compatible = "thead,c920", "riscv"; - reg = <1>; + reg = <0x1>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -60,10 +60,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache0>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -86,7 +86,7 @@ cpu2: cpu@2 { compatible = "thead,c920", "riscv"; - reg = <2>; + reg = <0x2>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -96,10 +96,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache0>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -122,7 +122,7 @@ cpu3: cpu@3 { compatible = "thead,c920", "riscv"; - reg = <3>; + reg = <0x3>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -132,10 +132,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache0>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -158,7 +158,7 @@ cpu4: cpu@4 { compatible = "thead,c920", "riscv"; - reg = <4>; + reg = <0x4>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -168,10 +168,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache1>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -194,7 +194,7 @@ cpu5: cpu@5 { compatible = "thead,c920", "riscv"; - reg = <5>; + reg = <0x5>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -204,10 +204,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache1>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -230,7 +230,7 @@ cpu6: cpu@6 { compatible = "thead,c920", "riscv"; - reg = <6>; + reg = <0x6>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -240,10 +240,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache1>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -266,7 +266,7 @@ cpu7: cpu@7 { compatible = "thead,c920", "riscv"; - reg = <7>; + reg = <0x7>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -276,10 +276,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache1>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -302,7 +302,7 @@ cpu8: cpu@8 { compatible = "thead,c920", "riscv"; - reg = <8>; + reg = <0x8>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -312,10 +312,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache2>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -338,7 +338,7 @@ cpu9: cpu@9 { compatible = "thead,c920", "riscv"; - reg = <9>; + reg = <0x9>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -348,10 +348,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache2>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -372,9 +372,9 @@ }; }; - cpu10: cpu@10 { + cpu10: cpu@a { compatible = "thead,c920", "riscv"; - reg = <10>; + reg = <0xa>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -384,10 +384,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache2>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -408,9 +408,9 @@ }; }; - cpu11: cpu@11 { + cpu11: cpu@b { compatible = "thead,c920", "riscv"; - reg = <11>; + reg = <0xb>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -420,10 +420,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache2>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -444,9 +444,9 @@ }; }; - cpu12: cpu@12 { + cpu12: cpu@c { compatible = "thead,c920", "riscv"; - reg = <12>; + reg = <0xc>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -456,10 +456,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache3>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -480,9 +480,9 @@ }; }; - cpu13: cpu@13 { + cpu13: cpu@d { compatible = "thead,c920", "riscv"; - reg = <13>; + reg = <0xd>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -492,10 +492,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache3>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -516,9 +516,9 @@ }; }; - cpu14: cpu@14 { + cpu14: cpu@e { compatible = "thead,c920", "riscv"; - reg = <14>; + reg = <0xe>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -528,10 +528,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache3>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -552,9 +552,9 @@ }; }; - cpu15: cpu@15 { + cpu15: cpu@f { compatible = "thead,c920", "riscv"; - reg = <15>; + reg = <0xf>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -564,10 +564,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache3>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -588,9 +588,9 @@ }; }; - cpu16: cpu@16 { + cpu16: cpu@10 { compatible = "thead,c920", "riscv"; - reg = <16>; + reg = <0x10>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -600,10 +600,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache4>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -624,9 +624,9 @@ }; }; - cpu17: cpu@17 { + cpu17: cpu@11 { compatible = "thead,c920", "riscv"; - reg = <17>; + reg = <0x11>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -636,10 +636,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache4>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -660,9 +660,9 @@ }; }; - cpu18: cpu@18 { + cpu18: cpu@12 { compatible = "thead,c920", "riscv"; - reg = <18>; + reg = <0x12>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -672,10 +672,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache4>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -696,9 +696,9 @@ }; }; - cpu19: cpu@19 { + cpu19: cpu@13 { compatible = "thead,c920", "riscv"; - reg = <19>; + reg = <0x13>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -708,10 +708,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache4>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -732,9 +732,9 @@ }; }; - cpu20: cpu@20 { + cpu20: cpu@14 { compatible = "thead,c920", "riscv"; - reg = <20>; + reg = <0x14>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -744,10 +744,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache5>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -768,9 +768,9 @@ }; }; - cpu21: cpu@21 { + cpu21: cpu@15 { compatible = "thead,c920", "riscv"; - reg = <21>; + reg = <0x15>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -780,10 +780,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache5>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -804,9 +804,9 @@ }; }; - cpu22: cpu@22 { + cpu22: cpu@16 { compatible = "thead,c920", "riscv"; - reg = <22>; + reg = <0x16>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -816,10 +816,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache5>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -840,9 +840,9 @@ }; }; - cpu23: cpu@23 { + cpu23: cpu@17 { compatible = "thead,c920", "riscv"; - reg = <23>; + reg = <0x17>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -852,10 +852,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache5>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -876,9 +876,9 @@ }; }; - cpu24: cpu@24 { + cpu24: cpu@18 { compatible = "thead,c920", "riscv"; - reg = <24>; + reg = <0x18>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -888,10 +888,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache6>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -912,9 +912,9 @@ }; }; - cpu25: cpu@25 { + cpu25: cpu@19 { compatible = "thead,c920", "riscv"; - reg = <25>; + reg = <0x19>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -924,10 +924,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache6>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -948,9 +948,9 @@ }; }; - cpu26: cpu@26 { + cpu26: cpu@1a { compatible = "thead,c920", "riscv"; - reg = <26>; + reg = <0x1a>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -960,10 +960,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache6>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -984,9 +984,9 @@ }; }; - cpu27: cpu@27 { + cpu27: cpu@1b { compatible = "thead,c920", "riscv"; - reg = <27>; + reg = <0x1b>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -996,10 +996,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache6>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1020,9 +1020,9 @@ }; }; - cpu28: cpu@28 { + cpu28: cpu@1c { compatible = "thead,c920", "riscv"; - reg = <28>; + reg = <0x1c>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1032,10 +1032,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache7>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1056,9 +1056,9 @@ }; }; - cpu29: cpu@29 { + cpu29: cpu@1d { compatible = "thead,c920", "riscv"; - reg = <29>; + reg = <0x1d>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1068,10 +1068,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache7>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1092,9 +1092,9 @@ }; }; - cpu30: cpu@30 { + cpu30: cpu@1e { compatible = "thead,c920", "riscv"; - reg = <30>; + reg = <0x1e>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1104,10 +1104,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache7>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1128,9 +1128,9 @@ }; }; - cpu31: cpu@31 { + cpu31: cpu@1f { compatible = "thead,c920", "riscv"; - reg = <31>; + reg = <0x1f>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1140,10 +1140,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache7>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1164,9 +1164,9 @@ }; }; - cpu32: cpu@32 { + cpu32: cpu@20 { compatible = "thead,c920", "riscv"; - reg = <32>; + reg = <0x20>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1176,10 +1176,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache8>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1200,9 +1200,9 @@ }; }; - cpu33: cpu@33 { + cpu33: cpu@21 { compatible = "thead,c920", "riscv"; - reg = <33>; + reg = <0x21>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1212,10 +1212,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache8>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1236,9 +1236,9 @@ }; }; - cpu34: cpu@34 { + cpu34: cpu@22 { compatible = "thead,c920", "riscv"; - reg = <34>; + reg = <0x22>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1248,10 +1248,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache8>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1272,9 +1272,9 @@ }; }; - cpu35: cpu@35 { + cpu35: cpu@23 { compatible = "thead,c920", "riscv"; - reg = <35>; + reg = <0x23>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1284,10 +1284,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache8>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1308,9 +1308,9 @@ }; }; - cpu36: cpu@36 { + cpu36: cpu@24 { compatible = "thead,c920", "riscv"; - reg = <36>; + reg = <0x24>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1320,10 +1320,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache9>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1344,9 +1344,9 @@ }; }; - cpu37: cpu@37 { + cpu37: cpu@25 { compatible = "thead,c920", "riscv"; - reg = <37>; + reg = <0x25>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1356,10 +1356,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache9>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1380,9 +1380,9 @@ }; }; - cpu38: cpu@38 { + cpu38: cpu@26 { compatible = "thead,c920", "riscv"; - reg = <38>; + reg = <0x26>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1392,10 +1392,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache9>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1416,9 +1416,9 @@ }; }; - cpu39: cpu@39 { + cpu39: cpu@27 { compatible = "thead,c920", "riscv"; - reg = <39>; + reg = <0x27>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1428,10 +1428,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache9>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1452,9 +1452,9 @@ }; }; - cpu40: cpu@40 { + cpu40: cpu@28 { compatible = "thead,c920", "riscv"; - reg = <40>; + reg = <0x28>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1464,10 +1464,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache10>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1488,9 +1488,9 @@ }; }; - cpu41: cpu@41 { + cpu41: cpu@29 { compatible = "thead,c920", "riscv"; - reg = <41>; + reg = <0x29>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1500,10 +1500,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache10>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1524,9 +1524,9 @@ }; }; - cpu42: cpu@42 { + cpu42: cpu@2a { compatible = "thead,c920", "riscv"; - reg = <42>; + reg = <0x2a>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1536,10 +1536,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache10>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1560,9 +1560,9 @@ }; }; - cpu43: cpu@43 { + cpu43: cpu@2b { compatible = "thead,c920", "riscv"; - reg = <43>; + reg = <0x2b>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1572,10 +1572,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache10>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1596,9 +1596,9 @@ }; }; - cpu44: cpu@44 { + cpu44: cpu@2c { compatible = "thead,c920", "riscv"; - reg = <44>; + reg = <0x2c>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1608,10 +1608,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache11>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1632,9 +1632,9 @@ }; }; - cpu45: cpu@45 { + cpu45: cpu@2d { compatible = "thead,c920", "riscv"; - reg = <45>; + reg = <0x2d>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1644,10 +1644,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache11>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1668,9 +1668,9 @@ }; }; - cpu46: cpu@46 { + cpu46: cpu@2e { compatible = "thead,c920", "riscv"; - reg = <46>; + reg = <0x2e>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1680,10 +1680,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache11>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1704,9 +1704,9 @@ }; }; - cpu47: cpu@47 { + cpu47: cpu@2f { compatible = "thead,c920", "riscv"; - reg = <47>; + reg = <0x2f>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1716,10 +1716,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache11>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1740,9 +1740,9 @@ }; }; - cpu48: cpu@48 { + cpu48: cpu@30 { compatible = "thead,c920", "riscv"; - reg = <48>; + reg = <0x30>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1752,10 +1752,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache12>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1776,9 +1776,9 @@ }; }; - cpu49: cpu@49 { + cpu49: cpu@31 { compatible = "thead,c920", "riscv"; - reg = <49>; + reg = <0x31>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1788,10 +1788,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache12>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1812,9 +1812,9 @@ }; }; - cpu50: cpu@50 { + cpu50: cpu@32 { compatible = "thead,c920", "riscv"; - reg = <50>; + reg = <0x32>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1824,10 +1824,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache12>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1848,9 +1848,9 @@ }; }; - cpu51: cpu@51 { + cpu51: cpu@33 { compatible = "thead,c920", "riscv"; - reg = <51>; + reg = <0x33>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1860,10 +1860,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache12>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1884,9 +1884,9 @@ }; }; - cpu52: cpu@52 { + cpu52: cpu@34 { compatible = "thead,c920", "riscv"; - reg = <52>; + reg = <0x34>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1896,10 +1896,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache13>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1920,9 +1920,9 @@ }; }; - cpu53: cpu@53 { + cpu53: cpu@35 { compatible = "thead,c920", "riscv"; - reg = <53>; + reg = <0x35>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1932,10 +1932,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache13>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1956,9 +1956,9 @@ }; }; - cpu54: cpu@54 { + cpu54: cpu@36 { compatible = "thead,c920", "riscv"; - reg = <54>; + reg = <0x36>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -1968,10 +1968,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache13>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -1992,9 +1992,9 @@ }; }; - cpu55: cpu@55 { + cpu55: cpu@37 { compatible = "thead,c920", "riscv"; - reg = <55>; + reg = <0x37>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -2004,10 +2004,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache13>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -2028,9 +2028,9 @@ }; }; - cpu56: cpu@56 { + cpu56: cpu@38 { compatible = "thead,c920", "riscv"; - reg = <56>; + reg = <0x38>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -2040,10 +2040,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache14>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -2064,9 +2064,9 @@ }; }; - cpu57: cpu@57 { + cpu57: cpu@39 { compatible = "thead,c920", "riscv"; - reg = <57>; + reg = <0x39>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -2076,10 +2076,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache14>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -2100,9 +2100,9 @@ }; }; - cpu58: cpu@58 { + cpu58: cpu@3a { compatible = "thead,c920", "riscv"; - reg = <58>; + reg = <0x3a>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -2112,10 +2112,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache14>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -2136,9 +2136,9 @@ }; }; - cpu59: cpu@59 { + cpu59: cpu@3b { compatible = "thead,c920", "riscv"; - reg = <59>; + reg = <0x3b>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -2148,10 +2148,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache14>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -2172,9 +2172,9 @@ }; }; - cpu60: cpu@60 { + cpu60: cpu@3c { compatible = "thead,c920", "riscv"; - reg = <60>; + reg = <0x3c>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -2184,10 +2184,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache15>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -2208,9 +2208,9 @@ }; }; - cpu61: cpu@61 { + cpu61: cpu@3d { compatible = "thead,c920", "riscv"; - reg = <61>; + reg = <0x3d>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -2220,10 +2220,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache15>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -2244,9 +2244,9 @@ }; }; - cpu62: cpu@62 { + cpu62: cpu@3e { compatible = "thead,c920", "riscv"; - reg = <62>; + reg = <0x3e>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -2256,10 +2256,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache15>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", @@ -2280,9 +2280,9 @@ }; }; - cpu63: cpu@63 { + cpu63: cpu@3f { compatible = "thead,c920", "riscv"; - reg = <63>; + reg = <0x3f>; i-cache-block-size = <64>; i-cache-size = <65536>; i-cache-sets = <512>; @@ -2292,10 +2292,10 @@ device_type = "cpu"; mmu-type = "riscv,sv48"; next-level-cache = <&l2_cache15>; - riscv,isa = "rv64imafdcv"; + riscv,isa = "rv64imafdcbv"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "v", "sscofpmf", "sstc", + "b", "v", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", diff --git a/arch/riscv/boot/dts/spacemit/Makefile b/arch/riscv/boot/dts/spacemit/Makefile index 95889e7269d1..851b701e5da4 100644 --- a/arch/riscv/boot/dts/spacemit/Makefile +++ b/arch/riscv/boot/dts/spacemit/Makefile @@ -1,6 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 +dtb-$(CONFIG_ARCH_SPACEMIT) += k1-bananapi-cm6-io.dtb dtb-$(CONFIG_ARCH_SPACEMIT) += k1-bananapi-f3.dtb dtb-$(CONFIG_ARCH_SPACEMIT) += k1-milkv-jupiter.dtb dtb-$(CONFIG_ARCH_SPACEMIT) += k1-musepi-pro.dtb dtb-$(CONFIG_ARCH_SPACEMIT) += k1-orangepi-r2s.dtb dtb-$(CONFIG_ARCH_SPACEMIT) += k1-orangepi-rv2.dtb +dtb-$(CONFIG_ARCH_SPACEMIT) += k3-com260-ifx.dtb +dtb-$(CONFIG_ARCH_SPACEMIT) += k3-deepcomputing-fml13v05.dtb +dtb-$(CONFIG_ARCH_SPACEMIT) += k3-pico-itx.dtb diff --git a/arch/riscv/boot/dts/spacemit/k1-bananapi-cm6-io.dts b/arch/riscv/boot/dts/spacemit/k1-bananapi-cm6-io.dts new file mode 100644 index 000000000000..2c2d78aee98a --- /dev/null +++ b/arch/riscv/boot/dts/spacemit/k1-bananapi-cm6-io.dts @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2026 Junhui Liu <junhui.liu@pigmoral.tech> + */ + +#include "k1-bananapi-cm6.dtsi" + +#include <dt-bindings/leds/common.h> + +/ { + model = "Banana Pi BPI-CM6 IO Board"; + compatible = "bananapi,bpi-cm6-io", "bananapi,bpi-cm6", "spacemit,k1"; + + aliases { + ethernet0 = ð0; + ethernet1 = ð1; + serial0 = &uart0; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + leds { + compatible = "gpio-leds"; + + led0 { + color = <LED_COLOR_ID_RED>; + gpios = <&gpio K1_GPIO(96) GPIO_ACTIVE_LOW>; + }; + + led1 { + color = <LED_COLOR_ID_BLUE>; + gpios = <&gpio K1_GPIO(97) GPIO_ACTIVE_LOW>; + }; + }; + + pcie_vcc_3v3: regulator-pcie-vcc-3v3 { + compatible = "regulator-fixed"; + regulator-name = "NGFF_KEYM_VDD"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + regulator-always-on; + }; + + usb_vbus_5v: regulator-usb-vbus-5v { + compatible = "regulator-fixed"; + regulator-name = "VBUS_A_B"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + gpio = <&gpio K1_GPIO(124) GPIO_ACTIVE_HIGH>; + enable-active-high; + }; + + sd_vcc_3v3: regulator-sd-vcc-3v3 { + compatible = "regulator-fixed"; + regulator-name = "3.3VS_CARD"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + gpio = <&gpio K1_GPIO(127) GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&combo_phy { + status = "okay"; +}; + +ð0 { + status = "okay"; +}; + +ð1 { + nvmem-cells = <&mac_address 1>; + nvmem-cell-names = "mac-address"; + phy-handle = <&rgmii1>; + phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&gmac1_cfg>; + rx-internal-delay-ps = <0>; + tx-internal-delay-ps = <250>; + status = "okay"; + + mdio-bus { + #address-cells = <0x1>; + #size-cells = <0x0>; + + reset-gpios = <&gpio K1_GPIO(46) GPIO_ACTIVE_LOW>; + reset-delay-us = <10000>; + reset-post-delay-us = <100000>; + + rgmii1: phy@1 { + reg = <0x1>; + }; + }; +}; + +&i2c2 { + eeprom@54 { + compatible = "atmel,24c08"; + reg = <0x54>; + vcc-supply = <&buck3_1v8>; + pagesize = <16>; + size = <1024>; + }; +}; + +&pcie1_phy { + pinctrl-names = "default"; + pinctrl-0 = <&pcie1_3_cfg>; + status = "okay"; +}; + +&pcie1_port { + phys = <&pcie1_phy>; + vpcie3v3-supply = <&pcie_vcc_3v3>; +}; + +&pcie1 { + status = "okay"; +}; + +&pcie2_phy { + pinctrl-names = "default"; + pinctrl-0 = <&pcie2_4_cfg>; + status = "okay"; +}; + +&pcie2_port { + phys = <&pcie2_phy>; + vpcie3v3-supply = <&pcie_vcc_3v3>; +}; + +&pcie2 { + status = "okay"; +}; + +&qspi { + pinctrl-names = "default"; + pinctrl-0 = <&qspi_cfg>; + status = "okay"; +}; + +&sdhci0 { + pinctrl-names = "default", "uhs"; + pinctrl-0 = <&mmc1_cfg>; + pinctrl-1 = <&mmc1_uhs_cfg>; + bus-width = <4>; + cd-gpios = <&gpio K1_GPIO(80) (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>; + no-mmc; + no-sdio; + disable-wp; + cap-sd-highspeed; + vmmc-supply = <&sd_vcc_3v3>; + vqmmc-supply = <&aldo1>; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + status = "okay"; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_2_cfg>; + status = "okay"; +}; + +&usbphy2 { + status = "okay"; +}; + +&usb_dwc3 { + dr_mode = "host"; + vbus-supply = <&usb_vbus_5v>; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/spacemit/k1-bananapi-cm6.dtsi b/arch/riscv/boot/dts/spacemit/k1-bananapi-cm6.dtsi new file mode 100644 index 000000000000..81c40df2a366 --- /dev/null +++ b/arch/riscv/boot/dts/spacemit/k1-bananapi-cm6.dtsi @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2026 Junhui Liu <junhui.liu@pigmoral.tech> + */ + +#include "k1.dtsi" +#include "k1-pinctrl.dtsi" + +/ { + model = "Banana Pi BPI-CM6 Module"; + compatible = "bananapi,bpi-cm6", "spacemit,k1"; + + aliases { + i2c2 = &i2c2; + i2c8 = &i2c8; + }; + + reg_vcc_4v: regulator-vcc-4v { + compatible = "regulator-fixed"; + regulator-name = "VCC4V0_SYS"; + regulator-min-microvolt = <4000000>; + regulator-max-microvolt = <4000000>; + regulator-boot-on; + regulator-always-on; + }; +}; + +&emmc { + bus-width = <8>; + mmc-hs400-1_8v; + mmc-hs400-enhanced-strobe; + non-removable; + no-sd; + no-sdio; + status = "okay"; +}; + +ð0 { + nvmem-cells = <&mac_address 0>; + nvmem-cell-names = "mac-address"; + phy-handle = <&rgmii0>; + phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&gmac0_cfg>; + rx-internal-delay-ps = <0>; + tx-internal-delay-ps = <0>; + + mdio-bus { + #address-cells = <0x1>; + #size-cells = <0x0>; + + reset-gpios = <&gpio K1_GPIO(45) GPIO_ACTIVE_LOW>; + reset-delay-us = <10000>; + reset-post-delay-us = <100000>; + + rgmii0: phy@1 { + reg = <0x1>; + }; + }; +}; + +&pdma { + status = "okay"; +}; + +&i2c2 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c2_0_cfg>; + status = "okay"; + + eeprom@50 { + compatible = "atmel,24c02"; + reg = <0x50>; + vcc-supply = <&buck3_1v8>; + pagesize = <16>; + read-only; + size = <256>; + + nvmem-layout { + compatible = "onie,tlv-layout"; + + mac_address: mac-address { + #nvmem-cell-cells = <1>; + }; + + num-macs { + }; + + serial-number { + }; + }; + }; +}; + +&i2c8 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c8_cfg>; + status = "okay"; + + pmic@41 { + compatible = "spacemit,p1"; + reg = <0x41>; + interrupts = <64>; + vin1-supply = <®_vcc_4v>; + vin2-supply = <®_vcc_4v>; + vin3-supply = <®_vcc_4v>; + vin4-supply = <®_vcc_4v>; + vin5-supply = <®_vcc_4v>; + vin6-supply = <®_vcc_4v>; + aldoin-supply = <®_vcc_4v>; + dldoin1-supply = <&buck5>; + dldoin2-supply = <&buck5>; + + regulators { + buck1 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1050000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck2 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1050000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck3_1v8: buck3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1800000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck4 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3300000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck5: buck5 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3450000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck6 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3450000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + aldo1: aldo1 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-boot-on; + }; + + aldo2 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + aldo3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + aldo4 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo1 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-boot-on; + }; + + dldo2 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo4 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-always-on; + }; + + dldo5 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo6 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-always-on; + }; + + dldo7 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + }; + }; +}; diff --git a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts index 02f218a16318..58035d886bf6 100644 --- a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts +++ b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts @@ -4,6 +4,7 @@ */ #include "k1.dtsi" +#include "k1-opp.dtsi" #include "k1-pinctrl.dtsi" / { @@ -14,12 +15,15 @@ ethernet0 = ð0; ethernet1 = ð1; serial0 = &uart0; + spi3 = &spi3; i2c2 = &i2c2; i2c8 = &i2c8; + mmc0 = &emmc; + mmc1 = &sdhci0; }; chosen { - stdout-path = "serial0"; + stdout-path = "serial0:115200n8"; }; leds { @@ -33,7 +37,15 @@ }; }; - reg_dc_in: dc-in-12v { + pcie_vcc_3v3: regulator-pcie-vcc3v3 { + compatible = "regulator-fixed"; + regulator-name = "PCIE_VCC3V3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + }; + + reg_dc_in: regulator-dc-in-12v { compatible = "regulator-fixed"; regulator-name = "dc_in_12v"; regulator-min-microvolt = <12000000>; @@ -42,7 +54,7 @@ regulator-always-on; }; - reg_vcc_4v: vcc-4v { + reg_vcc_4v: regulator-vcc-4v { compatible = "regulator-fixed"; regulator-name = "vcc_4v"; regulator-min-microvolt = <4000000>; @@ -51,6 +63,61 @@ regulator-always-on; vin-supply = <®_dc_in>; }; + + regulator-usb3-vbus-5v { + compatible = "regulator-fixed"; + regulator-name = "USB30_VBUS"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-always-on; + gpio = <&gpio K1_GPIO(97) GPIO_ACTIVE_HIGH>; + enable-active-high; + }; + + usb3_hub_5v: regulator-usb3-hub-5v { + compatible = "regulator-fixed"; + regulator-name = "USB30_HUB"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + gpio = <&gpio K1_GPIO(123) GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&combo_phy { + status = "okay"; +}; + +&cpu_0 { + cpu-supply = <&buck1_0v9>; +}; + +&cpu_1 { + cpu-supply = <&buck1_0v9>; +}; + +&cpu_2 { + cpu-supply = <&buck1_0v9>; +}; + +&cpu_3 { + cpu-supply = <&buck1_0v9>; +}; + +&cpu_4 { + cpu-supply = <&buck1_0v9>; +}; + +&cpu_5 { + cpu-supply = <&buck1_0v9>; +}; + +&cpu_6 { + cpu-supply = <&buck1_0v9>; +}; + +&cpu_7 { + cpu-supply = <&buck1_0v9>; }; &emmc { @@ -67,7 +134,7 @@ phy-handle = <&rgmii0>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac0_cfg>; + pinctrl-0 = <&gmac0_cfg>, <&gmac0_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <0>; status = "okay"; @@ -90,7 +157,7 @@ phy-handle = <&rgmii1>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac1_cfg>; + pinctrl-0 = <&gmac1_cfg>, <&gmac1_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <250>; status = "okay"; @@ -157,19 +224,27 @@ compatible = "spacemit,p1"; reg = <0x41>; interrupts = <64>; - vin-supply = <®_vcc_4v>; + vin1-supply = <®_vcc_4v>; + vin2-supply = <®_vcc_4v>; + vin3-supply = <®_vcc_4v>; + vin4-supply = <®_vcc_4v>; + vin5-supply = <®_vcc_4v>; + vin6-supply = <®_vcc_4v>; + aldoin-supply = <®_vcc_4v>; + dldoin1-supply = <&buck5>; + dldoin2-supply = <&buck5>; regulators { - buck1 { + buck1_0v9: buck1 { regulator-min-microvolt = <500000>; - regulator-max-microvolt = <3450000>; + regulator-max-microvolt = <1050000>; regulator-ramp-delay = <5000>; regulator-always-on; }; buck2 { regulator-min-microvolt = <500000>; - regulator-max-microvolt = <3450000>; + regulator-max-microvolt = <1050000>; regulator-ramp-delay = <5000>; regulator-always-on; }; @@ -181,14 +256,14 @@ regulator-always-on; }; - buck4 { + buck4: buck4 { regulator-min-microvolt = <500000>; regulator-max-microvolt = <3300000>; regulator-ramp-delay = <5000>; regulator-always-on; }; - buck5 { + buck5: buck5 { regulator-min-microvolt = <500000>; regulator-max-microvolt = <3450000>; regulator-ramp-delay = <5000>; @@ -202,7 +277,7 @@ regulator-always-on; }; - aldo1 { + aldo1: aldo1 { regulator-min-microvolt = <500000>; regulator-max-microvolt = <3400000>; regulator-boot-on; @@ -264,8 +339,91 @@ }; }; +&pcie1_phy { + pinctrl-names = "default"; + pinctrl-0 = <&pcie1_3_cfg>; + status = "okay"; +}; + +&pcie1_port { + phys = <&pcie1_phy>; + vpcie3v3-supply = <&pcie_vcc_3v3>; +}; + +&pcie1 { + vpcie3v3-supply = <&pcie_vcc_3v3>; + status = "okay"; +}; + +&pcie2_phy { + pinctrl-names = "default"; + pinctrl-0 = <&pcie2_4_cfg>; + status = "okay"; +}; + +&pcie2_port { + phys = <&pcie2_phy>; + vpcie3v3-supply = <&pcie_vcc_3v3>; +}; + +&pcie2 { + vpcie3v3-supply = <&pcie_vcc_3v3>; + status = "okay"; +}; + +&spi3 { + pinctrl-0 = <&ssp3_0_cfg>; + pinctrl-names = "default"; + status = "okay"; +}; + &uart0 { pinctrl-names = "default"; pinctrl-0 = <&uart0_2_cfg>; status = "okay"; }; + +&usbphy2 { + status = "okay"; +}; + +&usb_dwc3 { + dr_mode = "host"; + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + hub_2_0: hub@1 { + compatible = "usb2109,2817"; + reg = <0x1>; + vdd-supply = <&usb3_hub_5v>; + peer-hub = <&hub_3_0>; + reset-gpios = <&gpio K1_GPIO(124) GPIO_ACTIVE_LOW>; + }; + + hub_3_0: hub@2 { + compatible = "usb2109,817"; + reg = <0x2>; + vdd-supply = <&usb3_hub_5v>; + peer-hub = <&hub_2_0>; + reset-gpios = <&gpio K1_GPIO(124) GPIO_ACTIVE_LOW>; + }; +}; + +&sdhci0 { + pinctrl-names = "default", "uhs"; + pinctrl-0 = <&mmc1_cfg>; + pinctrl-1 = <&mmc1_uhs_cfg>; + bus-width = <4>; + cd-gpios = <&gpio K1_GPIO(80) (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>; + no-mmc; + no-sdio; + disable-wp; + cap-sd-highspeed; + vmmc-supply = <&buck4>; + vqmmc-supply = <&aldo1>; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts b/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts index 28afd39b28da..635e95aad14a 100644 --- a/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts +++ b/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts @@ -5,6 +5,7 @@ */ #include "k1.dtsi" +#include "k1-opp.dtsi" #include "k1-pinctrl.dtsi" / { @@ -14,19 +15,146 @@ aliases { ethernet0 = ð0; ethernet1 = ð1; + i2c2 = &i2c2; + i2c8 = &i2c8; + mmc0 = &emmc; + mmc1 = &sdhci0; serial0 = &uart0; }; chosen { - stdout-path = "serial0"; + stdout-path = "serial0:115200n8"; }; + + leds { + compatible = "gpio-leds"; + + led1 { + label = "pwr-led"; + gpios = <&gpio K1_GPIO(96) GPIO_ACTIVE_HIGH>; + linux,default-trigger = "default-on"; + default-state = "on"; + }; + + led2 { + label = "hdd-led"; + gpios = <&gpio K1_GPIO(92) GPIO_ACTIVE_HIGH>; + linux,default-trigger = "disk-activity"; + }; + }; + + pcie_vcc_3v3: regulator-pcie-vcc3v3 { + compatible = "regulator-fixed"; + regulator-name = "pcie_vcc3v3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + regulator-always-on; + vin-supply = <®_dc_in>; + }; + + reg_dc_in: regulator-dc-in-12v { + compatible = "regulator-fixed"; + regulator-name = "dc_in_12v"; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + regulator-boot-on; + regulator-always-on; + }; + + reg_vcc_4v: regulator-vcc-4v { + compatible = "regulator-fixed"; + regulator-name = "vcc_4v"; + regulator-min-microvolt = <4000000>; + regulator-max-microvolt = <4000000>; + regulator-boot-on; + regulator-always-on; + vin-supply = <®_dc_in>; + }; + + reg_vcc_5v: regulator-vcc-5v { + compatible = "regulator-fixed"; + regulator-name = "vcc_5v"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-boot-on; + regulator-always-on; + vin-supply = <®_dc_in>; + }; + + regulator-usb3-vbus-5v { + compatible = "regulator-fixed"; + regulator-name = "USB30_VBUS"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-always-on; + vin-supply = <®_vcc_5v>; + gpio = <&gpio K1_GPIO(97) GPIO_ACTIVE_HIGH>; + enable-active-high; + }; + + usb3_hub_5v: regulator-usb3-hub-5v { + compatible = "regulator-fixed"; + regulator-name = "USB30_HUB"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + vin-supply = <®_vcc_5v>; + gpio = <&gpio K1_GPIO(123) GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&combo_phy { + status = "okay"; +}; + +&cpu_0 { + cpu-supply = <&buck1_0v9>; +}; + +&cpu_1 { + cpu-supply = <&buck1_0v9>; +}; + +&cpu_2 { + cpu-supply = <&buck1_0v9>; +}; + +&cpu_3 { + cpu-supply = <&buck1_0v9>; +}; + +&cpu_4 { + cpu-supply = <&buck1_0v9>; +}; + +&cpu_5 { + cpu-supply = <&buck1_0v9>; +}; + +&cpu_6 { + cpu-supply = <&buck1_0v9>; +}; + +&cpu_7 { + cpu-supply = <&buck1_0v9>; +}; + +&emmc { + bus-width = <8>; + mmc-hs400-1_8v; + mmc-hs400-enhanced-strobe; + non-removable; + no-sd; + no-sdio; + status = "okay"; }; ð0 { phy-handle = <&rgmii0>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac0_cfg>; + pinctrl-0 = <&gmac0_cfg>, <&gmac0_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <0>; status = "okay"; @@ -49,7 +177,7 @@ phy-handle = <&rgmii1>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac1_cfg>; + pinctrl-0 = <&gmac1_cfg>, <&gmac1_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <250>; status = "okay"; @@ -72,8 +200,272 @@ status = "okay"; }; +&i2c2 { + pinctrl-0 = <&i2c2_0_cfg>; + pinctrl-names = "default"; + status = "okay"; + + eeprom@50 { + compatible = "atmel,24c04"; + reg = <0x50>; + vcc-supply = <&buck3_1v8>; /* EEPROM_VCC18 */ + pagesize = <16>; + read-only; + size = <512>; + + nvmem-layout { + compatible = "onie,tlv-layout"; + + product-name { + }; + }; + }; +}; + +&i2c8 { + pinctrl-0 = <&i2c8_cfg>; + pinctrl-names = "default"; + status = "okay"; + + pmic@41 { + compatible = "spacemit,p1"; + reg = <0x41>; + interrupts = <64>; + vin1-supply = <®_vcc_4v>; + vin2-supply = <®_vcc_4v>; + vin3-supply = <®_vcc_4v>; + vin4-supply = <®_vcc_4v>; + vin5-supply = <®_vcc_4v>; + vin6-supply = <®_vcc_4v>; + aldoin-supply = <®_vcc_4v>; + dldoin1-supply = <&buck5>; + dldoin2-supply = <&buck5>; + + regulators { + buck1_0v9: buck1 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1050000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck2 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1050000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck3_1v8: buck3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1800000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck4_3v3: buck4 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3300000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck5: buck5 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3450000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck6 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3450000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + aldo1: aldo1 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-boot-on; + }; + + aldo2 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + aldo3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + aldo4 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo1 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-boot-on; + }; + + dldo2 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo4 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-always-on; + }; + + dldo5 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo6 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-always-on; + }; + + dldo7 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + }; + }; +}; + +&pcie1_phy { + pinctrl-names = "default"; + pinctrl-0 = <&pcie1_3_cfg>; + status = "okay"; +}; + +&pcie1_port { + phys = <&pcie1_phy>; + vpcie3v3-supply = <&pcie_vcc_3v3>; +}; + +&pcie1 { + vpcie3v3-supply = <&pcie_vcc_3v3>; + status = "okay"; +}; + +&pcie2_phy { + pinctrl-names = "default"; + pinctrl-0 = <&pcie2_4_cfg>; + status = "okay"; +}; + +&pcie2_port { + phys = <&pcie2_phy>; + vpcie3v3-supply = <&pcie_vcc_3v3>; +}; + +&pcie2 { + vpcie3v3-supply = <&pcie_vcc_3v3>; + status = "okay"; +}; + +&qspi { + pinctrl-names = "default"; + pinctrl-0 = <&qspi_cfg>; + status = "okay"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <26500000>; + spi-rx-bus-width = <4>; + spi-tx-bus-width = <4>; + vcc-supply = <&buck4_3v3>; /* QSPI_VCC1833 */ + m25p,fast-read; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + bootinfo@0 { + reg = <0x0 0x10000>; + }; + private@10000 { + reg = <0x10000 0x10000>; + }; + fsbl@20000 { + reg = <0x20000 0x40000>; + }; + env@60000 { + reg = <0x60000 0x10000>; + }; + opensbi@70000 { + reg = <0x70000 0x30000>; + }; + uboot@a0000 { + reg = <0xa0000 0x760000>; + }; + }; + }; +}; + +&sdhci0 { + pinctrl-names = "default", "uhs"; + pinctrl-0 = <&mmc1_cfg>; + pinctrl-1 = <&mmc1_uhs_cfg>; + bus-width = <4>; + cd-gpios = <&gpio K1_GPIO(80) (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>; + no-mmc; + no-sdio; + disable-wp; + cap-sd-highspeed; + vmmc-supply = <&buck4_3v3>; + vqmmc-supply = <&aldo1>; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + status = "okay"; +}; + &uart0 { pinctrl-names = "default"; pinctrl-0 = <&uart0_2_cfg>; status = "okay"; }; + +&usbphy2 { + status = "okay"; +}; + +&usb_dwc3 { + dr_mode = "host"; + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + hub_2_0: hub@1 { + compatible = "usb2109,2817"; + reg = <0x1>; + vdd-supply = <&usb3_hub_5v>; + peer-hub = <&hub_3_0>; + reset-gpios = <&gpio K1_GPIO(124) GPIO_ACTIVE_LOW>; + }; + + hub_3_0: hub@2 { + compatible = "usb2109,817"; + reg = <0x2>; + vdd-supply = <&usb3_hub_5v>; + peer-hub = <&hub_2_0>; + reset-gpios = <&gpio K1_GPIO(124) GPIO_ACTIVE_LOW>; + }; +}; diff --git a/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts b/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts index 29e333b670cf..246f8f2ab62b 100644 --- a/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts +++ b/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts @@ -17,10 +17,14 @@ aliases { ethernet0 = ð0; serial0 = &uart0; + i2c2 = &i2c2; + i2c8 = &i2c8; + mmc0 = &emmc; + mmc1 = &sdhci0; }; chosen { - stdout-path = "serial0"; + stdout-path = "serial0:115200n8"; }; leds { @@ -33,6 +37,52 @@ default-state = "on"; }; }; + + reg_pcie_vcc_3v3: regulator-pcie-vcc3v3 { + compatible = "regulator-fixed"; + regulator-name = "PCIE_VCC3V3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + }; + + reg_vcc_4v: regulator-vcc-4v { + compatible = "regulator-fixed"; + regulator-name = "VCC4V0"; + regulator-min-microvolt = <4000000>; + regulator-max-microvolt = <4000000>; + regulator-boot-on; + regulator-always-on; + }; + + reg_qspi_vcc1v833: regulator-qspi-vcc1v833 { + compatible = "regulator-fixed"; + regulator-name = "QSPI_VCC1833"; + regulator-min-microvolt = <1833000>; + regulator-max-microvolt = <1833000>; + regulator-always-on; + vin-supply = <&buck4_3v3>; + }; + + reg_5v_vbus: regulator-5v-vbus { + compatible = "regulator-fixed"; + regulator-name = "5V_VBUS"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-always-on; + gpio = <&gpio K1_GPIO(79) GPIO_ACTIVE_HIGH>; /* USB3_PWREN */ + enable-active-high; + }; + + reg_vcc5v_hub: regulator-vcc5v-hub { + compatible = "regulator-fixed"; + regulator-name = "VCC5V0_HUB"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-always-on; + gpio = <&gpio K1_GPIO(127) GPIO_ACTIVE_HIGH>; /* HUB_PWREN */ + enable-active-high; + }; }; &emmc { @@ -45,10 +95,14 @@ status = "okay"; }; +&combo_phy { + status = "okay"; +}; + ð0 { phy-handle = <&rgmii0>; phy-mode = "rgmii-id"; - pinctrl-0 = <&gmac0_cfg>; + pinctrl-0 = <&gmac0_cfg>, <&gmac0_clk_ref_cfg>; pinctrl-names = "default"; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <0>; @@ -72,8 +126,281 @@ status = "okay"; }; +&i2c2 { + pinctrl-0 = <&i2c2_0_cfg>; + pinctrl-names = "default"; + status = "okay"; + + eeprom@50 { + compatible = "atmel,24c02"; + reg = <0x50>; + vcc-supply = <&buck3_1v8>; /* EEPROM_VCC1V8 */ + pagesize = <8>; + read-only; + size = <256>; + + nvmem-layout { + compatible = "onie,tlv-layout"; + + product-name { + }; + + serial-number { + }; + + mac-address { + #nvmem-cell-cells = <1>; + }; + + num-macs { + }; + }; + }; +}; + +&i2c8 { + pinctrl-0 = <&i2c8_cfg>; + pinctrl-names = "default"; + status = "okay"; + + pmic@41 { + compatible = "spacemit,p1"; + reg = <0x41>; + interrupts = <64>; + vin1-supply = <®_vcc_4v>; + vin2-supply = <®_vcc_4v>; + vin3-supply = <®_vcc_4v>; + vin4-supply = <®_vcc_4v>; + vin5-supply = <®_vcc_4v>; + vin6-supply = <®_vcc_4v>; + aldoin-supply = <®_vcc_4v>; + dldoin1-supply = <&buck5>; + dldoin2-supply = <&buck5>; + + regulators { + buck1 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1050000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck2 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1050000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck3_1v8: buck3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1800000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck4_3v3: buck4 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3300000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck5: buck5 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3450000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck6 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3450000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + aldo1: aldo1 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-boot-on; + }; + + aldo2 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + aldo3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + aldo4 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo1 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-boot-on; + }; + + dldo2 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo4 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-always-on; + }; + + dldo5 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo6 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-always-on; + }; + + dldo7 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + }; + }; +}; + +&pcie1_phy { + pinctrl-names = "default"; + pinctrl-0 = <&pcie1_3_cfg>; + status = "okay"; +}; + +&pcie1_port { + phys = <&pcie1_phy>; + vpcie3v3-supply = <®_pcie_vcc_3v3>; +}; + +&pcie1 { + status = "okay"; +}; + +&pcie2_phy { + pinctrl-names = "default"; + pinctrl-0 = <&pcie2_4_cfg>; + status = "okay"; +}; + +&pcie2_port { + phys = <&pcie2_phy>; + vpcie3v3-supply = <®_pcie_vcc_3v3>; +}; + +&pcie2 { + status = "okay"; +}; + +&qspi { + pinctrl-names = "default"; + pinctrl-0 = <&qspi_cfg>; + status = "okay"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <26500000>; + spi-rx-bus-width = <4>; + spi-tx-bus-width = <4>; + vcc-supply = <®_qspi_vcc1v833>; + m25p,fast-read; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + bootinfo@0 { + reg = <0x0 0x10000>; + }; + private@10000 { + reg = <0x10000 0x10000>; + }; + fsbl@20000 { + reg = <0x20000 0x40000>; + }; + env@60000 { + reg = <0x60000 0x10000>; + }; + opensbi@70000 { + reg = <0x70000 0x30000>; + }; + uboot@a0000 { + reg = <0xa0000 0x760000>; + }; + }; + }; +}; + &uart0 { pinctrl-0 = <&uart0_2_cfg>; pinctrl-names = "default"; status = "okay"; }; + +&usbphy2 { + status = "okay"; +}; + +&usb_dwc3 { + dr_mode = "host"; + #address-cells = <1>; + #size-cells = <0>; + vbus-supply = <®_5v_vbus>; + status = "okay"; + + hub_2_0: hub@1 { + compatible = "usb2109,2817"; + reg = <0x1>; + vdd-supply = <®_vcc5v_hub>; + peer-hub = <&hub_3_0>; + reset-gpios = <&gpio K1_GPIO(123) GPIO_ACTIVE_LOW>; /* HUB_RST */ + }; + + hub_3_0: hub@2 { + compatible = "usb2109,817"; + reg = <0x2>; + vdd-supply = <®_vcc5v_hub>; + peer-hub = <&hub_2_0>; + reset-gpios = <&gpio K1_GPIO(123) GPIO_ACTIVE_LOW>; /* HUB_RST */ + }; +}; + +&sdhci0 { + pinctrl-names = "default", "uhs"; + pinctrl-0 = <&mmc1_cfg>; + pinctrl-1 = <&mmc1_uhs_cfg>; + bus-width = <4>; + cd-gpios = <&gpio K1_GPIO(80) (GPIO_ACTIVE_HIGH | GPIO_PULL_UP)>; + no-mmc; + no-sdio; + disable-wp; + cap-sd-highspeed; + vmmc-supply = <&buck4_3v3>; + vqmmc-supply = <&aldo1>; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/spacemit/k1-opp.dtsi b/arch/riscv/boot/dts/spacemit/k1-opp.dtsi new file mode 100644 index 000000000000..4cebfcd87485 --- /dev/null +++ b/arch/riscv/boot/dts/spacemit/k1-opp.dtsi @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) + +/ { + cluster0_opp_table: opp-table-cluster0 { + compatible = "operating-points-v2"; + opp-shared; + + opp-614400000 { + opp-hz = /bits/ 64 <614400000>; + opp-microvolt = <950000 950000 1050000>; + clock-latency-ns = <200000>; + }; + + opp-819000000 { + opp-hz = /bits/ 64 <819000000>; + opp-microvolt = <950000 950000 1050000>; + clock-latency-ns = <200000>; + }; + + opp-1000000000 { + opp-hz = /bits/ 64 <1000000000>; + opp-microvolt = <950000 950000 1050000>; + clock-latency-ns = <200000>; + }; + + opp-1228800000 { + opp-hz = /bits/ 64 <1228800000>; + opp-microvolt = <950000 950000 1050000>; + clock-latency-ns = <200000>; + }; + + opp-1600000000 { + opp-hz = /bits/ 64 <1600000000>; + opp-microvolt = <1050000 1050000 1050000>; + clock-latency-ns = <200000>; + }; + }; + + cluster1_opp_table: opp-table-cluster1 { + compatible = "operating-points-v2"; + opp-shared; + + opp-614400000 { + opp-hz = /bits/ 64 <614400000>; + opp-microvolt = <950000 950000 1050000>; + clock-latency-ns = <200000>; + }; + + opp-819000000 { + opp-hz = /bits/ 64 <819000000>; + opp-microvolt = <950000 950000 1050000>; + clock-latency-ns = <200000>; + }; + + opp-1000000000 { + opp-hz = /bits/ 64 <1000000000>; + opp-microvolt = <950000 950000 1050000>; + clock-latency-ns = <200000>; + }; + + opp-1228800000 { + opp-hz = /bits/ 64 <1228800000>; + opp-microvolt = <950000 950000 1050000>; + clock-latency-ns = <200000>; + }; + + opp-1600000000 { + opp-hz = /bits/ 64 <1600000000>; + opp-microvolt = <1050000 1050000 1050000>; + clock-latency-ns = <200000>; + }; + }; +}; + +&cpu_0 { + operating-points-v2 = <&cluster0_opp_table>; +}; + +&cpu_1 { + operating-points-v2 = <&cluster0_opp_table>; +}; + +&cpu_2 { + operating-points-v2 = <&cluster0_opp_table>; +}; + +&cpu_3 { + operating-points-v2 = <&cluster0_opp_table>; +}; + +&cpu_4 { + operating-points-v2 = <&cluster1_opp_table>; +}; + +&cpu_5 { + operating-points-v2 = <&cluster1_opp_table>; +}; + +&cpu_6 { + operating-points-v2 = <&cluster1_opp_table>; +}; + +&cpu_7 { + operating-points-v2 = <&cluster1_opp_table>; +}; diff --git a/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts b/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts index 58098c4a2aab..2d1ba34cb00e 100644 --- a/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts +++ b/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts @@ -13,6 +13,7 @@ compatible = "xunlong,orangepi-r2s", "spacemit,k1"; aliases { + i2c8 = &i2c8; serial0 = &uart0; ethernet0 = ð0; ethernet1 = ð1; @@ -21,6 +22,36 @@ chosen { stdout-path = "serial0"; }; + + pcie_vcc3v3: regulator-pcie-vcc3v3 { + compatible = "regulator-fixed"; + regulator-name = "pcie_vcc3v3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + }; + + vcc4v0: regulator-vcc4v0 { + compatible = "regulator-fixed"; + regulator-name = "vcc4v0"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <4000000>; + regulator-max-microvolt = <4000000>; + }; + + vcc5v0_usb: regulator-vcc5v0-usb { + compatible = "regulator-fixed"; + enable-active-high; + gpios = <&gpio K1_GPIO(126) GPIO_ACTIVE_HIGH>; + regulator-name = "vcc5v0_usb"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; +}; + +&combo_phy { + status = "okay"; }; &emmc { @@ -37,7 +68,7 @@ phy-handle = <&rgmii0>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac0_cfg>; + pinctrl-0 = <&gmac0_cfg>, <&gmac0_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <0>; status = "okay"; @@ -52,6 +83,7 @@ rgmii0: phy@1 { reg = <0x1>; + motorcomm,auto-sleep-disabled; }; }; }; @@ -60,7 +92,7 @@ phy-handle = <&rgmii1>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac1_cfg>; + pinctrl-0 = <&gmac1_cfg>, <&gmac1_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <250>; status = "okay"; @@ -75,10 +107,165 @@ rgmii1: phy@1 { reg = <0x1>; + motorcomm,auto-sleep-disabled; }; }; }; +&i2c8 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c8_cfg>; + status = "okay"; + + pmic@41 { + compatible = "spacemit,p1"; + reg = <0x41>; + interrupts = <64>; + vin1-supply = <&vcc4v0>; + vin2-supply = <&vcc4v0>; + vin3-supply = <&vcc4v0>; + vin4-supply = <&vcc4v0>; + vin5-supply = <&vcc4v0>; + vin6-supply = <&vcc4v0>; + aldoin-supply = <&vcc4v0>; + dldoin1-supply = <&buck5>; + dldoin2-supply = <&buck5>; + + regulators { + buck1 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1050000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck2 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1050000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck3_1v8: buck3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1800000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck4_3v3: buck4 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3300000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck5: buck5 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3450000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck6 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3450000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + aldo1_3v3: aldo1 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-boot-on; + }; + + aldo2 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + aldo3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + aldo4 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo1 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-boot-on; + }; + + dldo2 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo4 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-always-on; + }; + + dldo5 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo6 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-always-on; + }; + + dldo7 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + }; + }; +}; + +&pcie1_phy { + pinctrl-names = "default"; + pinctrl-0 = <&pcie1_3_cfg>; + status = "okay"; +}; + +&pcie1_port { + phys = <&pcie1_phy>; + vpcie3v3-supply = <&pcie_vcc3v3>; +}; + +&pcie1 { + status = "okay"; +}; + +&pcie2_phy { + pinctrl-names = "default"; + pinctrl-0 = <&pcie2_4_cfg>; + status = "okay"; +}; + +&pcie2_port { + phys = <&pcie2_phy>; + vpcie3v3-supply = <&pcie_vcc3v3>; +}; + +&pcie2 { + status = "okay"; +}; + &pdma { status = "okay"; }; @@ -88,3 +275,13 @@ pinctrl-0 = <&uart0_2_cfg>; status = "okay"; }; + +&usbphy2 { + status = "okay"; +}; + +&usb_dwc3 { + dr_mode = "host"; + vbus-supply = <&vcc5v0_usb>; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/spacemit/k1-orangepi-rv2.dts b/arch/riscv/boot/dts/spacemit/k1-orangepi-rv2.dts index 41dc8e35e6eb..b907a1396b21 100644 --- a/arch/riscv/boot/dts/spacemit/k1-orangepi-rv2.dts +++ b/arch/riscv/boot/dts/spacemit/k1-orangepi-rv2.dts @@ -17,12 +17,53 @@ serial0 = &uart0; ethernet0 = ð0; ethernet1 = ð1; + mmc0 = &emmc; + mmc1 = &sdhci0; }; chosen { stdout-path = "serial0"; }; + pcie_vcc3v3: regulator-pcie-vcc3v3 { + compatible = "regulator-fixed"; + enable-active-high; + gpios = <&gpio K1_GPIO(116) GPIO_ACTIVE_HIGH>; + regulator-name = "pcie_vcc3v3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc_5v0>; + }; + + vcc_5v0: regulator-vcc-5v0 { + compatible = "regulator-fixed"; + regulator-name = "vcc_5v0"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + vcc4v0: regulator-vcc4v0 { + compatible = "regulator-fixed"; + regulator-name = "vcc4v0"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <4000000>; + regulator-max-microvolt = <4000000>; + vin-supply = <&vcc_5v0>; + }; + + vcc5v0_usb30: regulator-vcc5v0-usb30 { + compatible = "regulator-fixed"; + enable-active-high; + gpios = <&gpio K1_GPIO(123) GPIO_ACTIVE_HIGH>; + regulator-name = "vcc5v0_usb30"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + vin-supply = <&vcc_5v0>; + }; + leds { compatible = "gpio-leds"; @@ -35,6 +76,20 @@ }; }; +&combo_phy { + status = "okay"; +}; + +&emmc { + bus-width = <8>; + mmc-hs400-1_8v; + mmc-hs400-enhanced-strobe; + no-sd; + no-sdio; + non-removable; + status = "okay"; +}; + ð0 { phy-handle = <&rgmii0>; phy-mode = "rgmii-id"; @@ -54,6 +109,7 @@ rgmii0: phy@1 { reg = <0x1>; + motorcomm,auto-sleep-disabled; }; }; }; @@ -77,6 +133,7 @@ rgmii1: phy@1 { reg = <0x1>; + motorcomm,auto-sleep-disabled; }; }; }; @@ -85,8 +142,245 @@ status = "okay"; }; +&i2c8 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c8_cfg>; + status = "okay"; + + pmic@41 { + compatible = "spacemit,p1"; + reg = <0x41>; + interrupts = <64>; + vin1-supply = <&vcc4v0>; + vin2-supply = <&vcc4v0>; + vin3-supply = <&vcc4v0>; + vin4-supply = <&vcc4v0>; + vin5-supply = <&vcc4v0>; + vin6-supply = <&vcc4v0>; + aldoin-supply = <&vcc4v0>; + dldoin1-supply = <&buck5>; + dldoin2-supply = <&buck5>; + + regulators { + buck1 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1050000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck2 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1050000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck3_1v8: buck3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1800000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck4_3v3: buck4 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3300000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck5: buck5 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3450000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck6 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3450000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + aldo1: aldo1 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-boot-on; + }; + + aldo2 { + /* not connected */ + }; + + aldo3 { + /* not connected */ + }; + + aldo4 { + /* not connected */ + }; + + dldo1 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-boot-on; + }; + + dldo2 { + /* not connected */ + }; + + dldo3 { + /* not connected */ + }; + + dldo4 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-always-on; + }; + + dldo5 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo6 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-always-on; + }; + + dldo7 { + /* not connected */ + }; + }; + }; +}; + +&pcie1_phy { + pinctrl-names = "default"; + pinctrl-0 = <&pcie1_3_cfg>; + status = "okay"; +}; + +&pcie1_port { + phys = <&pcie1_phy>; + vpcie3v3-supply = <&pcie_vcc3v3>; +}; + +&pcie1 { + status = "okay"; +}; + +&pcie2_phy { + pinctrl-names = "default"; + pinctrl-0 = <&pcie2_4_cfg>; + status = "okay"; +}; + +&pcie2_port { + phys = <&pcie2_phy>; + vpcie3v3-supply = <&pcie_vcc3v3>; +}; + +&pcie2 { + status = "okay"; +}; + +&qspi { + pinctrl-names = "default"; + pinctrl-0 = <&qspi_cfg>; + status = "okay"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <26500000>; + spi-rx-bus-width = <4>; + spi-tx-bus-width = <4>; + vcc-supply = <&buck3_1v8>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + bootinfo@0 { + reg = <0x00000 0x010000>; + }; + + private@10000 { + reg = <0x10000 0x010000>; + }; + + fsbl@20000 { + reg = <0x20000 0x040000>; + }; + + env@60000 { + reg = <0x60000 0x010000>; + }; + + opensbi@70000 { + reg = <0x70000 0x030000>; + }; + + uboot@a0000 { + reg = <0xa0000 0x760000>; + }; + }; + }; +}; + &uart0 { pinctrl-names = "default"; pinctrl-0 = <&uart0_2_cfg>; status = "okay"; }; + +&usbphy2 { + status = "okay"; +}; + +&usb_dwc3 { + dr_mode = "host"; + #address-cells = <1>; + #size-cells = <0>; + vbus-supply = <&vcc5v0_usb30>; + status = "okay"; + + hub_2_0: hub@1 { + compatible = "usb5e3,610"; + reg = <0x1>; + peer-hub = <&hub_3_0>; + vdd-supply = <&vcc_5v0>; + }; + + hub_3_0: hub@2 { + compatible = "usb5e3,620"; + reg = <0x2>; + peer-hub = <&hub_2_0>; + vdd-supply = <&vcc_5v0>; + }; +}; + +&sdhci0 { + pinctrl-names = "default", "uhs"; + pinctrl-0 = <&mmc1_cfg>; + pinctrl-1 = <&mmc1_uhs_cfg>; + bus-width = <4>; + cd-gpios = <&gpio K1_GPIO(80) GPIO_ACTIVE_LOW>; + no-mmc; + no-sdio; + disable-wp; + cap-sd-highspeed; + vmmc-supply = <&buck4_3v3>; + vqmmc-supply = <&aldo1>; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/spacemit/k1-pdma.h b/arch/riscv/boot/dts/spacemit/k1-pdma.h new file mode 100644 index 000000000000..7e5ad3d7111d --- /dev/null +++ b/arch/riscv/boot/dts/spacemit/k1-pdma.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * DMA request number (DRQ) definitions for non-secure peripherals of + * the SpacemiT K1 PDMA. + * + * Copyright (c) 2026 Guodong Xu <docular.xu@gmail.com> + */ + +#ifndef _DTS_SPACEMIT_K1_PDMA_H +#define _DTS_SPACEMIT_K1_PDMA_H + +#define K1_PDMA_UART0_TX 3 +#define K1_PDMA_UART0_RX 4 +#define K1_PDMA_UART2_TX 5 +#define K1_PDMA_UART2_RX 6 +#define K1_PDMA_UART3_TX 7 +#define K1_PDMA_UART3_RX 8 +#define K1_PDMA_UART4_TX 9 +#define K1_PDMA_UART4_RX 10 +#define K1_PDMA_I2C0_TX 11 +#define K1_PDMA_I2C0_RX 12 +#define K1_PDMA_I2C1_TX 13 +#define K1_PDMA_I2C1_RX 14 +#define K1_PDMA_I2C2_TX 15 +#define K1_PDMA_I2C2_RX 16 +#define K1_PDMA_I2C4_TX 17 +#define K1_PDMA_I2C4_RX 18 +#define K1_PDMA_SPI3_TX 19 +#define K1_PDMA_SPI3_RX 20 +#define K1_PDMA_I2S0_TX 21 +#define K1_PDMA_I2S0_RX 22 +#define K1_PDMA_I2S1_TX 23 +#define K1_PDMA_I2S1_RX 24 +#define K1_PDMA_UART5_TX 25 +#define K1_PDMA_UART5_RX 26 +#define K1_PDMA_UART6_TX 27 +#define K1_PDMA_UART6_RX 28 +#define K1_PDMA_UART7_TX 29 +#define K1_PDMA_UART7_RX 30 +#define K1_PDMA_UART8_TX 31 +#define K1_PDMA_UART8_RX 32 +#define K1_PDMA_UART9_TX 33 +#define K1_PDMA_UART9_RX 34 +#define K1_PDMA_I2C5_TX 35 +#define K1_PDMA_I2C5_RX 36 +#define K1_PDMA_I2C6_TX 37 +#define K1_PDMA_I2C6_RX 38 +#define K1_PDMA_I2C7_TX 39 +#define K1_PDMA_I2C7_RX 40 +#define K1_PDMA_I2C8_TX 41 +#define K1_PDMA_I2C8_RX 42 +#define K1_PDMA_CAN0_RX 43 +#define K1_PDMA_QSPI_RX 44 +#define K1_PDMA_QSPI_TX 45 + +#endif /* _DTS_SPACEMIT_K1_PDMA_H */ diff --git a/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi b/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi index e922e05ff856..8c57ca05dabd 100644 --- a/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi +++ b/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi @@ -27,8 +27,16 @@ <K1_PADCONF(11, 1)>, /* gmac0_tx_en */ <K1_PADCONF(12, 1)>, /* gmac0_mdc */ <K1_PADCONF(13, 1)>, /* gmac0_mdio */ - <K1_PADCONF(14, 1)>, /* gmac0_int_n */ - <K1_PADCONF(45, 1)>; /* gmac0_clk_ref */ + <K1_PADCONF(14, 1)>; /* gmac0_int_n */ + + bias-pull-up = <0>; + drive-strength = <21>; + }; + }; + + gmac0_clk_ref_cfg: gmac0-clk-ref-cfg { + gmac0-clk-ref-pins { + pinmux = <K1_PADCONF(45, 1)>; /* gmac0_clk_ref */ bias-pull-up = <0>; drive-strength = <21>; @@ -51,8 +59,16 @@ <K1_PADCONF(40, 1)>, /* gmac1_tx_en */ <K1_PADCONF(41, 1)>, /* gmac1_mdc */ <K1_PADCONF(42, 1)>, /* gmac1_mdio */ - <K1_PADCONF(43, 1)>, /* gmac1_int_n */ - <K1_PADCONF(46, 1)>; /* gmac1_clk_ref */ + <K1_PADCONF(43, 1)>; /* gmac1_int_n */ + + bias-pull-up = <0>; + drive-strength = <21>; + }; + }; + + gmac1_clk_ref_cfg: gmac1-clk-ref-cfg { + gmac1-clk-ref-pins { + pinmux = <K1_PADCONF(46, 1)>; /* gmac1_clk_ref */ bias-pull-up = <0>; drive-strength = <21>; @@ -530,6 +546,39 @@ }; }; + pcie0_3_cfg: pcie0-3-cfg { + pcie0-3-pins { + pinmux = <K1_PADCONF(54, 3)>, /* PERST# */ + <K1_PADCONF(55, 3)>, /* WAKE# */ + <K1_PADCONF(53, 3)>; /* CLKREQ# */ + + bias-pull-up = <0>; + drive-strength = <21>; + }; + }; + + pcie1_3_cfg: pcie1-3-cfg { + pcie1-3-pins { + pinmux = <K1_PADCONF(59, 4)>, /* PERST# */ + <K1_PADCONF(60, 4)>, /* WAKE# */ + <K1_PADCONF(61, 4)>; /* CLKREQ# */ + + bias-pull-up = <0>; + drive-strength = <21>; + }; + }; + + pcie2_4_cfg: pcie2-4-cfg { + pcie2-4-pins { + pinmux = <K1_PADCONF(62, 4)>, /* PERST# */ + <K1_PADCONF(112, 3)>, /* WAKE# */ + <K1_PADCONF(117, 4)>; /* CLKREQ# */ + + bias-pull-up = <0>; + drive-strength = <21>; + }; + }; + pwm14_1_cfg: pwm14-1-cfg { pwm14-1-pins { pinmux = <K1_PADCONF(44, 4)>; @@ -537,4 +586,64 @@ drive-strength = <32>; }; }; + + ssp3_0_cfg: ssp3-0-cfg { + ssp3-0-pins { + pinmux = <K1_PADCONF(75, 2)>, /* SCLK */ + <K1_PADCONF(77, 2)>, /* MOSI */ + <K1_PADCONF(78, 2)>; /* MISO */ + + bias-disable; + drive-strength = <19>; + power-source = <3300>; + }; + + ssp3-0-frm-pins { + pinmux = <K1_PADCONF(76, 2)>; /* FRM (frame) */ + + bias-pull-up = <0>; + drive-strength = <19>; + power-source = <3300>; + }; + }; + + mmc1_cfg: mmc1-cfg { + mmc1-data-cmd-pins { + pinmux = <K1_PADCONF(104, 0)>, /* mmc1_d3 */ + <K1_PADCONF(105, 0)>, /* mmc1_d2 */ + <K1_PADCONF(106, 0)>, /* mmc1_d1 */ + <K1_PADCONF(107, 0)>, /* mmc1_d0 */ + <K1_PADCONF(108, 0)>; /* mmc1_cmd */ + bias-pull-up = <1>; + drive-strength = <19>; + power-source = <3300>; + }; + + mmc1-clk-pins { + pinmux = <K1_PADCONF(109, 0)>; /* mmc1_clk */ + bias-pull-down = <1>; + drive-strength = <19>; + power-source = <3300>; + }; + }; + + mmc1_uhs_cfg: mmc1-uhs-cfg { + mmc1-data-cmd-pins { + pinmux = <K1_PADCONF(104, 0)>, /* mmc1_d3 */ + <K1_PADCONF(105, 0)>, /* mmc1_d2 */ + <K1_PADCONF(106, 0)>, /* mmc1_d1 */ + <K1_PADCONF(107, 0)>, /* mmc1_d0 */ + <K1_PADCONF(108, 0)>; /* mmc1_cmd */ + bias-pull-up = <1>; + drive-strength = <42>; + power-source = <1800>; + }; + + mmc1-clk-pins { + pinmux = <K1_PADCONF(109, 0)>; /* mmc1_clk */ + bias-pull-down = <1>; + drive-strength = <42>; + power-source = <1800>; + }; + }; }; diff --git a/arch/riscv/boot/dts/spacemit/k1.dtsi b/arch/riscv/boot/dts/spacemit/k1.dtsi index 7818ca4979b6..3fabfd34114b 100644 --- a/arch/riscv/boot/dts/spacemit/k1.dtsi +++ b/arch/riscv/boot/dts/spacemit/k1.dtsi @@ -4,6 +4,9 @@ */ #include <dt-bindings/clock/spacemit,k1-syscon.h> +#include <dt-bindings/phy/phy.h> + +#include "k1-pdma.h" /dts-v1/; / { @@ -53,9 +56,10 @@ compatible = "spacemit,x60", "riscv"; device_type = "cpu"; reg = <0>; - riscv,isa = "rv64imafdcv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + clocks = <&syscon_apmu CLK_CPU_C0_CORE>; + riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "zicbom", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", "zicbop", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", @@ -83,9 +87,10 @@ compatible = "spacemit,x60", "riscv"; device_type = "cpu"; reg = <1>; - riscv,isa = "rv64imafdcv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + clocks = <&syscon_apmu CLK_CPU_C0_CORE>; + riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "zicbom", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", "zicbop", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", @@ -113,9 +118,10 @@ compatible = "spacemit,x60", "riscv"; device_type = "cpu"; reg = <2>; - riscv,isa = "rv64imafdcv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + clocks = <&syscon_apmu CLK_CPU_C0_CORE>; + riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "zicbom", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", "zicbop", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", @@ -143,9 +149,10 @@ compatible = "spacemit,x60", "riscv"; device_type = "cpu"; reg = <3>; - riscv,isa = "rv64imafdcv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + clocks = <&syscon_apmu CLK_CPU_C0_CORE>; + riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "zicbom", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", "zicbop", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", @@ -173,9 +180,10 @@ compatible = "spacemit,x60", "riscv"; device_type = "cpu"; reg = <4>; - riscv,isa = "rv64imafdcv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + clocks = <&syscon_apmu CLK_CPU_C1_CORE>; + riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "zicbom", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", "zicbop", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", @@ -203,9 +211,10 @@ compatible = "spacemit,x60", "riscv"; device_type = "cpu"; reg = <5>; - riscv,isa = "rv64imafdcv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + clocks = <&syscon_apmu CLK_CPU_C1_CORE>; + riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "zicbom", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", "zicbop", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", @@ -233,9 +242,10 @@ compatible = "spacemit,x60", "riscv"; device_type = "cpu"; reg = <6>; - riscv,isa = "rv64imafdcv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + clocks = <&syscon_apmu CLK_CPU_C1_CORE>; + riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "zicbom", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", "zicbop", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", @@ -263,9 +273,10 @@ compatible = "spacemit,x60", "riscv"; device_type = "cpu"; reg = <7>; - riscv,isa = "rv64imafdcv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + clocks = <&syscon_apmu CLK_CPU_C1_CORE>; + riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "zicbom", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", "zicbop", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", @@ -332,12 +343,102 @@ osc_32k: clock-32k { compatible = "fixed-clock"; - clock-frequency = <32000>; + clock-frequency = <32768>; clock-output-names = "osc_32k"; #clock-cells = <0>; }; }; + thermal-zones { + soc-thermal { + polling-delay-passive = <0>; + polling-delay = <0>; + thermal-sensors = <&thermal 0>; + + trips { + soc-crit { + temperature = <115000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + package-thermal { + polling-delay-passive = <0>; + polling-delay = <0>; + thermal-sensors = <&thermal 1>; + + trips { + package-crit { + temperature = <115000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + gpu-thermal { + polling-delay-passive = <100>; + polling-delay = <0>; + thermal-sensors = <&thermal 2>; + + trips { + gpu-alert { + temperature = <85000>; + hysteresis = <2000>; + type = "passive"; + }; + + gpu-crit { + temperature = <115000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cluster0-thermal { + polling-delay-passive = <100>; + polling-delay = <0>; + thermal-sensors = <&thermal 3>; + + trips { + cluster0-alert { + temperature = <85000>; + hysteresis = <2000>; + type = "passive"; + }; + + cluster0-crit { + temperature = <115000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + + cluster1-thermal { + polling-delay-passive = <100>; + polling-delay = <0>; + thermal-sensors = <&thermal 4>; + + trips { + cluster1-alert { + temperature = <85000>; + hysteresis = <2000>; + type = "passive"; + }; + + cluster1-crit { + temperature = <115000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + }; + soc { compatible = "simple-bus"; interrupt-parent = <&plic>; @@ -358,6 +459,60 @@ #reset-cells = <1>; }; + usbphy2: phy@c0a30000 { + compatible = "spacemit,k1-usb2-phy"; + reg = <0x0 0xc0a30000 0x0 0x200>; + clocks = <&syscon_apmu CLK_USB30>; + #phy-cells = <0>; + status = "disabled"; + }; + + combo_phy: phy@c0b10000 { + compatible = "spacemit,k1-combo-phy"; + reg = <0x0 0xc0b10000 0x0 0x1000>; + clocks = <&vctcxo_24m>, + <&syscon_apmu CLK_PCIE0_DBI>, + <&syscon_apmu CLK_PCIE0_MASTER>, + <&syscon_apmu CLK_PCIE0_SLAVE>; + clock-names = "refclk", + "dbi", + "mstr", + "slv"; + resets = <&syscon_apmu RESET_PCIE0_GLOBAL>, + <&syscon_apmu RESET_PCIE0_DBI>, + <&syscon_apmu RESET_PCIE0_MASTER>, + <&syscon_apmu RESET_PCIE0_SLAVE>; + reset-names = "phy", + "dbi", + "mstr", + "slv"; + #phy-cells = <1>; + spacemit,apmu = <&syscon_apmu>; + status = "disabled"; + }; + + pcie1_phy: phy@c0c10000 { + compatible = "spacemit,k1-pcie-phy"; + reg = <0x0 0xc0c10000 0x0 0x1000>; + clocks = <&vctcxo_24m>; + clock-names = "refclk"; + resets = <&syscon_apmu RESET_PCIE1_GLOBAL>; + reset-names = "phy"; + #phy-cells = <0>; + status = "disabled"; + }; + + pcie2_phy: phy@c0d10000 { + compatible = "spacemit,k1-pcie-phy"; + reg = <0x0 0xc0d10000 0x0 0x1000>; + clocks = <&vctcxo_24m>; + clock-names = "refclk"; + resets = <&syscon_apmu RESET_PCIE2_GLOBAL>; + reset-names = "phy"; + #phy-cells = <0>; + status = "disabled"; + }; + i2c0: i2c@d4010800 { compatible = "spacemit,k1-i2c"; reg = <0x0 0xd4010800 0x0 0x38>; @@ -367,6 +522,7 @@ <&syscon_apbc CLK_TWSI0_BUS>; clock-names = "func", "bus"; clock-frequency = <400000>; + resets = <&syscon_apbc RESET_TWSI0>; interrupts = <36>; status = "disabled"; }; @@ -380,6 +536,7 @@ <&syscon_apbc CLK_TWSI1_BUS>; clock-names = "func", "bus"; clock-frequency = <400000>; + resets = <&syscon_apbc RESET_TWSI1>; interrupts = <37>; status = "disabled"; }; @@ -393,6 +550,7 @@ <&syscon_apbc CLK_TWSI2_BUS>; clock-names = "func", "bus"; clock-frequency = <400000>; + resets = <&syscon_apbc RESET_TWSI2>; interrupts = <38>; status = "disabled"; }; @@ -406,6 +564,7 @@ <&syscon_apbc CLK_TWSI4_BUS>; clock-names = "func", "bus"; clock-frequency = <400000>; + resets = <&syscon_apbc RESET_TWSI4>; interrupts = <40>; status = "disabled"; }; @@ -419,6 +578,7 @@ <&syscon_apbc CLK_TWSI5_BUS>; clock-names = "func", "bus"; clock-frequency = <400000>; + resets = <&syscon_apbc RESET_TWSI5>; interrupts = <41>; status = "disabled"; }; @@ -434,6 +594,17 @@ #reset-cells = <1>; }; + thermal: thermal@d4018000 { + compatible = "spacemit,k1-tsensor"; + reg = <0x0 0xd4018000 0x0 0x100>; + clocks = <&syscon_apbc CLK_TSEN>, + <&syscon_apbc CLK_TSEN_BUS>; + clock-names = "core", "bus"; + interrupts = <61>; + resets = <&syscon_apbc RESET_TSEN>; + #thermal-sensor-cells = <1>; + }; + i2c6: i2c@d4018800 { compatible = "spacemit,k1-i2c"; reg = <0x0 0xd4018800 0x0 0x38>; @@ -443,6 +614,7 @@ <&syscon_apbc CLK_TWSI6_BUS>; clock-names = "func", "bus"; clock-frequency = <400000>; + resets = <&syscon_apbc RESET_TWSI6>; interrupts = <70>; status = "disabled"; }; @@ -546,6 +718,7 @@ <&syscon_apbc CLK_TWSI7_BUS>; clock-names = "func", "bus"; clock-frequency = <400000>; + resets = <&syscon_apbc RESET_TWSI7>; interrupts = <18>; status = "disabled"; }; @@ -559,16 +732,18 @@ <&syscon_apbc CLK_TWSI8_BUS>; clock-names = "func", "bus"; clock-frequency = <400000>; + resets = <&syscon_apbc RESET_TWSI8>; interrupts = <19>; status = "disabled"; }; pinctrl: pinctrl@d401e000 { compatible = "spacemit,k1-pinctrl"; - reg = <0x0 0xd401e000 0x0 0x400>; + reg = <0x0 0xd401e000 0x0 0x1000>; clocks = <&syscon_apbc CLK_AIB>, <&syscon_apbc CLK_AIB_BUS>; clock-names = "func", "bus"; + spacemit,apbc = <&syscon_apbc>; }; pwm8: pwm@d4020000 { @@ -919,6 +1094,21 @@ status = "disabled"; }; + spi3: spi@d401c000 { + compatible = "spacemit,k1-spi"; + reg = <0x0 0xd401c000 0x0 0x30>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&syscon_apbc CLK_SSP3>, + <&syscon_apbc CLK_SSP3_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_SSP3>; + interrupts = <55>; + dmas = <&pdma K1_PDMA_SPI3_RX>, <&pdma K1_PDMA_SPI3_TX>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + /* sec_uart1: 0xf0612000, not available from Linux */ }; @@ -969,6 +1159,139 @@ #size-cells = <2>; dma-ranges = <0x0 0x00000000 0x0 0x00000000 0x0 0x80000000>, <0x0 0xb8000000 0x1 0x38000000 0x3 0x48000000>; + + pcie0: pcie@ca000000 { + device_type = "pci"; + compatible = "spacemit,k1-pcie"; + reg = <0x0 0xca000000 0x0 0x00001000>, + <0x0 0xca300000 0x0 0x0001ff24>, + <0x0 0x8f000000 0x0 0x00002000>, + <0x0 0xc0b20000 0x0 0x00001000>; + reg-names = "dbi", + "atu", + "config", + "link"; + linux,pci-domain = <0>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x01000000 0x0 0x00000000 0x0 0x8f002000 0x0 0x00100000>, + <0x02000000 0x0 0x80000000 0x0 0x80000000 0x0 0x0f000000>; + interrupts = <141>; + interrupt-names = "msi"; + clocks = <&syscon_apmu CLK_PCIE0_DBI>, + <&syscon_apmu CLK_PCIE0_MASTER>, + <&syscon_apmu CLK_PCIE0_SLAVE>; + clock-names = "dbi", + "mstr", + "slv"; + resets = <&syscon_apmu RESET_PCIE0_DBI>, + <&syscon_apmu RESET_PCIE0_MASTER>, + <&syscon_apmu RESET_PCIE0_SLAVE>; + reset-names = "dbi", + "mstr", + "slv"; + spacemit,apmu = <&syscon_apmu 0x03cc>; + status = "disabled"; + + pcie0_port: pcie@0 { + device_type = "pci"; + compatible = "pciclass,0604"; + reg = <0x0 0x0 0x0 0x0 0x0>; + bus-range = <0x01 0xff>; + #address-cells = <3>; + #size-cells = <2>; + ranges; + }; + }; + + pcie1: pcie@ca400000 { + device_type = "pci"; + compatible = "spacemit,k1-pcie"; + reg = <0x0 0xca400000 0x0 0x00001000>, + <0x0 0xca700000 0x0 0x0001ff24>, + <0x0 0x9f000000 0x0 0x00002000>, + <0x0 0xc0c20000 0x0 0x00001000>; + reg-names = "dbi", + "atu", + "config", + "link"; + linux,pci-domain = <1>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x01000000 0x0 0x00000000 0x0 0x9f002000 0x0 0x00100000>, + <0x02000000 0x0 0x90000000 0x0 0x90000000 0x0 0x0f000000>; + interrupts = <142>; + interrupt-names = "msi"; + clocks = <&syscon_apmu CLK_PCIE1_DBI>, + <&syscon_apmu CLK_PCIE1_MASTER>, + <&syscon_apmu CLK_PCIE1_SLAVE>; + clock-names = "dbi", + "mstr", + "slv"; + resets = <&syscon_apmu RESET_PCIE1_DBI>, + <&syscon_apmu RESET_PCIE1_MASTER>, + <&syscon_apmu RESET_PCIE1_SLAVE>; + reset-names = "dbi", + "mstr", + "slv"; + spacemit,apmu = <&syscon_apmu 0x3d4>; + status = "disabled"; + + pcie1_port: pcie@0 { + device_type = "pci"; + compatible = "pciclass,0604"; + reg = <0x0 0x0 0x0 0x0 0x0>; + bus-range = <0x01 0xff>; + #address-cells = <3>; + #size-cells = <2>; + ranges; + }; + }; + + pcie2: pcie@ca800000 { + device_type = "pci"; + compatible = "spacemit,k1-pcie"; + reg = <0x0 0xca800000 0x0 0x00001000>, + <0x0 0xcab00000 0x0 0x0001ff24>, + <0x0 0xb7000000 0x0 0x00002000>, + <0x0 0xc0d20000 0x0 0x00001000>; + reg-names = "dbi", + "atu", + "config", + "link"; + linux,pci-domain = <2>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x01000000 0x0 0x00000000 0x0 0xb7002000 0x0 0x00100000>, + <0x42000000 0x0 0xa0000000 0x0 0xa0000000 0x0 0x10000000>, + <0x02000000 0x0 0xb0000000 0x0 0xb0000000 0x0 0x07000000>; + interrupts = <143>; + interrupt-names = "msi"; + clocks = <&syscon_apmu CLK_PCIE2_DBI>, + <&syscon_apmu CLK_PCIE2_MASTER>, + <&syscon_apmu CLK_PCIE2_SLAVE>; + clock-names = "dbi", + "mstr", + "slv"; + resets = <&syscon_apmu RESET_PCIE2_DBI>, + <&syscon_apmu RESET_PCIE2_MASTER>, + <&syscon_apmu RESET_PCIE2_SLAVE>; + reset-names = "dbi", + "mstr", + "slv"; + spacemit,apmu = <&syscon_apmu 0x3dc>; + status = "disabled"; + + pcie2_port: pcie@0 { + device_type = "pci"; + compatible = "pciclass,0604"; + reg = <0x0 0x0 0x0 0x0 0x0>; + bus-range = <0x01 0xff>; + #address-cells = <3>; + #size-cells = <2>; + ranges; + }; + }; }; storage-bus { @@ -978,12 +1301,52 @@ #size-cells = <2>; dma-ranges = <0x0 0x00000000 0x0 0x00000000 0x0 0x80000000>; + usb_dwc3: usb@c0a00000 { + compatible = "spacemit,k1-dwc3"; + reg = <0x0 0xc0a00000 0x0 0x10000>; + clocks = <&syscon_apmu CLK_USB30>; + clock-names = "usbdrd30"; + interrupts = <125>; + phys = <&usbphy2>, <&combo_phy PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + phy_type = "utmi"; + resets = <&syscon_apmu RESET_USB30_AHB>, + <&syscon_apmu RESET_USB30_VCC>, + <&syscon_apmu RESET_USB30_PHY>; + reset-names = "ahb", "vcc", "phy"; + reset-delay = <2>; + snps,hsphy_interface = "utmi"; + snps,dis_enblslpm_quirk; + snps,dis-u2-freeclk-exists-quirk; + snps,dis-del-phy-power-chg-quirk; + snps,dis_u2_susphy_quirk; + snps,dis_u3_susphy_quirk; + snps,dis_rxdet_inp3_quirk; + status = "disabled"; + }; + + sdhci0: mmc@d4280000 { + compatible = "spacemit,k1-sdhci"; + reg = <0x0 0xd4280000 0x0 0x200>; + clocks = <&syscon_apmu CLK_SDH_AXI>, + <&syscon_apmu CLK_SDH0>; + clock-names = "core", "io"; + resets = <&syscon_apmu RESET_SDH_AXI>, + <&syscon_apmu RESET_SDH0>; + reset-names = "axi", "sdh"; + interrupts = <99>; + status = "disabled"; + }; + emmc: mmc@d4281000 { compatible = "spacemit,k1-sdhci"; reg = <0x0 0xd4281000 0x0 0x200>; clocks = <&syscon_apmu CLK_SDH_AXI>, <&syscon_apmu CLK_SDH2>; clock-names = "core", "io"; + resets = <&syscon_apmu RESET_SDH_AXI>, + <&syscon_apmu RESET_SDH2>; + reset-names = "axi", "sdh"; interrupts = <101>; status = "disabled"; }; diff --git a/arch/riscv/boot/dts/spacemit/k3-com260-ifx.dts b/arch/riscv/boot/dts/spacemit/k3-com260-ifx.dts new file mode 100644 index 000000000000..a5c08a7a781a --- /dev/null +++ b/arch/riscv/boot/dts/spacemit/k3-com260-ifx.dts @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (c) 2026 SpacemiT (Hangzhou) Technology Co. Ltd + * Copyright (c) 2026 Yixun Lan <dlan@kernel.org> + */ + +#include "k3-com260.dtsi" + +/ { + model = "SpacemiT K3 CoM260 IFX"; + compatible = "spacemit,k3-com260-ifx", "spacemit,k3-com260", "spacemit,k3"; + + aliases { + serial0 = &uart0; + ethernet0 = ð1; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; +}; + +&i2c1 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_1_cfg>; + status = "okay"; + + eeprom@50 { + compatible = "atmel,24c02"; + pagesize = <8>; + reg = <0x50>; + read-only; + size = <256>; + status = "okay"; + }; +}; + +&sdhci0 { + pinctrl-names = "default","uhs"; + pinctrl-0 = <&mmc1_cfg>; + pinctrl-1 = <&mmc1_uhs_cfg>; + bus-width = <4>; + cd-gpios = <&gpio K3_GPIO(4) GPIO_ACTIVE_HIGH>; + no-mmc; + no-sdio; + disable-wp; + cap-sd-highspeed; + vmmc-supply = <&aldo4>; + vqmmc-supply = <&aldo1>; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/spacemit/k3-com260.dtsi b/arch/riscv/boot/dts/spacemit/k3-com260.dtsi new file mode 100644 index 000000000000..37627963a649 --- /dev/null +++ b/arch/riscv/boot/dts/spacemit/k3-com260.dtsi @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (c) 2026 SpacemiT (Hangzhou) Technology Co. Ltd + * Copyright (c) 2026 Yixun Lan <dlan@kernel.org> + */ +#include <dt-bindings/gpio/gpio.h> + +#include "k3.dtsi" +#include "k3-pinctrl.dtsi" + +/ { + model = "SpacemiT K3 CoM260 Module"; + compatible = "spacemit,k3-com260", "spacemit,k3"; + + memory@100000000 { + device_type = "memory"; + reg = <0x1 0x00000000 0x4 0x00000000>; + }; + + reg_5v_sys: regulator-5v-sys { + compatible = "regulator-fixed"; + regulator-name = "P5V0_SYS"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-always-on; + }; +}; + +&i2c2 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c2_1_cfg>; + status = "okay"; + + eeprom@50 { + compatible = "atmel,24c02"; + pagesize = <8>; + reg = <0x50>; + read-only; + size = <256>; + status = "okay"; + + nvmem-layout { + compatible = "onie,tlv-layout"; + + mac-address { + #nvmem-cell-cells = <1>; + }; + + num-macs { + }; + + product-name { + }; + + serial-number { + }; + }; + }; +}; + +&i2c8 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c8_cfg>; + status = "okay"; + + pmic@41 { + compatible = "spacemit,p1"; + reg = <0x41>; + interrupts = <64 IRQ_TYPE_LEVEL_HIGH>; + vin1-supply = <®_5v_sys>; + vin2-supply = <®_5v_sys>; + vin3-supply = <®_5v_sys>; + vin4-supply = <®_5v_sys>; + vin5-supply = <®_5v_sys>; + vin6-supply = <®_5v_sys>; + aldoin-supply = <®_5v_sys>; + dldoin1-supply = <&buck4>; + dldoin2-supply = <&buck4>; + + regulators { + buck1: buck1 { + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1050000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck2: buck2 { + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1050000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck3: buck3 { + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <800000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck4: buck4 { + regulator-min-microvolt = <2100000>; + regulator-max-microvolt = <2100000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck5: buck5 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck6: buck6 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <500000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + aldo1: aldo1 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + regulator-boot-on; + }; + + aldo2: aldo2 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + }; + + aldo3: aldo3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + aldo4: aldo4 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + regulator-boot-on; + }; + + dldo1: dldo1 { + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-always-on; + regulator-boot-on; + }; + + dldo2: dldo2 { + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-always-on; + regulator-boot-on; + }; + + dldo3: dldo3 { + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <800000>; + regulator-always-on; + regulator-boot-on; + }; + + dldo4: dldo4 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-boot-on; + }; + + dldo5: dldo5 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + }; + + dldo6: dldo6 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + }; + + dldo7: dldo7 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + }; + }; + }; +}; + +ð1 { + pinctrl-names = "default"; + pinctrl-0 = <&gmac1_rgmii_0_cfg>, <&gmac1_phy_0_cfg>; + phy-mode = "rgmii-id"; + phy-handle = <&phy1>; + status = "okay"; + + mdio { + phy1: ethernet-phy@1 { + compatible = "ethernet-phy-id001c.c916"; + reg = <1>; + reset-gpios = <&gpio 1 5 GPIO_ACTIVE_LOW>; + reset-assert-us = <10000>; + reset-deassert-us = <50000>; + }; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_0_cfg>; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/spacemit/k3-deepcomputing-fml13v05.dts b/arch/riscv/boot/dts/spacemit/k3-deepcomputing-fml13v05.dts new file mode 100644 index 000000000000..b5d5112de358 --- /dev/null +++ b/arch/riscv/boot/dts/spacemit/k3-deepcomputing-fml13v05.dts @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (C) 2026 DeepComputing (HK) Limited + */ + +#include "k3.dtsi" +#include "k3-pinctrl.dtsi" + +/ { + model = "DeepComputing FML13V05"; + compatible = "deepcomputing,fml13v05", "spacemit,k3"; + + aliases { + serial0 = &uart0; + }; + + chosen { + stdout-path = "serial0"; + }; + + memory@100000000 { + device_type = "memory"; + reg = <0x1 0x00000000 0x4 0x00000000>; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_0_cfg>; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/spacemit/k3-pdma.h b/arch/riscv/boot/dts/spacemit/k3-pdma.h new file mode 100644 index 000000000000..a5b723ad616a --- /dev/null +++ b/arch/riscv/boot/dts/spacemit/k3-pdma.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * DMA request number (DRQ) definitions for non-secure peripherals of + * the SpacemiT K3 PDMA. + * + * Copyright (c) 2025 SpacemiT + * Copyright (c) 2026 Guodong Xu <docular.xu@gmail.com> + */ + +#ifndef _DTS_SPACEMIT_K3_PDMA_H +#define _DTS_SPACEMIT_K3_PDMA_H + +/* UART DMA request numbers */ +#define K3_PDMA_UART0_TX 3 +#define K3_PDMA_UART0_RX 4 +#define K3_PDMA_UART2_TX 5 +#define K3_PDMA_UART2_RX 6 +#define K3_PDMA_UART3_TX 7 +#define K3_PDMA_UART3_RX 8 +#define K3_PDMA_UART4_TX 9 +#define K3_PDMA_UART4_RX 10 +#define K3_PDMA_UART5_TX 25 +#define K3_PDMA_UART5_RX 26 +#define K3_PDMA_UART6_TX 27 +#define K3_PDMA_UART6_RX 28 +#define K3_PDMA_UART7_TX 29 +#define K3_PDMA_UART7_RX 30 +#define K3_PDMA_UART8_TX 31 +#define K3_PDMA_UART8_RX 32 +#define K3_PDMA_UART9_TX 33 +#define K3_PDMA_UART9_RX 34 +#define K3_PDMA_UART10_TX 53 +#define K3_PDMA_UART10_RX 54 + +/* I2C DMA request numbers */ +#define K3_PDMA_I2C0_TX 11 +#define K3_PDMA_I2C0_RX 12 +#define K3_PDMA_I2C1_TX 13 +#define K3_PDMA_I2C1_RX 14 +#define K3_PDMA_I2C2_TX 15 +#define K3_PDMA_I2C2_RX 16 +#define K3_PDMA_I2C4_TX 17 +#define K3_PDMA_I2C4_RX 18 +#define K3_PDMA_I2C5_TX 35 +#define K3_PDMA_I2C5_RX 36 +#define K3_PDMA_I2C6_TX 37 +#define K3_PDMA_I2C6_RX 38 +#define K3_PDMA_I2C8_TX 41 +#define K3_PDMA_I2C8_RX 42 + +/* SSP/SPI DMA request numbers */ +#define K3_PDMA_SSP3_TX 19 +#define K3_PDMA_SSP3_RX 20 +#define K3_PDMA_SSPA0_TX 21 +#define K3_PDMA_SSPA0_RX 22 +#define K3_PDMA_SSPA1_TX 23 +#define K3_PDMA_SSPA1_RX 24 +#define K3_PDMA_SSPA2_TX 56 +#define K3_PDMA_SSPA2_RX 57 +#define K3_PDMA_SSPA3_TX 58 +#define K3_PDMA_SSPA3_RX 59 +#define K3_PDMA_SSPA4_TX 60 +#define K3_PDMA_SSPA4_RX 61 +#define K3_PDMA_SSPA5_TX 62 +#define K3_PDMA_SSPA5_RX 63 + +/* CAN DMA request numbers */ +#define K3_PDMA_CAN0_RX 43 +#define K3_PDMA_CAN1_RX 44 +#define K3_PDMA_CAN2_RX 51 +#define K3_PDMA_CAN3_RX 52 + +/* SSP0/1 DMA request numbers */ +#define K3_PDMA_SSP0_TX 64 +#define K3_PDMA_SSP0_RX 65 +#define K3_PDMA_SSP1_TX 66 +#define K3_PDMA_SSP1_RX 67 + +/* QSPI DMA request numbers */ +#define K3_PDMA_QSPI_RX 84 +#define K3_PDMA_QSPI_TX 85 + +#endif /* _DTS_SPACEMIT_K3_PDMA_H */ diff --git a/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts b/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts new file mode 100644 index 000000000000..b084a35aa00e --- /dev/null +++ b/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts @@ -0,0 +1,332 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (c) 2026 SpacemiT (Hangzhou) Technology Co. Ltd + * Copyright (c) 2026 Guodong Xu <guodong@riscstar.com> + */ +#include <dt-bindings/gpio/gpio.h> + +#include "k3.dtsi" +#include "k3-pinctrl.dtsi" + +/ { + model = "SpacemiT K3 Pico-ITX"; + compatible = "spacemit,k3-pico-itx", "spacemit,k3"; + + aliases { + ethernet0 = ð0; + i2c2 = &i2c2; + i2c8 = &i2c8; + serial0 = &uart0; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + memory@100000000 { + device_type = "memory"; + reg = <0x1 0x00000000 0x4 0x00000000>; + }; + + reg_aux_vcc3v3: regulator-aux-vcc3v3 { + compatible = "regulator-fixed"; + regulator-name = "AUX_VCC3V3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + }; + + reg_aux_vcc5v: regulator-aux-vcc5v { + compatible = "regulator-fixed"; + regulator-name = "AUX_VCC5V"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-always-on; + }; +}; + +&i2c2 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c2_1_cfg>; + status = "okay"; + + eeprom@50 { + compatible = "atmel,24c02"; + reg = <0x50>; + pagesize = <8>; + read-only; + size = <256>; + status = "okay"; + + nvmem-layout { + compatible = "onie,tlv-layout"; + + mac-address { + #nvmem-cell-cells = <1>; + }; + + num-macs { + }; + + part-number { + }; + + product-name { + }; + + serial-number { + }; + + }; + }; +}; + +&i2c8 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c8_cfg>; + status = "okay"; + + p1@41 { + compatible = "spacemit,p1"; + reg = <0x41>; + interrupts = <64 IRQ_TYPE_LEVEL_HIGH>; + vin1-supply = <®_aux_vcc5v>; + vin2-supply = <®_aux_vcc5v>; + vin3-supply = <®_aux_vcc5v>; + vin4-supply = <®_aux_vcc5v>; + vin5-supply = <®_aux_vcc5v>; + vin6-supply = <®_aux_vcc5v>; + aldoin-supply = <®_aux_vcc5v>; + dldoin1-supply = <&buck4>; + dldoin2-supply = <&buck4>; + + regulators { + buck1: buck1 { + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1050000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck2: buck2 { + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1050000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck3: buck3 { + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <800000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck4: buck4 { + regulator-min-microvolt = <2100000>; + regulator-max-microvolt = <2100000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck5: buck5 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck6: buck6 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <500000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + aldo1: aldo1 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + regulator-boot-on; + }; + + aldo2: aldo2 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + }; + + aldo3: aldo3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + aldo4: aldo4 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + regulator-boot-on; + }; + + dldo1: dldo1 { + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-always-on; + regulator-boot-on; + }; + + dldo2: dldo2 { + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-always-on; + regulator-boot-on; + }; + + dldo3: dldo3 { + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <800000>; + regulator-always-on; + regulator-boot-on; + }; + + dldo4: dldo4 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-boot-on; + }; + + dldo5: dldo5 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + }; + + dldo6: dldo6 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + }; + + dldo7: dldo7 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + }; + }; + }; +}; + +&combophy { + spacemit,apmu = <&syscon_apmu 0x11>; + status = "okay"; +}; + +ð0 { + pinctrl-names = "default"; + pinctrl-0 = <&gmac0_rgmii_0_cfg>, <&gmac0_phy_0_cfg>; + phy-mode = "rgmii-id"; + phy-handle = <&phy0>; + status = "okay"; + + mdio { + phy0: ethernet-phy@1 { + compatible = "ethernet-phy-id001c.c916"; + reg = <1>; + reset-gpios = <&gpio 0 15 GPIO_ACTIVE_LOW>; + reset-assert-us = <10000>; + reset-deassert-us = <50000>; + realtek,aldps-enable; + realtek,clkout-disable; + }; + }; +}; + +&qspi_cfg { + qspi-pins { + power-source = <1800>; + }; + + qspi-cs0-pins { + power-source = <1800>; + }; +}; + +&qspi { + pinctrl-names = "default"; + pinctrl-0 = <&qspi_cfg>; + status = "okay"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <26500000>; + spi-rx-bus-width = <4>; + spi-tx-bus-width = <4>; + vcc-supply = <&aldo2>; /* PMIC_VCC1V8_QSPI */ + m25p,fast-read; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + bootinfo@0 { + reg = <0x0 0x20000>; + }; + + fsbl@20000 { + reg = <0x20000 0x80000>; + }; + + env@a0000 { + reg = <0xa0000 0x10000>; + }; + + esos@b0000 { + reg = <0xb0000 0x100000>; + }; + + opensbi@1b0000 { + reg = <0x1b0000 0x60000>; + }; + + uboot@210000 { + reg = <0x210000 0x5f0000>; + }; + }; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_0_cfg>; + status = "okay"; +}; + +&usb2_host { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + hub@1 { + compatible = "usb1a40,0101"; + reg = <1>; + vdd-supply = <®_aux_vcc3v3>; + }; +}; + +&usb2_phy { + status = "okay"; +}; + +&usb3d_u2phy { + status = "okay"; +}; + +&usb3d { + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi b/arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi new file mode 100644 index 000000000000..6482d24be139 --- /dev/null +++ b/arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi @@ -0,0 +1,780 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (c) 2026 Yixun Lan <dlan@kernel.org> + */ + +#include <dt-bindings/gpio/gpio.h> + +#define K3_PADCONF(pin, func) (((pin) << 16) | (func)) + +/* Map GPIO pin to each bank's <index, offset> */ +#define K3_GPIO(x) (x / 32) (x % 32) + +&pinctrl { + gmac0_rgmii_0_cfg: gmac0-rgmii-0-cfg { + gmac0-rgmii-0-pins { + pinmux = <K3_PADCONF(0, 1)>, /* gmac0_rxdv */ + <K3_PADCONF(1, 1)>, /* gmac0_rx_d0 */ + <K3_PADCONF(2, 1)>, /* gmac0_rx_d1 */ + <K3_PADCONF(3, 1)>, /* gmac0_rx_clk */ + <K3_PADCONF(4, 1)>, /* gmac0_rx_d2 */ + <K3_PADCONF(5, 1)>, /* gmac0_rx_d3 */ + <K3_PADCONF(6, 1)>, /* gmac0_tx_d0 */ + <K3_PADCONF(7, 1)>, /* gmac0_tx_d1 */ + <K3_PADCONF(8, 1)>, /* gmac0_tx_clk */ + <K3_PADCONF(9, 1)>, /* gmac0_tx_d2 */ + <K3_PADCONF(10, 1)>, /* gmac0_tx_d3 */ + <K3_PADCONF(11, 1)>, /* gmac0_tx_en */ + <K3_PADCONF(12, 1)>, /* gmac0_mdc */ + <K3_PADCONF(13, 1)>; /* gmac0_mdio */ + + bias-disable; + drive-strength = <25>; + power-source = <1800>; + }; + + }; + + gmac0_phy_0_cfg: gmac0-phy-0-cfg { + gmac0-phy-0-pins { + pinmux = <K3_PADCONF(14, 1)>; /* gmac0_int */ + + bias-disable; + drive-strength = <25>; + power-source = <1800>; + }; + }; + + gmac1_rgmii_0_cfg: gmac1-rgmii-0-cfg { + gmac1-rgmii-0-pins { + pinmux = <K3_PADCONF(21, 1)>, /* gmac1_rxdv */ + <K3_PADCONF(22, 1)>, /* gmac1 rx d0 */ + <K3_PADCONF(23, 1)>, /* gmac1 rx d1 */ + <K3_PADCONF(24, 1)>, /* gmac1 rx_clk */ + <K3_PADCONF(25, 1)>, /* gmac1 rx d2 */ + <K3_PADCONF(26, 1)>, /* gmac1 rx d3 */ + <K3_PADCONF(27, 1)>, /* gmac1 tx d0 */ + <K3_PADCONF(28, 1)>, /* gmac1 tx d1 */ + <K3_PADCONF(29, 1)>, /* gmac1 tx clk */ + <K3_PADCONF(30, 1)>, /* gmac1 tx d2 */ + <K3_PADCONF(31, 1)>, /* gmac1 tx d3 */ + <K3_PADCONF(32, 1)>, /* gmac1 tx_en */ + <K3_PADCONF(33, 1)>, /* gmac1 mdc */ + <K3_PADCONF(34, 1)>; /* gmac1 mdio */ + + bias-disable; + drive-strength = <25>; + power-source = <1800>; + }; + }; + + gmac1_phy_0_cfg: gmac1-phy-0-cfg { + gmac1-phy-0-pins { + pinmux = <K3_PADCONF(35, 1)>; /* gmac1 int */ + + bias-disable; + drive-strength = <25>; + power-source = <1800>; + }; + }; + + /omit-if-no-ref/ + i2c1_1_cfg: i2c1-1-cfg { + i2c1-1-pins { + pinmux = <K3_PADCONF(54, 5)>, /* i2c1 scl */ + <K3_PADCONF(55, 5)>; /* i2c1 sda */ + + bias-pull-up = <1>; + drive-strength = <25>; + power-source = <1800>; + }; + }; + + /omit-if-no-ref/ + i2c2_1_cfg: i2c2-1-cfg { + i2c2-1-pins { + pinmux = <K3_PADCONF(46, 5)>, /* i2c2 scl */ + <K3_PADCONF(47, 5)>; /* i2c2 sda */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + i2c8_cfg: i2c8-cfg { + i2c8-pins { + pinmux = <K3_PADCONF(128, 0)>, /* i2c8 scl */ + <K3_PADCONF(129, 0)>; /* i2c8 sda */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm0_0_cfg: pwm0-0-cfg { + pwm0-0-pins { + pinmux = <K3_PADCONF(0, 3)>; /* pwm0 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm0_1_cfg: pwm0-1-cfg { + pwm0-1-pins { + pinmux = <K3_PADCONF(42, 6)>; /* pwm0 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm1_0_cfg: pwm1-0-cfg { + pwm1-0-pins { + pinmux = <K3_PADCONF(1, 3)>; /* pwm1 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm1_1_cfg: pwm1-1-cfg { + pwm1-1-pins { + pinmux = <K3_PADCONF(43, 6)>; /* pwm1 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm1_2_cfg: pwm1-2-cfg { + pwm1-2-pins { + pinmux = <K3_PADCONF(95, 6)>; /* pwm1 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm2_0_cfg: pwm2-0-cfg { + pwm2-0-pins { + pinmux = <K3_PADCONF(2, 3)>; /* pwm2 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm2_1_cfg: pwm2-1-cfg { + pwm2-1-pins { + pinmux = <K3_PADCONF(44, 6)>; /* pwm2 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm2_2_cfg: pwm2-2-cfg { + pwm2-2-pins { + pinmux = <K3_PADCONF(96, 6)>; /* pwm2 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm2_3_cfg: pwm2-3-cfg { + pwm2-3-pins { + pinmux = <K3_PADCONF(134, 4)>; /* pwm2 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm3_0_cfg: pwm3-0-cfg { + pwm3-0-pins { + pinmux = <K3_PADCONF(3, 3)>; /* pwm3 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm3_1_cfg: pwm3-1-cfg { + pwm3-1-pins { + pinmux = <K3_PADCONF(45, 6)>; /* pwm3 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm3_2_cfg: pwm3-2-cfg { + pwm3-2-pins { + pinmux = <K3_PADCONF(97, 6)>; /* pwm3 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm3_3_cfg: pwm3-3-cfg { + pwm3-3-pins { + pinmux = <K3_PADCONF(135, 4)>; /* pwm3 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm4_0_cfg: pwm4-0-cfg { + pwm4-0-pins { + pinmux = <K3_PADCONF(4, 3)>; /* pwm4 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm4_1_cfg: pwm4-1-cfg { + pwm4-1-pins { + pinmux = <K3_PADCONF(46, 6)>; /* pwm4 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm4_2_cfg: pwm4-2-cfg { + pwm4-2-pins { + pinmux = <K3_PADCONF(136, 4)>; /* pwm4 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm5_0_cfg: pwm5-0-cfg { + pwm5-0-pins { + pinmux = <K3_PADCONF(5, 3)>; /* pwm5 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm5_1_cfg: pwm5-1-cfg { + pwm5-1-pins { + pinmux = <K3_PADCONF(47, 6)>; /* pwm5 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm5_2_cfg: pwm5-2-cfg { + pwm5-2-pins { + pinmux = <K3_PADCONF(137, 4)>; /* pwm5 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm6_0_cfg: pwm6-0-cfg { + pwm6-0-pins { + pinmux = <K3_PADCONF(6, 3)>; /* pwm6 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm6_1_cfg: pwm6-1-cfg { + pwm6-1-pins { + pinmux = <K3_PADCONF(145, 3)>; /* pwm6 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm6_2_cfg: pwm6-2-cfg { + pwm6-2-pins { + pinmux = <K3_PADCONF(48, 6)>; /* pwm6 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm7_0_cfg: pwm7-0-cfg { + pwm7-0-pins { + pinmux = <K3_PADCONF(7, 3)>; /* pwm7 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm7_1_cfg: pwm7-1-cfg { + pwm7-1-pins { + pinmux = <K3_PADCONF(146, 3)>; /* pwm7 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm7_2_cfg: pwm7-2-cfg { + pwm7-2-pins { + pinmux = <K3_PADCONF(49, 6)>; /* pwm7 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm8_0_cfg: pwm8-0-cfg { + pwm8-0-pins { + pinmux = <K3_PADCONF(8, 3)>; /* pwm8 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm8_1_cfg: pwm8-1-cfg { + pwm8-1-pins { + pinmux = <K3_PADCONF(147, 3)>; /* pwm8 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm8_2_cfg: pwm8-2-cfg { + pwm8-2-pins { + pinmux = <K3_PADCONF(50, 6)>; /* pwm8 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm9_0_cfg: pwm9-0-cfg { + pwm9-0-pins { + pinmux = <K3_PADCONF(9, 3)>; /* pwm9 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm9_1_cfg: pwm9-1-cfg { + pwm9-1-pins { + pinmux = <K3_PADCONF(148, 3)>; /* pwm9 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm9_2_cfg: pwm9-2-cfg { + pwm9-2-pins { + pinmux = <K3_PADCONF(51, 6)>; /* pwm9 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm10_0_cfg: pwm10-0-cfg { + pwm10-0-pins { + pinmux = <K3_PADCONF(10, 3)>; /* pwm10 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm10_1_cfg: pwm10-1-cfg { + pwm10-1-pins { + pinmux = <K3_PADCONF(52, 6)>; /* pwm10 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm10_2_cfg: pwm10-2-cfg { + pwm10-2-pins { + pinmux = <K3_PADCONF(141, 4)>; /* pwm10 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm11_0_cfg: pwm11-0-cfg { + pwm11-0-pins { + pinmux = <K3_PADCONF(53, 6)>; /* pwm11 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm11_1_cfg: pwm11-1-cfg { + pwm11-1-pins { + pinmux = <K3_PADCONF(140, 4)>; /* pwm11 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm12_0_cfg: pwm12-0-cfg { + pwm12-0-pins { + pinmux = <K3_PADCONF(54, 6)>; /* pwm12 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm12_1_cfg: pwm12-1-cfg { + pwm12-1-pins { + pinmux = <K3_PADCONF(139, 4)>; /* pwm12 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm13_0_cfg: pwm13-0-cfg { + pwm13-0-pins { + pinmux = <K3_PADCONF(55, 6)>; /* pwm13 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm13_1_cfg: pwm13-1-cfg { + pwm13-1-pins { + pinmux = <K3_PADCONF(138, 4)>; /* pwm13 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm13_2_cfg: pwm13-2-cfg { + pwm13-2-pins { + pinmux = <K3_PADCONF(13, 3)>; /* pwm13 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm14_0_cfg: pwm14-0-cfg { + pwm14-0-pins { + pinmux = <K3_PADCONF(56, 6)>; /* pwm14 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm14_1_cfg: pwm14-1-cfg { + pwm14-1-pins { + pinmux = <K3_PADCONF(144, 4)>; /* pwm14 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm14_2_cfg: pwm14-2-cfg { + pwm14-2-pins { + pinmux = <K3_PADCONF(14, 3)>; /* pwm14 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm15_0_cfg: pwm15-0-cfg { + pwm15-0-pins { + pinmux = <K3_PADCONF(57, 6)>; /* pwm15 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm15_1_cfg: pwm15-1-cfg { + pwm15-1-pins { + pinmux = <K3_PADCONF(142, 4)>; /* pwm15 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm15_2_cfg: pwm15-2-cfg { + pwm15-2-pins { + pinmux = <K3_PADCONF(21, 3)>; /* pwm15 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm16_0_cfg: pwm16-0-cfg { + pwm16-0-pins { + pinmux = <K3_PADCONF(58, 6)>; /* pwm16 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm16_1_cfg: pwm16-1-cfg { + pwm16-1-pins { + pinmux = <K3_PADCONF(143, 4)>; /* pwm16 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm16_2_cfg: pwm16-2-cfg { + pwm16-2-pins { + pinmux = <K3_PADCONF(22, 3)>; /* pwm16 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm17_0_cfg: pwm17-0-cfg { + pwm17-0-pins { + pinmux = <K3_PADCONF(23, 3)>; /* pwm17 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm17_1_cfg: pwm17-1-cfg { + pwm17-1-pins { + pinmux = <K3_PADCONF(59, 6)>; /* pwm17 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm17_2_cfg: pwm17-2-cfg { + pwm17-2-pins { + pinmux = <K3_PADCONF(105, 6)>; /* pwm17 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm18_0_cfg: pwm18-0-cfg { + pwm18-0-pins { + pinmux = <K3_PADCONF(24, 3)>; /* pwm18 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm18_1_cfg: pwm18-1-cfg { + pwm18-1-pins { + pinmux = <K3_PADCONF(60, 6)>; /* pwm18 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm18_2_cfg: pwm18-2-cfg { + pwm18-2-pins { + pinmux = <K3_PADCONF(106, 6)>; /* pwm18 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm19_0_cfg: pwm19-0-cfg { + pwm19-0-pins { + pinmux = <K3_PADCONF(25, 3)>; /* pwm19 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm19_1_cfg: pwm19-1-cfg { + pwm19-1-pins { + pinmux = <K3_PADCONF(61, 6)>; /* pwm19 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + pwm19_2_cfg: pwm19-2-cfg { + pwm19-2-pins { + pinmux = <K3_PADCONF(107, 6)>; /* pwm19 */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + qspi_cfg: qspi-cfg { + qspi-pins { + pinmux = <K3_PADCONF(138, 0)>, /* qspi dat0 */ + <K3_PADCONF(139, 0)>, /* qspi dat1 */ + <K3_PADCONF(140, 0)>, /* qspi dat2 */ + <K3_PADCONF(141, 0)>, /* qspi dat3 */ + <K3_PADCONF(144, 0)>; /* qspi clk */ + + bias-disable; + drive-strength = <25>; + }; + + qspi-cs0-pins { + pinmux = <K3_PADCONF(142, 0)>; /* qspi cs0 */ + + bias-disable; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + uart0_0_cfg: uart0-0-cfg { + uart0-0-pins { + pinmux = <K3_PADCONF(149, 2)>, /* uart0 tx */ + <K3_PADCONF(150, 2)>; /* uart0 rx */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + mmc1_cfg: mmc1-cfg { + mmc1-data-cmd-pins { + pinmux = <K3_PADCONF(132, 0)>, /* mmc1 dat3 */ + <K3_PADCONF(133, 0)>, /* mmc1 dat2 */ + <K3_PADCONF(134, 0)>, /* mmc1 dat1 */ + <K3_PADCONF(135, 0)>, /* mmc1 dat0 */ + <K3_PADCONF(136, 0)>; /* mmc1 cmd */ + + bias-pull-up = <0>; + drive-strength = <25>; + power-source = <3300>; + }; + + mmc1-clk-pins { + pinmux = <K3_PADCONF(137, 0)>; /* mmc1 clk */ + + bias-pull-down; + drive-strength = <25>; + power-source = <3300>; + }; + }; + + mmc1_uhs_cfg: mmc1-uhs-cfg { + mmc1-uhs-data-cmd-pins { + pinmux = <K3_PADCONF(132, 0)>, /* mmc1 dat3 */ + <K3_PADCONF(133, 0)>, /* mmc1 dat2 */ + <K3_PADCONF(134, 0)>, /* mmc1 dat1 */ + <K3_PADCONF(135, 0)>, /* mmc1 dat0 */ + <K3_PADCONF(136, 0)>; /* mmc1 cmd */ + + bias-pull-up = <0>; + drive-strength = <21>; + power-source = <1800>; + }; + + mmc1-uhs-clk-pins { + pinmux = <K3_PADCONF(137, 0)>; /* mmc1 clk */ + + bias-pull-down; + drive-strength = <21>; + power-source = <1800>; + }; + }; +}; diff --git a/arch/riscv/boot/dts/spacemit/k3.dtsi b/arch/riscv/boot/dts/spacemit/k3.dtsi new file mode 100644 index 000000000000..c3f2dce0969c --- /dev/null +++ b/arch/riscv/boot/dts/spacemit/k3.dtsi @@ -0,0 +1,1309 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (c) 2026 SpacemiT (Hangzhou) Technology Co. Ltd + * Copyright (c) 2026 Guodong Xu <guodong@riscstar.com> + */ + +#include <dt-bindings/clock/spacemit,k3-clocks.h> +#include <dt-bindings/interrupt-controller/irq.h> +#include <dt-bindings/reset/spacemit,k3-resets.h> +#include <dt-bindings/phy/phy.h> + +#include "k3-pdma.h" + +/dts-v1/; + +/ { + #address-cells = <2>; + #size-cells = <2>; + model = "SpacemiT K3"; + compatible = "spacemit,k3"; + + cpus: cpus { + #address-cells = <1>; + #size-cells = <0>; + timebase-frequency = <24000000>; + + cpu_0: cpu@0 { + compatible = "spacemit,x100", "riscv"; + device_type = "cpu"; + reg = <0>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "h", + "sha", "shcounterenw", "shgatpa", "shtvala", + "shvsatpa", "shvstvala", "shvstvecd", "smaia", + "smstateen", "ssaia", "ssccptr", "sscofpmf", + "sscounterenw", "ssnpm", "ssstateen", "sstc", + "sstvala", "sstvecd", "ssu64xl", "svade", + "svinval", "svnapot", "svpbmt", "za64rs", + "zawrs", "zba", "zbb", "zbc", "zbs", "zca", + "zcb", "zcd", "zcmop", "zfa", "zfbfmin", + "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", + "ziccamoa", "ziccif", "zicclsm", "ziccrse", "zicntr", + "zicond", "zicsr", "zifencei", "zihintntl", + "zihintpause", "zihpm", "zimop", "zkt", "zvbb", + "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", + "zvkned", "zvkng", "zvknha", "zvknhb", "zvks", + "zvksc", "zvksed", "zvksg", "zvksh", "zvkt"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <256>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <256>; + next-level-cache = <&l2_cache0>; + mmu-type = "riscv,sv39"; + + cpu0_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu_1: cpu@1 { + compatible = "spacemit,x100", "riscv"; + device_type = "cpu"; + reg = <1>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "h", + "sha", "shcounterenw", "shgatpa", "shtvala", + "shvsatpa", "shvstvala", "shvstvecd", "smaia", + "smstateen", "ssaia", "ssccptr", "sscofpmf", + "sscounterenw", "ssnpm", "ssstateen", "sstc", + "sstvala", "sstvecd", "ssu64xl", "svade", + "svinval", "svnapot", "svpbmt", "za64rs", + "zawrs", "zba", "zbb", "zbc", "zbs", "zca", + "zcb", "zcd", "zcmop", "zfa", "zfbfmin", + "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", + "ziccamoa", "ziccif", "zicclsm", "ziccrse", "zicntr", + "zicond", "zicsr", "zifencei", "zihintntl", + "zihintpause", "zihpm", "zimop", "zkt", "zvbb", + "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", + "zvkned", "zvkng", "zvknha", "zvknhb", "zvks", + "zvksc", "zvksed", "zvksg", "zvksh", "zvkt"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <256>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <256>; + next-level-cache = <&l2_cache0>; + mmu-type = "riscv,sv39"; + + cpu1_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu_2: cpu@2 { + compatible = "spacemit,x100", "riscv"; + device_type = "cpu"; + reg = <2>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "h", + "sha", "shcounterenw", "shgatpa", "shtvala", + "shvsatpa", "shvstvala", "shvstvecd", "smaia", + "smstateen", "ssaia", "ssccptr", "sscofpmf", + "sscounterenw", "ssnpm", "ssstateen", "sstc", + "sstvala", "sstvecd", "ssu64xl", "svade", + "svinval", "svnapot", "svpbmt", "za64rs", + "zawrs", "zba", "zbb", "zbc", "zbs", "zca", + "zcb", "zcd", "zcmop", "zfa", "zfbfmin", + "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", + "ziccamoa", "ziccif", "zicclsm", "ziccrse", "zicntr", + "zicond", "zicsr", "zifencei", "zihintntl", + "zihintpause", "zihpm", "zimop", "zkt", "zvbb", + "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", + "zvkned", "zvkng", "zvknha", "zvknhb", "zvks", + "zvksc", "zvksed", "zvksg", "zvksh", "zvkt"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <256>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <256>; + next-level-cache = <&l2_cache0>; + mmu-type = "riscv,sv39"; + + cpu2_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu_3: cpu@3 { + compatible = "spacemit,x100", "riscv"; + device_type = "cpu"; + reg = <3>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "h", + "sha", "shcounterenw", "shgatpa", "shtvala", + "shvsatpa", "shvstvala", "shvstvecd", "smaia", + "smstateen", "ssaia", "ssccptr", "sscofpmf", + "sscounterenw", "ssnpm", "ssstateen", "sstc", + "sstvala", "sstvecd", "ssu64xl", "svade", + "svinval", "svnapot", "svpbmt", "za64rs", + "zawrs", "zba", "zbb", "zbc", "zbs", "zca", + "zcb", "zcd", "zcmop", "zfa", "zfbfmin", + "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", + "ziccamoa", "ziccif", "zicclsm", "ziccrse", "zicntr", + "zicond", "zicsr", "zifencei", "zihintntl", + "zihintpause", "zihpm", "zimop", "zkt", "zvbb", + "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", + "zvkned", "zvkng", "zvknha", "zvknhb", "zvks", + "zvksc", "zvksed", "zvksg", "zvksh", "zvkt"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <256>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <256>; + next-level-cache = <&l2_cache0>; + mmu-type = "riscv,sv39"; + + cpu3_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu_4: cpu@4 { + compatible = "spacemit,x100", "riscv"; + device_type = "cpu"; + reg = <4>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "h", + "sha", "shcounterenw", "shgatpa", "shtvala", + "shvsatpa", "shvstvala", "shvstvecd", "smaia", + "smstateen", "ssaia", "ssccptr", "sscofpmf", + "sscounterenw", "ssnpm", "ssstateen", "sstc", + "sstvala", "sstvecd", "ssu64xl", "svade", + "svinval", "svnapot", "svpbmt", "za64rs", + "zawrs", "zba", "zbb", "zbc", "zbs", "zca", + "zcb", "zcd", "zcmop", "zfa", "zfbfmin", + "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", + "ziccamoa", "ziccif", "zicclsm", "ziccrse", "zicntr", + "zicond", "zicsr", "zifencei", "zihintntl", + "zihintpause", "zihpm", "zimop", "zkt", "zvbb", + "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", + "zvkned", "zvkng", "zvknha", "zvknhb", "zvks", + "zvksc", "zvksed", "zvksg", "zvksh", "zvkt"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <256>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <256>; + next-level-cache = <&l2_cache1>; + mmu-type = "riscv,sv39"; + + cpu4_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu_5: cpu@5 { + compatible = "spacemit,x100", "riscv"; + device_type = "cpu"; + reg = <5>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "h", + "sha", "shcounterenw", "shgatpa", "shtvala", + "shvsatpa", "shvstvala", "shvstvecd", "smaia", + "smstateen", "ssaia", "ssccptr", "sscofpmf", + "sscounterenw", "ssnpm", "ssstateen", "sstc", + "sstvala", "sstvecd", "ssu64xl", "svade", + "svinval", "svnapot", "svpbmt", "za64rs", + "zawrs", "zba", "zbb", "zbc", "zbs", "zca", + "zcb", "zcd", "zcmop", "zfa", "zfbfmin", + "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", + "ziccamoa", "ziccif", "zicclsm", "ziccrse", "zicntr", + "zicond", "zicsr", "zifencei", "zihintntl", + "zihintpause", "zihpm", "zimop", "zkt", "zvbb", + "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", + "zvkned", "zvkng", "zvknha", "zvknhb", "zvks", + "zvksc", "zvksed", "zvksg", "zvksh", "zvkt"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <256>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <256>; + next-level-cache = <&l2_cache1>; + mmu-type = "riscv,sv39"; + + cpu5_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu_6: cpu@6 { + compatible = "spacemit,x100", "riscv"; + device_type = "cpu"; + reg = <6>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "h", + "sha", "shcounterenw", "shgatpa", "shtvala", + "shvsatpa", "shvstvala", "shvstvecd", "smaia", + "smstateen", "ssaia", "ssccptr", "sscofpmf", + "sscounterenw", "ssnpm", "ssstateen", "sstc", + "sstvala", "sstvecd", "ssu64xl", "svade", + "svinval", "svnapot", "svpbmt", "za64rs", + "zawrs", "zba", "zbb", "zbc", "zbs", "zca", + "zcb", "zcd", "zcmop", "zfa", "zfbfmin", + "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", + "ziccamoa", "ziccif", "zicclsm", "ziccrse", "zicntr", + "zicond", "zicsr", "zifencei", "zihintntl", + "zihintpause", "zihpm", "zimop", "zkt", "zvbb", + "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", + "zvkned", "zvkng", "zvknha", "zvknhb", "zvks", + "zvksc", "zvksed", "zvksg", "zvksh", "zvkt"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <256>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <256>; + next-level-cache = <&l2_cache1>; + mmu-type = "riscv,sv39"; + + cpu6_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu_7: cpu@7 { + compatible = "spacemit,x100", "riscv"; + device_type = "cpu"; + reg = <7>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "h", + "sha", "shcounterenw", "shgatpa", "shtvala", + "shvsatpa", "shvstvala", "shvstvecd", "smaia", + "smstateen", "ssaia", "ssccptr", "sscofpmf", + "sscounterenw", "ssnpm", "ssstateen", "sstc", + "sstvala", "sstvecd", "ssu64xl", "svade", + "svinval", "svnapot", "svpbmt", "za64rs", + "zawrs", "zba", "zbb", "zbc", "zbs", "zca", + "zcb", "zcd", "zcmop", "zfa", "zfbfmin", + "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", + "ziccamoa", "ziccif", "zicclsm", "ziccrse", "zicntr", + "zicond", "zicsr", "zifencei", "zihintntl", + "zihintpause", "zihpm", "zimop", "zkt", "zvbb", + "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", + "zvkned", "zvkng", "zvknha", "zvknhb", "zvks", + "zvksc", "zvksed", "zvksg", "zvksh", "zvkt"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <256>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <256>; + next-level-cache = <&l2_cache1>; + mmu-type = "riscv,sv39"; + + cpu7_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + l2_cache0: cache-controller-0 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-size = <4194304>; + cache-sets = <4096>; + cache-unified; + }; + + l2_cache1: cache-controller-1 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-size = <4194304>; + cache-sets = <4096>; + cache-unified; + }; + + cpu-map { + cluster0 { + core0 { + cpu = <&cpu_0>; + }; + core1 { + cpu = <&cpu_1>; + }; + core2 { + cpu = <&cpu_2>; + }; + core3 { + cpu = <&cpu_3>; + }; + }; + + cluster1 { + core0 { + cpu = <&cpu_4>; + }; + core1 { + cpu = <&cpu_5>; + }; + core2 { + cpu = <&cpu_6>; + }; + core3 { + cpu = <&cpu_7>; + }; + }; + }; + }; + + clocks { + vctcxo_1m: clock-1m { + compatible = "fixed-clock"; + clock-frequency = <1000000>; + clock-output-names = "vctcxo_1m"; + #clock-cells = <0>; + }; + + vctcxo_24m: clock-24m { + compatible = "fixed-clock"; + clock-frequency = <24000000>; + clock-output-names = "vctcxo_24m"; + #clock-cells = <0>; + }; + + vctcxo_3m: clock-3m { + compatible = "fixed-clock"; + clock-frequency = <3000000>; + clock-output-names = "vctcxo_3m"; + #clock-cells = <0>; + }; + + osc_32k: clock-32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "osc_32k"; + #clock-cells = <0>; + }; + }; + + soc: soc { + compatible = "simple-bus"; + interrupt-parent = <&saplic>; + #address-cells = <2>; + #size-cells = <2>; + dma-noncoherent; + ranges; + + usb3d: usb@81a00000 { + compatible = "spacemit,k3-dwc3"; + reg = <0x0 0x81a00000 0x0 0x10000>; + interrupts = <149 IRQ_TYPE_LEVEL_HIGH>; + interrupt-parent = <&saplic>; + clocks = <&syscon_apmu CLK_APMU_USB3_PORTD_BUS>; + clock-names = "usbdrd30"; + resets = <&syscon_apmu RESET_APMU_USB3_D_AHB>, + <&syscon_apmu RESET_APMU_USB3_D_VCC>, + <&syscon_apmu RESET_APMU_USB3_D_PHY>; + reset-names = "ahb", "vcc", "phy"; + phys = <&usb3d_u2phy>, + <&combophy 4 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + phy_type = "utmi"; + snps,dis_enblslpm_quirk; + snps,dis_u2_susphy_quirk; + snps,dis_u3_susphy_quirk; + snps,dis-del-phy-power-chg-quirk; + snps,dis-tx-ipgap-linecheck-quirk; + snps,parkmode-disable-ss-quirk; + dr_mode = "host"; + status = "disabled"; + }; + + usb3d_u2phy: phy@81b00000 { + compatible = "spacemit,k3-usb2-phy"; + reg = <0x0 0x81b00000 0x0 0x200>; + clocks = <&syscon_apmu CLK_APMU_USB3_PORTD_BUS>; + #phy-cells = <0>; + status = "disabled"; + }; + + combophy: phy@81d00000 { + compatible = "spacemit,k3-combo-phy"; + reg = <0x0 0x81d00000 0x0 0x600000>; + #phy-cells = <2>; + spacemit,apb-spare = <&pll>; + status = "disabled"; + }; + + usb2_host: usb@c0a00000 { + compatible = "spacemit,k3-dwc3"; + reg = <0x0 0xc0a00000 0x0 0x10000>; + clocks = <&syscon_apmu CLK_APMU_USB2_BUS>; + clock-names = "usbdrd30"; + resets = <&syscon_apmu RESET_APMU_USB2_AHB>, + <&syscon_apmu RESET_APMU_USB2_VCC>, + <&syscon_apmu RESET_APMU_USB2_PHY>; + reset-names = "ahb", "vcc", "phy"; + interrupts = <105 IRQ_TYPE_LEVEL_HIGH>; + interrupt-parent = <&saplic>; + phys = <&usb2_phy>; + phy-names = "usb2-phy"; + phy_type = "utmi"; + snps,dis_enblslpm_quirk; + snps,dis_u2_susphy_quirk; + snps,dis-del-phy-power-chg-quirk; + snps,dis-tx-ipgap-linecheck-quirk; + dr_mode = "host"; + maximum-speed = "high-speed"; + status = "disabled"; + }; + + usb2_phy: phy@c0a20000 { + compatible = "spacemit,k3-usb2-phy"; + reg = <0x0 0xc0a20000 0x0 0x200>; + clocks = <&syscon_apmu CLK_APMU_USB2_BUS>; + #phy-cells = <0>; + status = "disabled"; + }; + + eth0: ethernet@cac80000 { + compatible = "spacemit,k3-dwmac", "snps,dwmac-5.40a"; + reg = <0x0 0xcac80000 0x0 0x2000>; + clocks = <&syscon_apmu CLK_APMU_EMAC0_BUS>, + <&syscon_apmu CLK_APMU_EMAC0_1588>, + <&syscon_apmu CLK_APMU_EMAC0_RGMII_TX>; + clock-names = "stmmaceth", "ptp_ref", "tx"; + interrupts = <131 IRQ_TYPE_LEVEL_HIGH>, + <276 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "macirq", "eth_wake_irq"; + resets = <&syscon_apmu RESET_APMU_EMAC0>; + reset-names = "stmmaceth"; + rx-fifo-depth = <8192>; + tx-fifo-depth = <8192>; + snps,multicast-filter-bins = <64>; + snps,perfect-filter-entries = <32>; + snps,aal; + snps,tso; + snps,txpbl = <8>; + snps,rxpbl = <8>; + snps,force_sf_dma_mode; + snps,axi-config = <&gmac0_axi_setup>; + spacemit,apmu = <&syscon_apmu 0x3e4 0x3e8>; + status = "disabled"; + + mdio { + compatible = "snps,dwmac-mdio"; + #address-cells = <1>; + #size-cells = <0>; + }; + + gmac0_axi_setup: stmmac-axi-config { + snps,wr_osr_lmt = <0xf>; + snps,rd_osr_lmt = <0xf>; + /* max axi burst len is 256 */ + snps,blen = <256 128 64 32 16 0 0>; + }; + }; + + eth1: ethernet@cac82000 { + compatible = "spacemit,k3-dwmac", "snps,dwmac-5.40a"; + reg = <0x0 0xcac82000 0x0 0x2000>; + clocks = <&syscon_apmu CLK_APMU_EMAC1_BUS>, + <&syscon_apmu CLK_APMU_EMAC1_1588>, + <&syscon_apmu CLK_APMU_EMAC1_RGMII_TX>; + clock-names = "stmmaceth", "ptp_ref", "tx"; + interrupts = <133 IRQ_TYPE_LEVEL_HIGH>, + <277 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "macirq", "eth_wake_irq"; + resets = <&syscon_apmu RESET_APMU_EMAC1>; + reset-names = "stmmaceth"; + rx-fifo-depth = <8192>; + tx-fifo-depth = <8192>; + snps,multicast-filter-bins = <64>; + snps,perfect-filter-entries = <32>; + snps,aal; + snps,tso; + snps,txpbl = <8>; + snps,rxpbl = <8>; + snps,force_sf_dma_mode; + snps,axi-config = <&gmac1_axi_setup>; + spacemit,apmu = <&syscon_apmu 0x3ec 0x3f0>; + status = "disabled"; + + mdio { + compatible = "snps,dwmac-mdio"; + #address-cells = <1>; + #size-cells = <0>; + }; + + gmac1_axi_setup: stmmac-axi-config { + snps,wr_osr_lmt = <0xf>; + snps,rd_osr_lmt = <0xf>; + /* max axi burst len is 256 */ + snps,blen = <256 128 64 32 16 0 0>; + }; + }; + + eth2: ethernet@cac8e000 { + compatible = "spacemit,k3-dwmac", "snps,dwmac-5.40a"; + reg = <0x0 0xcac8e000 0x0 0x2000>; + clocks = <&syscon_apmu CLK_APMU_EMAC2_BUS>, + <&syscon_apmu CLK_APMU_EMAC2_1588>, + <&syscon_apmu CLK_APMU_EMAC2_RGMII_TX>; + clock-names = "stmmaceth", "ptp_ref", "tx"; + interrupts = <130 IRQ_TYPE_LEVEL_HIGH>, + <278 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "macirq", "eth_wake_irq"; + resets = <&syscon_apmu RESET_APMU_EMAC2>; + reset-names = "stmmaceth"; + rx-fifo-depth = <4096>; + tx-fifo-depth = <4096>; + snps,multicast-filter-bins = <64>; + snps,perfect-filter-entries = <32>; + snps,aal; + snps,tso; + snps,txpbl = <8>; + snps,rxpbl = <8>; + snps,force_sf_dma_mode; + snps,axi-config = <&gmac2_axi_setup>; + spacemit,apmu = <&syscon_apmu 0x248 0x24c>; + status = "disabled"; + + mdio { + compatible = "snps,dwmac-mdio"; + #address-cells = <1>; + #size-cells = <0>; + }; + + gmac2_axi_setup: stmmac-axi-config { + snps,wr_osr_lmt = <0xf>; + snps,rd_osr_lmt = <0xf>; + /* max axi burst len is 256 */ + snps,blen = <256 128 64 32 16 0 0>; + }; + }; + + pdma: dma-controller@d4000000 { + compatible = "spacemit,k3-pdma"; + reg = <0x0 0xd4000000 0x0 0x4000>; + clocks = <&syscon_apmu CLK_APMU_DMA>; + resets = <&syscon_apmu RESET_APMU_DMA>; + interrupts = <72 IRQ_TYPE_LEVEL_HIGH>; + dma-channels = <16>; + #dma-cells = <1>; + status = "disabled"; + }; + + i2c0: i2c@d4010800 { + compatible = "spacemit,k3-i2c", "spacemit,k1-i2c"; + reg = <0x0 0xd4010800 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <36 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&syscon_apbc CLK_APBC_TWSI0>, + <&syscon_apbc CLK_APBC_TWSI0_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_APBC_TWSI0>; + status = "disabled"; + }; + + i2c1: i2c@d4011000 { + compatible = "spacemit,k3-i2c", "spacemit,k1-i2c"; + reg = <0x0 0xd4011000 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <37 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&syscon_apbc CLK_APBC_TWSI1>, + <&syscon_apbc CLK_APBC_TWSI1_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_APBC_TWSI1>; + status = "disabled"; + }; + + i2c2: i2c@d4012000 { + compatible = "spacemit,k3-i2c", "spacemit,k1-i2c"; + reg = <0x0 0xd4012000 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <38 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&syscon_apbc CLK_APBC_TWSI2>, + <&syscon_apbc CLK_APBC_TWSI2_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_APBC_TWSI2>; + status = "disabled"; + }; + + i2c4: i2c@d4012800 { + compatible = "spacemit,k3-i2c", "spacemit,k1-i2c"; + reg = <0x0 0xd4012800 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <40 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&syscon_apbc CLK_APBC_TWSI4>, + <&syscon_apbc CLK_APBC_TWSI4_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_APBC_TWSI4>; + status = "disabled"; + }; + + i2c5: i2c@d4013800 { + compatible = "spacemit,k3-i2c", "spacemit,k1-i2c"; + reg = <0x0 0xd4013800 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <41 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&syscon_apbc CLK_APBC_TWSI5>, + <&syscon_apbc CLK_APBC_TWSI5_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_APBC_TWSI5>; + status = "disabled"; + }; + + syscon_apbc: system-controller@d4015000 { + compatible = "spacemit,k3-syscon-apbc"; + reg = <0x0 0xd4015000 0x0 0x1000>; + clocks = <&osc_32k>, <&vctcxo_1m>, <&vctcxo_3m>, <&vctcxo_24m>; + clock-names = "osc", "vctcxo_1m", "vctcxo_3m", "vctcxo_24m"; + #clock-cells = <1>; + #reset-cells = <1>; + }; + + uart0: serial@d4017000 { + compatible = "spacemit,k3-uart", "intel,xscale-uart"; + reg = <0x0 0xd4017000 0x0 0x100>; + reg-shift = <2>; + reg-io-width = <4>; + clocks = <&syscon_apbc CLK_APBC_UART0>, + <&syscon_apbc CLK_APBC_UART0_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_UART0>; + interrupts = <42 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + uart2: serial@d4017100 { + compatible = "spacemit,k3-uart", "intel,xscale-uart"; + reg = <0x0 0xd4017100 0x0 0x100>; + reg-shift = <2>; + reg-io-width = <4>; + clocks = <&syscon_apbc CLK_APBC_UART2>, + <&syscon_apbc CLK_APBC_UART2_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_UART2>; + interrupts = <44 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + uart3: serial@d4017200 { + compatible = "spacemit,k3-uart", "intel,xscale-uart"; + reg = <0x0 0xd4017200 0x0 0x100>; + reg-shift = <2>; + reg-io-width = <4>; + clocks = <&syscon_apbc CLK_APBC_UART3>, + <&syscon_apbc CLK_APBC_UART3_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_UART3>; + interrupts = <45 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + uart4: serial@d4017300 { + compatible = "spacemit,k3-uart", "intel,xscale-uart"; + reg = <0x0 0xd4017300 0x0 0x100>; + reg-shift = <2>; + reg-io-width = <4>; + clocks = <&syscon_apbc CLK_APBC_UART4>, + <&syscon_apbc CLK_APBC_UART4_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_UART4>; + interrupts = <46 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + uart5: serial@d4017400 { + compatible = "spacemit,k3-uart", "intel,xscale-uart"; + reg = <0x0 0xd4017400 0x0 0x100>; + reg-shift = <2>; + reg-io-width = <4>; + clocks = <&syscon_apbc CLK_APBC_UART5>, + <&syscon_apbc CLK_APBC_UART5_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_UART5>; + interrupts = <47 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + uart6: serial@d4017500 { + compatible = "spacemit,k3-uart", "intel,xscale-uart"; + reg = <0x0 0xd4017500 0x0 0x100>; + reg-shift = <2>; + reg-io-width = <4>; + clocks = <&syscon_apbc CLK_APBC_UART6>, + <&syscon_apbc CLK_APBC_UART6_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_UART6>; + interrupts = <48 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + uart7: serial@d4017600 { + compatible = "spacemit,k3-uart", "intel,xscale-uart"; + reg = <0x0 0xd4017600 0x0 0x100>; + reg-shift = <2>; + reg-io-width = <4>; + clocks = <&syscon_apbc CLK_APBC_UART7>, + <&syscon_apbc CLK_APBC_UART7_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_UART7>; + interrupts = <49 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + uart8: serial@d4017700 { + compatible = "spacemit,k3-uart", "intel,xscale-uart"; + reg = <0x0 0xd4017700 0x0 0x100>; + reg-shift = <2>; + reg-io-width = <4>; + clocks = <&syscon_apbc CLK_APBC_UART8>, + <&syscon_apbc CLK_APBC_UART8_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_UART8>; + interrupts = <50 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + uart9: serial@d4017800 { + compatible = "spacemit,k3-uart", "intel,xscale-uart"; + reg = <0x0 0xd4017800 0x0 0x100>; + reg-shift = <2>; + reg-io-width = <4>; + clocks = <&syscon_apbc CLK_APBC_UART9>, + <&syscon_apbc CLK_APBC_UART9_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_UART9>; + interrupts = <51 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + i2c6: i2c@d4018800 { + compatible = "spacemit,k3-i2c", "spacemit,k1-i2c"; + reg = <0x0 0xd4018800 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <70 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&syscon_apbc CLK_APBC_TWSI6>, + <&syscon_apbc CLK_APBC_TWSI6_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_APBC_TWSI6>; + status = "disabled"; + }; + + gpio: gpio@d4019000 { + compatible = "spacemit,k3-gpio"; + reg = <0x0 0xd4019000 0x0 0x100>; + clocks = <&syscon_apbc CLK_APBC_GPIO>, + <&syscon_apbc CLK_APBC_GPIO_BUS>; + clock-names = "core", "bus"; + gpio-controller; + #gpio-cells = <3>; + interrupts = <58 IRQ_TYPE_LEVEL_HIGH>; + interrupt-parent = <&saplic>; + interrupt-controller; + #interrupt-cells = <3>; + gpio-ranges = <&pinctrl 0 0 0 32>, + <&pinctrl 1 0 32 32>, + <&pinctrl 2 0 64 32>, + <&pinctrl 3 0 96 32>; + }; + + i2c8: i2c@d401d800 { + compatible = "spacemit,k3-i2c", "spacemit,k1-i2c"; + reg = <0x0 0xd401d800 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <19 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&syscon_apbc CLK_APBC_TWSI8>, + <&syscon_apbc CLK_APBC_TWSI8_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_APBC_TWSI8>; + status = "disabled"; + }; + + pwm0: pwm@d401a000 { + compatible = "spacemit,k3-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd401a000 0x0 0x10>; + clocks = <&syscon_apbc CLK_APBC_PWM0>, + <&syscon_apbc CLK_APBC_PWM0_BUS>; + clock-names = "func", "bus"; + resets = <&syscon_apbc RESET_APBC_PWM0>; + #pwm-cells = <3>; + status = "disabled"; + }; + + pwm1: pwm@d401a400 { + compatible = "spacemit,k3-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd401a400 0x0 0x10>; + clocks = <&syscon_apbc CLK_APBC_PWM1>, + <&syscon_apbc CLK_APBC_PWM1_BUS>; + clock-names = "func", "bus"; + resets = <&syscon_apbc RESET_APBC_PWM1>; + #pwm-cells = <3>; + status = "disabled"; + }; + + pwm2: pwm@d401a800 { + compatible = "spacemit,k3-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd401a800 0x0 0x10>; + clocks = <&syscon_apbc CLK_APBC_PWM2>, + <&syscon_apbc CLK_APBC_PWM2_BUS>; + clock-names = "func", "bus"; + resets = <&syscon_apbc RESET_APBC_PWM2>; + #pwm-cells = <3>; + status = "disabled"; + }; + + pwm3: pwm@d401ac00 { + compatible = "spacemit,k3-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd401ac00 0x0 0x10>; + clocks = <&syscon_apbc CLK_APBC_PWM3>, + <&syscon_apbc CLK_APBC_PWM3_BUS>; + clock-names = "func", "bus"; + resets = <&syscon_apbc RESET_APBC_PWM3>; + #pwm-cells = <3>; + status = "disabled"; + }; + + pwm4: pwm@d401b000 { + compatible = "spacemit,k3-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd401b000 0x0 0x10>; + clocks = <&syscon_apbc CLK_APBC_PWM4>, + <&syscon_apbc CLK_APBC_PWM4_BUS>; + clock-names = "func", "bus"; + resets = <&syscon_apbc RESET_APBC_PWM4>; + #pwm-cells = <3>; + status = "disabled"; + }; + + pwm5: pwm@d401b400 { + compatible = "spacemit,k3-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd401b400 0x0 0x10>; + clocks = <&syscon_apbc CLK_APBC_PWM5>, + <&syscon_apbc CLK_APBC_PWM5_BUS>; + clock-names = "func", "bus"; + resets = <&syscon_apbc RESET_APBC_PWM5>; + #pwm-cells = <3>; + status = "disabled"; + }; + + pwm6: pwm@d401b800 { + compatible = "spacemit,k3-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd401b800 0x0 0x10>; + clocks = <&syscon_apbc CLK_APBC_PWM6>, + <&syscon_apbc CLK_APBC_PWM6_BUS>; + clock-names = "func", "bus"; + resets = <&syscon_apbc RESET_APBC_PWM6>; + #pwm-cells = <3>; + status = "disabled"; + }; + + pwm7: pwm@d401bc00 { + compatible = "spacemit,k3-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd401bc00 0x0 0x10>; + clocks = <&syscon_apbc CLK_APBC_PWM7>, + <&syscon_apbc CLK_APBC_PWM7_BUS>; + clock-names = "func", "bus"; + resets = <&syscon_apbc RESET_APBC_PWM7>; + #pwm-cells = <3>; + status = "disabled"; + }; + + spi3: spi@d401c000 { + compatible = "spacemit,k3-spi", "spacemit,k1-spi"; + reg = <0x0 0xd401c000 0x0 0x30>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&syscon_apbc CLK_APBC_SPI3>, + <&syscon_apbc CLK_APBC_SPI3_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_SPI3>; + interrupts = <55 IRQ_TYPE_LEVEL_HIGH>; + dmas = <&pdma K3_PDMA_SSP3_RX>, <&pdma K3_PDMA_SSP3_TX>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + pwm8: pwm@d4020000 { + compatible = "spacemit,k3-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4020000 0x0 0x10>; + clocks = <&syscon_apbc CLK_APBC_PWM8>, + <&syscon_apbc CLK_APBC_PWM8_BUS>; + clock-names = "func", "bus"; + resets = <&syscon_apbc RESET_APBC_PWM8>; + #pwm-cells = <3>; + status = "disabled"; + }; + + pwm9: pwm@d4020400 { + compatible = "spacemit,k3-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4020400 0x0 0x10>; + clocks = <&syscon_apbc CLK_APBC_PWM9>, + <&syscon_apbc CLK_APBC_PWM9_BUS>; + clock-names = "func", "bus"; + resets = <&syscon_apbc RESET_APBC_PWM9>; + #pwm-cells = <3>; + status = "disabled"; + }; + + pwm10: pwm@d4020800 { + compatible = "spacemit,k3-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4020800 0x0 0x10>; + clocks = <&syscon_apbc CLK_APBC_PWM10>, + <&syscon_apbc CLK_APBC_PWM10_BUS>; + clock-names = "func", "bus"; + resets = <&syscon_apbc RESET_APBC_PWM10>; + #pwm-cells = <3>; + status = "disabled"; + }; + + pwm11: pwm@d4020c00 { + compatible = "spacemit,k3-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4020c00 0x0 0x10>; + clocks = <&syscon_apbc CLK_APBC_PWM11>, + <&syscon_apbc CLK_APBC_PWM11_BUS>; + clock-names = "func", "bus"; + resets = <&syscon_apbc RESET_APBC_PWM11>; + #pwm-cells = <3>; + status = "disabled"; + }; + + pwm12: pwm@d4021000 { + compatible = "spacemit,k3-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4021000 0x0 0x10>; + clocks = <&syscon_apbc CLK_APBC_PWM12>, + <&syscon_apbc CLK_APBC_PWM12_BUS>; + clock-names = "func", "bus"; + resets = <&syscon_apbc RESET_APBC_PWM12>; + #pwm-cells = <3>; + status = "disabled"; + }; + + pwm13: pwm@d4021400 { + compatible = "spacemit,k3-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4021400 0x0 0x10>; + clocks = <&syscon_apbc CLK_APBC_PWM13>, + <&syscon_apbc CLK_APBC_PWM13_BUS>; + clock-names = "func", "bus"; + resets = <&syscon_apbc RESET_APBC_PWM13>; + #pwm-cells = <3>; + status = "disabled"; + }; + + pwm14: pwm@d4021800 { + compatible = "spacemit,k3-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4021800 0x0 0x10>; + clocks = <&syscon_apbc CLK_APBC_PWM14>, + <&syscon_apbc CLK_APBC_PWM14_BUS>; + clock-names = "func", "bus"; + resets = <&syscon_apbc RESET_APBC_PWM14>; + #pwm-cells = <3>; + status = "disabled"; + }; + + pwm15: pwm@d4021c00 { + compatible = "spacemit,k3-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4021c00 0x0 0x10>; + clocks = <&syscon_apbc CLK_APBC_PWM15>, + <&syscon_apbc CLK_APBC_PWM15_BUS>; + clock-names = "func", "bus"; + resets = <&syscon_apbc RESET_APBC_PWM15>; + #pwm-cells = <3>; + status = "disabled"; + }; + + pwm16: pwm@d4022000 { + compatible = "spacemit,k3-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4022000 0x0 0x10>; + clocks = <&syscon_apbc CLK_APBC_PWM16>, + <&syscon_apbc CLK_APBC_PWM16_BUS>; + clock-names = "func", "bus"; + resets = <&syscon_apbc RESET_APBC_PWM16>; + #pwm-cells = <3>; + status = "disabled"; + }; + + pwm17: pwm@d4022400 { + compatible = "spacemit,k3-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4022400 0x0 0x10>; + clocks = <&syscon_apbc CLK_APBC_PWM17>, + <&syscon_apbc CLK_APBC_PWM17_BUS>; + clock-names = "func", "bus"; + resets = <&syscon_apbc RESET_APBC_PWM17>; + #pwm-cells = <3>; + status = "disabled"; + }; + + pwm18: pwm@d4022800 { + compatible = "spacemit,k3-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4022800 0x0 0x10>; + clocks = <&syscon_apbc CLK_APBC_PWM18>, + <&syscon_apbc CLK_APBC_PWM18_BUS>; + clock-names = "func", "bus"; + resets = <&syscon_apbc RESET_APBC_PWM18>; + #pwm-cells = <3>; + status = "disabled"; + }; + + pwm19: pwm@d4022c00 { + compatible = "spacemit,k3-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4022c00 0x0 0x10>; + clocks = <&syscon_apbc CLK_APBC_PWM19>, + <&syscon_apbc CLK_APBC_PWM19_BUS>; + clock-names = "func", "bus"; + resets = <&syscon_apbc RESET_APBC_PWM19>; + #pwm-cells = <3>; + status = "disabled"; + }; + + pinctrl: pinctrl@d401e000 { + compatible = "spacemit,k3-pinctrl"; + reg = <0x0 0xd401e000 0x0 0x1000>; + clocks = <&syscon_apbc CLK_APBC_AIB>, + <&syscon_apbc CLK_APBC_AIB_BUS>; + clock-names = "func", "bus"; + spacemit,apbc = <&syscon_apbc>; + }; + + uart10: serial@d401f000 { + compatible = "spacemit,k3-uart", "intel,xscale-uart"; + reg = <0x0 0xd401f000 0x0 0x100>; + reg-shift = <2>; + reg-io-width = <4>; + clocks = <&syscon_apbc CLK_APBC_UART10>, + <&syscon_apbc CLK_APBC_UART10_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_UART10>; + interrupts = <281 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + spi0: spi@d4040000 { + compatible = "spacemit,k3-spi", "spacemit,k1-spi"; + reg = <0x0 0xd4040000 0x0 0x30>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&syscon_apbc CLK_APBC_SPI0>, + <&syscon_apbc CLK_APBC_SPI0_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_SPI0>; + interrupts = <283 IRQ_TYPE_LEVEL_HIGH>; + dmas = <&pdma K3_PDMA_SSP0_RX>, <&pdma K3_PDMA_SSP0_TX>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + spi1: spi@d4040800 { + compatible = "spacemit,k3-spi", "spacemit,k1-spi"; + reg = <0x0 0xd4040800 0x0 0x30>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&syscon_apbc CLK_APBC_SPI1>, + <&syscon_apbc CLK_APBC_SPI1_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_SPI1>; + interrupts = <282 IRQ_TYPE_LEVEL_HIGH>; + dmas = <&pdma K3_PDMA_SSP1_RX>, <&pdma K3_PDMA_SSP1_TX>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + syscon_mpmu: system-controller@d4050000 { + compatible = "spacemit,k3-syscon-mpmu"; + reg = <0x0 0xd4050000 0x0 0x10000>; + clocks = <&osc_32k>, <&vctcxo_1m>, <&vctcxo_3m>, <&vctcxo_24m>; + clock-names = "osc", "vctcxo_1m", "vctcxo_3m", "vctcxo_24m"; + #clock-cells = <1>; + #power-domain-cells = <1>; + #reset-cells = <1>; + }; + + pll: clock-controller@d4090000 { + compatible = "spacemit,k3-pll"; + reg = <0x0 0xd4090000 0x0 0x10000>; + clocks = <&vctcxo_24m>; + spacemit,mpmu = <&syscon_mpmu>; + #clock-cells = <1>; + }; + + qspi: spi@d420c000 { + compatible = "spacemit,k3-qspi", + "spacemit,k1-qspi"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0xd420c000 0x0 0x1000>, + <0x0 0xb8000000 0x0 0xc00000>; + reg-names = "QuadSPI", "QuadSPI-memory"; + clocks = <&syscon_apmu CLK_APMU_QSPI_BUS>, + <&syscon_apmu CLK_APMU_QSPI>; + clock-names = "qspi_en", "qspi"; + resets = <&syscon_apmu RESET_APMU_QSPI>, + <&syscon_apmu RESET_APMU_QSPI_BUS>; + interrupts = <117 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + sdhci0: mmc@d4280000 { + compatible = "spacemit,k3-sdhci"; + reg = <0x0 0xd4280000 0x0 0x200>; + clocks = <&syscon_apmu CLK_APMU_SDH_AXI>, + <&syscon_apmu CLK_APMU_SDH0>; + clock-names = "core", "io"; + resets = <&syscon_apmu RESET_APMU_SDH_AXI>, + <&syscon_apmu RESET_APMU_SDH0>; + reset-names = "axi", "sdh"; + interrupts = <99 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + syscon_apmu: system-controller@d4282800 { + compatible = "spacemit,k3-syscon-apmu"; + reg = <0x0 0xd4282800 0x0 0x400>; + clocks = <&osc_32k>, <&vctcxo_1m>, <&vctcxo_3m>, <&vctcxo_24m>; + clock-names = "osc", "vctcxo_1m", "vctcxo_3m", "vctcxo_24m"; + #clock-cells = <1>; + #power-domain-cells = <1>; + #reset-cells = <1>; + }; + + syscon_dciu: system-controller@d8440000 { + compatible = "spacemit,k3-syscon-dciu"; + reg = <0x0 0xd8440000 0x0 0xc000>; + #clock-cells = <1>; + #reset-cells = <1>; + }; + + simsic: interrupt-controller@e0400000 { + compatible = "spacemit,k3-imsics", "riscv,imsics"; + reg = <0x0 0xe0400000 0x0 0x200000>; + #interrupt-cells = <0>; + #msi-cells = <0>; + interrupt-controller; + interrupts-extended = <&cpu0_intc 9>, <&cpu1_intc 9>, + <&cpu2_intc 9>, <&cpu3_intc 9>, + <&cpu4_intc 9>, <&cpu5_intc 9>, + <&cpu6_intc 9>, <&cpu7_intc 9>; + msi-controller; + riscv,guest-index-bits = <6>; + riscv,hart-index-bits = <4>; + riscv,num-guest-ids = <511>; + riscv,num-ids = <511>; + }; + + saplic: interrupt-controller@e0804000 { + compatible = "spacemit,k3-aplic", "riscv,aplic"; + reg = <0x0 0xe0804000 0x0 0x4000>; + #interrupt-cells = <2>; + interrupt-controller; + msi-parent = <&simsic>; + riscv,num-sources = <512>; + }; + + clint: timer@e081c000 { + compatible = "spacemit,k3-clint", "sifive,clint0"; + reg = <0x0 0xe081c000 0x0 0x4000>; + interrupts-extended = <&cpu0_intc 3>, <&cpu0_intc 7>, + <&cpu1_intc 3>, <&cpu1_intc 7>, + <&cpu2_intc 3>, <&cpu2_intc 7>, + <&cpu3_intc 3>, <&cpu3_intc 7>, + <&cpu4_intc 3>, <&cpu4_intc 7>, + <&cpu5_intc 3>, <&cpu5_intc 7>, + <&cpu6_intc 3>, <&cpu6_intc 7>, + <&cpu7_intc 3>, <&cpu7_intc 7>; + }; + + /* sec_i2c3: 0xf0614000, not available from Linux */ + + mimsic: interrupt-controller@f1000000 { + compatible = "spacemit,k3-imsics", "riscv,imsics"; + reg = <0x0 0xf1000000 0x0 0x10000>; + #interrupt-cells = <0>; + #msi-cells = <0>; + interrupt-controller; + interrupts-extended = <&cpu0_intc 11>, <&cpu1_intc 11>, + <&cpu2_intc 11>, <&cpu3_intc 11>, + <&cpu4_intc 11>, <&cpu5_intc 11>, + <&cpu6_intc 11>, <&cpu7_intc 11>; + msi-controller; + riscv,guest-index-bits = <6>; + riscv,hart-index-bits = <4>; + riscv,num-guest-ids = <511>; + riscv,num-ids = <511>; + status = "reserved"; + }; + + maplic: interrupt-controller@f1800000 { + compatible = "spacemit,k3-aplic", "riscv,aplic"; + reg = <0x0 0xf1800000 0x0 0x4000>; + #interrupt-cells = <2>; + interrupt-controller; + msi-parent = <&mimsic>; + riscv,children = <&saplic>; + riscv,delegation = <&saplic 1 512>; + riscv,num-sources = <512>; + status = "reserved"; + }; + }; +}; diff --git a/arch/riscv/boot/dts/starfive/jh7110-common.dtsi b/arch/riscv/boot/dts/starfive/jh7110-common.dtsi index 8cfe8033305d..a7a1c09a2c90 100644 --- a/arch/riscv/boot/dts/starfive/jh7110-common.dtsi +++ b/arch/riscv/boot/dts/starfive/jh7110-common.dtsi @@ -135,29 +135,6 @@ clock-frequency = <49152000>; }; -&camss { - assigned-clocks = <&ispcrg JH7110_ISPCLK_DOM4_APB_FUNC>, - <&ispcrg JH7110_ISPCLK_MIPI_RX0_PXL>; - assigned-clock-rates = <49500000>, <198000000>; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - }; - - port@1 { - reg = <1>; - - camss_from_csi2rx: endpoint { - remote-endpoint = <&csi2rx_to_camss>; - }; - }; - }; -}; - &csi2rx { assigned-clocks = <&ispcrg JH7110_ISPCLK_VIN_SYS>; assigned-clock-rates = <297000000>; @@ -175,9 +152,7 @@ port@1 { reg = <1>; - csi2rx_to_camss: endpoint { - remote-endpoint = <&camss_from_csi2rx>; - }; + /* remote CAMSS endpoint */ }; }; }; diff --git a/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2-lite-emmc.dts b/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2-lite-emmc.dts index e27a662d4022..7544efa95de4 100644 --- a/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2-lite-emmc.dts +++ b/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2-lite-emmc.dts @@ -9,7 +9,7 @@ / { model = "StarFive VisionFive 2 Lite eMMC"; - compatible = "starfive,visionfive-2-lite-emmc", "starfive,jh7110s"; + compatible = "starfive,visionfive-2-lite-emmc", "starfive,jh7110s", "starfive,jh7110"; }; &mmc0 { diff --git a/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2-lite.dts b/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2-lite.dts index b96eea4fa7d5..b9913991a1b7 100644 --- a/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2-lite.dts +++ b/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2-lite.dts @@ -9,7 +9,7 @@ / { model = "StarFive VisionFive 2 Lite"; - compatible = "starfive,visionfive-2-lite", "starfive,jh7110s"; + compatible = "starfive,visionfive-2-lite", "starfive,jh7110s", "starfive,jh7110"; }; &mmc0 { diff --git a/arch/riscv/boot/dts/starfive/jh7110.dtsi b/arch/riscv/boot/dts/starfive/jh7110.dtsi index 6e56e9d20bb0..9c3e4598747e 100644 --- a/arch/riscv/boot/dts/starfive/jh7110.dtsi +++ b/arch/riscv/boot/dts/starfive/jh7110.dtsi @@ -1199,34 +1199,6 @@ #phy-cells = <0>; }; - camss: isp@19840000 { - compatible = "starfive,jh7110-camss"; - reg = <0x0 0x19840000 0x0 0x10000>, - <0x0 0x19870000 0x0 0x30000>; - reg-names = "syscon", "isp"; - clocks = <&ispcrg JH7110_ISPCLK_DOM4_APB_FUNC>, - <&ispcrg JH7110_ISPCLK_ISPV2_TOP_WRAPPER_C>, - <&ispcrg JH7110_ISPCLK_DVP_INV>, - <&ispcrg JH7110_ISPCLK_VIN_P_AXI_WR>, - <&ispcrg JH7110_ISPCLK_MIPI_RX0_PXL>, - <&syscrg JH7110_SYSCLK_ISP_TOP_CORE>, - <&syscrg JH7110_SYSCLK_ISP_TOP_AXI>; - clock-names = "apb_func", "wrapper_clk_c", "dvp_inv", - "axiwr", "mipi_rx0_pxl", "ispcore_2x", - "isp_axi"; - resets = <&ispcrg JH7110_ISPRST_ISPV2_TOP_WRAPPER_P>, - <&ispcrg JH7110_ISPRST_ISPV2_TOP_WRAPPER_C>, - <&ispcrg JH7110_ISPRST_VIN_P_AXI_RD>, - <&ispcrg JH7110_ISPRST_VIN_P_AXI_WR>, - <&syscrg JH7110_SYSRST_ISP_TOP>, - <&syscrg JH7110_SYSRST_ISP_TOP_AXI>; - reset-names = "wrapper_p", "wrapper_c", "axird", - "axiwr", "isp_top_n", "isp_top_axi"; - power-domains = <&pwrc JH7110_PD_ISP>; - interrupts = <92>, <87>, <90>, <88>; - status = "disabled"; - }; - voutcrg: clock-controller@295c0000 { compatible = "starfive,jh7110-voutcrg"; reg = <0x0 0x295c0000 0x0 0x10000>; diff --git a/arch/riscv/boot/dts/tenstorrent/blackhole.dtsi b/arch/riscv/boot/dts/tenstorrent/blackhole.dtsi index 6408810d8d80..5f709e45d9b2 100644 --- a/arch/riscv/boot/dts/tenstorrent/blackhole.dtsi +++ b/arch/riscv/boot/dts/tenstorrent/blackhole.dtsi @@ -77,6 +77,54 @@ }; }; + pmu { + compatible = "riscv,pmu"; + riscv,event-to-mhpmevent = + /* SBI_PMU_HW_CPU_CYCLES -> CPU cycles */ + <0x00001 0x00000000 0x0001>, + /* SBI_PMU_HW_INSTRUCTIONS -> Instructions executed */ + <0x00002 0x00000000 0x0004>, + /* SBI_PMU_HW_CACHE_REFERENCES -> I-cache/ITIM busy | D-cache/DTIM busy */ + <0x00003 0x00000000 0x1801>, + /* SBI_PMU_HW_CACHE_MISSES -> I-cache miss | D-cache miss */ + <0x00004 0x00000000 0x0302>, + /* SBI_PMU_HW_BRANCH_INSTRUCTIONS -> Conditional branch retired */ + <0x00005 0x00000000 0x4000>, + /* + * SBI_PMU_HW_BRANCH_MISSES -> + * Branch direction misprediction | Branch/jump target misprediction + */ + <0x00006 0x00000000 0x6001>, + /* L1D_READ_MISS -> Data cache miss or MMIO access */ + <0x10001 0x00000000 0x0202>, + /* L1D_WRITE_ACCESS -> Data cache write-back */ + <0x10002 0x00000000 0x0402>, + /* L1I_READ_MISS -> Instruction cache miss */ + <0x10009 0x00000000 0x0102>, + /* LL_READ_MISS -> UTLB miss */ + <0x10011 0x00000000 0x2002>, + /* DTLB_READ_MISS -> Data TLB miss */ + <0x10019 0x00000000 0x1002>, + /* ITLB_READ_MISS -> Instruction TLB miss */ + <0x10021 0x00000000 0x0802>; + riscv,event-to-mhpmcounters = + <0x00001 0x00001 0x01>, + <0x00002 0x00002 0x04>, + <0x00003 0x00006 0x78>, + <0x10001 0x10002 0x78>, + <0x10009 0x10009 0x78>, + <0x10011 0x10011 0x78>, + <0x10019 0x10019 0x78>, + <0x10021 0x10021 0x78>; + riscv,raw-event-to-mhpmcounters = + /* Class 0: Instruction Commit Events, bits 8-25 variant */ + <0x0 0x0 0xffffffff 0xfc0000ff 0x78>, + /* Class 1: Microarchitectural Events, bits 8-18 variant */ + <0x0 0x1 0xffffffff 0xfff800ff 0x78>, + /* Class 2: Memory System Events, bits 8-13 variant */ + <0x0 0x2 0xffffffff 0xffffc0ff 0x78>; + }; + soc { #address-cells = <2>; #size-cells = <2>; diff --git a/arch/riscv/boot/dts/thead/th1520-beaglev-ahead.dts b/arch/riscv/boot/dts/thead/th1520-beaglev-ahead.dts index 21c33f165ba9..e16484a47653 100644 --- a/arch/riscv/boot/dts/thead/th1520-beaglev-ahead.dts +++ b/arch/riscv/boot/dts/thead/th1520-beaglev-ahead.dts @@ -75,6 +75,22 @@ label = "led5"; }; }; + + hdmi-connector { + compatible = "hdmi-connector"; + type = "d"; + + port { + hdmi_con_in: endpoint { + remote-endpoint = <&hdmi_out_con>; + }; + }; + }; + + wifi_pwrseq: wifi-pwrseq { + compatible = "mmc-pwrseq-simple"; + reset-gpios = <&gpio2 31 GPIO_ACTIVE_LOW>; /* WL-REG-ON */ + }; }; &osc { @@ -228,6 +244,28 @@ slew-rate = <0>; }; }; + + wifi_pins: wifi-0 { + host-wake-pins { + pins = "GPIO2_25"; + function = "gpio"; + bias-disable; + drive-strength = <1>; + input-enable; + input-schmitt-disable; + slew-rate = <0>; + }; + + reg-on-pins { + pins = "GPIO2_31"; + function = "gpio"; + bias-disable; + drive-strength = <3>; + input-disable; + input-schmitt-disable; + slew-rate = <0>; + }; + }; }; &sdio0 { @@ -236,6 +274,41 @@ status = "okay"; }; +&sdio1 { + bus-width = <4>; + max-frequency = <198000000>; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_pins>; + #address-cells = <1>; + #size-cells = <0>; + non-removable; + keep-power-in-suspend; + mmc-pwrseq = <&wifi_pwrseq>; + status = "okay"; + + wifi@1 { + compatible = "cypress,cyw43012-fmac", "brcm,bcm4329-fmac"; + reg = <1>; + interrupt-parent = <&gpio2>; + interrupts = <25 IRQ_TYPE_EDGE_RISING>; + interrupt-names = "host-wake"; + }; +}; + +&dpu { + status = "okay"; +}; + +&hdmi { + status = "okay"; +}; + +&hdmi_out_port { + hdmi_out_con: endpoint { + remote-endpoint = <&hdmi_con_in>; + }; +}; + &uart0 { pinctrl-names = "default"; pinctrl-0 = <&uart0_pins>; diff --git a/arch/riscv/boot/dts/thead/th1520-lichee-pi-4a.dts b/arch/riscv/boot/dts/thead/th1520-lichee-pi-4a.dts index c58c2085ca92..1eb575aa9a74 100644 --- a/arch/riscv/boot/dts/thead/th1520-lichee-pi-4a.dts +++ b/arch/riscv/boot/dts/thead/th1520-lichee-pi-4a.dts @@ -4,6 +4,7 @@ */ #include "th1520-lichee-module-4a.dtsi" +#include <dt-bindings/gpio/gpio.h> / { model = "Sipeed Lichee Pi 4A"; @@ -29,6 +30,23 @@ stdout-path = "serial0:115200n8"; }; + hdmi-connector { + compatible = "hdmi-connector"; + type = "a"; + + port { + hdmi_con_in: endpoint { + remote-endpoint = <&hdmi_out_con>; + }; + }; + }; + + wifi_pwrseq: wifi-pwrseq { + compatible = "mmc-pwrseq-simple"; + reset-gpios = <&ioexp2 4 GPIO_ACTIVE_LOW>; + post-power-on-delay-ms = <200>; + }; + thermal-zones { cpu-thermal { polling-delay = <1000>; @@ -85,6 +103,81 @@ }; +&dpu { + status = "okay"; +}; + +&hdmi { + status = "okay"; +}; + +&hdmi_out_port { + hdmi_out_con: endpoint { + remote-endpoint = <&hdmi_con_in>; + }; +}; + +&i2c0 { + clock-frequency = <400000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_pins>; + status = "okay"; + + ioexp1: gpio@18 { + compatible = "nxp,pca9557"; + reg = <0x18>; + gpio-controller; + #gpio-cells = <2>; + gpio-line-names = "cam0_dvdd12", + "cam0_avdd28", + "cam0_dovdd18"; + }; +}; + +&i2c1 { + clock-frequency = <400000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_pins>; + status = "okay"; + + ioexp2: gpio@18 { + compatible = "nxp,pca9557"; + reg = <0x18>; + gpio-controller; + #gpio-cells = <2>; + gpio-line-names = "", + "cam0_reset", + "cam1_reset", + "cam2_reset", + "wl_host_wake_dev", + "bt_resetn", + "", + "bt_host_wake_dev"; + }; +}; + +&i2c3 { + clock-frequency = <400000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c3_pins>; + status = "okay"; + + ioexp3: gpio@18 { + compatible = "nxp,pca9557"; + reg = <0x18>; + gpio-controller; + #gpio-cells = <2>; + gpio-line-names = "tp0_rst", + "", + "", + "vcc5v_usb", + "vdd28_tp0", + "vdd33_lcd0", + "vdd18_lcd0", + "lcd0_reset"; + }; +}; + &padctrl0_apsys { fan_pins: fan-0 { pwm1-pins { @@ -98,6 +191,18 @@ }; }; + i2c3_pins: i2c3-0 { + i2c-pins { + pins = "I2C3_SCL", "I2C3_SDA"; + function = "i2c"; + bias-pull-up = <2100>; + drive-strength = <7>; + input-enable; + input-schmitt-enable; + slew-rate = <0>; + }; + }; + uart0_pins: uart0-0 { tx-pins { pins = "UART0_TXD"; @@ -121,6 +226,44 @@ }; }; +&padctrl1_apsys { + i2c0_pins: i2c0-0 { + i2c-pins { + pins = "I2C0_SCL", "I2C0_SDA"; + function = "i2c"; + bias-pull-up = <2100>; + drive-strength = <7>; + input-enable; + input-schmitt-enable; + slew-rate = <0>; + }; + }; + + i2c1_pins: i2c1-0 { + i2c-pins { + pins = "I2C1_SCL", "I2C1_SDA"; + function = "i2c"; + bias-pull-up = <2100>; + drive-strength = <7>; + input-enable; + input-schmitt-enable; + slew-rate = <0>; + }; + }; +}; + +&sdio1 { + #address-cells = <1>; + #size-cells = <0>; + bus-width = <4>; + max-frequency = <198000000>; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&wifi_pwrseq>; + non-removable; + status = "okay"; +}; + &uart0 { pinctrl-names = "default"; pinctrl-0 = <&uart0_pins>; diff --git a/arch/riscv/boot/dts/thead/th1520.dtsi b/arch/riscv/boot/dts/thead/th1520.dtsi index bd5d33840884..b01a94785d87 100644 --- a/arch/riscv/boot/dts/thead/th1520.dtsi +++ b/arch/riscv/boot/dts/thead/th1520.dtsi @@ -411,6 +411,36 @@ status = "disabled"; }; + i2c0: i2c@ffe7f20000 { + compatible = "thead,th1520-i2c", "snps,designware-i2c"; + reg = <0xff 0xe7f20000 0x0 0x4000>; + interrupts = <44 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&clk CLK_I2C0>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c1: i2c@ffe7f24000 { + compatible = "thead,th1520-i2c", "snps,designware-i2c"; + reg = <0xff 0xe7f24000 0x0 0x4000>; + interrupts = <45 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&clk CLK_I2C1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c4: i2c@ffe7f28000 { + compatible = "thead,th1520-i2c", "snps,designware-i2c"; + reg = <0xff 0xe7f28000 0x0 0x4000>; + interrupts = <48 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&clk CLK_I2C4>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + gpio@ffe7f34000 { compatible = "snps,dw-apb-gpio"; reg = <0xff 0xe7f34000 0x0 0x1000>; @@ -509,6 +539,16 @@ thead,pad-group = <3>; }; + i2c2: i2c@ffec00c000 { + compatible = "thead,th1520-i2c", "snps,designware-i2c"; + reg = <0xff 0xec00c000 0x0 0x4000>; + interrupts = <46 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&clk CLK_I2C2>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + uart2: serial@ffec010000 { compatible = "snps,dw-apb-uart"; reg = <0xff 0xec010000 0x0 0x4000>; @@ -520,6 +560,16 @@ status = "disabled"; }; + i2c3: i2c@ffec014000 { + compatible = "thead,th1520-i2c", "snps,designware-i2c"; + reg = <0xff 0xec014000 0x0 0x4000>; + interrupts = <47 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&clk CLK_I2C3>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + pwm: pwm@ffec01c000 { compatible = "thead,th1520-pwm"; reg = <0xff 0xec01c000 0x0 0x4000>; @@ -585,6 +635,72 @@ #clock-cells = <1>; }; + hdmi: hdmi@ffef540000 { + compatible = "thead,th1520-dw-hdmi"; + reg = <0xff 0xef540000 0x0 0x40000>; + reg-io-width = <4>; + interrupts = <111 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&clk_vo CLK_HDMI_PCLK>, + <&clk_vo CLK_HDMI_SFR>, + <&clk_vo CLK_HDMI_CEC>, + <&clk_vo CLK_HDMI_PIXCLK>; + clock-names = "iahb", "isfr", "cec", "pix"; + resets = <&rst TH1520_RESET_ID_HDMI>, + <&rst TH1520_RESET_ID_HDMI_APB>; + reset-names = "main", "apb"; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + hdmi_in: endpoint { + remote-endpoint = <&dpu_out_dp1>; + }; + }; + + hdmi_out_port: port@1 { + reg = <1>; + }; + }; + }; + + dpu: display@ffef600000 { + compatible = "thead,th1520-dc8200", "verisilicon,dc"; + reg = <0xff 0xef600000 0x0 0x100000>; + interrupts = <93 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&clk_vo CLK_DPU_CCLK>, + <&clk_vo CLK_DPU_ACLK>, + <&clk_vo CLK_DPU_HCLK>, + <&clk_vo CLK_DPU_PIXELCLK0>, + <&clk_vo CLK_DPU_PIXELCLK1>; + clock-names = "core", "axi", "ahb", "pix0", "pix1"; + resets = <&rst TH1520_RESET_ID_DPU_CORE>, + <&rst TH1520_RESET_ID_DPU_AXI>, + <&rst TH1520_RESET_ID_DPU_AHB>; + reset-names = "core", "axi", "ahb"; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + dpu_port1: port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dpu_out_dp1: endpoint@1 { + reg = <1>; + remote-endpoint = <&hdmi_in>; + }; + }; + }; + }; + dmac0: dma-controller@ffefc00000 { compatible = "snps,axi-dma-1.01a"; reg = <0xff 0xefc00000 0x0 0x1000>; @@ -659,6 +775,16 @@ status = "disabled"; }; + i2c5: i2c@fff7f2c000 { + compatible = "thead,th1520-i2c", "snps,designware-i2c"; + reg = <0xff 0xf7f2c000 0x0 0x4000>; + interrupts = <49 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&clk CLK_I2C5>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + timer4: timer@ffffc33000 { compatible = "snps,dw-apb-timer"; reg = <0xff 0xffc33000 0x0 0x14>; @@ -753,6 +879,10 @@ reg-names = "common", "ts", "pd", "vm"; clocks = <&aonsys_clk>; #thermal-sensor-cells = <1>; + moortec,ts-coeff-g = <42740>; + moortec,ts-coeff-h = <220500>; + moortec,ts-coeff-j = <(-160)>; + moortec,ts-coeff-cal5 = <4094>; }; gpio@fffff52000 { diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index cd736a1d657e..04ae305d5511 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -24,6 +24,7 @@ CONFIG_BLK_DEV_INITRD=y CONFIG_PROFILING=y CONFIG_ARCH_ANDES=y CONFIG_ARCH_ANLOGIC=y +CONFIG_ARCH_ESWIN=y CONFIG_ARCH_MICROCHIP=y CONFIG_ARCH_SIFIVE=y CONFIG_ARCH_SOPHGO=y @@ -32,6 +33,7 @@ CONFIG_SOC_STARFIVE=y CONFIG_ARCH_SUNXI=y CONFIG_ARCH_TENSTORRENT=y CONFIG_ARCH_THEAD=y +CONFIG_ARCH_ULTRARISC=y CONFIG_ARCH_VIRT=y CONFIG_ARCH_CANAAN=y CONFIG_SMP=y @@ -173,6 +175,8 @@ CONFIG_PINCTRL_SOPHGO_SG2002=y CONFIG_GPIO_DWAPB=y CONFIG_GPIO_SIFIVE=y CONFIG_GPIO_SPACEMIT_K1=y +CONFIG_GPIO_PCA953X=m +CONFIG_GPIO_PCA953X_IRQ=y CONFIG_POWER_RESET_GPIO_RESTART=y CONFIG_SENSORS_SFCTEMP=m CONFIG_CPU_THERMAL=y @@ -255,8 +259,8 @@ CONFIG_CLK_SOPHGO_CV1800=y CONFIG_CLK_SOPHGO_SG2042_PLL=y CONFIG_CLK_SOPHGO_SG2042_CLKGEN=y CONFIG_CLK_SOPHGO_SG2042_RPGATE=y -CONFIG_SPACEMIT_CCU=y CONFIG_SPACEMIT_K1_CCU=y +CONFIG_SPACEMIT_K3_CCU=y CONFIG_SUN8I_DE2_CCU=m CONFIG_SUN50I_IOMMU=y CONFIG_RPMSG_CHAR=y @@ -295,7 +299,7 @@ CONFIG_NFS_V4_2=y CONFIG_ROOT_NFS=y CONFIG_9P_FS=y CONFIG_NLS_CODEPAGE_437=y -CONFIG_NLS_ISO8859_1=m +CONFIG_NLS_ISO8859_1=y CONFIG_SECURITY=y CONFIG_SECURITY_SELINUX=y CONFIG_SECURITY_APPARMOR=y diff --git a/arch/riscv/configs/hardening.config b/arch/riscv/configs/hardening.config new file mode 100644 index 000000000000..089f4cee82f4 --- /dev/null +++ b/arch/riscv/configs/hardening.config @@ -0,0 +1,4 @@ +# RISCV specific kernel hardening options + +# Enable control flow integrity support for usermode. +CONFIG_RISCV_USER_CFI=y diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig index 14c5acb935e9..6905232ddb03 100644 --- a/arch/riscv/crypto/Kconfig +++ b/arch/riscv/crypto/Kconfig @@ -6,11 +6,9 @@ config CRYPTO_AES_RISCV64 tristate "Ciphers: AES, modes: ECB, CBC, CTS, CTR, XTS" depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS - select CRYPTO_ALGAPI select CRYPTO_LIB_AES select CRYPTO_SKCIPHER help - Block cipher: AES cipher algorithms Length-preserving ciphers: AES with ECB, CBC, CTS, CTR, XTS Architecture: riscv64 using: @@ -19,30 +17,6 @@ config CRYPTO_AES_RISCV64 - Zvkb vector crypto extension (CTR) - Zvkg vector crypto extension (XTS) -config CRYPTO_GHASH_RISCV64 - tristate "Hash functions: GHASH" - depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ - RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS - select CRYPTO_GCM - help - GCM GHASH function (NIST SP 800-38D) - - Architecture: riscv64 using: - - Zvkg vector crypto extension - -config CRYPTO_SM3_RISCV64 - tristate "Hash functions: SM3 (ShangMi 3)" - depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ - RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS - select CRYPTO_HASH - select CRYPTO_LIB_SM3 - help - SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012) - - Architecture: riscv64 using: - - Zvksh vector crypto extension - - Zvkb vector crypto extension - config CRYPTO_SM4_RISCV64 tristate "Ciphers: SM4 (ShangMi 4)" depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile index 183495a95cc0..8cf31db57fc4 100644 --- a/arch/riscv/crypto/Makefile +++ b/arch/riscv/crypto/Makefile @@ -4,11 +4,5 @@ obj-$(CONFIG_CRYPTO_AES_RISCV64) += aes-riscv64.o aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \ aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o -obj-$(CONFIG_CRYPTO_GHASH_RISCV64) += ghash-riscv64.o -ghash-riscv64-y := ghash-riscv64-glue.o ghash-riscv64-zvkg.o - -obj-$(CONFIG_CRYPTO_SM3_RISCV64) += sm3-riscv64.o -sm3-riscv64-y := sm3-riscv64-glue.o sm3-riscv64-zvksh-zvkb.o - obj-$(CONFIG_CRYPTO_SM4_RISCV64) += sm4-riscv64.o sm4-riscv64-y := sm4-riscv64-glue.o sm4-riscv64-zvksed-zvkb.o diff --git a/arch/riscv/crypto/aes-macros.S b/arch/riscv/crypto/aes-macros.S index d1a258d04bc7..1384164621a5 100644 --- a/arch/riscv/crypto/aes-macros.S +++ b/arch/riscv/crypto/aes-macros.S @@ -51,8 +51,10 @@ // - If AES-256, loads round keys into v1-v15 and continues onwards. // // Also sets vl=4 and vtype=e32,m1,ta,ma. Clobbers t0 and t1. -.macro aes_begin keyp, label128, label192 +.macro aes_begin keyp, label128, label192, key_len +.ifb \key_len lwu t0, 480(\keyp) // t0 = key length in bytes +.endif li t1, 24 // t1 = key length for AES-192 vsetivli zero, 4, e32, m1, ta, ma vle32.v v1, (\keyp) @@ -76,12 +78,20 @@ vle32.v v10, (\keyp) addi \keyp, \keyp, 16 vle32.v v11, (\keyp) +.ifb \key_len blt t0, t1, \label128 // If AES-128, goto label128. +.else + blt \key_len, t1, \label128 // If AES-128, goto label128. +.endif addi \keyp, \keyp, 16 vle32.v v12, (\keyp) addi \keyp, \keyp, 16 vle32.v v13, (\keyp) +.ifb \key_len beq t0, t1, \label192 // If AES-192, goto label192. +.else + beq \key_len, t1, \label192 // If AES-192, goto label192. +.endif // Else, it's AES-256. addi \keyp, \keyp, 16 vle32.v v14, (\keyp) diff --git a/arch/riscv/crypto/aes-riscv64-glue.c b/arch/riscv/crypto/aes-riscv64-glue.c index f814ee048555..bbd920c9e29d 100644 --- a/arch/riscv/crypto/aes-riscv64-glue.c +++ b/arch/riscv/crypto/aes-riscv64-glue.c @@ -1,7 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * AES using the RISC-V vector crypto extensions. Includes the bare block - * cipher and the ECB, CBC, CBC-CTS, CTR, and XTS modes. + * AES modes using the RISC-V vector crypto extensions * * Copyright (C) 2023 VRULL GmbH * Author: Heiko Stuebner <heiko.stuebner@vrull.eu> @@ -15,21 +14,14 @@ #include <asm/simd.h> #include <asm/vector.h> #include <crypto/aes.h> -#include <crypto/internal/cipher.h> #include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> #include <crypto/xts.h> #include <linux/linkage.h> +#include <linux/minmax.h> #include <linux/module.h> -asmlinkage void aes_encrypt_zvkned(const struct crypto_aes_ctx *key, - const u8 in[AES_BLOCK_SIZE], - u8 out[AES_BLOCK_SIZE]); -asmlinkage void aes_decrypt_zvkned(const struct crypto_aes_ctx *key, - const u8 in[AES_BLOCK_SIZE], - u8 out[AES_BLOCK_SIZE]); - asmlinkage void aes_ecb_encrypt_zvkned(const struct crypto_aes_ctx *key, const u8 *in, u8 *out, size_t len); asmlinkage void aes_ecb_decrypt_zvkned(const struct crypto_aes_ctx *key, @@ -86,14 +78,6 @@ static int riscv64_aes_setkey(struct crypto_aes_ctx *ctx, return aes_expandkey(ctx, key, keylen); } -static int riscv64_aes_setkey_cipher(struct crypto_tfm *tfm, - const u8 *key, unsigned int keylen) -{ - struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); - - return riscv64_aes_setkey(ctx, key, keylen); -} - static int riscv64_aes_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { @@ -102,34 +86,6 @@ static int riscv64_aes_setkey_skcipher(struct crypto_skcipher *tfm, return riscv64_aes_setkey(ctx, key, keylen); } -/* Bare AES, without a mode of operation */ - -static void riscv64_aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) -{ - const struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); - - if (crypto_simd_usable()) { - kernel_vector_begin(); - aes_encrypt_zvkned(ctx, src, dst); - kernel_vector_end(); - } else { - aes_encrypt(ctx, dst, src); - } -} - -static void riscv64_aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) -{ - const struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); - - if (crypto_simd_usable()) { - kernel_vector_begin(); - aes_decrypt_zvkned(ctx, src, dst); - kernel_vector_end(); - } else { - aes_decrypt(ctx, dst, src); - } -} - /* AES-ECB */ static inline int riscv64_aes_ecb_crypt(struct skcipher_request *req, bool enc) @@ -311,8 +267,7 @@ static int riscv64_aes_ctr_crypt(struct skcipher_request *req) * operation into two at the point where the overflow * will occur. After the first part, add the carry bit. */ - p1_nbytes = min_t(unsigned int, nbytes, - (nblocks - ctr32) * AES_BLOCK_SIZE); + p1_nbytes = min(nbytes, (nblocks - ctr32) * AES_BLOCK_SIZE); aes_ctr32_crypt_zvkned_zvkb(ctx, walk.src.virt.addr, walk.dst.virt.addr, p1_nbytes, req->iv); @@ -338,7 +293,7 @@ static int riscv64_aes_ctr_crypt(struct skcipher_request *req) struct riscv64_aes_xts_ctx { struct crypto_aes_ctx ctx1; - struct crypto_aes_ctx ctx2; + struct aes_enckey tweak_key; }; static int riscv64_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, @@ -348,7 +303,7 @@ static int riscv64_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, return xts_verify_key(tfm, key, keylen) ?: riscv64_aes_setkey(&ctx->ctx1, key, keylen / 2) ?: - riscv64_aes_setkey(&ctx->ctx2, key + keylen / 2, keylen / 2); + aes_prepareenckey(&ctx->tweak_key, key + keylen / 2, keylen / 2); } static int riscv64_aes_xts_crypt(struct skcipher_request *req, bool enc) @@ -366,9 +321,7 @@ static int riscv64_aes_xts_crypt(struct skcipher_request *req, bool enc) return -EINVAL; /* Encrypt the IV with the tweak key to get the first tweak. */ - kernel_vector_begin(); - aes_encrypt_zvkned(&ctx->ctx2, req->iv, req->iv); - kernel_vector_end(); + aes_encrypt(&ctx->tweak_key, req->iv, req->iv); err = skcipher_walk_virt(&walk, req, false); @@ -456,23 +409,6 @@ static int riscv64_aes_xts_decrypt(struct skcipher_request *req) /* Algorithm definitions */ -static struct crypto_alg riscv64_zvkned_aes_cipher_alg = { - .cra_flags = CRYPTO_ALG_TYPE_CIPHER, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct crypto_aes_ctx), - .cra_priority = 300, - .cra_name = "aes", - .cra_driver_name = "aes-riscv64-zvkned", - .cra_cipher = { - .cia_min_keysize = AES_MIN_KEY_SIZE, - .cia_max_keysize = AES_MAX_KEY_SIZE, - .cia_setkey = riscv64_aes_setkey_cipher, - .cia_encrypt = riscv64_aes_encrypt, - .cia_decrypt = riscv64_aes_decrypt, - }, - .cra_module = THIS_MODULE, -}; - static struct skcipher_alg riscv64_zvkned_aes_skcipher_algs[] = { { .setkey = riscv64_aes_setkey_skcipher, @@ -574,15 +510,11 @@ static int __init riscv64_aes_mod_init(void) if (riscv_isa_extension_available(NULL, ZVKNED) && riscv_vector_vlen() >= 128) { - err = crypto_register_alg(&riscv64_zvkned_aes_cipher_alg); - if (err) - return err; - err = crypto_register_skciphers( riscv64_zvkned_aes_skcipher_algs, ARRAY_SIZE(riscv64_zvkned_aes_skcipher_algs)); if (err) - goto unregister_zvkned_cipher_alg; + return err; if (riscv_isa_extension_available(NULL, ZVKB)) { err = crypto_register_skcipher( @@ -607,8 +539,6 @@ unregister_zvkned_zvkb_skcipher_alg: unregister_zvkned_skcipher_algs: crypto_unregister_skciphers(riscv64_zvkned_aes_skcipher_algs, ARRAY_SIZE(riscv64_zvkned_aes_skcipher_algs)); -unregister_zvkned_cipher_alg: - crypto_unregister_alg(&riscv64_zvkned_aes_cipher_alg); return err; } @@ -620,7 +550,6 @@ static void __exit riscv64_aes_mod_exit(void) crypto_unregister_skcipher(&riscv64_zvkned_zvkb_aes_skcipher_alg); crypto_unregister_skciphers(riscv64_zvkned_aes_skcipher_algs, ARRAY_SIZE(riscv64_zvkned_aes_skcipher_algs)); - crypto_unregister_alg(&riscv64_zvkned_aes_cipher_alg); } module_init(riscv64_aes_mod_init); diff --git a/arch/riscv/crypto/aes-riscv64-zvkned.S b/arch/riscv/crypto/aes-riscv64-zvkned.S index 23d063f94ce6..d0fc4581a380 100644 --- a/arch/riscv/crypto/aes-riscv64-zvkned.S +++ b/arch/riscv/crypto/aes-riscv64-zvkned.S @@ -56,33 +56,6 @@ #define LEN a3 #define IVP a4 -.macro __aes_crypt_zvkned enc, keylen - vle32.v v16, (INP) - aes_crypt v16, \enc, \keylen - vse32.v v16, (OUTP) - ret -.endm - -.macro aes_crypt_zvkned enc - aes_begin KEYP, 128f, 192f - __aes_crypt_zvkned \enc, 256 -128: - __aes_crypt_zvkned \enc, 128 -192: - __aes_crypt_zvkned \enc, 192 -.endm - -// void aes_encrypt_zvkned(const struct crypto_aes_ctx *key, -// const u8 in[16], u8 out[16]); -SYM_FUNC_START(aes_encrypt_zvkned) - aes_crypt_zvkned 1 -SYM_FUNC_END(aes_encrypt_zvkned) - -// Same prototype and calling convention as the encryption function -SYM_FUNC_START(aes_decrypt_zvkned) - aes_crypt_zvkned 0 -SYM_FUNC_END(aes_decrypt_zvkned) - .macro __aes_ecb_crypt enc, keylen srli t0, LEN, 2 // t0 is the remaining length in 32-bit words. It's a multiple of 4. diff --git a/arch/riscv/crypto/ghash-riscv64-glue.c b/arch/riscv/crypto/ghash-riscv64-glue.c deleted file mode 100644 index d86073d25387..000000000000 --- a/arch/riscv/crypto/ghash-riscv64-glue.c +++ /dev/null @@ -1,146 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * GHASH using the RISC-V vector crypto extensions - * - * Copyright (C) 2023 VRULL GmbH - * Author: Heiko Stuebner <heiko.stuebner@vrull.eu> - * - * Copyright (C) 2023 SiFive, Inc. - * Author: Jerry Shih <jerry.shih@sifive.com> - */ - -#include <asm/simd.h> -#include <asm/vector.h> -#include <crypto/b128ops.h> -#include <crypto/gf128mul.h> -#include <crypto/ghash.h> -#include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> -#include <crypto/utils.h> -#include <linux/errno.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/string.h> - -asmlinkage void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data, - size_t len); - -struct riscv64_ghash_tfm_ctx { - be128 key; -}; - -struct riscv64_ghash_desc_ctx { - be128 accumulator; -}; - -static int riscv64_ghash_setkey(struct crypto_shash *tfm, const u8 *key, - unsigned int keylen) -{ - struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(tfm); - - if (keylen != GHASH_BLOCK_SIZE) - return -EINVAL; - - memcpy(&tctx->key, key, GHASH_BLOCK_SIZE); - - return 0; -} - -static int riscv64_ghash_init(struct shash_desc *desc) -{ - struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - *dctx = (struct riscv64_ghash_desc_ctx){}; - - return 0; -} - -static inline void -riscv64_ghash_blocks(const struct riscv64_ghash_tfm_ctx *tctx, - struct riscv64_ghash_desc_ctx *dctx, - const u8 *src, size_t srclen) -{ - /* The srclen is nonzero and a multiple of 16. */ - if (crypto_simd_usable()) { - kernel_vector_begin(); - ghash_zvkg(&dctx->accumulator, &tctx->key, src, srclen); - kernel_vector_end(); - } else { - do { - crypto_xor((u8 *)&dctx->accumulator, src, - GHASH_BLOCK_SIZE); - gf128mul_lle(&dctx->accumulator, &tctx->key); - src += GHASH_BLOCK_SIZE; - srclen -= GHASH_BLOCK_SIZE; - } while (srclen); - } -} - -static int riscv64_ghash_update(struct shash_desc *desc, const u8 *src, - unsigned int srclen) -{ - const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - riscv64_ghash_blocks(tctx, dctx, src, - round_down(srclen, GHASH_BLOCK_SIZE)); - return srclen - round_down(srclen, GHASH_BLOCK_SIZE); -} - -static int riscv64_ghash_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *out) -{ - const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - if (len) { - u8 buf[GHASH_BLOCK_SIZE] = {}; - - memcpy(buf, src, len); - riscv64_ghash_blocks(tctx, dctx, buf, GHASH_BLOCK_SIZE); - memzero_explicit(buf, sizeof(buf)); - } - - memcpy(out, &dctx->accumulator, GHASH_DIGEST_SIZE); - return 0; -} - -static struct shash_alg riscv64_ghash_alg = { - .init = riscv64_ghash_init, - .update = riscv64_ghash_update, - .finup = riscv64_ghash_finup, - .setkey = riscv64_ghash_setkey, - .descsize = sizeof(struct riscv64_ghash_desc_ctx), - .digestsize = GHASH_DIGEST_SIZE, - .base = { - .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct riscv64_ghash_tfm_ctx), - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_name = "ghash", - .cra_driver_name = "ghash-riscv64-zvkg", - .cra_module = THIS_MODULE, - }, -}; - -static int __init riscv64_ghash_mod_init(void) -{ - if (riscv_isa_extension_available(NULL, ZVKG) && - riscv_vector_vlen() >= 128) - return crypto_register_shash(&riscv64_ghash_alg); - - return -ENODEV; -} - -static void __exit riscv64_ghash_mod_exit(void) -{ - crypto_unregister_shash(&riscv64_ghash_alg); -} - -module_init(riscv64_ghash_mod_init); -module_exit(riscv64_ghash_mod_exit); - -MODULE_DESCRIPTION("GHASH (RISC-V accelerated)"); -MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>"); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("ghash"); diff --git a/arch/riscv/crypto/ghash-riscv64-zvkg.S b/arch/riscv/crypto/ghash-riscv64-zvkg.S deleted file mode 100644 index f2b43fb4d434..000000000000 --- a/arch/riscv/crypto/ghash-riscv64-zvkg.S +++ /dev/null @@ -1,72 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */ -// -// This file is dual-licensed, meaning that you can use it under your -// choice of either of the following two licenses: -// -// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. -// -// Licensed under the Apache License 2.0 (the "License"). You can obtain -// a copy in the file LICENSE in the source distribution or at -// https://www.openssl.org/source/license.html -// -// or -// -// Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu> -// Copyright (c) 2023, Jerry Shih <jerry.shih@sifive.com> -// Copyright 2024 Google LLC -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// The generated code of this file depends on the following RISC-V extensions: -// - RV64I -// - RISC-V Vector ('V') with VLEN >= 128 -// - RISC-V Vector GCM/GMAC extension ('Zvkg') - -#include <linux/linkage.h> - -.text -.option arch, +zvkg - -#define ACCUMULATOR a0 -#define KEY a1 -#define DATA a2 -#define LEN a3 - -// void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data, -// size_t len); -// -// |len| must be nonzero and a multiple of 16 (GHASH_BLOCK_SIZE). -SYM_FUNC_START(ghash_zvkg) - vsetivli zero, 4, e32, m1, ta, ma - vle32.v v1, (ACCUMULATOR) - vle32.v v2, (KEY) -.Lnext_block: - vle32.v v3, (DATA) - vghsh.vv v1, v2, v3 - addi DATA, DATA, 16 - addi LEN, LEN, -16 - bnez LEN, .Lnext_block - - vse32.v v1, (ACCUMULATOR) - ret -SYM_FUNC_END(ghash_zvkg) diff --git a/arch/riscv/crypto/sm3-riscv64-glue.c b/arch/riscv/crypto/sm3-riscv64-glue.c deleted file mode 100644 index abdfe4a63a27..000000000000 --- a/arch/riscv/crypto/sm3-riscv64-glue.c +++ /dev/null @@ -1,97 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * SM3 using the RISC-V vector crypto extensions - * - * Copyright (C) 2023 VRULL GmbH - * Author: Heiko Stuebner <heiko.stuebner@vrull.eu> - * - * Copyright (C) 2023 SiFive, Inc. - * Author: Jerry Shih <jerry.shih@sifive.com> - */ - -#include <asm/simd.h> -#include <asm/vector.h> -#include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> -#include <crypto/sm3.h> -#include <crypto/sm3_base.h> -#include <linux/kernel.h> -#include <linux/module.h> - -/* - * Note: the asm function only uses the 'state' field of struct sm3_state. - * It is assumed to be the first field. - */ -asmlinkage void sm3_transform_zvksh_zvkb( - struct sm3_state *state, const u8 *data, int num_blocks); - -static void sm3_block(struct sm3_state *state, const u8 *data, - int num_blocks) -{ - /* - * Ensure struct sm3_state begins directly with the SM3 - * 256-bit internal state, as this is what the asm function expects. - */ - BUILD_BUG_ON(offsetof(struct sm3_state, state) != 0); - - if (crypto_simd_usable()) { - kernel_vector_begin(); - sm3_transform_zvksh_zvkb(state, data, num_blocks); - kernel_vector_end(); - } else { - sm3_block_generic(state, data, num_blocks); - } -} - -static int riscv64_sm3_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sm3_base_do_update_blocks(desc, data, len, sm3_block); -} - -static int riscv64_sm3_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sm3_base_do_finup(desc, data, len, sm3_block); - return sm3_base_finish(desc, out); -} - -static struct shash_alg riscv64_sm3_alg = { - .init = sm3_base_init, - .update = riscv64_sm3_update, - .finup = riscv64_sm3_finup, - .descsize = SM3_STATE_SIZE, - .digestsize = SM3_DIGEST_SIZE, - .base = { - .cra_blocksize = SM3_BLOCK_SIZE, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_priority = 300, - .cra_name = "sm3", - .cra_driver_name = "sm3-riscv64-zvksh-zvkb", - .cra_module = THIS_MODULE, - }, -}; - -static int __init riscv64_sm3_mod_init(void) -{ - if (riscv_isa_extension_available(NULL, ZVKSH) && - riscv_isa_extension_available(NULL, ZVKB) && - riscv_vector_vlen() >= 128) - return crypto_register_shash(&riscv64_sm3_alg); - - return -ENODEV; -} - -static void __exit riscv64_sm3_mod_exit(void) -{ - crypto_unregister_shash(&riscv64_sm3_alg); -} - -module_init(riscv64_sm3_mod_init); -module_exit(riscv64_sm3_mod_exit); - -MODULE_DESCRIPTION("SM3 (RISC-V accelerated)"); -MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>"); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("sm3"); diff --git a/arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S b/arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S deleted file mode 100644 index 4fe754846f65..000000000000 --- a/arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S +++ /dev/null @@ -1,123 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */ -// -// This file is dual-licensed, meaning that you can use it under your -// choice of either of the following two licenses: -// -// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. -// -// Licensed under the Apache License 2.0 (the "License"). You can obtain -// a copy in the file LICENSE in the source distribution or at -// https://www.openssl.org/source/license.html -// -// or -// -// Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu> -// Copyright (c) 2023, Jerry Shih <jerry.shih@sifive.com> -// Copyright 2024 Google LLC -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// The generated code of this file depends on the following RISC-V extensions: -// - RV64I -// - RISC-V Vector ('V') with VLEN >= 128 -// - RISC-V Vector SM3 Secure Hash extension ('Zvksh') -// - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb') - -#include <linux/linkage.h> - -.text -.option arch, +zvksh, +zvkb - -#define STATEP a0 -#define DATA a1 -#define NUM_BLOCKS a2 - -#define STATE v0 // LMUL=2 -#define PREV_STATE v2 // LMUL=2 -#define W0 v4 // LMUL=2 -#define W1 v6 // LMUL=2 -#define VTMP v8 // LMUL=2 - -.macro sm3_8rounds i, w0, w1 - // Do 4 rounds using W_{0+i}..W_{7+i}. - vsm3c.vi STATE, \w0, \i + 0 - vslidedown.vi VTMP, \w0, 2 - vsm3c.vi STATE, VTMP, \i + 1 - - // Compute W_{4+i}..W_{11+i}. - vslidedown.vi VTMP, \w0, 4 - vslideup.vi VTMP, \w1, 4 - - // Do 4 rounds using W_{4+i}..W_{11+i}. - vsm3c.vi STATE, VTMP, \i + 2 - vslidedown.vi VTMP, VTMP, 2 - vsm3c.vi STATE, VTMP, \i + 3 - -.if \i < 28 - // Compute W_{16+i}..W_{23+i}. - vsm3me.vv \w0, \w1, \w0 -.endif - // For the next 8 rounds, w0 and w1 are swapped. -.endm - -// void sm3_transform_zvksh_zvkb(u32 state[8], const u8 *data, int num_blocks); -SYM_FUNC_START(sm3_transform_zvksh_zvkb) - - // Load the state and endian-swap each 32-bit word. - vsetivli zero, 8, e32, m2, ta, ma - vle32.v STATE, (STATEP) - vrev8.v STATE, STATE - -.Lnext_block: - addi NUM_BLOCKS, NUM_BLOCKS, -1 - - // Save the previous state, as it's needed later. - vmv.v.v PREV_STATE, STATE - - // Load the next 512-bit message block into W0-W1. - vle32.v W0, (DATA) - addi DATA, DATA, 32 - vle32.v W1, (DATA) - addi DATA, DATA, 32 - - // Do the 64 rounds of SM3. - sm3_8rounds 0, W0, W1 - sm3_8rounds 4, W1, W0 - sm3_8rounds 8, W0, W1 - sm3_8rounds 12, W1, W0 - sm3_8rounds 16, W0, W1 - sm3_8rounds 20, W1, W0 - sm3_8rounds 24, W0, W1 - sm3_8rounds 28, W1, W0 - - // XOR in the previous state. - vxor.vv STATE, STATE, PREV_STATE - - // Repeat if more blocks remain. - bnez NUM_BLOCKS, .Lnext_block - - // Store the new state and return. - vrev8.v STATE, STATE - vse32.v STATE, (STATEP) - ret -SYM_FUNC_END(sm3_transform_zvksh_zvkb) diff --git a/arch/riscv/errata/mips/errata.c b/arch/riscv/errata/mips/errata.c index e984a8152208..2c3dc2259e93 100644 --- a/arch/riscv/errata/mips/errata.c +++ b/arch/riscv/errata/mips/errata.c @@ -57,7 +57,7 @@ void mips_errata_patch_func(struct alt_entry *begin, struct alt_entry *end, } tmp = (1U << alt->patch_id); - if (cpu_req_errata && tmp) { + if (cpu_req_errata & tmp) { mutex_lock(&text_mutex); patch_text_nosync(ALT_OLD_PTR(alt), ALT_ALT_PTR(alt), alt->alt_len); diff --git a/arch/riscv/errata/sifive/errata.c b/arch/riscv/errata/sifive/errata.c index 38aac2c47845..df80c9614df1 100644 --- a/arch/riscv/errata/sifive/errata.c +++ b/arch/riscv/errata/sifive/errata.c @@ -75,26 +75,12 @@ static u32 __init_or_module sifive_errata_probe(unsigned long archid, return cpu_req_errata; } -static void __init_or_module warn_miss_errata(u32 miss_errata) -{ - int i; - - pr_warn("----------------------------------------------------------------\n"); - pr_warn("WARNING: Missing the following errata may cause potential issues\n"); - for (i = 0; i < ERRATA_SIFIVE_NUMBER; i++) - if (miss_errata & 0x1 << i) - pr_warn("\tSiFive Errata[%d]:%s\n", i, errata_list[i].name); - pr_warn("Please enable the corresponding Kconfig to apply them\n"); - pr_warn("----------------------------------------------------------------\n"); -} - void sifive_errata_patch_func(struct alt_entry *begin, struct alt_entry *end, unsigned long archid, unsigned long impid, unsigned int stage) { struct alt_entry *alt; u32 cpu_req_errata; - u32 cpu_apply_errata = 0; u32 tmp; BUILD_BUG_ON(ERRATA_SIFIVE_NUMBER >= RISCV_VENDOR_EXT_ALTERNATIVES_BASE); @@ -107,10 +93,8 @@ void sifive_errata_patch_func(struct alt_entry *begin, struct alt_entry *end, for (alt = begin; alt < end; alt++) { if (alt->vendor_id != SIFIVE_VENDOR_ID) continue; - if (alt->patch_id >= ERRATA_SIFIVE_NUMBER) { - WARN(1, "This errata id:%d is not in kernel errata list", alt->patch_id); + if (alt->patch_id >= ERRATA_SIFIVE_NUMBER) continue; - } tmp = (1U << alt->patch_id); if (cpu_req_errata & tmp) { @@ -118,10 +102,6 @@ void sifive_errata_patch_func(struct alt_entry *begin, struct alt_entry *end, patch_text_nosync(ALT_OLD_PTR(alt), ALT_ALT_PTR(alt), alt->alt_len); mutex_unlock(&text_mutex); - cpu_apply_errata |= tmp; } } - if (stage != RISCV_ALTERNATIVES_MODULE && - cpu_apply_errata != cpu_req_errata) - warn_miss_errata(cpu_req_errata - cpu_apply_errata); } diff --git a/arch/riscv/errata/thead/errata.c b/arch/riscv/errata/thead/errata.c index 0b942183f708..6f1c683f0ec0 100644 --- a/arch/riscv/errata/thead/errata.c +++ b/arch/riscv/errata/thead/errata.c @@ -153,7 +153,7 @@ static bool errata_probe_ghostwrite(unsigned int stage, * target-c9xx cores report arch_id and impid as 0 * * While ghostwrite may not affect all c9xx cores that implement - * xtheadvector, there is no futher granularity than c9xx. Assume + * xtheadvector, there is no further granularity than c9xx. Assume * vulnerable for this entire class of processors when xtheadvector is * enabled. */ diff --git a/arch/riscv/include/asm/Kbuild b/arch/riscv/include/asm/Kbuild index bd5fc9403295..7721b63642f4 100644 --- a/arch/riscv/include/asm/Kbuild +++ b/arch/riscv/include/asm/Kbuild @@ -14,5 +14,6 @@ generic-y += ticket_spinlock.h generic-y += qrwlock.h generic-y += qrwlock_types.h generic-y += qspinlock.h +generic-y += ring_buffer.h generic-y += user.h generic-y += vmlinux.lds.h diff --git a/arch/riscv/include/asm/acpi.h b/arch/riscv/include/asm/acpi.h index 6e13695120bc..cdefa1f2d67d 100644 --- a/arch/riscv/include/asm/acpi.h +++ b/arch/riscv/include/asm/acpi.h @@ -12,6 +12,8 @@ #ifndef _ASM_ACPI_H #define _ASM_ACPI_H +#include <linux/cpuidle.h> + /* Basic configuration for ACPI */ #ifdef CONFIG_ACPI @@ -61,16 +63,30 @@ static inline void arch_fix_phys_package_id(int num, u32 slot) { } void acpi_init_rintc_map(void); struct acpi_madt_rintc *acpi_cpu_get_madt_rintc(int cpu); -static inline u32 get_acpi_id_for_cpu(int cpu) -{ - return acpi_cpu_get_madt_rintc(cpu)->uid; -} int acpi_get_riscv_isa(struct acpi_table_header *table, unsigned int cpu, const char **isa); void acpi_get_cbo_block_size(struct acpi_table_header *table, u32 *cbom_size, u32 *cboz_size, u32 *cbop_size); +acpi_handle acpi_get_riscv_gsi_handle(u32 gsi); + +/* + * RISC-V Functional Fixed Hardware Specification Version v1.0.1, + * Chapter 3.1.2, Table 4: Arch. Context Lost Flags + */ +#define RISCV_LPI_HART_TIMER_CTXT_LOST BIT(0) + +static inline unsigned int arch_get_idle_state_flags(u32 arch_flags) +{ + if (arch_flags & RISCV_LPI_HART_TIMER_CTXT_LOST) + return CPUIDLE_FLAG_TIMER_STOP; + + return 0; +} + +#define arch_get_idle_state_flags arch_get_idle_state_flags + #else static inline void acpi_init_rintc_map(void) { } static inline struct acpi_madt_rintc *acpi_cpu_get_madt_rintc(int cpu) @@ -96,4 +112,6 @@ void acpi_map_cpus_to_nodes(void); static inline void acpi_map_cpus_to_nodes(void) { } #endif /* CONFIG_ACPI_NUMA */ +#define ACPI_TABLE_UPGRADE_MAX_PHYS MEMBLOCK_ALLOC_ACCESSIBLE + #endif /*_ASM_ACPI_H*/ diff --git a/arch/riscv/include/asm/asm-prototypes.h b/arch/riscv/include/asm/asm-prototypes.h index a9988bf21ec8..a0ca9efff267 100644 --- a/arch/riscv/include/asm/asm-prototypes.h +++ b/arch/riscv/include/asm/asm-prototypes.h @@ -5,6 +5,10 @@ #include <linux/ftrace.h> #include <asm-generic/asm-prototypes.h> +long long __lshrdi3(long long a, int b); +long long __ashrdi3(long long a, int b); +long long __ashldi3(long long a, int b); + long long __lshrti3(long long a, int b); long long __ashrti3(long long a, int b); long long __ashlti3(long long a, int b); @@ -40,6 +44,7 @@ asmlinkage void riscv_v_context_nesting_end(struct pt_regs *regs); #define DECLARE_DO_ERROR_INFO(name) asmlinkage void name(struct pt_regs *regs) DECLARE_DO_ERROR_INFO(do_trap_unknown); +DECLARE_DO_ERROR_INFO(do_trap_hardware_error); DECLARE_DO_ERROR_INFO(do_trap_insn_misaligned); DECLARE_DO_ERROR_INFO(do_trap_insn_fault); DECLARE_DO_ERROR_INFO(do_trap_insn_illegal); @@ -51,6 +56,7 @@ DECLARE_DO_ERROR_INFO(do_trap_ecall_u); DECLARE_DO_ERROR_INFO(do_trap_ecall_s); DECLARE_DO_ERROR_INFO(do_trap_ecall_m); DECLARE_DO_ERROR_INFO(do_trap_break); +DECLARE_DO_ERROR_INFO(do_trap_software_check); asmlinkage void ret_from_fork_kernel(void *fn_arg, int (*fn)(void *), struct pt_regs *regs); asmlinkage void ret_from_fork_user(struct pt_regs *regs); diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h index e9e8ba83e632..b8bf842d4c13 100644 --- a/arch/riscv/include/asm/asm.h +++ b/arch/riscv/include/asm/asm.h @@ -34,6 +34,7 @@ #define SZREG __REG_SEL(8, 4) #define LGREG __REG_SEL(3, 2) #define SRLI __REG_SEL(srliw, srli) +#define SLLI __REG_SEL(slliw, slli) #if __SIZEOF_POINTER__ == 8 #ifdef __ASSEMBLER__ diff --git a/arch/riscv/include/asm/assembler.h b/arch/riscv/include/asm/assembler.h index 16931712beab..a8df1999118b 100644 --- a/arch/riscv/include/asm/assembler.h +++ b/arch/riscv/include/asm/assembler.h @@ -80,3 +80,47 @@ .endm #endif /* __ASM_ASSEMBLER_H */ + +#if defined(VDSO_CFI) && (__riscv_xlen == 64) +.macro vdso_lpad, label = 0 +lpad \label +.endm +#else +.macro vdso_lpad, label = 0 +.endm +#endif + +/* + * This macro emits a program property note section identifying + * architecture features which require special handling, mainly for + * use in assembly files included in the VDSO. + */ +#define NT_GNU_PROPERTY_TYPE_0 5 +#define GNU_PROPERTY_RISCV_FEATURE_1_AND 0xc0000000 + +#define GNU_PROPERTY_RISCV_FEATURE_1_ZICFILP BIT(0) +#define GNU_PROPERTY_RISCV_FEATURE_1_ZICFISS BIT(1) + +#if defined(VDSO_CFI) && (__riscv_xlen == 64) +#define GNU_PROPERTY_RISCV_FEATURE_1_DEFAULT \ + (GNU_PROPERTY_RISCV_FEATURE_1_ZICFILP | GNU_PROPERTY_RISCV_FEATURE_1_ZICFISS) +#endif + +#ifdef GNU_PROPERTY_RISCV_FEATURE_1_DEFAULT +.macro emit_riscv_feature_1_and, feat = GNU_PROPERTY_RISCV_FEATURE_1_DEFAULT + .pushsection .note.gnu.property, "a" + .p2align 3 + .word 4 + .word 16 + .word NT_GNU_PROPERTY_TYPE_0 + .asciz "GNU" + .word GNU_PROPERTY_RISCV_FEATURE_1_AND + .word 4 + .word \feat + .word 0 + .popsection +.endm +#else +.macro emit_riscv_feature_1_and, feat = 0 +.endm +#endif diff --git a/arch/riscv/include/asm/atomic.h b/arch/riscv/include/asm/atomic.h index 3f33dc54f94b..616b8b332ac5 100644 --- a/arch/riscv/include/asm/atomic.h +++ b/arch/riscv/include/asm/atomic.h @@ -46,7 +46,7 @@ static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i) #endif /* - * First, the atomic ops that have no ordering constraints and therefor don't + * First, the atomic ops that have no ordering constraints and therefore don't * have the AQ or RL bits set. These don't return anything, so there's only * one version to worry about. */ @@ -81,7 +81,7 @@ ATOMIC_OPS(xor, xor, i) /* * Atomic ops that have ordered, relaxed, acquire, and release variants. - * There's two flavors of these: the arithmatic ops have both fetch and return + * There's two flavors of these: the arithmetic ops have both fetch and return * versions, while the logical ops only have fetch versions. */ #define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix) \ diff --git a/arch/riscv/include/asm/bitrev.h b/arch/riscv/include/asm/bitrev.h new file mode 100644 index 000000000000..4b9b8d34cc3b --- /dev/null +++ b/arch/riscv/include/asm/bitrev.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_BITREV_H +#define __ASM_BITREV_H + +#include <linux/types.h> +#include <asm/cpufeature-macros.h> +#include <asm/hwcap.h> +#include <asm-generic/bitops/__bitrev.h> + +static __always_inline __attribute_const__ u32 __arch_bitrev32(u32 x) +{ + unsigned long result; + + if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBKB)) + return generic___bitrev32(x); + + asm volatile( + ".option push\n" + ".option arch,+zbkb\n" + "rev8 %0, %1\n" + "brev8 %0, %0\n" + ".option pop" + : "=r" (result) : "r" ((long)x) + ); + + return result >> (__riscv_xlen - 32); +} + +static __always_inline __attribute_const__ u16 __arch_bitrev16(u16 x) +{ + return __arch_bitrev32(x) >> 16; +} + +static __always_inline __attribute_const__ u8 __arch_bitrev8(u8 x) +{ + unsigned long result; + + if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBKB)) + return generic___bitrev8(x); + + asm volatile( + ".option push\n" + ".option arch,+zbkb\n" + "brev8 %0, %1\n" + ".option pop" + : "=r" (result) : "r" ((long)x) + ); + + return result; +} +#endif diff --git a/arch/riscv/include/asm/bug.h b/arch/riscv/include/asm/bug.h index 6f581b84d8fc..699c0cf3e4ef 100644 --- a/arch/riscv/include/asm/bug.h +++ b/arch/riscv/include/asm/bug.h @@ -29,13 +29,8 @@ typedef u32 bug_insn_t; -#ifdef CONFIG_GENERIC_BUG_RELATIVE_POINTERS #define __BUG_ENTRY_ADDR RISCV_INT " 1b - ." #define __BUG_ENTRY_FILE(file) RISCV_INT " " file " - ." -#else -#define __BUG_ENTRY_ADDR RISCV_PTR " 1b" -#define __BUG_ENTRY_FILE(file) RISCV_PTR " " file -#endif #ifdef CONFIG_DEBUG_BUGVERBOSE #define __BUG_ENTRY(file, line, flags) \ diff --git a/arch/riscv/include/asm/cacheflush.h b/arch/riscv/include/asm/cacheflush.h index 0092513c3376..c2b0a2928f06 100644 --- a/arch/riscv/include/asm/cacheflush.h +++ b/arch/riscv/include/asm/cacheflush.h @@ -40,23 +40,25 @@ do { \ flush_icache_mm(vma->vm_mm, 0); \ } while (0) -#ifdef CONFIG_64BIT -extern u64 new_vmalloc[NR_CPUS / sizeof(u64) + 1]; +#if defined(CONFIG_64BIT) && defined(CONFIG_MMU) +/* This is accessed in assembly code. cpumask_var_t would be too complex. */ +extern DECLARE_BITMAP(new_valid_map_cpus, NR_CPUS); extern char _end[]; +static inline void mark_new_valid_map(void) +{ + /* + * We don't care if concurrently a cpu resets this value since + * the only place this can happen is in handle_exception() where + * an sfence.vma is emitted. + */ + bitmap_fill(new_valid_map_cpus, NR_CPUS); +} #define flush_cache_vmap flush_cache_vmap static inline void flush_cache_vmap(unsigned long start, unsigned long end) { - if (is_vmalloc_or_module_addr((void *)start)) { - int i; - - /* - * We don't care if concurrently a cpu resets this value since - * the only place this can happen is in handle_exception() where - * an sfence.vma is emitted. - */ - for (i = 0; i < ARRAY_SIZE(new_vmalloc); ++i) - new_vmalloc[i] = -1ULL; - } + if (is_vmalloc_or_module_addr((void *)start) || + (start >= VMEMMAP_START && end <= VMEMMAP_END)) + mark_new_valid_map(); } #define flush_cache_vmap_early(start, end) local_flush_tlb_kernel_range(start, end) #endif diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h index 8712cf9c69dc..662e160b0522 100644 --- a/arch/riscv/include/asm/cmpxchg.h +++ b/arch/riscv/include/asm/cmpxchg.h @@ -23,7 +23,10 @@ riscv_has_extension_unlikely(RISCV_ISA_EXT_ZABHA)) { \ __asm__ __volatile__ ( \ prepend \ + " .option push\n" \ + " .option arch, +zabha\n" \ " amoswap" swap_sfx " %0, %z2, %1\n" \ + " .option pop\n" \ swap_append \ : "=&r" (r), "+A" (*(p)) \ : "rJ" (n) \ @@ -141,7 +144,10 @@ \ __asm__ __volatile__ ( \ cas_prepend \ + " .option push\n" \ + " .option arch, +zacas, +zabha\n" \ " amocas" cas_sfx " %0, %z2, %1\n" \ + " .option pop\n" \ cas_append \ : "+&r" (r), "+A" (*(p)) \ : "rJ" (n) \ @@ -188,7 +194,10 @@ \ __asm__ __volatile__ ( \ cas_prepend \ + " .option push\n" \ + " .option arch, +zacas\n" \ " amocas" cas_sfx " %0, %z2, %1\n" \ + " .option pop\n" \ cas_append \ : "+&r" (r), "+A" (*(p)) \ : "rJ" (n) \ @@ -340,7 +349,10 @@ union __u128_halves { register unsigned long t4 asm ("t4") = __ho.high; \ \ __asm__ __volatile__ ( \ - " amocas.q" cas_sfx " %0, %z3, %2" \ + " .option push\n" \ + " .option arch, +zacas\n" \ + " amocas.q" cas_sfx " %0, %z3, %2\n" \ + " .option pop\n" \ : "+&r" (t3), "+&r" (t4), "+A" (*(p)) \ : "rJ" (t1), "rJ" (t2) \ : "memory"); \ diff --git a/arch/riscv/include/asm/compat.h b/arch/riscv/include/asm/compat.h index 6081327e55f5..28e115eed218 100644 --- a/arch/riscv/include/asm/compat.h +++ b/arch/riscv/include/asm/compat.h @@ -2,7 +2,7 @@ #ifndef __ASM_COMPAT_H #define __ASM_COMPAT_H -#define COMPAT_UTS_MACHINE "riscv\0\0" +#define COMPAT_UTS_MACHINE "riscv32\0\0" /* * Architecture specific compatibility types diff --git a/arch/riscv/include/asm/cpu_ops.h b/arch/riscv/include/asm/cpu_ops.h index 176b570ef982..065811fca594 100644 --- a/arch/riscv/include/asm/cpu_ops.h +++ b/arch/riscv/include/asm/cpu_ops.h @@ -24,7 +24,7 @@ struct cpu_operations { struct task_struct *tidle); #ifdef CONFIG_HOTPLUG_CPU void (*cpu_stop)(void); - int (*cpu_is_stopped)(unsigned int cpu); + bool (*cpu_is_stopped)(unsigned int cpu); #endif }; diff --git a/arch/riscv/include/asm/cpufeature.h b/arch/riscv/include/asm/cpufeature.h index 62837fa981e8..739fcc84bf7b 100644 --- a/arch/riscv/include/asm/cpufeature.h +++ b/arch/riscv/include/asm/cpufeature.h @@ -152,4 +152,16 @@ static __always_inline bool riscv_cpu_has_extension_unlikely(int cpu, const unsi return __riscv_isa_extension_available(hart_isa[cpu].isa, ext); } +static inline bool cpu_supports_shadow_stack(void) +{ + return (IS_ENABLED(CONFIG_RISCV_USER_CFI) && + riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICFISS)); +} + +static inline bool cpu_supports_indirect_br_lp_instr(void) +{ + return (IS_ENABLED(CONFIG_RISCV_USER_CFI) && + riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICFILP)); +} + #endif diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h index 4a37a98398ad..621f84d1898f 100644 --- a/arch/riscv/include/asm/csr.h +++ b/arch/riscv/include/asm/csr.h @@ -18,6 +18,15 @@ #define SR_MPP _AC(0x00001800, UL) /* Previously Machine */ #define SR_SUM _AC(0x00040000, UL) /* Supervisor User Memory Access */ +/* zicfilp landing pad status bit */ +#define SR_SPELP _AC(0x00800000, UL) +#define SR_MPELP _AC(0x020000000000, UL) +#ifdef CONFIG_RISCV_M_MODE +#define SR_ELP SR_MPELP +#else +#define SR_ELP SR_SPELP +#endif + #define SR_FS _AC(0x00006000, UL) /* Floating-point Status */ #define SR_FS_OFF _AC(0x00000000, UL) #define SR_FS_INITIAL _AC(0x00002000, UL) @@ -75,6 +84,10 @@ #define SATP_ASID_MASK _AC(0xFFFF, UL) #endif +/* SRMCFG fields */ +#define SRMCFG_RCID_MASK GENMASK(11, 0) +#define SRMCFG_MCID_MASK GENMASK(27, 16) + /* Exception cause high bit - is an interrupt if set */ #define CAUSE_IRQ_FLAG (_AC(1, UL) << (__riscv_xlen - 1)) @@ -108,6 +121,7 @@ #define EXC_INST_PAGE_FAULT 12 #define EXC_LOAD_PAGE_FAULT 13 #define EXC_STORE_PAGE_FAULT 15 +#define EXC_SOFTWARE_CHECK 18 #define EXC_INST_GUEST_PAGE_FAULT 20 #define EXC_LOAD_GUEST_PAGE_FAULT 21 #define EXC_VIRTUAL_INST_FAULT 22 @@ -174,12 +188,24 @@ #define HGATP_MODE_SHIFT HGATP32_MODE_SHIFT #endif -/* VSIP & HVIP relation */ +/* + * VSIP & HVIP relation + * + * The bit positions are same between VSIP and HVIP for interrupt + * numbers 13-63, where there's a shift for the SSI, STI and SEI. + */ #define VSIP_TO_HVIP_SHIFT (IRQ_VS_SOFT - IRQ_S_SOFT) -#define VSIP_VALID_MASK ((_AC(1, UL) << IRQ_S_SOFT) | \ +#define VSIP_BIAS_MASK ((_AC(1, UL) << IRQ_S_SOFT) | \ (_AC(1, UL) << IRQ_S_TIMER) | \ - (_AC(1, UL) << IRQ_S_EXT) | \ - (_AC(1, UL) << IRQ_PMU_OVF)) + (_AC(1, UL) << IRQ_S_EXT)) +#define VSIP_NO_BIAS_MASK (_AC(1, UL) << IRQ_PMU_OVF) +#define VSIP_VALID_MASK (VSIP_BIAS_MASK | VSIP_NO_BIAS_MASK) +#define vsip_to_hvip(_vsip) ((((_vsip) & VSIP_BIAS_MASK) << \ + VSIP_TO_HVIP_SHIFT) | \ + ((_vsip) & VSIP_NO_BIAS_MASK)) +#define hvip_to_vsip(_hvip) ((((_hvip) >> VSIP_TO_HVIP_SHIFT) & \ + VSIP_BIAS_MASK) | \ + ((_hvip) & VSIP_NO_BIAS_MASK)) /* AIA CSR bits */ #define TOPI_IID_SHIFT 16 @@ -212,6 +238,8 @@ #define ENVCFG_PMM_PMLEN_16 (_AC(0x3, ULL) << 32) #define ENVCFG_CBZE (_AC(1, UL) << 7) #define ENVCFG_CBCFE (_AC(1, UL) << 6) +#define ENVCFG_LPE (_AC(1, UL) << 2) +#define ENVCFG_SSE (_AC(1, UL) << 3) #define ENVCFG_CBIE_SHIFT 4 #define ENVCFG_CBIE (_AC(0x3, UL) << ENVCFG_CBIE_SHIFT) #define ENVCFG_CBIE_ILL _AC(0x0, UL) @@ -230,6 +258,23 @@ #define SMSTATEEN0_HSENVCFG (_ULL(1) << SMSTATEEN0_HSENVCFG_SHIFT) #define SMSTATEEN0_SSTATEEN0_SHIFT 63 #define SMSTATEEN0_SSTATEEN0 (_ULL(1) << SMSTATEEN0_SSTATEEN0_SHIFT) +/* HPMEVENT bits. These are accessible in S-mode via Smcdeleg/Ssccfg */ +#define HPMEVENT_OF (BIT_ULL(63)) +#define HPMEVENT_MINH (BIT_ULL(62)) +#define HPMEVENT_SINH (BIT_ULL(61)) +#define HPMEVENT_UINH (BIT_ULL(60)) +#define HPMEVENT_VSINH (BIT_ULL(59)) +#define HPMEVENT_VUINH (BIT_ULL(58)) +#ifndef CONFIG_64BIT +#define HPMEVENTH_OF (BIT(31)) +#define HPMEVENTH_MINH (BIT(30)) +#define HPMEVENTH_SINH (BIT(29)) +#define HPMEVENTH_UINH (BIT(28)) +#define HPMEVENTH_VSINH (BIT(27)) +#define HPMEVENTH_VUINH (BIT(26)) +#endif + +#define SISELECT_SSCCFG_BASE 0x40 /* mseccfg bits */ #define MSECCFG_PMM ENVCFG_PMM @@ -311,16 +356,21 @@ #define CSR_SCOUNTEREN 0x106 #define CSR_SENVCFG 0x10a #define CSR_SSTATEEN0 0x10c +#define CSR_SCOUNTINHIBIT 0x120 #define CSR_SSCRATCH 0x140 #define CSR_SEPC 0x141 #define CSR_SCAUSE 0x142 #define CSR_STVAL 0x143 #define CSR_SIP 0x144 #define CSR_SATP 0x180 +#define CSR_SRMCFG 0x181 #define CSR_STIMECMP 0x14D #define CSR_STIMECMPH 0x15D +/* zicfiss user mode csr. CSR_SSP holds current shadow stack pointer */ +#define CSR_SSP 0x011 + /* xtheadvector symbolic CSR names */ #define CSR_VXSAT 0x9 #define CSR_VXRM 0xa @@ -333,6 +383,12 @@ /* Supervisor-Level Window to Indirectly Accessed Registers (AIA) */ #define CSR_SISELECT 0x150 #define CSR_SIREG 0x151 +/* Supervisor-Level Window to Indirectly Accessed Registers (Sscsrind) */ +#define CSR_SIREG2 0x152 +#define CSR_SIREG3 0x153 +#define CSR_SIREG4 0x155 +#define CSR_SIREG5 0x156 +#define CSR_SIREG6 0x157 /* Supervisor-Level Interrupts (AIA) */ #define CSR_STOPEI 0x15c @@ -380,6 +436,14 @@ /* VS-Level Window to Indirectly Accessed Registers (H-extension with AIA) */ #define CSR_VSISELECT 0x250 #define CSR_VSIREG 0x251 +/* + * VS-Level Window to Indirectly Accessed Registers (H-extension with Sscsrind) + */ +#define CSR_VSIREG2 0x252 +#define CSR_VSIREG3 0x253 +#define CSR_VSIREG4 0x255 +#define CSR_VSIREG5 0x256 +#define CSR_VSIREG6 0x257 /* VS-Level Interrupts (H-extension with AIA) */ #define CSR_VSTOPEI 0x25c @@ -422,6 +486,12 @@ /* Machine-Level Window to Indirectly Accessed Registers (AIA) */ #define CSR_MISELECT 0x350 #define CSR_MIREG 0x351 +/* Machine-Level Window to Indirectly Accessed Registers (Smcsrind) */ +#define CSR_MIREG2 0x352 +#define CSR_MIREG3 0x353 +#define CSR_MIREG4 0x355 +#define CSR_MIREG5 0x356 +#define CSR_MIREG6 0x357 /* Machine-Level Interrupts (AIA) */ #define CSR_MTOPEI 0x35c @@ -444,6 +514,23 @@ #define CSR_VTYPE 0xc21 #define CSR_VLENB 0xc22 +#define VTYPE_VLMUL _AC(7, UL) +#define VTYPE_VLMUL_FRAC _AC(4, UL) +#define VTYPE_VSEW_SHIFT 3 +#define VTYPE_VSEW (_AC(7, UL) << VTYPE_VSEW_SHIFT) +#define VTYPE_VTA_SHIFT 6 +#define VTYPE_VTA (_AC(1, UL) << VTYPE_VTA_SHIFT) +#define VTYPE_VMA_SHIFT 7 +#define VTYPE_VMA (_AC(1, UL) << VTYPE_VMA_SHIFT) +#define VTYPE_VILL_SHIFT (__riscv_xlen - 1) +#define VTYPE_VILL (_AC(1, UL) << VTYPE_VILL_SHIFT) + +#define VTYPE_VLMUL_THEAD _AC(3, UL) +#define VTYPE_VSEW_THEAD_SHIFT 2 +#define VTYPE_VSEW_THEAD (_AC(7, UL) << VTYPE_VSEW_THEAD_SHIFT) +#define VTYPE_VEDIV_THEAD_SHIFT 5 +#define VTYPE_VEDIV_THEAD (_AC(3, UL) << VTYPE_VEDIV_THEAD_SHIFT) + /* Scalar Crypto Extension - Entropy */ #define CSR_SEED 0x015 #define SEED_OPST_MASK _AC(0xC0000000, UL) @@ -467,6 +554,11 @@ # define CSR_IEH CSR_MIEH # define CSR_ISELECT CSR_MISELECT # define CSR_IREG CSR_MIREG +# define CSR_IREG2 CSR_MIREG2 +# define CSR_IREG3 CSR_MIREG3 +# define CSR_IREG4 CSR_MIREG4 +# define CSR_IREG5 CSR_MIREG5 +# define CSR_IREG6 CSR_MIREG6 # define CSR_IPH CSR_MIPH # define CSR_TOPEI CSR_MTOPEI # define CSR_TOPI CSR_MTOPI @@ -492,6 +584,11 @@ # define CSR_IEH CSR_SIEH # define CSR_ISELECT CSR_SISELECT # define CSR_IREG CSR_SIREG +# define CSR_IREG2 CSR_SIREG2 +# define CSR_IREG3 CSR_SIREG3 +# define CSR_IREG4 CSR_SIREG4 +# define CSR_IREG5 CSR_SIREG5 +# define CSR_IREG6 CSR_SIREG6 # define CSR_IPH CSR_SIPH # define CSR_TOPEI CSR_STOPEI # define CSR_TOPI CSR_STOPI diff --git a/arch/riscv/include/asm/csr_indirect.h b/arch/riscv/include/asm/csr_indirect.h new file mode 100644 index 000000000000..0f558fac8f5f --- /dev/null +++ b/arch/riscv/include/asm/csr_indirect.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _ASM_RISCV_CSR_INDIRECT_H +#define _ASM_RISCV_CSR_INDIRECT_H + +#include <linux/irqflags.h> + +#include <asm/csr.h> + +/* + * These have to be macros rather than functions: csr_read()/csr_write() + * stringify their CSR argument into the inline asm template via __ASM_STR(), + * so the CSR number must be a literal token at preprocessing time. Passing it + * as a function parameter emits "csrr %0, iregcsr", which no assembler can + * resolve. RISC-V has no register-indirect form of csrr/csrw - the CSR is a + * 12-bit immediate - so the sireg CSR selecting the indirect window cannot + * itself be a variable. + */ +#define csr_indirect_read(iregcsr, iselbase, iseloff) ({ \ + unsigned long __value = 0; \ + unsigned long __flags; \ + local_irq_save(__flags); \ + csr_write(CSR_ISELECT, (iselbase) + (iseloff)); \ + __value = csr_read(iregcsr); \ + local_irq_restore(__flags); \ + __value; \ +}) + +#define csr_indirect_write(iregcsr, iselbase, iseloff, value) ({ \ + unsigned long __flags; \ + local_irq_save(__flags); \ + csr_write(CSR_ISELECT, (iselbase) + (iseloff)); \ + csr_write(iregcsr, (value)); \ + local_irq_restore(__flags); \ +}) + +#define csr_indirect_warl(iregcsr, iselbase, iseloff, warl_val) ({ \ + unsigned long __old_val = 0, __value = 0; \ + unsigned long __flags; \ + local_irq_save(__flags); \ + csr_write(CSR_ISELECT, (iselbase) + (iseloff)); \ + __old_val = csr_read(iregcsr); \ + csr_write(iregcsr, (warl_val)); \ + __value = csr_read(iregcsr); \ + csr_write(iregcsr, __old_val); \ + local_irq_restore(__flags); \ + __value; \ +}) + +#endif diff --git a/arch/riscv/include/asm/elf.h b/arch/riscv/include/asm/elf.h index c7aea7886d22..aa4961b0e208 100644 --- a/arch/riscv/include/asm/elf.h +++ b/arch/riscv/include/asm/elf.h @@ -59,8 +59,8 @@ extern bool compat_elf_check_arch(Elf32_Ehdr *hdr); #endif /* - * Provides information on the availiable set of ISA extensions to userspace, - * via a bitmap that coorespends to each single-letter ISA extension. This is + * Provides information on the available set of ISA extensions to userspace, + * via a bitmap that corresponds to each single-letter ISA extension. This is * essentially defunct, but will remain for compatibility with userspace. */ #define ELF_HWCAP riscv_get_elf_hwcap() diff --git a/arch/riscv/include/asm/entry-common.h b/arch/riscv/include/asm/entry-common.h index b28ccc6cdeea..34ed149af5d1 100644 --- a/arch/riscv/include/asm/entry-common.h +++ b/arch/riscv/include/asm/entry-common.h @@ -40,4 +40,6 @@ static inline int handle_misaligned_store(struct pt_regs *regs) } #endif +bool handle_user_cfi_violation(struct pt_regs *regs); + #endif /* _ASM_RISCV_ENTRY_COMMON_H */ diff --git a/arch/riscv/include/asm/fixmap.h b/arch/riscv/include/asm/fixmap.h index 0a55099bb734..a81823c5ee4a 100644 --- a/arch/riscv/include/asm/fixmap.h +++ b/arch/riscv/include/asm/fixmap.h @@ -55,9 +55,6 @@ enum fixed_addresses { #define __early_set_fixmap __set_fixmap -#define __late_set_fixmap __set_fixmap -#define __late_clear_fixmap(idx) __set_fixmap((idx), 0, FIXMAP_PAGE_CLEAR) - extern void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot); diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 4369a2338541..f8db798b2654 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -10,15 +10,15 @@ #include <uapi/asm/hwcap.h> -#define RISCV_ISA_EXT_a ('a' - 'a') -#define RISCV_ISA_EXT_c ('c' - 'a') -#define RISCV_ISA_EXT_d ('d' - 'a') -#define RISCV_ISA_EXT_f ('f' - 'a') -#define RISCV_ISA_EXT_h ('h' - 'a') -#define RISCV_ISA_EXT_i ('i' - 'a') -#define RISCV_ISA_EXT_m ('m' - 'a') -#define RISCV_ISA_EXT_q ('q' - 'a') -#define RISCV_ISA_EXT_v ('v' - 'a') +#define RISCV_ISA_EXT_A ('a' - 'a') +#define RISCV_ISA_EXT_C ('c' - 'a') +#define RISCV_ISA_EXT_D ('d' - 'a') +#define RISCV_ISA_EXT_F ('f' - 'a') +#define RISCV_ISA_EXT_H ('h' - 'a') +#define RISCV_ISA_EXT_I ('i' - 'a') +#define RISCV_ISA_EXT_M ('m' - 'a') +#define RISCV_ISA_EXT_Q ('q' - 'a') +#define RISCV_ISA_EXT_V ('v' - 'a') /* * These macros represent the logical IDs of each multi-letter RISC-V ISA @@ -110,6 +110,18 @@ #define RISCV_ISA_EXT_ZALASR 101 #define RISCV_ISA_EXT_ZILSD 102 #define RISCV_ISA_EXT_ZCLSD 103 +#define RISCV_ISA_EXT_ZICFILP 104 +#define RISCV_ISA_EXT_ZICFISS 105 +#define RISCV_ISA_EXT_SSCSRIND 106 +#define RISCV_ISA_EXT_SMCSRIND 107 +#define RISCV_ISA_EXT_SMCNTRPMF 108 +#define RISCV_ISA_EXT_SSCCFG 109 +#define RISCV_ISA_EXT_SMCDELEG 110 +#define RISCV_ISA_EXT_SSQOSID 111 +#define RISCV_ISA_EXT_ZICCLSM 112 +#define RISCV_ISA_EXT_ZICCAMOA 113 +#define RISCV_ISA_EXT_ZICCIF 114 +#define RISCV_ISA_EXT_ZA64RS 115 #define RISCV_ISA_EXT_XLINUXENVCFG 127 @@ -119,9 +131,11 @@ #ifdef CONFIG_RISCV_M_MODE #define RISCV_ISA_EXT_SxAIA RISCV_ISA_EXT_SMAIA #define RISCV_ISA_EXT_SUPM RISCV_ISA_EXT_SMNPM +#define RISCV_ISA_EXT_SxCSRIND RISCV_ISA_EXT_SMCSRIND #else #define RISCV_ISA_EXT_SxAIA RISCV_ISA_EXT_SSAIA #define RISCV_ISA_EXT_SUPM RISCV_ISA_EXT_SSNPM +#define RISCV_ISA_EXT_SxCSRIND RISCV_ISA_EXT_SSCSRIND #endif #endif /* _ASM_RISCV_HWCAP_H */ diff --git a/arch/riscv/include/asm/hwprobe.h b/arch/riscv/include/asm/hwprobe.h index 8c572a464719..9b04377c0f98 100644 --- a/arch/riscv/include/asm/hwprobe.h +++ b/arch/riscv/include/asm/hwprobe.h @@ -8,7 +8,7 @@ #include <uapi/asm/hwprobe.h> -#define RISCV_HWPROBE_MAX_KEY 15 +#define RISCV_HWPROBE_MAX_KEY 16 static inline bool riscv_hwprobe_key_is_valid(__s64 key) { @@ -20,6 +20,7 @@ static inline bool hwprobe_key_is_bitmask(__s64 key) switch (key) { case RISCV_HWPROBE_KEY_BASE_BEHAVIOR: case RISCV_HWPROBE_KEY_IMA_EXT_0: + case RISCV_HWPROBE_KEY_IMA_EXT_1: case RISCV_HWPROBE_KEY_CPUPERF_0: case RISCV_HWPROBE_KEY_VENDOR_EXT_THEAD_0: case RISCV_HWPROBE_KEY_VENDOR_EXT_MIPS_0: @@ -42,11 +43,4 @@ static inline bool riscv_hwprobe_pair_cmp(struct riscv_hwprobe *pair, return pair->value == other_pair->value; } -#ifdef CONFIG_MMU -void riscv_hwprobe_register_async_probe(void); -void riscv_hwprobe_complete_async_probe(void); -#else -static inline void riscv_hwprobe_register_async_probe(void) {} -static inline void riscv_hwprobe_complete_async_probe(void) {} -#endif #endif diff --git a/arch/riscv/include/asm/io.h b/arch/riscv/include/asm/io.h index 09bb5f57a9d3..92d5f831f349 100644 --- a/arch/riscv/include/asm/io.h +++ b/arch/riscv/include/asm/io.h @@ -102,12 +102,14 @@ __io_reads_ins(reads, u32, l, __io_br(), __io_ar(addr)) #define readsw(addr, buffer, count) __readsw(addr, buffer, count) #define readsl(addr, buffer, count) __readsl(addr, buffer, count) +#ifdef CONFIG_HAS_IOPORT __io_reads_ins(ins, u8, b, __io_pbr(), __io_par(addr)) __io_reads_ins(ins, u16, w, __io_pbr(), __io_par(addr)) __io_reads_ins(ins, u32, l, __io_pbr(), __io_par(addr)) #define insb(addr, buffer, count) __insb(PCI_IOBASE + (addr), buffer, count) #define insw(addr, buffer, count) __insw(PCI_IOBASE + (addr), buffer, count) #define insl(addr, buffer, count) __insl(PCI_IOBASE + (addr), buffer, count) +#endif __io_writes_outs(writes, u8, b, __io_bw(), __io_aw()) __io_writes_outs(writes, u16, w, __io_bw(), __io_aw()) @@ -116,26 +118,32 @@ __io_writes_outs(writes, u32, l, __io_bw(), __io_aw()) #define writesw(addr, buffer, count) __writesw(addr, buffer, count) #define writesl(addr, buffer, count) __writesl(addr, buffer, count) +#ifdef CONFIG_HAS_IOPORT __io_writes_outs(outs, u8, b, __io_pbw(), __io_paw()) __io_writes_outs(outs, u16, w, __io_pbw(), __io_paw()) __io_writes_outs(outs, u32, l, __io_pbw(), __io_paw()) #define outsb(addr, buffer, count) __outsb(PCI_IOBASE + (addr), buffer, count) #define outsw(addr, buffer, count) __outsw(PCI_IOBASE + (addr), buffer, count) #define outsl(addr, buffer, count) __outsl(PCI_IOBASE + (addr), buffer, count) +#endif #ifdef CONFIG_64BIT __io_reads_ins(reads, u64, q, __io_br(), __io_ar(addr)) #define readsq(addr, buffer, count) __readsq(addr, buffer, count) +#ifdef CONFIG_HAS_IOPORT __io_reads_ins(ins, u64, q, __io_pbr(), __io_par(addr)) #define insq(addr, buffer, count) __insq(PCI_IOBASE + (addr), buffer, count) +#endif __io_writes_outs(writes, u64, q, __io_bw(), __io_aw()) #define writesq(addr, buffer, count) __writesq(addr, buffer, count) +#ifdef CONFIG_HAS_IOPORT __io_writes_outs(outs, u64, q, __io_pbr(), __io_paw()) #define outsq(addr, buffer, count) __outsq(PCI_IOBASE + (addr), buffer, count) #endif +#endif #include <asm-generic/io.h> diff --git a/arch/riscv/include/asm/kfence.h b/arch/riscv/include/asm/kfence.h index d08bf7fb3aee..29cb3a6ee113 100644 --- a/arch/riscv/include/asm/kfence.h +++ b/arch/riscv/include/asm/kfence.h @@ -6,6 +6,7 @@ #include <linux/kfence.h> #include <linux/pfn.h> #include <asm-generic/pgalloc.h> +#include <asm/cacheflush.h> #include <asm/pgtable.h> static inline bool arch_kfence_init_pool(void) @@ -17,10 +18,12 @@ static inline bool kfence_protect_page(unsigned long addr, bool protect) { pte_t *pte = virt_to_kpte(addr); - if (protect) + if (protect) { set_pte(pte, __pte(pte_val(ptep_get(pte)) & ~_PAGE_PRESENT)); - else + } else { set_pte(pte, __pte(pte_val(ptep_get(pte)) | _PAGE_PRESENT)); + mark_new_valid_map(); + } preempt_disable(); local_flush_tlb_kernel_range(addr, addr + PAGE_SIZE); diff --git a/arch/riscv/include/asm/kvm_aia.h b/arch/riscv/include/asm/kvm_aia.h index b04ecdd1a860..ab03f853e507 100644 --- a/arch/riscv/include/asm/kvm_aia.h +++ b/arch/riscv/include/asm/kvm_aia.h @@ -79,7 +79,7 @@ struct kvm_vcpu_aia { #define irqchip_in_kernel(k) ((k)->arch.aia.in_kernel) -extern unsigned int kvm_riscv_aia_nr_hgei; +extern atomic_t kvm_riscv_aia_nr_hgei; extern unsigned int kvm_riscv_aia_max_ids; DECLARE_STATIC_KEY_FALSE(kvm_riscv_aia_available); #define kvm_riscv_aia_available() \ @@ -165,6 +165,9 @@ int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner, void __iomem **hgei_va, phys_addr_t *hgei_pa); void kvm_riscv_aia_free_hgei(int cpu, int hgei); +void kvm_riscv_aia_pm_exit(void); +void kvm_riscv_aia_pm_enter(void); + void kvm_riscv_aia_enable(void); void kvm_riscv_aia_disable(void); int kvm_riscv_aia_init(void); diff --git a/arch/riscv/include/asm/kvm_gstage.h b/arch/riscv/include/asm/kvm_gstage.h index 595e2183173e..aaf080ba1b77 100644 --- a/arch/riscv/include/asm/kvm_gstage.h +++ b/arch/riscv/include/asm/kvm_gstage.h @@ -15,6 +15,7 @@ struct kvm_gstage { #define KVM_GSTAGE_FLAGS_LOCAL BIT(0) unsigned long vmid; pgd_t *pgd; + unsigned long pgd_levels; }; struct kvm_gstage_mapping { @@ -29,16 +30,22 @@ struct kvm_gstage_mapping { #define kvm_riscv_gstage_index_bits 10 #endif -extern unsigned long kvm_riscv_gstage_mode; -extern unsigned long kvm_riscv_gstage_pgd_levels; +extern unsigned long kvm_riscv_gstage_max_pgd_levels; #define kvm_riscv_gstage_pgd_xbits 2 #define kvm_riscv_gstage_pgd_size (1UL << (HGATP_PAGE_SHIFT + kvm_riscv_gstage_pgd_xbits)) -#define kvm_riscv_gstage_gpa_bits (HGATP_PAGE_SHIFT + \ - (kvm_riscv_gstage_pgd_levels * \ - kvm_riscv_gstage_index_bits) + \ - kvm_riscv_gstage_pgd_xbits) -#define kvm_riscv_gstage_gpa_size ((gpa_t)(1ULL << kvm_riscv_gstage_gpa_bits)) + +static inline unsigned long kvm_riscv_gstage_gpa_bits(unsigned long pgd_levels) +{ + return (HGATP_PAGE_SHIFT + + pgd_levels * kvm_riscv_gstage_index_bits + + kvm_riscv_gstage_pgd_xbits); +} + +static inline gpa_t kvm_riscv_gstage_gpa_size(unsigned long pgd_levels) +{ + return BIT_ULL(kvm_riscv_gstage_gpa_bits(pgd_levels)); +} bool kvm_riscv_gstage_get_leaf(struct kvm_gstage *gstage, gpa_t addr, pte_t **ptepp, u32 *ptep_level); @@ -47,26 +54,63 @@ int kvm_riscv_gstage_set_pte(struct kvm_gstage *gstage, struct kvm_mmu_memory_cache *pcache, const struct kvm_gstage_mapping *map); +bool kvm_riscv_gstage_try_update_pte(struct kvm_gstage *gstage, u32 level, + gpa_t addr, pte_t *ptep, + pte_t old_pte, pte_t new_pte); + int kvm_riscv_gstage_map_page(struct kvm_gstage *gstage, struct kvm_mmu_memory_cache *pcache, gpa_t gpa, phys_addr_t hpa, unsigned long page_size, bool page_rdonly, bool page_exec, struct kvm_gstage_mapping *out_map); +bool kvm_riscv_gstage_split_huge(struct kvm_gstage *gstage, + struct kvm_mmu_memory_cache *pcache, + gpa_t addr, u32 target_level, bool flush); + enum kvm_riscv_gstage_op { GSTAGE_OP_NOP = 0, /* Nothing */ GSTAGE_OP_CLEAR, /* Clear/Unmap */ GSTAGE_OP_WP, /* Write-protect */ }; -void kvm_riscv_gstage_op_pte(struct kvm_gstage *gstage, gpa_t addr, +bool kvm_riscv_gstage_op_pte(struct kvm_gstage *gstage, gpa_t addr, pte_t *ptep, u32 ptep_level, enum kvm_riscv_gstage_op op); -void kvm_riscv_gstage_unmap_range(struct kvm_gstage *gstage, +bool kvm_riscv_gstage_unmap_range(struct kvm_gstage *gstage, gpa_t start, gpa_t size, bool may_block); -void kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end); +bool kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end); + +bool kvm_riscv_gstage_wp_pt_masked(struct kvm_gstage *gstage, gfn_t base_gfn, + unsigned long mask); void kvm_riscv_gstage_mode_detect(void); +static inline unsigned long kvm_riscv_gstage_mode(unsigned long pgd_levels) +{ + switch (pgd_levels) { + case 2: + return HGATP_MODE_SV32X4; + case 3: + return HGATP_MODE_SV39X4; + case 4: + return HGATP_MODE_SV48X4; + case 5: + return HGATP_MODE_SV57X4; + default: + WARN_ON_ONCE(1); + return HGATP_MODE_OFF; + } +} + +static inline void kvm_riscv_gstage_init(struct kvm_gstage *gstage, struct kvm *kvm) +{ + gstage->kvm = kvm; + gstage->flags = 0; + gstage->vmid = READ_ONCE(kvm->arch.vmid.vmid); + gstage->pgd = kvm->arch.pgd; + gstage->pgd_levels = kvm->arch.pgd_levels; +} + #endif diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index 24585304c02b..a30600579231 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -18,6 +18,7 @@ #include <asm/ptrace.h> #include <asm/kvm_tlb.h> #include <asm/kvm_vmid.h> +#include <asm/kvm_vcpu_config.h> #include <asm/kvm_vcpu_fp.h> #include <asm/kvm_vcpu_insn.h> #include <asm/kvm_vcpu_sbi.h> @@ -47,17 +48,7 @@ #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE -#define KVM_HEDELEG_DEFAULT (BIT(EXC_INST_MISALIGNED) | \ - BIT(EXC_INST_ILLEGAL) | \ - BIT(EXC_BREAKPOINT) | \ - BIT(EXC_SYSCALL) | \ - BIT(EXC_INST_PAGE_FAULT) | \ - BIT(EXC_LOAD_PAGE_FAULT) | \ - BIT(EXC_STORE_PAGE_FAULT)) - -#define KVM_HIDELEG_DEFAULT (BIT(IRQ_VS_SOFT) | \ - BIT(IRQ_VS_TIMER) | \ - BIT(IRQ_VS_EXT)) +#define KVM_HAVE_MMU_RWLOCK #define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ KVM_DIRTY_LOG_INITIALLY_SET) @@ -94,6 +85,8 @@ struct kvm_arch { /* G-stage page table */ pgd_t *pgd; phys_addr_t pgd_phys; + unsigned long pgd_levels; + struct kvm_mmu_memory_cache pgd_split_page_cache; /* Guest Timer */ struct kvm_guest_timer timer; @@ -167,16 +160,14 @@ struct kvm_vcpu_csr { unsigned long senvcfg; }; -struct kvm_vcpu_config { - u64 henvcfg; - u64 hstateen0; - unsigned long hedeleg; -}; - struct kvm_vcpu_smstateen_csr { unsigned long sstateen0; }; +struct kvm_vcpu_zicfiss_csr { + unsigned long ssp; +}; + struct kvm_vcpu_reset_state { spinlock_t lock; unsigned long pc; @@ -217,19 +208,22 @@ struct kvm_vcpu_arch { /* CPU Smstateen CSR context of Guest VCPU */ struct kvm_vcpu_smstateen_csr smstateen_csr; + /* CPU Zicfiss CSR context of Guest VCPU */ + struct kvm_vcpu_zicfiss_csr zicfiss_csr; + /* CPU reset state of Guest VCPU */ struct kvm_vcpu_reset_state reset_state; /* * VCPU interrupts * - * We have a lockless approach for tracking pending VCPU interrupts - * implemented using atomic bitops. The irqs_pending bitmap represent - * pending interrupts whereas irqs_pending_mask represent bits changed - * in irqs_pending. Our approach is modeled around multiple producer - * and single consumer problem where the consumer is the VCPU itself. + * The irqs_pending bitmap represents pending interrupts whereas + * irqs_pending_mask represents bits changed in irqs_pending. Updates + * to these bitmaps are serialized so vcpu interrupt sync/flush cannot + * drop a newly injected interrupt while syncing guest-visible HVIP. */ #define KVM_RISCV_VCPU_NR_IRQS 64 + raw_spinlock_t irqs_pending_lock; DECLARE_BITMAP(irqs_pending, KVM_RISCV_VCPU_NR_IRQS); DECLARE_BITMAP(irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS); @@ -273,6 +267,9 @@ struct kvm_vcpu_arch { /* 'static' configurations which are set only once */ struct kvm_vcpu_config cfg; + /* Indicates modified guest CSRs */ + bool csr_dirty; + /* SBI steal-time accounting */ struct { gpa_t shmem; @@ -293,6 +290,8 @@ static inline bool kvm_arch_pmi_in_guest(struct kvm_vcpu *vcpu) static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} +void kvm_riscv_clear_former_vcpu(void); + int kvm_riscv_setup_default_irq_routing(struct kvm *kvm, u32 lines); void __kvm_riscv_unpriv_trap(void); diff --git a/arch/riscv/include/asm/kvm_isa.h b/arch/riscv/include/asm/kvm_isa.h new file mode 100644 index 000000000000..bc4b956d5f17 --- /dev/null +++ b/arch/riscv/include/asm/kvm_isa.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2026 Qualcomm Technologies, Inc. + */ + +#ifndef __KVM_RISCV_ISA_H +#define __KVM_RISCV_ISA_H + +#include <linux/types.h> + +unsigned long kvm_riscv_base2isa_ext(unsigned long base_ext); + +int __kvm_riscv_isa_check_host(unsigned long ext, unsigned long *base_ext); +#define kvm_riscv_isa_check_host(ext) \ + __kvm_riscv_isa_check_host(KVM_RISCV_ISA_EXT_##ext, NULL) + +bool kvm_riscv_isa_enable_allowed(unsigned long ext); +bool kvm_riscv_isa_disable_allowed(unsigned long ext); + +#endif diff --git a/arch/riscv/include/asm/kvm_nacl.h b/arch/riscv/include/asm/kvm_nacl.h index 4124d5e06a0f..f45407bcaa26 100644 --- a/arch/riscv/include/asm/kvm_nacl.h +++ b/arch/riscv/include/asm/kvm_nacl.h @@ -60,9 +60,11 @@ int kvm_riscv_nacl_init(void); #ifdef CONFIG_32BIT #define lelong_to_cpu(__x) le32_to_cpu(__x) #define cpu_to_lelong(__x) cpu_to_le32(__x) +#define __lelong __le32 #else #define lelong_to_cpu(__x) le64_to_cpu(__x) #define cpu_to_lelong(__x) cpu_to_le64(__x) +#define __lelong __le64 #endif #define nacl_shmem() \ @@ -70,7 +72,7 @@ int kvm_riscv_nacl_init(void); #define nacl_scratch_read_long(__shmem, __offset) \ ({ \ - unsigned long *__p = (__shmem) + \ + __lelong *__p = (__shmem) + \ SBI_NACL_SHMEM_SCRATCH_OFFSET + \ (__offset); \ lelong_to_cpu(*__p); \ @@ -78,7 +80,7 @@ int kvm_riscv_nacl_init(void); #define nacl_scratch_write_long(__shmem, __offset, __val) \ do { \ - unsigned long *__p = (__shmem) + \ + __lelong *__p = (__shmem) + \ SBI_NACL_SHMEM_SCRATCH_OFFSET + \ (__offset); \ *__p = cpu_to_lelong(__val); \ @@ -87,7 +89,7 @@ do { \ #define nacl_scratch_write_longs(__shmem, __offset, __array, __count) \ do { \ unsigned int __i; \ - unsigned long *__p = (__shmem) + \ + __lelong *__p = (__shmem) + \ SBI_NACL_SHMEM_SCRATCH_OFFSET + \ (__offset); \ for (__i = 0; __i < (__count); __i++) \ @@ -168,7 +170,7 @@ __kvm_riscv_nacl_hfence(__shmem, \ #define nacl_csr_read(__shmem, __csr) \ ({ \ - unsigned long *__a = (__shmem) + SBI_NACL_SHMEM_CSR_OFFSET; \ + __lelong *__a = (__shmem) + SBI_NACL_SHMEM_CSR_OFFSET; \ lelong_to_cpu(__a[SBI_NACL_SHMEM_CSR_INDEX(__csr)]); \ }) @@ -176,7 +178,7 @@ __kvm_riscv_nacl_hfence(__shmem, \ do { \ void *__s = (__shmem); \ unsigned int __i = SBI_NACL_SHMEM_CSR_INDEX(__csr); \ - unsigned long *__a = (__s) + SBI_NACL_SHMEM_CSR_OFFSET; \ + __lelong *__a = (__s) + SBI_NACL_SHMEM_CSR_OFFSET; \ u8 *__b = (__s) + SBI_NACL_SHMEM_DBITMAP_OFFSET; \ __a[__i] = cpu_to_lelong(__val); \ __b[__i >> 3] |= 1U << (__i & 0x7); \ @@ -186,7 +188,7 @@ do { \ ({ \ void *__s = (__shmem); \ unsigned int __i = SBI_NACL_SHMEM_CSR_INDEX(__csr); \ - unsigned long *__a = (__s) + SBI_NACL_SHMEM_CSR_OFFSET; \ + __lelong *__a = (__s) + SBI_NACL_SHMEM_CSR_OFFSET; \ u8 *__b = (__s) + SBI_NACL_SHMEM_DBITMAP_OFFSET; \ unsigned long __r = lelong_to_cpu(__a[__i]); \ __a[__i] = cpu_to_lelong(__val); \ diff --git a/arch/riscv/include/asm/kvm_vcpu_config.h b/arch/riscv/include/asm/kvm_vcpu_config.h new file mode 100644 index 000000000000..fcc15a0296b3 --- /dev/null +++ b/arch/riscv/include/asm/kvm_vcpu_config.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2026 Qualcomm Technologies, Inc. + */ + +#ifndef __KVM_VCPU_RISCV_CONFIG_H +#define __KVM_VCPU_RISCV_CONFIG_H + +#include <linux/types.h> + +struct kvm_vcpu; + +struct kvm_vcpu_config { + u64 henvcfg; + u64 hstateen0; + unsigned long hedeleg; + unsigned long hideleg; +}; + +void kvm_riscv_vcpu_config_init(struct kvm_vcpu *vcpu); +void kvm_riscv_vcpu_config_guest_debug(struct kvm_vcpu *vcpu); +void kvm_riscv_vcpu_config_ran_once(struct kvm_vcpu *vcpu); +void kvm_riscv_vcpu_config_load(struct kvm_vcpu *vcpu); + +#endif diff --git a/arch/riscv/include/asm/kvm_vcpu_insn.h b/arch/riscv/include/asm/kvm_vcpu_insn.h index 350011c83581..a5c50dd4a884 100644 --- a/arch/riscv/include/asm/kvm_vcpu_insn.h +++ b/arch/riscv/include/asm/kvm_vcpu_insn.h @@ -38,10 +38,10 @@ int kvm_riscv_vcpu_virtual_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, struct kvm_cpu_trap *trap); int kvm_riscv_vcpu_mmio_load(struct kvm_vcpu *vcpu, struct kvm_run *run, - unsigned long fault_addr, + gpa_t fault_addr, unsigned long htinst); int kvm_riscv_vcpu_mmio_store(struct kvm_vcpu *vcpu, struct kvm_run *run, - unsigned long fault_addr, + gpa_t fault_addr, unsigned long htinst); int kvm_riscv_vcpu_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run); diff --git a/arch/riscv/include/asm/kvm_vcpu_sbi.h b/arch/riscv/include/asm/kvm_vcpu_sbi.h index c1a7e3b40d9c..f01a2860c751 100644 --- a/arch/riscv/include/asm/kvm_vcpu_sbi.h +++ b/arch/riscv/include/asm/kvm_vcpu_sbi.h @@ -60,6 +60,9 @@ struct kvm_vcpu_sbi_extension { void (*reset)(struct kvm_vcpu *vcpu); + /* Allow the extension to correct its parameters before the first run */ + void (*validate)(struct kvm_vcpu *vcpu); + unsigned long state_reg_subtype; unsigned long (*get_state_reg_count)(struct kvm_vcpu *vcpu); int (*get_state_reg_id)(struct kvm_vcpu *vcpu, int index, u64 *reg_id); @@ -93,6 +96,7 @@ int kvm_riscv_vcpu_sbi_ecall(struct kvm_vcpu *vcpu, struct kvm_run *run); void kvm_riscv_vcpu_sbi_init(struct kvm_vcpu *vcpu); void kvm_riscv_vcpu_sbi_deinit(struct kvm_vcpu *vcpu); void kvm_riscv_vcpu_sbi_reset(struct kvm_vcpu *vcpu); +void kvm_riscv_vcpu_sbi_validate(struct kvm_vcpu *vcpu); #ifdef CONFIG_RISCV_SBI_V01 extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_v01; diff --git a/arch/riscv/include/asm/kvm_vcpu_vector.h b/arch/riscv/include/asm/kvm_vcpu_vector.h index 57a798a4cb0d..6371d5ea5392 100644 --- a/arch/riscv/include/asm/kvm_vcpu_vector.h +++ b/arch/riscv/include/asm/kvm_vcpu_vector.h @@ -35,6 +35,22 @@ void kvm_riscv_vcpu_host_vector_save(struct kvm_cpu_context *cntx); void kvm_riscv_vcpu_host_vector_restore(struct kvm_cpu_context *cntx); int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu); void kvm_riscv_vcpu_free_vector_context(struct kvm_vcpu *vcpu); +void kvm_riscv_register_vctx_callback(void (*func)(void)); +void kvm_riscv_unregister_vctx_callback(void); +void kvm_riscv_vcpu_flush_vector(void); + +static inline void kvm_riscv_v_init(void) +{ + if (has_vector()) + kvm_riscv_register_vctx_callback(&kvm_riscv_vcpu_flush_vector); +} + +static inline void kvm_riscv_v_exit(void) +{ + if (has_vector()) + kvm_riscv_unregister_vctx_callback(); +} + #else struct kvm_cpu_context; @@ -69,6 +85,14 @@ static inline int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu) static inline void kvm_riscv_vcpu_free_vector_context(struct kvm_vcpu *vcpu) { } + +static inline void kvm_riscv_v_init(void) +{ +} + +static inline void kvm_riscv_v_exit(void) +{ +} #endif int kvm_riscv_vcpu_get_reg_vector(struct kvm_vcpu *vcpu, diff --git a/arch/riscv/include/asm/linkage.h b/arch/riscv/include/asm/linkage.h index 9e88ba23cd2b..7e0210ef4eb4 100644 --- a/arch/riscv/include/asm/linkage.h +++ b/arch/riscv/include/asm/linkage.h @@ -9,4 +9,6 @@ #define __ALIGN .balign 4 #define __ALIGN_STR ".balign 4" +#define _THIS_IP_ ({ unsigned long __ip; asm volatile("auipc %0, 0" : "=r" (__ip)); __ip; }) + #endif /* _ASM_RISCV_LINKAGE_H */ diff --git a/arch/riscv/include/asm/mman.h b/arch/riscv/include/asm/mman.h new file mode 100644 index 000000000000..0ad1d19832eb --- /dev/null +++ b/arch/riscv/include/asm/mman.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_MMAN_H__ +#define __ASM_MMAN_H__ + +#include <linux/compiler.h> +#include <linux/types.h> +#include <linux/mm.h> +#include <uapi/asm/mman.h> + +static inline unsigned long arch_calc_vm_prot_bits(unsigned long prot, + unsigned long pkey __always_unused) +{ + unsigned long ret = 0; + + /* + * If PROT_WRITE was specified, force it to VM_READ | VM_WRITE. + * Only VM_WRITE means shadow stack. + */ + if (prot & PROT_WRITE) + ret = (VM_READ | VM_WRITE); + return ret; +} + +#define arch_calc_vm_prot_bits(prot, pkey) arch_calc_vm_prot_bits(prot, pkey) + +#endif /* ! __ASM_MMAN_H__ */ diff --git a/arch/riscv/include/asm/mmu_context.h b/arch/riscv/include/asm/mmu_context.h index 8c4bc49a3a0f..dbf27a78df6c 100644 --- a/arch/riscv/include/asm/mmu_context.h +++ b/arch/riscv/include/asm/mmu_context.h @@ -48,6 +48,13 @@ static inline unsigned long mm_untag_mask(struct mm_struct *mm) } #endif +#define deactivate_mm deactivate_mm +static inline void deactivate_mm(struct task_struct *tsk, + struct mm_struct *mm) +{ + shstk_release(tsk); +} + #include <asm-generic/mmu_context.h> #endif /* _ASM_RISCV_MMU_CONTEXT_H */ diff --git a/arch/riscv/include/asm/module.lds.h b/arch/riscv/include/asm/module.lds.h index 1075beae1ac6..9ced27c8ccb6 100644 --- a/arch/riscv/include/asm/module.lds.h +++ b/arch/riscv/include/asm/module.lds.h @@ -2,8 +2,8 @@ /* Copyright (C) 2017 Andes Technology Corporation */ #ifdef CONFIG_MODULE_SECTIONS SECTIONS { - .plt : { BYTE(0) } - .got : { BYTE(0) } - .got.plt : { BYTE(0) } + .plt 0 : { BYTE(0) } + .got 0 : { BYTE(0) } + .got.plt 0 : { BYTE(0) } } #endif diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h index ffe213ad65a4..709a36fb4323 100644 --- a/arch/riscv/include/asm/page.h +++ b/arch/riscv/include/asm/page.h @@ -29,11 +29,7 @@ #define PAGE_OFFSET_L5 _AC(0xff60000000000000, UL) #define PAGE_OFFSET_L4 _AC(0xffffaf8000000000, UL) #define PAGE_OFFSET_L3 _AC(0xffffffd600000000, UL) -#ifdef CONFIG_XIP_KERNEL -#define PAGE_OFFSET PAGE_OFFSET_L3 -#else #define PAGE_OFFSET kernel_map.page_offset -#endif /* CONFIG_XIP_KERNEL */ #else #define PAGE_OFFSET _AC(0xc0000000, UL) #endif /* CONFIG_64BIT */ @@ -50,9 +46,7 @@ void clear_page(void *page); #endif #define copy_page(to, from) memcpy((to), (from), PAGE_SIZE) -#define clear_user_page(pgaddr, vaddr, page) clear_page(pgaddr) -#define copy_user_page(vto, vfrom, vaddr, topg) \ - memcpy((vto), (vfrom), PAGE_SIZE) +#define copy_user_page(vto, vfrom, vaddr, topg) copy_page(vto, vfrom) /* * Use struct definitions to apply C type checking @@ -106,15 +100,8 @@ struct kernel_mapping { /* Offset between linear mapping virtual address and kernel load address */ unsigned long va_pa_offset; /* Offset between kernel mapping virtual address and kernel load address */ -#ifdef CONFIG_XIP_KERNEL - unsigned long va_kernel_xip_text_pa_offset; - unsigned long va_kernel_xip_data_pa_offset; - uintptr_t xiprom; - uintptr_t xiprom_sz; -#else unsigned long page_offset; unsigned long va_kernel_pa_offset; -#endif }; extern struct kernel_mapping kernel_map; @@ -133,16 +120,7 @@ extern unsigned long vmemmap_start_pfn; void *linear_mapping_pa_to_va(unsigned long x); #endif -#ifdef CONFIG_XIP_KERNEL -#define kernel_mapping_pa_to_va(y) ({ \ - unsigned long _y = (unsigned long)(y); \ - (_y < phys_ram_base) ? \ - (void *)(_y + kernel_map.va_kernel_xip_text_pa_offset) : \ - (void *)(_y + kernel_map.va_kernel_xip_data_pa_offset); \ - }) -#else #define kernel_mapping_pa_to_va(y) ((void *)((unsigned long)(y) + kernel_map.va_kernel_pa_offset)) -#endif #define __pa_to_va_nodebug(x) linear_mapping_pa_to_va(x) @@ -152,16 +130,7 @@ void *linear_mapping_pa_to_va(unsigned long x); phys_addr_t linear_mapping_va_to_pa(unsigned long x); #endif -#ifdef CONFIG_XIP_KERNEL -#define kernel_mapping_va_to_pa(y) ({ \ - unsigned long _y = (unsigned long)(y); \ - (_y < kernel_map.virt_addr + kernel_map.xiprom_sz) ? \ - (_y - kernel_map.va_kernel_xip_text_pa_offset) : \ - (_y - kernel_map.va_kernel_xip_data_pa_offset); \ - }) -#else #define kernel_mapping_va_to_pa(y) ((unsigned long)(y) - kernel_map.va_kernel_pa_offset) -#endif #define __va_to_pa_nodebug(x) ({ \ unsigned long _x = x; \ @@ -192,7 +161,10 @@ extern phys_addr_t __phys_addr_symbol(unsigned long x); #define sym_to_pfn(x) __phys_to_pfn(__pa_symbol(x)) -unsigned long kaslr_offset(void); +static inline unsigned long kaslr_offset(void) +{ + return kernel_map.virt_offset; +} static __always_inline void *pfn_to_kaddr(unsigned long pfn) { @@ -206,7 +178,7 @@ static __always_inline void *pfn_to_kaddr(unsigned long pfn) (unsigned long)(_addr) >= PAGE_OFFSET && pfn_valid(virt_to_pfn(_addr)); \ }) -#define VM_DATA_DEFAULT_FLAGS VM_DATA_FLAGS_NON_EXEC +#define VMA_DATA_DEFAULT_FLAGS VMA_DATA_FLAGS_NON_EXEC #include <asm-generic/memory_model.h> #include <asm-generic/getorder.h> diff --git a/arch/riscv/include/asm/paravirt.h b/arch/riscv/include/asm/paravirt.h index c0abde70fc2c..c49c55b266f3 100644 --- a/arch/riscv/include/asm/paravirt.h +++ b/arch/riscv/include/asm/paravirt.h @@ -3,20 +3,6 @@ #define _ASM_RISCV_PARAVIRT_H #ifdef CONFIG_PARAVIRT -#include <linux/static_call_types.h> - -struct static_key; -extern struct static_key paravirt_steal_enabled; -extern struct static_key paravirt_steal_rq_enabled; - -u64 dummy_steal_clock(int cpu); - -DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); - -static inline u64 paravirt_steal_clock(int cpu) -{ - return static_call(pv_steal_clock)(cpu); -} int __init pv_time_init(void); diff --git a/arch/riscv/include/asm/paravirt_api_clock.h b/arch/riscv/include/asm/paravirt_api_clock.h deleted file mode 100644 index 65ac7cee0dad..000000000000 --- a/arch/riscv/include/asm/paravirt_api_clock.h +++ /dev/null @@ -1 +0,0 @@ -#include <asm/paravirt.h> diff --git a/arch/riscv/include/asm/pgtable-bits.h b/arch/riscv/include/asm/pgtable-bits.h index b422d9691e60..d5a86b4df3ce 100644 --- a/arch/riscv/include/asm/pgtable-bits.h +++ b/arch/riscv/include/asm/pgtable-bits.h @@ -40,20 +40,20 @@ #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -/* ext_svrsw60t59b: Bit(60) for uffd-wp tracking */ -#define _PAGE_UFFD_WP \ +/* ext_svrsw60t59b: Bit(60) for userfaultfd tracking */ +#define _PAGE_UFFD \ ((riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B)) ? \ (1UL << 60) : 0) /* * Bit 4 is not involved into swap entry computation, so we - * can borrow it for swap page uffd-wp tracking. + * can borrow it for swap page userfaultfd tracking. */ -#define _PAGE_SWP_UFFD_WP \ +#define _PAGE_SWP_UFFD \ ((riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B)) ? \ _PAGE_USER : 0) #else -#define _PAGE_UFFD_WP 0 -#define _PAGE_SWP_UFFD_WP 0 +#define _PAGE_UFFD 0 +#define _PAGE_SWP_UFFD 0 #endif #define _PAGE_TABLE _PAGE_PRESENT diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 9acd58a67123..40b1ed4f3ea8 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -93,6 +93,16 @@ */ #define vmemmap ((struct page *)VMEMMAP_START - vmemmap_start_pfn) +/* Needed to limit get_free_mem_region() */ +#if defined(CONFIG_FLATMEM) +#define DIRECT_MAP_PHYSMEM_END (phys_ram_base + KERN_VIRT_SIZE - 1) +#elif defined(CONFIG_SPARSEMEM_VMEMMAP) +#define DIRECT_MAP_PHYSMEM_END \ + ((vmemmap_start_pfn + VMEMMAP_SIZE / sizeof(struct page)) * PAGE_SIZE - 1) +#elif defined(CONFIG_SPARSEMEM) +/* DIRECT_MAP_PHYSMEM_END is not limited by VA space assignment in this case */ +#endif + #define PCI_IO_SIZE SZ_16M #define PCI_IO_END VMEMMAP_START #define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE) @@ -134,21 +144,6 @@ #include <linux/page_table_check.h> -#ifdef CONFIG_XIP_KERNEL -#define XIP_FIXUP(addr) ({ \ - extern char _sdata[], _start[], _end[]; \ - uintptr_t __rom_start_data = CONFIG_XIP_PHYS_ADDR \ - + (uintptr_t)&_sdata - (uintptr_t)&_start; \ - uintptr_t __rom_end_data = CONFIG_XIP_PHYS_ADDR \ - + (uintptr_t)&_end - (uintptr_t)&_start; \ - uintptr_t __a = (uintptr_t)(addr); \ - (__a >= __rom_start_data && __a < __rom_end_data) ? \ - __a - __rom_start_data + CONFIG_PHYS_RAM_BASE : __a; \ - }) -#else -#define XIP_FIXUP(addr) (addr) -#endif /* CONFIG_XIP_KERNEL */ - struct pt_alloc_ops { pte_t *(*get_pte_virt)(phys_addr_t pa); phys_addr_t (*alloc_pte)(uintptr_t va); @@ -178,6 +173,7 @@ extern struct pt_alloc_ops pt_ops __meminitdata; #define PAGE_READ_EXEC __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_EXEC) #define PAGE_WRITE_EXEC __pgprot(_PAGE_BASE | _PAGE_READ | \ _PAGE_EXEC | _PAGE_WRITE) +#define PAGE_SHADOWSTACK __pgprot(_PAGE_BASE | _PAGE_WRITE) #define PAGE_COPY PAGE_READ #define PAGE_COPY_EXEC PAGE_READ_EXEC @@ -410,51 +406,60 @@ static inline int pte_special(pte_t pte) static inline pte_t pte_wrprotect(pte_t pte) { - return __pte(pte_val(pte) & ~(_PAGE_WRITE)); + return __pte((pte_val(pte) & ~(_PAGE_WRITE)) | (_PAGE_READ)); } #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -#define pgtable_supports_uffd_wp() \ +#define pgtable_supports_uffd() \ riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B) -static inline bool pte_uffd_wp(pte_t pte) +static inline bool pte_uffd(pte_t pte) { - return !!(pte_val(pte) & _PAGE_UFFD_WP); + return !!(pte_val(pte) & _PAGE_UFFD); } -static inline pte_t pte_mkuffd_wp(pte_t pte) +static inline pte_t pte_mkuffd(pte_t pte) { - return pte_wrprotect(__pte(pte_val(pte) | _PAGE_UFFD_WP)); + return pte_wrprotect(__pte(pte_val(pte) | _PAGE_UFFD)); } -static inline pte_t pte_clear_uffd_wp(pte_t pte) +static inline pte_t pte_clear_uffd(pte_t pte) { - return __pte(pte_val(pte) & ~(_PAGE_UFFD_WP)); + return __pte(pte_val(pte) & ~(_PAGE_UFFD)); } -static inline bool pte_swp_uffd_wp(pte_t pte) +static inline bool pte_swp_uffd(pte_t pte) { - return !!(pte_val(pte) & _PAGE_SWP_UFFD_WP); + return !!(pte_val(pte) & _PAGE_SWP_UFFD); } -static inline pte_t pte_swp_mkuffd_wp(pte_t pte) +static inline pte_t pte_swp_mkuffd(pte_t pte) { - return __pte(pte_val(pte) | _PAGE_SWP_UFFD_WP); + return __pte(pte_val(pte) | _PAGE_SWP_UFFD); } -static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) +static inline pte_t pte_swp_clear_uffd(pte_t pte) { - return __pte(pte_val(pte) & ~(_PAGE_SWP_UFFD_WP)); + return __pte(pte_val(pte) & ~(_PAGE_SWP_UFFD)); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ /* static inline pte_t pte_mkread(pte_t pte) */ +struct vm_area_struct; +pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma); +#define pte_mkwrite pte_mkwrite + static inline pte_t pte_mkwrite_novma(pte_t pte) { return __pte(pte_val(pte) | _PAGE_WRITE); } +static inline pte_t pte_mkwrite_shstk(pte_t pte) +{ + return __pte((pte_val(pte) & ~(_PAGE_LEAF)) | _PAGE_WRITE); +} + /* static inline pte_t pte_mkexec(pte_t pte) */ static inline pte_t pte_mkdirty(pte_t pte) @@ -529,10 +534,7 @@ static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) PAGE_SIZE) #endif -#ifdef CONFIG_NUMA_BALANCING -/* - * See the comment in include/asm-generic/pgtable.h - */ +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pte_protnone(pte_t pte) { return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE)) == _PAGE_PROT_NONE; @@ -542,7 +544,7 @@ static inline int pmd_protnone(pmd_t pmd) { return pte_protnone(pmd_pte(pmd)); } -#endif +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ /* Modify page protection bits */ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) @@ -563,6 +565,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { + unsigned long asid = get_mm_asid(vma->vm_mm); + /* * Svvptc guarantees that the new valid pte will be visible within * a bounded timeframe, so when the uarch does not cache invalid @@ -571,6 +575,14 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC)) return; + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)) { + local_sfence_w_inval(); + while (nr--) + local_sinval_vma(address + nr * PAGE_SIZE, asid); + local_sfence_inval_ir(); + return; + } + /* * The kernel assumes that TLBs don't cache invalid entries, but * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a @@ -579,7 +591,7 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, * the extra traps reduce performance. So, eagerly SFENCE.VMA. */ while (nr--) - local_flush_tlb_page(address + nr * PAGE_SIZE); + local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid); } #define update_mmu_cache(vma, addr, ptep) \ @@ -627,7 +639,7 @@ static inline void __set_pte_at(struct mm_struct *mm, pte_t *ptep, pte_t pteval) static inline void set_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pteval, unsigned int nr) { - page_table_check_ptes_set(mm, ptep, pteval, nr); + page_table_check_ptes_set(mm, addr, ptep, pteval, nr); for (;;) { __set_pte_at(mm, ptep, pteval); @@ -649,8 +661,8 @@ static inline void pte_clear(struct mm_struct *mm, extern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, pte_t *ptep, pte_t entry, int dirty); #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG /* defined in mm/pgtable.c */ -extern int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, - pte_t *ptep); +bool ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep); #define __HAVE_ARCH_PTEP_GET_AND_CLEAR static inline pte_t ptep_get_and_clear(struct mm_struct *mm, @@ -664,7 +676,7 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, set_pte(ptep, __pte(0)); #endif - page_table_check_pte_clear(mm, pte); + page_table_check_pte_clear(mm, address, pte); return pte; } @@ -673,12 +685,20 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep) { - atomic_long_and(~(unsigned long)_PAGE_WRITE, (atomic_long_t *)ptep); + pte_t read_pte = READ_ONCE(*ptep); + /* + * ptep_set_wrprotect can be called for shadow stack ranges too. + * shadow stack memory is XWR = 010 and thus clearing _PAGE_WRITE will lead to + * encoding 000b which is wrong encoding with V = 1. This should lead to page fault + * but we dont want this wrong configuration to be set in page tables. + */ + atomic_long_set((atomic_long_t *)ptep, + ((pte_val(read_pte) & ~(unsigned long)_PAGE_WRITE) | _PAGE_READ)); } #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH -static inline int ptep_clear_flush_young(struct vm_area_struct *vma, - unsigned long address, pte_t *ptep) +static inline bool ptep_clear_flush_young(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep) { /* * This comment is borrowed from x86, but applies equally to RISC-V: @@ -833,11 +853,19 @@ static inline pmd_t pmd_mkyoung(pmd_t pmd) return pte_pmd(pte_mkyoung(pmd_pte(pmd))); } +pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma); +#define pmd_mkwrite pmd_mkwrite + static inline pmd_t pmd_mkwrite_novma(pmd_t pmd) { return pte_pmd(pte_mkwrite_novma(pmd_pte(pmd))); } +static inline pmd_t pmd_mkwrite_shstk(pmd_t pte) +{ + return __pmd((pmd_val(pte) & ~(_PAGE_LEAF)) | _PAGE_WRITE); +} + static inline pmd_t pmd_wrprotect(pmd_t pmd) { return pte_pmd(pte_wrprotect(pmd_pte(pmd))); @@ -878,34 +906,34 @@ static inline pud_t pud_mkspecial(pud_t pud) #endif #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static inline bool pmd_uffd_wp(pmd_t pmd) +static inline bool pmd_uffd(pmd_t pmd) { - return pte_uffd_wp(pmd_pte(pmd)); + return pte_uffd(pmd_pte(pmd)); } -static inline pmd_t pmd_mkuffd_wp(pmd_t pmd) +static inline pmd_t pmd_mkuffd(pmd_t pmd) { - return pte_pmd(pte_mkuffd_wp(pmd_pte(pmd))); + return pte_pmd(pte_mkuffd(pmd_pte(pmd))); } -static inline pmd_t pmd_clear_uffd_wp(pmd_t pmd) +static inline pmd_t pmd_clear_uffd(pmd_t pmd) { - return pte_pmd(pte_clear_uffd_wp(pmd_pte(pmd))); + return pte_pmd(pte_clear_uffd(pmd_pte(pmd))); } -static inline bool pmd_swp_uffd_wp(pmd_t pmd) +static inline bool pmd_swp_uffd(pmd_t pmd) { - return pte_swp_uffd_wp(pmd_pte(pmd)); + return pte_swp_uffd(pmd_pte(pmd)); } -static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_mkuffd(pmd_t pmd) { - return pte_pmd(pte_swp_mkuffd_wp(pmd_pte(pmd))); + return pte_pmd(pte_swp_mkuffd(pmd_pte(pmd))); } -static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_clear_uffd(pmd_t pmd) { - return pte_pmd(pte_swp_clear_uffd_wp(pmd_pte(pmd))); + return pte_pmd(pte_swp_clear_uffd(pmd_pte(pmd))); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ @@ -925,7 +953,7 @@ static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) return pte_pmd(pte_clear_soft_dirty(pmd_pte(pmd))); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static inline bool pmd_swp_soft_dirty(pmd_t pmd) { return pte_swp_soft_dirty(pmd_pte(pmd)); @@ -940,35 +968,35 @@ static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) { return pte_pmd(pte_swp_clear_soft_dirty(pmd_pte(pmd))); } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd) { - page_table_check_pmd_set(mm, pmdp, pmd); + page_table_check_pmd_set(mm, addr, pmdp, pmd); return __set_pte_at(mm, (pte_t *)pmdp, pmd_pte(pmd)); } static inline void set_pud_at(struct mm_struct *mm, unsigned long addr, pud_t *pudp, pud_t pud) { - page_table_check_pud_set(mm, pudp, pud); + page_table_check_pud_set(mm, addr, pudp, pud); return __set_pte_at(mm, (pte_t *)pudp, pud_pte(pud)); } #ifdef CONFIG_PAGE_TABLE_CHECK -static inline bool pte_user_accessible_page(pte_t pte) +static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte) { return pte_present(pte) && pte_user(pte); } -static inline bool pmd_user_accessible_page(pmd_t pmd) +static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd) { return pmd_leaf(pmd) && pmd_user(pmd); } -static inline bool pud_user_accessible_page(pud_t pud) +static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud) { return pud_leaf(pud) && pud_user(pud); } @@ -989,8 +1017,8 @@ static inline int pmdp_set_access_flags(struct vm_area_struct *vma, } #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG -static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, - unsigned long address, pmd_t *pmdp) +static inline bool pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp) { return ptep_test_and_clear_young(vma, address, (pte_t *)pmdp); } @@ -1007,7 +1035,7 @@ static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, pmd_clear(pmdp); #endif - page_table_check_pmd_clear(mm, pmd); + page_table_check_pmd_clear(mm, address, pmd); return pmd; } @@ -1023,7 +1051,7 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm, static inline pmd_t pmdp_establish(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp, pmd_t pmd) { - page_table_check_pmd_set(vma->vm_mm, pmdp, pmd); + page_table_check_pmd_set(vma->vm_mm, address, pmdp, pmd); return __pmd(atomic_long_xchg((atomic_long_t *)pmdp, pmd_val(pmd))); } @@ -1083,8 +1111,8 @@ static inline int pudp_set_access_flags(struct vm_area_struct *vma, return ptep_set_access_flags(vma, address, (pte_t *)pudp, pud_pte(entry), dirty); } -static inline int pudp_test_and_clear_young(struct vm_area_struct *vma, - unsigned long address, pud_t *pudp) +static inline bool pudp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, pud_t *pudp) { return ptep_test_and_clear_young(vma, address, (pte_t *)pudp); } @@ -1101,7 +1129,7 @@ static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm, pud_clear(pudp); #endif - page_table_check_pud_clear(mm, pud); + page_table_check_pud_clear(mm, address, pud); return pud; } @@ -1122,7 +1150,7 @@ static inline void update_mmu_cache_pud(struct vm_area_struct *vma, static inline pud_t pudp_establish(struct vm_area_struct *vma, unsigned long address, pud_t *pudp, pud_t pud) { - page_table_check_pud_set(vma->vm_mm, pudp, pud); + page_table_check_pud_set(vma->vm_mm, address, pudp, pud); return __pud(atomic_long_xchg((atomic_long_t *)pudp, pud_val(pud))); } @@ -1149,7 +1177,7 @@ static inline pud_t pud_modify(pud_t pud, pgprot_t newprot) * bit 0: _PAGE_PRESENT (zero) * bit 1 to 2: (zero) * bit 3: _PAGE_SWP_SOFT_DIRTY - * bit 4: _PAGE_SWP_UFFD_WP + * bit 4: _PAGE_SWP_UFFD * bit 5: _PAGE_PROT_NONE (zero) * bit 6: exclusive marker * bits 7 to 11: swap type @@ -1187,10 +1215,10 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) return __pte(pte_val(pte) & ~_PAGE_SWP_EXCLUSIVE); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES #define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) }) #define __swp_entry_to_pmd(swp) __pmd((swp).val) -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ /* * In the RV64 Linux scheme, we give the user half of the virtual-address space @@ -1246,26 +1274,14 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) extern char _start[]; extern void *_dtb_early_va; extern uintptr_t _dtb_early_pa; -#if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_MMU) -#define dtb_early_va (*(void **)XIP_FIXUP(&_dtb_early_va)) -#define dtb_early_pa (*(uintptr_t *)XIP_FIXUP(&_dtb_early_pa)) -#else #define dtb_early_va _dtb_early_va #define dtb_early_pa _dtb_early_pa -#endif /* CONFIG_XIP_KERNEL */ extern u64 satp_mode; void paging_init(void); void misc_mem_init(void); /* - * ZERO_PAGE is a global shared page that is always zero, - * used for zero-mapped memory areas, etc. - */ -extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; -#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) - -/* * Use set_p*_safe(), and elide TLB flushing, when confident that *no* * TLB flush will be required as a result of the "set". For example, use * in scenarios where it is known ahead of time that the routine is diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h index da5426122d28..815715c67f94 100644 --- a/arch/riscv/include/asm/processor.h +++ b/arch/riscv/include/asm/processor.h @@ -16,6 +16,7 @@ #include <asm/insn-def.h> #include <asm/alternative-macros.h> #include <asm/hwcap.h> +#include <asm/usercfi.h> #define arch_get_mmap_end(addr, len, flags) \ ({ \ @@ -66,6 +67,12 @@ struct pt_regs; * - bit 0: indicates whether the in-kernel Vector context is active. The * activation of this state disables the preemption. On a non-RT kernel, it * also disable bh. + * - bit 1: tells kvm that the vcpu process has guest context saved in vcpu's + * context memory and need to be restore upon returing back to the guest. + * - bit 2: represents that the vector context has now loaded and belongs to + * the guest kernel. Any non-scheduler context saving routing needs to save + * the register file to vcpu's context memory. The bit is set upon returing + * back to the guest and cleared after loading the host's vector context. * - bits 8: is used for tracking preemptible kernel-mode Vector, when * RISCV_ISA_V_PREEMPTIVE is enabled. Calling kernel_vector_begin() does not * disable the preemption if the thread's kernel_vstate.datap is allocated. @@ -85,7 +92,7 @@ struct pt_regs; * preempt_v. All preempt_v context should be dropped in such case because * V-regs are caller-saved. Only sstatus.VS=ON is persisted across a * schedule() call. - * - bit 30: The in-kernel preempt_v context is saved, and requries to be + * - bit 30: The in-kernel preempt_v context is saved, and is required to be * restored when returning to the context that owns the preempt_v. * - bit 31: The in-kernel preempt_v context is dirty, as signaled by the * trap entry code. Any context switches out-of current task need to save @@ -96,6 +103,8 @@ struct pt_regs; #define RISCV_V_CTX_UNIT_DEPTH 0x00010000 #define RISCV_KERNEL_MODE_V 0x00000001 +#define RISCV_V_VCPU_NEED_RESTORE 0x00000002 +#define RISCV_V_VCPU_CTX 0x00000004 #define RISCV_PREEMPT_V 0x00000100 #define RISCV_PREEMPT_V_DIRTY 0x80000000 #define RISCV_PREEMPT_V_NEED_RESTORE 0x40000000 @@ -122,6 +131,9 @@ struct thread_struct { /* A forced icache flush is not needed if migrating to the previous cpu. */ unsigned int prev_cpu; #endif +#ifdef CONFIG_RISCV_ISA_SSQOSID + u32 srmcfg; +#endif }; /* Whitelist the fstate from the task_struct for hardened usercopy */ diff --git a/arch/riscv/include/asm/purgatory.h b/arch/riscv/include/asm/purgatory.h new file mode 100644 index 000000000000..5a827e6752b7 --- /dev/null +++ b/arch/riscv/include/asm/purgatory.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_RISCV_PURGATORY_H +#define _ASM_RISCV_PURGATORY_H + +#ifndef __ASSEMBLER__ +#include <linux/purgatory.h> + +void purgatory(void); +#endif /* __ASSEMBLER__ */ + +#endif /* _ASM_RISCV_PURGATORY_H */ diff --git a/arch/riscv/include/asm/qos.h b/arch/riscv/include/asm/qos.h new file mode 100644 index 000000000000..cf19e8438bb9 --- /dev/null +++ b/arch/riscv/include/asm/qos.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_RISCV_QOS_H +#define _ASM_RISCV_QOS_H + +#include <linux/percpu-defs.h> + +#ifdef CONFIG_RISCV_ISA_SSQOSID + +#include <linux/bitfield.h> +#include <linux/cpufeature.h> +#include <linux/sched.h> + +#include <asm/csr.h> +#include <asm/hwcap.h> + +/* cached value of srmcfg csr for each cpu */ +DECLARE_PER_CPU(u32, cpu_srmcfg); + +/* default srmcfg value for each cpu, set via resctrl cpu assignment */ +DECLARE_PER_CPU(u32, cpu_srmcfg_default); + +static inline void __switch_to_srmcfg(struct task_struct *next) +{ + u32 thread_srmcfg, default_srmcfg; + + thread_srmcfg = READ_ONCE(next->thread.srmcfg); + default_srmcfg = __this_cpu_read(cpu_srmcfg_default); + + /* + * RCID and MCID inherit from cpu_srmcfg_default independently. + * RESCTRL_RESERVED_CLOSID and RESCTRL_RESERVED_RMID are both 0, so a + * zero field means "unassigned" and takes the CPU default. + */ + if (thread_srmcfg == 0) { + thread_srmcfg = default_srmcfg; + } else { + u32 rcid = FIELD_GET(SRMCFG_RCID_MASK, thread_srmcfg); + u32 mcid = FIELD_GET(SRMCFG_MCID_MASK, thread_srmcfg); + + if (rcid == 0 || mcid == 0) { + if (rcid == 0) + rcid = FIELD_GET(SRMCFG_RCID_MASK, default_srmcfg); + if (mcid == 0) + mcid = FIELD_GET(SRMCFG_MCID_MASK, default_srmcfg); + thread_srmcfg = FIELD_PREP(SRMCFG_RCID_MASK, rcid) | + FIELD_PREP(SRMCFG_MCID_MASK, mcid); + } + } + + if (thread_srmcfg != __this_cpu_read(cpu_srmcfg)) { + /* + * No fence around the csrw. Ssqosid is silent on srmcfg + * ordering versus memory accesses, so a few accesses at the + * switch boundary may carry the previous RCID/MCID. The + * tagging inaccuracy is bounded and acceptable for QoS. + */ + __this_cpu_write(cpu_srmcfg, thread_srmcfg); + csr_write(CSR_SRMCFG, thread_srmcfg); + } +} + +static __always_inline bool has_srmcfg(void) +{ + return riscv_has_extension_unlikely(RISCV_ISA_EXT_SSQOSID); +} + +#else /* ! CONFIG_RISCV_ISA_SSQOSID */ + +struct task_struct; +static __always_inline bool has_srmcfg(void) { return false; } +static inline void __switch_to_srmcfg(struct task_struct *next) { } + +#endif /* CONFIG_RISCV_ISA_SSQOSID */ +#endif /* _ASM_RISCV_QOS_H */ diff --git a/arch/riscv/include/asm/runtime-const.h b/arch/riscv/include/asm/runtime-const.h index d766e2b9e6df..ce2008bb5227 100644 --- a/arch/riscv/include/asm/runtime-const.h +++ b/arch/riscv/include/asm/runtime-const.h @@ -2,6 +2,10 @@ #ifndef _ASM_RISCV_RUNTIME_CONST_H #define _ASM_RISCV_RUNTIME_CONST_H +#ifdef MODULE + #error "Cannot use runtime-const infrastructure from modules" +#endif + #include <asm/asm.h> #include <asm/alternative.h> #include <asm/cacheflush.h> @@ -11,21 +15,24 @@ #include <linux/uaccess.h> +#define RUNTIME_MAGIC __ASM_STR(0x89ABCDEF) + #ifdef CONFIG_32BIT -#define runtime_const_ptr(sym) \ -({ \ - typeof(sym) __ret; \ - asm_inline(".option push\n\t" \ - ".option norvc\n\t" \ - "1:\t" \ - "lui %[__ret],0x89abd\n\t" \ - "addi %[__ret],%[__ret],-0x211\n\t" \ - ".option pop\n\t" \ - ".pushsection runtime_ptr_" #sym ",\"a\"\n\t" \ - ".long 1b - .\n\t" \ - ".popsection" \ - : [__ret] "=r" (__ret)); \ - __ret; \ +#define runtime_const_ptr(sym) \ +({ \ + typeof(sym) __ret; \ + asm_inline(".option push\n\t" \ + ".option norvc\n\t" \ + ".option norelax\n\t" \ + "1:\t" \ + "lui %[__ret], %%hi(" RUNTIME_MAGIC ")\n\t" \ + "addi %[__ret],%[__ret], %%lo(" RUNTIME_MAGIC ")\n\t" \ + ".option pop\n\t" \ + ".pushsection runtime_ptr_" #sym ",\"a\"\n\t" \ + ".long 1b - .\n\t" \ + ".popsection" \ + : [__ret] "=r" (__ret)); \ + __ret; \ }) #else /* @@ -41,11 +48,12 @@ #define RISCV_RUNTIME_CONST_64_PREAMBLE \ ".option push\n\t" \ ".option norvc\n\t" \ + ".option norelax\n\t" \ "1:\t" \ - "lui %[__ret],0x89abd\n\t" \ - "lui %[__tmp],0x1234\n\t" \ - "addiw %[__ret],%[__ret],-0x211\n\t" \ - "addiw %[__tmp],%[__tmp],0x567\n\t" \ + "lui %[__ret], %%hi(" RUNTIME_MAGIC ")\n\t" \ + "lui %[__tmp], %%hi(" RUNTIME_MAGIC ")\n\t" \ + "addiw %[__ret],%[__ret], %%lo(" RUNTIME_MAGIC ")\n\t" \ + "addiw %[__tmp],%[__tmp], %%lo(" RUNTIME_MAGIC ")\n\t" \ #define RISCV_RUNTIME_CONST_64_BASE \ "slli %[__tmp],%[__tmp],32\n\t" \ @@ -153,6 +161,23 @@ __ret; \ }) +#define runtime_const_mask_32(val, sym) \ +({ \ + u32 __ret; \ + asm_inline(".option push\n\t" \ + ".option norvc\n\t" \ + "1:\t" \ + SLLI " %[__ret],%[__val],12\n\t" \ + SRLI " %[__ret],%[__ret],12\n\t" \ + ".option pop\n\t" \ + ".pushsection runtime_mask_" #sym ",\"a\"\n\t" \ + ".long 1b - .\n\t" \ + ".popsection" \ + : [__ret] "=r" (__ret) \ + : [__val] "r" (val)); \ + __ret; \ +}) + #define runtime_const_init(type, sym) do { \ extern s32 __start_runtime_##type##_##sym[]; \ extern s32 __stop_runtime_##type##_##sym[]; \ @@ -256,6 +281,33 @@ static inline void __runtime_fixup_shift(void *where, unsigned long val) mutex_unlock(&text_mutex); } +static inline void __runtime_fixup_mask(void *where, unsigned long val) +{ + unsigned int width = (val) ? __fls(val) + 1 : 0; + + /* + * XXX: Current implementation only supports patching masks of + * form GENMASK(width, 0) (width >= 0) using a SRLI + SLLI + * sequence instead of LUI + ADDI + AND sequence to improve + * performance, density, and covers all the current use-cases. + * + * When the need arises to support any generic mask, and this + * BUG_ON() is tripped, consider using a: + * + * lui %[__ret], #imm16 + * addi %[__ret], #imm16 + * + * sequence to load the 32bit const mask, and perform a logical + * and outside the asm block before returning the result. Fixup + * can simply reuse the existing __runtime_fixup_32() to patch + * the LUI + ADDI sequence. + */ + BUG_ON(!val || width > 31 || (GENMASK(width - 1, 0) != val)); + + __runtime_fixup_shift(where, 32 - width); + __runtime_fixup_shift(where + 4, 32 - width); +} + static inline void runtime_const_fixup(void (*fn)(void *, unsigned long), unsigned long val, s32 *start, s32 *end) { diff --git a/arch/riscv/include/asm/scs.h b/arch/riscv/include/asm/scs.h index ab7714aa93bd..023a412fe38d 100644 --- a/arch/riscv/include/asm/scs.h +++ b/arch/riscv/include/asm/scs.h @@ -10,7 +10,6 @@ /* Load init_shadow_call_stack to gp. */ .macro scs_load_init_stack la gp, init_shadow_call_stack - XIP_FIXUP_OFFSET gp .endm /* Load the per-CPU IRQ shadow call stack to gp. */ diff --git a/arch/riscv/include/asm/set_memory.h b/arch/riscv/include/asm/set_memory.h index 87389e93325a..ef59e1716a2c 100644 --- a/arch/riscv/include/asm/set_memory.h +++ b/arch/riscv/include/asm/set_memory.h @@ -47,7 +47,7 @@ bool kernel_page_present(struct page *page); #endif /* __ASSEMBLER__ */ -#if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_XIP_KERNEL) +#if defined(CONFIG_STRICT_KERNEL_RWX) #ifdef CONFIG_64BIT #define SECTION_ALIGN (1 << 21) #else diff --git a/arch/riscv/include/asm/simd.h b/arch/riscv/include/asm/simd.h index adb50f3ec205..f176a8072c62 100644 --- a/arch/riscv/include/asm/simd.h +++ b/arch/riscv/include/asm/simd.h @@ -36,20 +36,10 @@ static __must_check inline bool may_use_simd(void) /* * Nesting is achieved in preempt_v by spreading the control for * preemptible and non-preemptible kernel-mode Vector into two fields. - * Always try to match with preempt_v if kernel V-context exists. Then, - * fallback to check non preempt_v if nesting happens, or if the config - * is not set. + * Only non-preempt_v can nest on top of preempt_v, if non-preempt_v is + * unavailable, then preempt_v is not allowed. */ - if (IS_ENABLED(CONFIG_RISCV_ISA_V_PREEMPTIVE) && current->thread.kernel_vstate.datap) { - if (!riscv_preempt_v_started(current)) - return true; - } - /* - * Non-preemptible kernel-mode Vector temporarily disables bh. So we - * must not return true on irq_disabled(). Otherwise we would fail the - * lockdep check calling local_bh_enable() - */ - return !irqs_disabled() && !(riscv_v_flags() & RISCV_KERNEL_MODE_V); + return !(riscv_v_flags() & RISCV_KERNEL_MODE_V); } #else /* ! CONFIG_RISCV_ISA_V */ diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h index 7ac80e9f2288..0ecc67641b09 100644 --- a/arch/riscv/include/asm/smp.h +++ b/arch/riscv/include/asm/smp.h @@ -105,7 +105,7 @@ static inline void riscv_ipi_set_virq_range(int virq, int nr) #endif /* CONFIG_SMP */ -#if defined(CONFIG_HOTPLUG_CPU) && (CONFIG_SMP) +#if defined(CONFIG_HOTPLUG_CPU) bool cpu_has_hotplug(unsigned int cpu); #else static inline bool cpu_has_hotplug(unsigned int cpu) diff --git a/arch/riscv/include/asm/string.h b/arch/riscv/include/asm/string.h index 5ba77f60bf0b..764ffe8f6479 100644 --- a/arch/riscv/include/asm/string.h +++ b/arch/riscv/include/asm/string.h @@ -28,6 +28,15 @@ extern asmlinkage __kernel_size_t strlen(const char *); #define __HAVE_ARCH_STRNCMP extern asmlinkage int strncmp(const char *cs, const char *ct, size_t count); + +#define __HAVE_ARCH_STRNLEN +extern asmlinkage __kernel_size_t strnlen(const char *, size_t); + +#define __HAVE_ARCH_STRCHR +extern asmlinkage char *strchr(const char *, int); + +#define __HAVE_ARCH_STRRCHR +extern asmlinkage char *strrchr(const char *, int); #endif /* For those files which don't want to check by kasan. */ diff --git a/arch/riscv/include/asm/switch_to.h b/arch/riscv/include/asm/switch_to.h index 0e71eb82f920..123c89b694e8 100644 --- a/arch/riscv/include/asm/switch_to.h +++ b/arch/riscv/include/asm/switch_to.h @@ -14,6 +14,7 @@ #include <asm/processor.h> #include <asm/ptrace.h> #include <asm/csr.h> +#include <asm/qos.h> #ifdef CONFIG_FPU extern void __fstate_save(struct task_struct *save_to); @@ -60,8 +61,8 @@ static inline void __switch_to_fpu(struct task_struct *prev, static __always_inline bool has_fpu(void) { - return riscv_has_extension_likely(RISCV_ISA_EXT_f) || - riscv_has_extension_likely(RISCV_ISA_EXT_d); + /* D extension depends on F, so checking D alone is sufficient. */ + return riscv_has_extension_likely(RISCV_ISA_EXT_D); } #else static __always_inline bool has_fpu(void) { return false; } @@ -119,6 +120,8 @@ do { \ __switch_to_fpu(__prev, __next); \ if (has_vector() || has_xtheadvector()) \ __switch_to_vector(__prev, __next); \ + if (has_srmcfg()) \ + __switch_to_srmcfg(__next); \ if (switch_to_should_flush_icache(__next)) \ local_flush_icache_all(); \ __switch_to_envcfg(__next); \ diff --git a/arch/riscv/include/asm/syscall.h b/arch/riscv/include/asm/syscall.h index 34313387f977..987c9a78806f 100644 --- a/arch/riscv/include/asm/syscall.h +++ b/arch/riscv/include/asm/syscall.h @@ -20,7 +20,7 @@ extern void * const sys_call_table[]; extern void * const compat_sys_call_table[]; /* - * Only the low 32 bits of orig_r0 are meaningful, so we return int. + * Only the low 32 bits of orig_a0 are meaningful, so we return int. * This importantly ignores the high bits on 64-bit, so comparisons * sign-extend the low 32 bits. */ @@ -112,11 +112,6 @@ static inline void syscall_handler(struct pt_regs *regs, ulong syscall) regs->a0 = fn(regs); } -static inline bool arch_syscall_is_vdso_sigreturn(struct pt_regs *regs) -{ - return false; -} - asmlinkage long sys_riscv_flush_icache(uintptr_t, uintptr_t, uintptr_t); asmlinkage long sys_riscv_hwprobe(struct riscv_hwprobe *, size_t, size_t, diff --git a/arch/riscv/include/asm/syscall_wrapper.h b/arch/riscv/include/asm/syscall_wrapper.h index ac80216549ff..226289c3b5c8 100644 --- a/arch/riscv/include/asm/syscall_wrapper.h +++ b/arch/riscv/include/asm/syscall_wrapper.h @@ -32,6 +32,10 @@ asmlinkage long __riscv_sys_ni_syscall(const struct pt_regs *); __diag_push(); \ __diag_ignore(GCC, 8, "-Wattribute-alias", \ "Type aliasing is used to sanitize syscall arguments"); \ + __diag_ignore(clang, 23, "-Wunknown-warning-option", \ + "Avoid breaking versions without -Wattribute-alias"); \ + __diag_ignore(clang, 23, "-Wattribute-alias", \ + "Type aliasing is used to sanitize syscall arguments"); \ static long __se_##prefix##name(ulong, ulong, ulong, ulong, ulong, ulong, \ ulong) \ __attribute__((alias(__stringify(___se_##prefix##name)))); \ diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index 836d80dd2921..55019fdfa9ec 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -73,6 +73,9 @@ struct thread_info { */ unsigned long a0, a1, a2; #endif +#ifdef CONFIG_RISCV_USER_CFI + struct cfi_state user_cfi_state; +#endif }; #ifdef CONFIG_SHADOW_CALL_STACK @@ -117,7 +120,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src); #include <asm-generic/thread_info_tif.h> #define TIF_32BIT 16 /* compat-mode 32bit process */ -#define TIF_RISCV_V_DEFER_RESTORE 17 /* restore Vector before returing to user */ +#define TIF_RISCV_V_DEFER_RESTORE 17 /* restore Vector before returning to user */ #define _TIF_RISCV_V_DEFER_RESTORE BIT(TIF_RISCV_V_DEFER_RESTORE) diff --git a/arch/riscv/include/asm/timex.h b/arch/riscv/include/asm/timex.h index a06697846e69..f660afa30530 100644 --- a/arch/riscv/include/asm/timex.h +++ b/arch/riscv/include/asm/timex.h @@ -22,13 +22,13 @@ static inline cycles_t get_cycles(void) #else /* !CONFIG_64BIT */ static inline u32 get_cycles(void) { - return readl_relaxed(((u32 *)clint_time_val)); + return readl_relaxed(((u32 __iomem *)clint_time_val)); } #define get_cycles get_cycles static inline u32 get_cycles_hi(void) { - return readl_relaxed(((u32 *)clint_time_val) + 1); + return readl_relaxed(((u32 __iomem *)clint_time_val) + 1); } #define get_cycles_hi get_cycles_hi #endif /* CONFIG_64BIT */ @@ -80,12 +80,4 @@ static inline u64 get_cycles64(void) return ((u64)hi << 32) | lo; } #endif /* CONFIG_64BIT */ - -#define ARCH_HAS_READ_CURRENT_TIMER -static inline int read_current_timer(unsigned long *timer_val) -{ - *timer_val = get_cycles(); - return 0; -} - #endif /* _ASM_RISCV_TIMEX_H */ diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h index eed0abc40514..9636d07fe9ee 100644 --- a/arch/riscv/include/asm/tlbflush.h +++ b/arch/riscv/include/asm/tlbflush.h @@ -15,6 +15,29 @@ #define FLUSH_TLB_NO_ASID ((unsigned long)-1) #ifdef CONFIG_MMU +static inline unsigned long get_mm_asid(struct mm_struct *mm) +{ + return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID; +} + +static inline void local_sfence_inval_ir(void) +{ + asm volatile(SFENCE_INVAL_IR() ::: "memory"); +} + +static inline void local_sfence_w_inval(void) +{ + asm volatile(SFENCE_W_INVAL() ::: "memory"); +} + +static inline void local_sinval_vma(unsigned long vma, unsigned long asid) +{ + if (asid != FLUSH_TLB_NO_ASID) + asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory"); + else + asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory"); +} + static inline void local_flush_tlb_all(void) { __asm__ __volatile__ ("sfence.vma" : : : "memory"); diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index 11c9886c3b70..5d4ec15584cf 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -112,7 +112,7 @@ do { \ _ASM_EXTABLE_UACCESS_ERR(1b, %l2, %0) \ : "=&r" (__tmp) \ : "m" (*(ptr)) : : label); \ - (x) = (__typeof__(x))(unsigned long)__tmp; \ + (x) = (__force __typeof__(x))(unsigned long)__tmp; \ } while (0) #else /* !CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ #define __get_user_asm(insn, x, ptr, label) \ diff --git a/arch/riscv/include/asm/usercfi.h b/arch/riscv/include/asm/usercfi.h new file mode 100644 index 000000000000..61ee02cee297 --- /dev/null +++ b/arch/riscv/include/asm/usercfi.h @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: GPL-2.0 + * Copyright (C) 2024 Rivos, Inc. + * Deepak Gupta <debug@rivosinc.com> + */ +#ifndef _ASM_RISCV_USERCFI_H +#define _ASM_RISCV_USERCFI_H + +#define CMDLINE_DISABLE_RISCV_USERCFI_FCFI 1 +#define CMDLINE_DISABLE_RISCV_USERCFI_BCFI 2 +#define CMDLINE_DISABLE_RISCV_USERCFI 3 + +#ifndef __ASSEMBLER__ +#include <linux/types.h> +#include <linux/prctl.h> +#include <linux/errno.h> + +struct task_struct; +struct kernel_clone_args; + +extern unsigned long riscv_nousercfi; + +#ifdef CONFIG_RISCV_USER_CFI +struct cfi_state { + unsigned long ubcfi_en : 1; /* Enable for backward cfi. */ + unsigned long ubcfi_locked : 1; + unsigned long ufcfi_en : 1; /* Enable for forward cfi. Note that ELP goes in sstatus */ + unsigned long ufcfi_locked : 1; + unsigned long user_shdw_stk; /* Current user shadow stack pointer */ + unsigned long shdw_stk_base; /* Base address of shadow stack */ + unsigned long shdw_stk_size; /* size of shadow stack */ +}; + +unsigned long shstk_alloc_thread_stack(struct task_struct *tsk, + const struct kernel_clone_args *args); +void shstk_release(struct task_struct *tsk); +void set_shstk_base(struct task_struct *task, unsigned long shstk_addr, unsigned long size); +unsigned long get_shstk_base(struct task_struct *task, unsigned long *size); +void set_active_shstk(struct task_struct *task, unsigned long shstk_addr); +bool is_shstk_enabled(struct task_struct *task); +bool is_shstk_locked(struct task_struct *task); +bool is_shstk_allocated(struct task_struct *task); +void set_shstk_lock(struct task_struct *task, bool lock); +void set_shstk_status(struct task_struct *task, bool enable); +unsigned long get_active_shstk(struct task_struct *task); +int restore_user_shstk(struct task_struct *tsk, unsigned long shstk_ptr); +int save_user_shstk(struct task_struct *tsk, unsigned long *saved_shstk_ptr); +bool is_indir_lp_enabled(struct task_struct *task); +bool is_indir_lp_locked(struct task_struct *task); +void set_indir_lp_status(struct task_struct *task, bool enable); +void set_indir_lp_lock(struct task_struct *task, bool lock); + +#define PR_SHADOW_STACK_SUPPORTED_STATUS_MASK (PR_SHADOW_STACK_ENABLE) +#define PR_CFI_SUPPORTED_STATUS_MASK (PR_CFI_ENABLE | PR_CFI_DISABLE | PR_CFI_LOCK) + +#else + +#define shstk_alloc_thread_stack(tsk, args) 0 + +#define shstk_release(tsk) + +#define get_shstk_base(task, size) 0UL + +#define set_shstk_base(task, shstk_addr, size) do {} while (0) + +#define set_active_shstk(task, shstk_addr) do {} while (0) + +#define is_shstk_enabled(task) false + +#define is_shstk_locked(task) false + +#define is_shstk_allocated(task) false + +#define set_shstk_lock(task, lock) do {} while (0) + +#define set_shstk_status(task, enable) do {} while (0) + +#define is_indir_lp_enabled(task) false + +#define is_indir_lp_locked(task) false + +#define set_indir_lp_status(task, enable) do {} while (0) + +#define set_indir_lp_lock(task, lock) do {} while (0) + +#define restore_user_shstk(tsk, shstk_ptr) -EINVAL + +#define save_user_shstk(tsk, saved_shstk_ptr) -EINVAL + +#define get_active_shstk(task) 0UL + +#endif /* CONFIG_RISCV_USER_CFI */ + +bool is_user_shstk_enabled(void); +bool is_user_lpad_enabled(void); + +#endif /* __ASSEMBLER__ */ + +#endif /* _ASM_RISCV_USERCFI_H */ diff --git a/arch/riscv/include/asm/vdso.h b/arch/riscv/include/asm/vdso.h index f80357fe24d1..0259bbc98e93 100644 --- a/arch/riscv/include/asm/vdso.h +++ b/arch/riscv/include/asm/vdso.h @@ -12,15 +12,32 @@ * All systems with an MMU have a VDSO, but systems without an MMU don't * support shared libraries and therefore don't have one. */ -#ifdef CONFIG_MMU #define __VDSO_PAGES 4 #ifndef __ASSEMBLER__ + +#ifdef CONFIG_MMU #include <generated/vdso-offsets.h> +#else +extern size_t __vdso_alternatives_start_offset, __vdso_alternatives_end_offset; +#endif +#ifdef CONFIG_RISCV_USER_CFI +#include <generated/vdso-cfi-offsets.h> +#else +extern size_t __vdso_alternatives_start_cfi_offset, __vdso_alternatives_end_cfi_offset; +#endif + +#ifdef CONFIG_RISCV_USER_CFI +#define VDSO_SYMBOL(base, name) \ + (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZIMOP) ? \ + (void __user *)((unsigned long)(base) + __vdso_##name##_cfi_offset) : \ + (void __user *)((unsigned long)(base) + __vdso_##name##_offset)) +#else #define VDSO_SYMBOL(base, name) \ - (void __user *)((unsigned long)(base) + __vdso_##name##_offset) + ((void __user *)((unsigned long)(base) + __vdso_##name##_offset)) +#endif #ifdef CONFIG_COMPAT #include <generated/compat_vdso-offsets.h> @@ -28,14 +45,14 @@ #define COMPAT_VDSO_SYMBOL(base, name) \ (void __user *)((unsigned long)(base) + compat__vdso_##name##_offset) -extern char compat_vdso_start[], compat_vdso_end[]; - +#else +extern size_t compat__vdso_alternatives_start_offset, compat__vdso_alternatives_end_offset; #endif /* CONFIG_COMPAT */ extern char vdso_start[], vdso_end[]; +extern char vdso_cfi_start[], vdso_cfi_end[]; +extern char compat_vdso_start[], compat_vdso_end[]; #endif /* !__ASSEMBLER__ */ -#endif /* CONFIG_MMU */ - #endif /* _ASM_RISCV_VDSO_H */ diff --git a/arch/riscv/include/asm/vdso/arch_data.h b/arch/riscv/include/asm/vdso/arch_data.h index 88b37af55175..da57a3786f7a 100644 --- a/arch/riscv/include/asm/vdso/arch_data.h +++ b/arch/riscv/include/asm/vdso/arch_data.h @@ -12,12 +12,6 @@ struct vdso_arch_data { /* Boolean indicating all CPUs have the same static hwprobe values. */ __u8 homogeneous_cpus; - - /* - * A gate to check and see if the hwprobe data is actually ready, as - * probing is deferred to avoid boot slowdowns. - */ - __u8 ready; }; #endif /* __RISCV_ASM_VDSO_ARCH_DATA_H */ diff --git a/arch/riscv/include/asm/vdso/gettimeofday.h b/arch/riscv/include/asm/vdso/gettimeofday.h index 9ec08fa04d35..61cb3cbab143 100644 --- a/arch/riscv/include/asm/vdso/gettimeofday.h +++ b/arch/riscv/include/asm/vdso/gettimeofday.h @@ -9,12 +9,6 @@ #include <asm/csr.h> #include <uapi/linux/time.h> -/* - * 32-bit land is lacking generic time vsyscalls as well as the legacy 32-bit - * time syscalls like gettimeofday. Skip these definitions since on 32-bit. - */ -#ifdef CONFIG_GENERIC_TIME_VSYSCALL - #define VDSO_HAS_CLOCK_GETRES 1 static __always_inline @@ -66,8 +60,6 @@ int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) return ret; } -#endif /* CONFIG_GENERIC_TIME_VSYSCALL */ - static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, const struct vdso_time_data *vd) { diff --git a/arch/riscv/include/asm/vector.h b/arch/riscv/include/asm/vector.h index 00cb9c0982b1..fffe72a77208 100644 --- a/arch/riscv/include/asm/vector.h +++ b/arch/riscv/include/asm/vector.h @@ -58,6 +58,11 @@ static inline u32 riscv_v_flags(void) return READ_ONCE(current->thread.riscv_v_flags); } +static inline void riscv_v_flags_set(u32 flags) +{ + WRITE_ONCE(current->thread.riscv_v_flags, flags); +} + static __always_inline bool has_vector(void) { return riscv_has_extension_unlikely(RISCV_ISA_EXT_ZVE32X); diff --git a/arch/riscv/include/asm/xip_fixup.h b/arch/riscv/include/asm/xip_fixup.h deleted file mode 100644 index f3d56299bc22..000000000000 --- a/arch/riscv/include/asm/xip_fixup.h +++ /dev/null @@ -1,49 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * XIP fixup macros, only useful in assembly. - */ -#ifndef _ASM_RISCV_XIP_FIXUP_H -#define _ASM_RISCV_XIP_FIXUP_H - -#include <linux/pgtable.h> - -#ifdef CONFIG_XIP_KERNEL -.macro XIP_FIXUP_OFFSET reg - /* Fix-up address in Flash into address in RAM early during boot before - * MMU is up. Because generated code "thinks" data is in Flash, but it - * is actually in RAM (actually data is also in Flash, but Flash is - * read-only, thus we need to use the data residing in RAM). - * - * The start of data in Flash is _sdata and the start of data in RAM is - * CONFIG_PHYS_RAM_BASE. So this fix-up essentially does this: - * reg += CONFIG_PHYS_RAM_BASE - _start - */ - li t0, CONFIG_PHYS_RAM_BASE - add \reg, \reg, t0 - la t0, _sdata - sub \reg, \reg, t0 -.endm -.macro XIP_FIXUP_FLASH_OFFSET reg - /* In linker script, at the transition from read-only section to - * writable section, the VMA is increased while LMA remains the same. - * (See in linker script how _sdata, __data_loc and LOAD_OFFSET is - * changed) - * - * Consequently, early during boot before MMU is up, the generated code - * reads the "writable" section at wrong addresses, because VMA is used - * by compiler to generate code, but the data is located in Flash using - * LMA. - */ - la t0, _sdata - sub \reg, \reg, t0 - la t0, __data_loc - add \reg, \reg, t0 -.endm -#else -.macro XIP_FIXUP_OFFSET reg -.endm -.macro XIP_FIXUP_FLASH_OFFSET reg -.endm -#endif /* CONFIG_XIP_KERNEL */ - -#endif diff --git a/arch/riscv/include/asm/xor.h b/arch/riscv/include/asm/xor.h deleted file mode 100644 index 96011861e46b..000000000000 --- a/arch/riscv/include/asm/xor.h +++ /dev/null @@ -1,68 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) 2021 SiFive - */ - -#include <linux/hardirq.h> -#include <asm-generic/xor.h> -#ifdef CONFIG_RISCV_ISA_V -#include <asm/vector.h> -#include <asm/switch_to.h> -#include <asm/asm-prototypes.h> - -static void xor_vector_2(unsigned long bytes, unsigned long *__restrict p1, - const unsigned long *__restrict p2) -{ - kernel_vector_begin(); - xor_regs_2_(bytes, p1, p2); - kernel_vector_end(); -} - -static void xor_vector_3(unsigned long bytes, unsigned long *__restrict p1, - const unsigned long *__restrict p2, - const unsigned long *__restrict p3) -{ - kernel_vector_begin(); - xor_regs_3_(bytes, p1, p2, p3); - kernel_vector_end(); -} - -static void xor_vector_4(unsigned long bytes, unsigned long *__restrict p1, - const unsigned long *__restrict p2, - const unsigned long *__restrict p3, - const unsigned long *__restrict p4) -{ - kernel_vector_begin(); - xor_regs_4_(bytes, p1, p2, p3, p4); - kernel_vector_end(); -} - -static void xor_vector_5(unsigned long bytes, unsigned long *__restrict p1, - const unsigned long *__restrict p2, - const unsigned long *__restrict p3, - const unsigned long *__restrict p4, - const unsigned long *__restrict p5) -{ - kernel_vector_begin(); - xor_regs_5_(bytes, p1, p2, p3, p4, p5); - kernel_vector_end(); -} - -static struct xor_block_template xor_block_rvv = { - .name = "rvv", - .do_2 = xor_vector_2, - .do_3 = xor_vector_3, - .do_4 = xor_vector_4, - .do_5 = xor_vector_5 -}; - -#undef XOR_TRY_TEMPLATES -#define XOR_TRY_TEMPLATES \ - do { \ - xor_speed(&xor_block_8regs); \ - xor_speed(&xor_block_32regs); \ - if (has_vector()) { \ - xor_speed(&xor_block_rvv);\ - } \ - } while (0) -#endif diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h index cd3c126730c3..e6a394fbe7fc 100644 --- a/arch/riscv/include/uapi/asm/hwprobe.h +++ b/arch/riscv/include/uapi/asm/hwprobe.h @@ -7,6 +7,7 @@ #define _UAPI_ASM_HWPROBE_H #include <linux/types.h> +#include <linux/const.h> /* * Interface for probing hardware capabilities from userspace, see @@ -21,79 +22,80 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_KEY_MARCHID 1 #define RISCV_HWPROBE_KEY_MIMPID 2 #define RISCV_HWPROBE_KEY_BASE_BEHAVIOR 3 -#define RISCV_HWPROBE_BASE_BEHAVIOR_IMA (1 << 0) +#define RISCV_HWPROBE_BASE_BEHAVIOR_IMA _BITULL(0) #define RISCV_HWPROBE_KEY_IMA_EXT_0 4 -#define RISCV_HWPROBE_IMA_FD (1 << 0) -#define RISCV_HWPROBE_IMA_C (1 << 1) -#define RISCV_HWPROBE_IMA_V (1 << 2) -#define RISCV_HWPROBE_EXT_ZBA (1 << 3) -#define RISCV_HWPROBE_EXT_ZBB (1 << 4) -#define RISCV_HWPROBE_EXT_ZBS (1 << 5) -#define RISCV_HWPROBE_EXT_ZICBOZ (1 << 6) -#define RISCV_HWPROBE_EXT_ZBC (1 << 7) -#define RISCV_HWPROBE_EXT_ZBKB (1 << 8) -#define RISCV_HWPROBE_EXT_ZBKC (1 << 9) -#define RISCV_HWPROBE_EXT_ZBKX (1 << 10) -#define RISCV_HWPROBE_EXT_ZKND (1 << 11) -#define RISCV_HWPROBE_EXT_ZKNE (1 << 12) -#define RISCV_HWPROBE_EXT_ZKNH (1 << 13) -#define RISCV_HWPROBE_EXT_ZKSED (1 << 14) -#define RISCV_HWPROBE_EXT_ZKSH (1 << 15) -#define RISCV_HWPROBE_EXT_ZKT (1 << 16) -#define RISCV_HWPROBE_EXT_ZVBB (1 << 17) -#define RISCV_HWPROBE_EXT_ZVBC (1 << 18) -#define RISCV_HWPROBE_EXT_ZVKB (1 << 19) -#define RISCV_HWPROBE_EXT_ZVKG (1 << 20) -#define RISCV_HWPROBE_EXT_ZVKNED (1 << 21) -#define RISCV_HWPROBE_EXT_ZVKNHA (1 << 22) -#define RISCV_HWPROBE_EXT_ZVKNHB (1 << 23) -#define RISCV_HWPROBE_EXT_ZVKSED (1 << 24) -#define RISCV_HWPROBE_EXT_ZVKSH (1 << 25) -#define RISCV_HWPROBE_EXT_ZVKT (1 << 26) -#define RISCV_HWPROBE_EXT_ZFH (1 << 27) -#define RISCV_HWPROBE_EXT_ZFHMIN (1 << 28) -#define RISCV_HWPROBE_EXT_ZIHINTNTL (1 << 29) -#define RISCV_HWPROBE_EXT_ZVFH (1 << 30) -#define RISCV_HWPROBE_EXT_ZVFHMIN (1ULL << 31) -#define RISCV_HWPROBE_EXT_ZFA (1ULL << 32) -#define RISCV_HWPROBE_EXT_ZTSO (1ULL << 33) -#define RISCV_HWPROBE_EXT_ZACAS (1ULL << 34) -#define RISCV_HWPROBE_EXT_ZICOND (1ULL << 35) -#define RISCV_HWPROBE_EXT_ZIHINTPAUSE (1ULL << 36) -#define RISCV_HWPROBE_EXT_ZVE32X (1ULL << 37) -#define RISCV_HWPROBE_EXT_ZVE32F (1ULL << 38) -#define RISCV_HWPROBE_EXT_ZVE64X (1ULL << 39) -#define RISCV_HWPROBE_EXT_ZVE64F (1ULL << 40) -#define RISCV_HWPROBE_EXT_ZVE64D (1ULL << 41) -#define RISCV_HWPROBE_EXT_ZIMOP (1ULL << 42) -#define RISCV_HWPROBE_EXT_ZCA (1ULL << 43) -#define RISCV_HWPROBE_EXT_ZCB (1ULL << 44) -#define RISCV_HWPROBE_EXT_ZCD (1ULL << 45) -#define RISCV_HWPROBE_EXT_ZCF (1ULL << 46) -#define RISCV_HWPROBE_EXT_ZCMOP (1ULL << 47) -#define RISCV_HWPROBE_EXT_ZAWRS (1ULL << 48) -#define RISCV_HWPROBE_EXT_SUPM (1ULL << 49) -#define RISCV_HWPROBE_EXT_ZICNTR (1ULL << 50) -#define RISCV_HWPROBE_EXT_ZIHPM (1ULL << 51) -#define RISCV_HWPROBE_EXT_ZFBFMIN (1ULL << 52) -#define RISCV_HWPROBE_EXT_ZVFBFMIN (1ULL << 53) -#define RISCV_HWPROBE_EXT_ZVFBFWMA (1ULL << 54) -#define RISCV_HWPROBE_EXT_ZICBOM (1ULL << 55) -#define RISCV_HWPROBE_EXT_ZAAMO (1ULL << 56) -#define RISCV_HWPROBE_EXT_ZALRSC (1ULL << 57) -#define RISCV_HWPROBE_EXT_ZABHA (1ULL << 58) -#define RISCV_HWPROBE_EXT_ZALASR (1ULL << 59) -#define RISCV_HWPROBE_EXT_ZICBOP (1ULL << 60) -#define RISCV_HWPROBE_EXT_ZILSD (1ULL << 61) -#define RISCV_HWPROBE_EXT_ZCLSD (1ULL << 62) +#define RISCV_HWPROBE_IMA_FD _BITULL(0) +#define RISCV_HWPROBE_IMA_C _BITULL(1) +#define RISCV_HWPROBE_IMA_V _BITULL(2) +#define RISCV_HWPROBE_EXT_ZBA _BITULL(3) +#define RISCV_HWPROBE_EXT_ZBB _BITULL(4) +#define RISCV_HWPROBE_EXT_ZBS _BITULL(5) +#define RISCV_HWPROBE_EXT_ZICBOZ _BITULL(6) +#define RISCV_HWPROBE_EXT_ZBC _BITULL(7) +#define RISCV_HWPROBE_EXT_ZBKB _BITULL(8) +#define RISCV_HWPROBE_EXT_ZBKC _BITULL(9) +#define RISCV_HWPROBE_EXT_ZBKX _BITULL(10) +#define RISCV_HWPROBE_EXT_ZKND _BITULL(11) +#define RISCV_HWPROBE_EXT_ZKNE _BITULL(12) +#define RISCV_HWPROBE_EXT_ZKNH _BITULL(13) +#define RISCV_HWPROBE_EXT_ZKSED _BITULL(14) +#define RISCV_HWPROBE_EXT_ZKSH _BITULL(15) +#define RISCV_HWPROBE_EXT_ZKT _BITULL(16) +#define RISCV_HWPROBE_EXT_ZVBB _BITULL(17) +#define RISCV_HWPROBE_EXT_ZVBC _BITULL(18) +#define RISCV_HWPROBE_EXT_ZVKB _BITULL(19) +#define RISCV_HWPROBE_EXT_ZVKG _BITULL(20) +#define RISCV_HWPROBE_EXT_ZVKNED _BITULL(21) +#define RISCV_HWPROBE_EXT_ZVKNHA _BITULL(22) +#define RISCV_HWPROBE_EXT_ZVKNHB _BITULL(23) +#define RISCV_HWPROBE_EXT_ZVKSED _BITULL(24) +#define RISCV_HWPROBE_EXT_ZVKSH _BITULL(25) +#define RISCV_HWPROBE_EXT_ZVKT _BITULL(26) +#define RISCV_HWPROBE_EXT_ZFH _BITULL(27) +#define RISCV_HWPROBE_EXT_ZFHMIN _BITULL(28) +#define RISCV_HWPROBE_EXT_ZIHINTNTL _BITULL(29) +#define RISCV_HWPROBE_EXT_ZVFH _BITULL(30) +#define RISCV_HWPROBE_EXT_ZVFHMIN _BITULL(31) +#define RISCV_HWPROBE_EXT_ZFA _BITULL(32) +#define RISCV_HWPROBE_EXT_ZTSO _BITULL(33) +#define RISCV_HWPROBE_EXT_ZACAS _BITULL(34) +#define RISCV_HWPROBE_EXT_ZICOND _BITULL(35) +#define RISCV_HWPROBE_EXT_ZIHINTPAUSE _BITULL(36) +#define RISCV_HWPROBE_EXT_ZVE32X _BITULL(37) +#define RISCV_HWPROBE_EXT_ZVE32F _BITULL(38) +#define RISCV_HWPROBE_EXT_ZVE64X _BITULL(39) +#define RISCV_HWPROBE_EXT_ZVE64F _BITULL(40) +#define RISCV_HWPROBE_EXT_ZVE64D _BITULL(41) +#define RISCV_HWPROBE_EXT_ZIMOP _BITULL(42) +#define RISCV_HWPROBE_EXT_ZCA _BITULL(43) +#define RISCV_HWPROBE_EXT_ZCB _BITULL(44) +#define RISCV_HWPROBE_EXT_ZCD _BITULL(45) +#define RISCV_HWPROBE_EXT_ZCF _BITULL(46) +#define RISCV_HWPROBE_EXT_ZCMOP _BITULL(47) +#define RISCV_HWPROBE_EXT_ZAWRS _BITULL(48) +#define RISCV_HWPROBE_EXT_SUPM _BITULL(49) +#define RISCV_HWPROBE_EXT_ZICNTR _BITULL(50) +#define RISCV_HWPROBE_EXT_ZIHPM _BITULL(51) +#define RISCV_HWPROBE_EXT_ZFBFMIN _BITULL(52) +#define RISCV_HWPROBE_EXT_ZVFBFMIN _BITULL(53) +#define RISCV_HWPROBE_EXT_ZVFBFWMA _BITULL(54) +#define RISCV_HWPROBE_EXT_ZICBOM _BITULL(55) +#define RISCV_HWPROBE_EXT_ZAAMO _BITULL(56) +#define RISCV_HWPROBE_EXT_ZALRSC _BITULL(57) +#define RISCV_HWPROBE_EXT_ZABHA _BITULL(58) +#define RISCV_HWPROBE_EXT_ZALASR _BITULL(59) +#define RISCV_HWPROBE_EXT_ZICBOP _BITULL(60) +#define RISCV_HWPROBE_EXT_ZILSD _BITULL(61) +#define RISCV_HWPROBE_EXT_ZCLSD _BITULL(62) +#define RISCV_HWPROBE_EXT_ZICFILP _BITULL(63) #define RISCV_HWPROBE_KEY_CPUPERF_0 5 -#define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0) -#define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0) -#define RISCV_HWPROBE_MISALIGNED_SLOW (2 << 0) -#define RISCV_HWPROBE_MISALIGNED_FAST (3 << 0) -#define RISCV_HWPROBE_MISALIGNED_UNSUPPORTED (4 << 0) -#define RISCV_HWPROBE_MISALIGNED_MASK (7 << 0) +#define RISCV_HWPROBE_MISALIGNED_UNKNOWN 0 +#define RISCV_HWPROBE_MISALIGNED_EMULATED 1 +#define RISCV_HWPROBE_MISALIGNED_SLOW 2 +#define RISCV_HWPROBE_MISALIGNED_FAST 3 +#define RISCV_HWPROBE_MISALIGNED_UNSUPPORTED 4 +#define RISCV_HWPROBE_MISALIGNED_MASK 7 #define RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE 6 #define RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS 7 #define RISCV_HWPROBE_KEY_TIME_CSR_FREQ 8 @@ -113,9 +115,17 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_KEY_VENDOR_EXT_SIFIVE_0 13 #define RISCV_HWPROBE_KEY_VENDOR_EXT_MIPS_0 14 #define RISCV_HWPROBE_KEY_ZICBOP_BLOCK_SIZE 15 +#define RISCV_HWPROBE_KEY_IMA_EXT_1 16 +#define RISCV_HWPROBE_EXT_ZICFISS _BITULL(0) +#define RISCV_HWPROBE_EXT_ZICCLSM _BITULL(1) +#define RISCV_HWPROBE_EXT_ZICCAMOA _BITULL(2) +#define RISCV_HWPROBE_EXT_ZICCIF _BITULL(3) +#define RISCV_HWPROBE_EXT_ZICCRSE _BITULL(4) +#define RISCV_HWPROBE_EXT_ZA64RS _BITULL(5) + /* Increase RISCV_HWPROBE_MAX_KEY when adding items. */ /* Flags */ -#define RISCV_HWPROBE_WHICH_CPUS (1 << 0) +#define RISCV_HWPROBE_WHICH_CPUS _BITUL(0) #endif diff --git a/arch/riscv/include/uapi/asm/kvm.h b/arch/riscv/include/uapi/asm/kvm.h index 54f3ad7ed2e4..20d9959ca44f 100644 --- a/arch/riscv/include/uapi/asm/kvm.h +++ b/arch/riscv/include/uapi/asm/kvm.h @@ -102,6 +102,11 @@ struct kvm_riscv_smstateen_csr { unsigned long sstateen0; }; +/* Zicfiss CSR for KVM_GET_ONE_REG and KVM_SET_ONE_REG */ +struct kvm_riscv_zicfiss_csr { + unsigned long ssp; +}; + /* TIMER registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */ struct kvm_riscv_timer { __u64 frequency; @@ -110,6 +115,10 @@ struct kvm_riscv_timer { __u64 state; }; +/* Possible states for kvm_riscv_timer */ +#define KVM_RISCV_TIMER_STATE_OFF 0 +#define KVM_RISCV_TIMER_STATE_ON 1 + /* * ISA extension IDs specific to KVM. This is not the same as the host ISA * extension IDs as that is internal to the host and should not be exposed @@ -192,6 +201,11 @@ enum KVM_RISCV_ISA_EXT_ID { KVM_RISCV_ISA_EXT_ZFBFMIN, KVM_RISCV_ISA_EXT_ZVFBFMIN, KVM_RISCV_ISA_EXT_ZVFBFWMA, + KVM_RISCV_ISA_EXT_ZCLSD, + KVM_RISCV_ISA_EXT_ZILSD, + KVM_RISCV_ISA_EXT_ZALASR, + KVM_RISCV_ISA_EXT_ZICFILP, + KVM_RISCV_ISA_EXT_ZICFISS, KVM_RISCV_ISA_EXT_MAX, }; @@ -233,12 +247,11 @@ struct kvm_riscv_sbi_fwft_feature { struct kvm_riscv_sbi_fwft { struct kvm_riscv_sbi_fwft_feature misaligned_deleg; struct kvm_riscv_sbi_fwft_feature pointer_masking; + struct kvm_riscv_sbi_fwft_feature pte_ad_hw_updating; + struct kvm_riscv_sbi_fwft_feature landing_pad; + struct kvm_riscv_sbi_fwft_feature shadow_stack; }; -/* Possible states for kvm_riscv_timer */ -#define KVM_RISCV_TIMER_STATE_OFF 0 -#define KVM_RISCV_TIMER_STATE_ON 1 - /* If you need to interpret the index values, here is the key: */ #define KVM_REG_RISCV_TYPE_MASK 0x00000000FF000000 #define KVM_REG_RISCV_TYPE_SHIFT 24 @@ -260,12 +273,15 @@ struct kvm_riscv_sbi_fwft { #define KVM_REG_RISCV_CSR_GENERAL (0x0 << KVM_REG_RISCV_SUBTYPE_SHIFT) #define KVM_REG_RISCV_CSR_AIA (0x1 << KVM_REG_RISCV_SUBTYPE_SHIFT) #define KVM_REG_RISCV_CSR_SMSTATEEN (0x2 << KVM_REG_RISCV_SUBTYPE_SHIFT) +#define KVM_REG_RISCV_CSR_ZICFISS (0x3 << KVM_REG_RISCV_SUBTYPE_SHIFT) #define KVM_REG_RISCV_CSR_REG(name) \ (offsetof(struct kvm_riscv_csr, name) / sizeof(unsigned long)) #define KVM_REG_RISCV_CSR_AIA_REG(name) \ (offsetof(struct kvm_riscv_aia_csr, name) / sizeof(unsigned long)) #define KVM_REG_RISCV_CSR_SMSTATEEN_REG(name) \ (offsetof(struct kvm_riscv_smstateen_csr, name) / sizeof(unsigned long)) +#define KVM_REG_RISCV_CSR_ZICFISS_REG(name) \ + (offsetof(struct kvm_riscv_zicfiss_csr, name) / sizeof(unsigned long)) /* Timer registers are mapped as type 4 */ #define KVM_REG_RISCV_TIMER (0x04 << KVM_REG_RISCV_TYPE_SHIFT) diff --git a/arch/riscv/include/uapi/asm/ptrace.h b/arch/riscv/include/uapi/asm/ptrace.h index beff8df80ac9..3de2b7124aff 100644 --- a/arch/riscv/include/uapi/asm/ptrace.h +++ b/arch/riscv/include/uapi/asm/ptrace.h @@ -9,6 +9,7 @@ #ifndef __ASSEMBLER__ #include <linux/types.h> +#include <linux/const.h> #define PTRACE_GETFDPIC 33 @@ -127,6 +128,42 @@ struct __riscv_v_regset_state { */ #define RISCV_MAX_VLENB (8192) +struct __sc_riscv_cfi_state { + unsigned long ss_ptr; /* shadow stack pointer */ +}; + +#define PTRACE_CFI_BRANCH_LANDING_PAD_EN_BIT 0 +#define PTRACE_CFI_BRANCH_LANDING_PAD_LOCK_BIT 1 +#define PTRACE_CFI_BRANCH_EXPECTED_LANDING_PAD_BIT 2 +#define PTRACE_CFI_SHADOW_STACK_EN_BIT 3 +#define PTRACE_CFI_SHADOW_STACK_LOCK_BIT 4 +#define PTRACE_CFI_SHADOW_STACK_PTR_BIT 5 + +#define PTRACE_CFI_BRANCH_LANDING_PAD_EN_STATE _BITUL(PTRACE_CFI_BRANCH_LANDING_PAD_EN_BIT) +#define PTRACE_CFI_BRANCH_LANDING_PAD_LOCK_STATE \ + _BITUL(PTRACE_CFI_BRANCH_LANDING_PAD_LOCK_BIT) +#define PTRACE_CFI_BRANCH_EXPECTED_LANDING_PAD_STATE \ + _BITUL(PTRACE_CFI_BRANCH_EXPECTED_LANDING_PAD_BIT) +#define PTRACE_CFI_SHADOW_STACK_EN_STATE _BITUL(PTRACE_CFI_SHADOW_STACK_EN_BIT) +#define PTRACE_CFI_SHADOW_STACK_LOCK_STATE _BITUL(PTRACE_CFI_SHADOW_STACK_LOCK_BIT) +#define PTRACE_CFI_SHADOW_STACK_PTR_STATE _BITUL(PTRACE_CFI_SHADOW_STACK_PTR_BIT) + +#define PTRACE_CFI_STATE_INVALID_MASK ~(PTRACE_CFI_BRANCH_LANDING_PAD_EN_STATE | \ + PTRACE_CFI_BRANCH_LANDING_PAD_LOCK_STATE | \ + PTRACE_CFI_BRANCH_EXPECTED_LANDING_PAD_STATE | \ + PTRACE_CFI_SHADOW_STACK_EN_STATE | \ + PTRACE_CFI_SHADOW_STACK_LOCK_STATE | \ + PTRACE_CFI_SHADOW_STACK_PTR_STATE) + +struct __cfi_status { + __u64 cfi_state; +}; + +struct user_cfi_state { + struct __cfi_status cfi_status; + __u64 shstk_ptr; +}; + #endif /* __ASSEMBLER__ */ #endif /* _UAPI_ASM_RISCV_PTRACE_H */ diff --git a/arch/riscv/include/uapi/asm/setup.h b/arch/riscv/include/uapi/asm/setup.h index 66b13a522880..eb4f0209c696 100644 --- a/arch/riscv/include/uapi/asm/setup.h +++ b/arch/riscv/include/uapi/asm/setup.h @@ -3,6 +3,6 @@ #ifndef _UAPI_ASM_RISCV_SETUP_H #define _UAPI_ASM_RISCV_SETUP_H -#define COMMAND_LINE_SIZE 1024 +#define COMMAND_LINE_SIZE 2048 #endif /* _UAPI_ASM_RISCV_SETUP_H */ diff --git a/arch/riscv/include/uapi/asm/sigcontext.h b/arch/riscv/include/uapi/asm/sigcontext.h index 748dffc9ae19..d22d0815d605 100644 --- a/arch/riscv/include/uapi/asm/sigcontext.h +++ b/arch/riscv/include/uapi/asm/sigcontext.h @@ -10,6 +10,7 @@ /* The Magic number for signal context frame header. */ #define RISCV_V_MAGIC 0x53465457 +#define RISCV_ZICFISS_MAGIC 0x9487 #define END_MAGIC 0x0 /* The size of END signal context header. */ diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index a01f6439d62b..ebe1c3588177 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -73,6 +73,7 @@ obj-y += vendor_extensions/ obj-y += probes/ obj-y += tests/ obj-$(CONFIG_MMU) += vdso.o vdso/ +obj-$(CONFIG_RISCV_USER_CFI) += vdso_cfi/ obj-$(CONFIG_RISCV_MISALIGNED) += traps_misaligned.o obj-$(CONFIG_RISCV_MISALIGNED) += unaligned_access_speed.o @@ -126,3 +127,6 @@ obj-$(CONFIG_ACPI) += acpi.o obj-$(CONFIG_ACPI_NUMA) += acpi_numa.o obj-$(CONFIG_GENERIC_CPU_VULNERABILITIES) += bugs.o +obj-$(CONFIG_RISCV_USER_CFI) += usercfi.o + +obj-$(CONFIG_RISCV_ISA_SSQOSID) += qos.o diff --git a/arch/riscv/kernel/acpi.c b/arch/riscv/kernel/acpi.c index 71698ee11621..efd52a5d05b5 100644 --- a/arch/riscv/kernel/acpi.c +++ b/arch/riscv/kernel/acpi.c @@ -69,7 +69,7 @@ static int __init acpi_fadt_sanity_check(void) /* * FADT is required on riscv; retrieve it to check its presence - * and carry out revision and ACPI HW reduced compliancy tests + * and carry out revision and ACPI HW reduced compliance tests */ status = acpi_get_table(ACPI_SIG_FADT, 0, &table); if (ACPI_FAILURE(status)) { @@ -85,12 +85,12 @@ static int __init acpi_fadt_sanity_check(void) * The revision in the table header is the FADT's Major revision. The * FADT also has a minor revision, which is stored in the FADT itself. * - * TODO: Currently, we check for 6.5 as the minimum version to check - * for HW_REDUCED flag. However, once RISC-V updates are released in - * the ACPI spec, we need to update this check for exact minor revision + * ACPI 6.6 is required for RISC-V as it introduces RISC-V specific + * tables such as RHCT (RISC-V Hart Capabilities Table) and RIMT + * (RISC-V I/O Mapping Table). */ - if (table->revision < 6 || (table->revision == 6 && fadt->minor_revision < 5)) - pr_err(FW_BUG "Unsupported FADT revision %d.%d, should be 6.5+\n", + if (table->revision < 6 || (table->revision == 6 && fadt->minor_revision < 6)) + pr_err(FW_BUG "Unsupported FADT revision %d.%d, should be 6.6+\n", table->revision, fadt->minor_revision); if (!(fadt->flags & ACPI_FADT_HW_REDUCED)) { @@ -337,3 +337,24 @@ int raw_pci_write(unsigned int domain, unsigned int bus, } #endif /* CONFIG_PCI */ + +int acpi_get_cpu_uid(unsigned int cpu, u32 *uid) +{ + struct acpi_madt_rintc *rintc; + + if (cpu >= nr_cpu_ids) + return -EINVAL; + + rintc = acpi_cpu_get_madt_rintc(cpu); + if (!rintc) + return -ENODEV; + + *uid = rintc->uid; + return 0; +} +EXPORT_SYMBOL_GPL(acpi_get_cpu_uid); + +void __init arch_reserve_mem_area(acpi_physical_address addr, size_t size) +{ + memblock_mark_nomap(addr, size); +} diff --git a/arch/riscv/kernel/acpi_numa.c b/arch/riscv/kernel/acpi_numa.c index 130769e3a99c..6a2d4289f806 100644 --- a/arch/riscv/kernel/acpi_numa.c +++ b/arch/riscv/kernel/acpi_numa.c @@ -37,11 +37,14 @@ static int __init acpi_numa_get_nid(unsigned int cpu) static inline int get_cpu_for_acpi_id(u32 uid) { - int cpu; + u32 cpu_uid; + int ret; - for (cpu = 0; cpu < nr_cpu_ids; cpu++) - if (uid == get_acpi_id_for_cpu(cpu)) + for (int cpu = 0; cpu < nr_cpu_ids; cpu++) { + ret = acpi_get_cpu_uid(cpu, &cpu_uid); + if (ret == 0 && uid == cpu_uid) return cpu; + } return -EINVAL; } diff --git a/arch/riscv/kernel/alternative.c b/arch/riscv/kernel/alternative.c index 7642704c7f18..c0c9306022c5 100644 --- a/arch/riscv/kernel/alternative.c +++ b/arch/riscv/kernel/alternative.c @@ -173,30 +173,16 @@ static void __init_or_module _apply_alternatives(struct alt_entry *begin, stage); } -#ifdef CONFIG_MMU -static void __init apply_vdso_alternatives(void) +static __always_inline +void __init apply_vdso_alternatives(void *base, size_t alternatives_begin, size_t alternatives_end) { - const Elf_Ehdr *hdr; - const Elf_Shdr *shdr; - const Elf_Shdr *alt; - struct alt_entry *begin, *end; - - hdr = (Elf_Ehdr *)vdso_start; - shdr = (void *)hdr + hdr->e_shoff; - alt = find_section(hdr, shdr, ".alternative"); - if (!alt) + if (alternatives_begin == alternatives_end) return; - begin = (void *)hdr + alt->sh_offset, - end = (void *)hdr + alt->sh_offset + alt->sh_size, - - _apply_alternatives((struct alt_entry *)begin, - (struct alt_entry *)end, + _apply_alternatives(base + alternatives_begin, + base + alternatives_end, RISCV_ALTERNATIVES_BOOT); } -#else -static void __init apply_vdso_alternatives(void) { } -#endif void __init apply_boot_alternatives(void) { @@ -207,7 +193,20 @@ void __init apply_boot_alternatives(void) (struct alt_entry *)__alt_end, RISCV_ALTERNATIVES_BOOT); - apply_vdso_alternatives(); + if (IS_ENABLED(CONFIG_MMU)) + apply_vdso_alternatives(vdso_start, + __vdso_alternatives_start_offset, + __vdso_alternatives_end_offset); + + if (IS_ENABLED(CONFIG_RISCV_USER_CFI)) + apply_vdso_alternatives(vdso_cfi_start, + __vdso_alternatives_start_cfi_offset, + __vdso_alternatives_end_cfi_offset); + + if (IS_ENABLED(CONFIG_COMPAT)) + apply_vdso_alternatives(compat_vdso_start, + compat__vdso_alternatives_start_offset, + compat__vdso_alternatives_end_offset); } /* diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index 7d42d3b8a32a..a75f0cfea1e9 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -51,6 +51,10 @@ void asm_offsets(void) #endif OFFSET(TASK_TI_CPU_NUM, task_struct, thread_info.cpu); +#ifdef CONFIG_RISCV_USER_CFI + OFFSET(TASK_TI_CFI_STATE, task_struct, thread_info.user_cfi_state); + OFFSET(TASK_TI_USER_SSP, task_struct, thread_info.user_cfi_state.user_shdw_stk); +#endif OFFSET(TASK_THREAD_F0, task_struct, thread.fstate.f[0]); OFFSET(TASK_THREAD_F1, task_struct, thread.fstate.f[1]); OFFSET(TASK_THREAD_F2, task_struct, thread.fstate.f[2]); @@ -497,8 +501,8 @@ void asm_offsets(void) OFFSET(SBI_HART_BOOT_STACK_PTR_OFFSET, sbi_hart_boot_data, stack_ptr); DEFINE(STACKFRAME_SIZE_ON_STACK, ALIGN(sizeof(struct stackframe), STACK_ALIGN)); - OFFSET(STACKFRAME_FP, stackframe, fp); - OFFSET(STACKFRAME_RA, stackframe, ra); + DEFINE(STACKFRAME_FP, offsetof(struct stackframe, fp) - sizeof(struct stackframe)); + DEFINE(STACKFRAME_RA, offsetof(struct stackframe, ra) - sizeof(struct stackframe)); #ifdef CONFIG_FUNCTION_TRACER DEFINE(FTRACE_OPS_FUNC, offsetof(struct ftrace_ops, func)); #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS @@ -529,4 +533,10 @@ void asm_offsets(void) DEFINE(FREGS_A6, offsetof(struct __arch_ftrace_regs, a6)); DEFINE(FREGS_A7, offsetof(struct __arch_ftrace_regs, a7)); #endif +#ifdef CONFIG_RISCV_SBI + DEFINE(SBI_EXT_FWFT, SBI_EXT_FWFT); + DEFINE(SBI_EXT_FWFT_SET, SBI_EXT_FWFT_SET); + DEFINE(SBI_FWFT_SHADOW_STACK, SBI_FWFT_SHADOW_STACK); + DEFINE(SBI_FWFT_SET_FLAG_LOCK, SBI_FWFT_SET_FLAG_LOCK); +#endif } diff --git a/arch/riscv/kernel/bugs.c b/arch/riscv/kernel/bugs.c index 3655fe7d678c..e5758e3f1c7e 100644 --- a/arch/riscv/kernel/bugs.c +++ b/arch/riscv/kernel/bugs.c @@ -5,7 +5,7 @@ #include <linux/cpu.h> #include <linux/device.h> -#include <linux/sprintf.h> +#include <linux/sysfs.h> #include <asm/bugs.h> #include <asm/vendor_extensions/thead.h> @@ -46,15 +46,15 @@ ssize_t cpu_show_ghostwrite(struct device *dev, struct device_attribute *attr, c if (IS_ENABLED(CONFIG_RISCV_ISA_XTHEADVECTOR)) { switch (ghostwrite_state) { case UNAFFECTED: - return sprintf(buf, "Not affected\n"); + return sysfs_emit(buf, "Not affected\n"); case MITIGATED: - return sprintf(buf, "Mitigation: xtheadvector disabled\n"); + return sysfs_emit(buf, "Mitigation: xtheadvector disabled\n"); case VULNERABLE: fallthrough; default: - return sprintf(buf, "Vulnerable\n"); + return sysfs_emit(buf, "Vulnerable\n"); } - } else { - return sprintf(buf, "Not affected\n"); } + + return sysfs_emit(buf, "Not affected\n"); } diff --git a/arch/riscv/kernel/cacheinfo.c b/arch/riscv/kernel/cacheinfo.c index 26b085dbdd07..6c9a1ef2d45a 100644 --- a/arch/riscv/kernel/cacheinfo.c +++ b/arch/riscv/kernel/cacheinfo.c @@ -133,7 +133,7 @@ int populate_cache_leaves(unsigned int cpu) ci_leaf_init(this_leaf++, CACHE_TYPE_DATA, level); levels = level; } - of_node_put(np); + of_node_put(prev); return 0; } diff --git a/arch/riscv/kernel/compat_signal.c b/arch/riscv/kernel/compat_signal.c index 6ec4e34255a9..cf3eb33a11e4 100644 --- a/arch/riscv/kernel/compat_signal.c +++ b/arch/riscv/kernel/compat_signal.c @@ -107,6 +107,8 @@ static long compat_restore_sigcontext(struct pt_regs *regs, /* sc_regs is structured the same as the start of pt_regs */ err = __copy_from_user(&cregs, &sc->sc_regs, sizeof(sc->sc_regs)); + if (unlikely(err)) + return err; cregs_to_regs(&cregs, regs); diff --git a/arch/riscv/kernel/copy-unaligned.S b/arch/riscv/kernel/copy-unaligned.S index 2b3d9398c113..90f3549621f7 100644 --- a/arch/riscv/kernel/copy-unaligned.S +++ b/arch/riscv/kernel/copy-unaligned.S @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (C) 2023 Rivos Inc. */ +#include <linux/cfi_types.h> #include <linux/linkage.h> #include <asm/asm.h> @@ -9,7 +10,7 @@ /* void __riscv_copy_words_unaligned(void *, const void *, size_t) */ /* Performs a memcpy without aligning buffers, using word loads and stores. */ /* Note: The size is truncated to a multiple of 8 * SZREG */ -SYM_FUNC_START(__riscv_copy_words_unaligned) +SYM_TYPED_FUNC_START(__riscv_copy_words_unaligned) andi a4, a2, ~((8*SZREG)-1) beqz a4, 2f add a3, a1, a4 @@ -41,7 +42,7 @@ SYM_FUNC_END(__riscv_copy_words_unaligned) /* void __riscv_copy_bytes_unaligned(void *, const void *, size_t) */ /* Performs a memcpy without aligning buffers, using only byte accesses. */ /* Note: The size is truncated to a multiple of 8 */ -SYM_FUNC_START(__riscv_copy_bytes_unaligned) +SYM_TYPED_FUNC_START(__riscv_copy_bytes_unaligned) andi a4, a2, ~(8-1) beqz a4, 2f add a3, a1, a4 diff --git a/arch/riscv/kernel/cpu-hotplug.c b/arch/riscv/kernel/cpu-hotplug.c index 3f50d3dd76c6..0bc56d8381b6 100644 --- a/arch/riscv/kernel/cpu-hotplug.c +++ b/arch/riscv/kernel/cpu-hotplug.c @@ -43,7 +43,6 @@ int __cpu_disable(void) return 0; } -#ifdef CONFIG_HOTPLUG_CPU /* * Called on the thread which is asking for a CPU to be shutdown, if the * CPU reported dead to the hotplug core. @@ -58,8 +57,8 @@ void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu) /* Verify from the firmware if the cpu is really stopped*/ if (cpu_ops->cpu_is_stopped) ret = cpu_ops->cpu_is_stopped(cpu); - if (ret) - pr_warn("CPU%u may not have stopped: %d\n", cpu, ret); + if (!ret) + pr_warn("CPU%u may not have stopped\n", cpu); } /* @@ -75,4 +74,3 @@ void __noreturn arch_cpu_idle_dead(void) /* It should never reach here */ BUG(); } -#endif diff --git a/arch/riscv/kernel/cpu_ops_sbi.c b/arch/riscv/kernel/cpu_ops_sbi.c index 00aff669f5f2..ee6e4b5cc39e 100644 --- a/arch/riscv/kernel/cpu_ops_sbi.c +++ b/arch/riscv/kernel/cpu_ops_sbi.c @@ -30,10 +30,8 @@ static int sbi_hsm_hart_start(unsigned long hartid, unsigned long saddr, ret = sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_START, hartid, saddr, priv, 0, 0, 0); - if (ret.error) - return sbi_err_map_linux_errno(ret.error); - else - return 0; + + return sbi_err_map_linux_errno(ret.error); } #ifdef CONFIG_HOTPLUG_CPU @@ -43,10 +41,7 @@ static int sbi_hsm_hart_stop(void) ret = sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_STOP, 0, 0, 0, 0, 0, 0); - if (ret.error) - return sbi_err_map_linux_errno(ret.error); - else - return 0; + return sbi_err_map_linux_errno(ret.error); } static int sbi_hsm_hart_get_status(unsigned long hartid) @@ -88,16 +83,19 @@ static void sbi_cpu_stop(void) pr_crit("Unable to stop the cpu %d (%d)\n", smp_processor_id(), ret); } -static int sbi_cpu_is_stopped(unsigned int cpuid) +static bool sbi_cpu_is_stopped(unsigned int cpuid) { int rc; unsigned long hartid = cpuid_to_hartid_map(cpuid); rc = sbi_hsm_hart_get_status(hartid); - if (rc == SBI_HSM_STATE_STOPPED) - return 0; - return rc; + if (rc != SBI_HSM_STATE_STOPPED) { + pr_warn("HART%lu isn't stopped; status %d\n", hartid, rc); + return false; + } + + return true; } #endif diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index fa591aff9d33..61d21f714830 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -28,6 +28,7 @@ #include <asm/vector.h> #include <asm/vendor_extensions.h> #include <asm/vendor_extensions/thead.h> +#include <asm/usercfi.h> #define NUM_ALPHA_EXTS ('z' - 'a' + 1) @@ -83,7 +84,7 @@ EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); static int riscv_ext_f_depends(const struct riscv_isa_ext_data *data, const unsigned long *isa_bitmap) { - if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_f)) + if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_F)) return 0; return -EPROBE_DEFER; @@ -145,7 +146,7 @@ static int riscv_ext_f_validate(const struct riscv_isa_ext_data *data, * Due to extension ordering, d is checked before f, so no deferral * is required. */ - if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d)) { + if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_D)) { pr_warn_once("This kernel does not support systems with F but not D\n"); return -EINVAL; } @@ -188,7 +189,7 @@ static int riscv_ext_vector_float_validate(const struct riscv_isa_ext_data *data * Since this function validates vector only, and v/Zve* are probed * after f/d, there's no need for a deferral here. */ - if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d)) + if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_D)) return -EINVAL; return 0; @@ -223,7 +224,7 @@ static int riscv_ext_zcd_validate(const struct riscv_isa_ext_data *data, const unsigned long *isa_bitmap) { if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA) && - __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d)) + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_D)) return 0; return -EPROBE_DEFER; @@ -236,7 +237,7 @@ static int riscv_ext_zcf_validate(const struct riscv_isa_ext_data *data, return -EINVAL; if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA) && - __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_f)) + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_F)) return 0; return -EPROBE_DEFER; @@ -296,6 +297,26 @@ static int riscv_ext_svadu_validate(const struct riscv_isa_ext_data *data, return 0; } +static int riscv_cfilp_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (!IS_ENABLED(CONFIG_RISCV_USER_CFI) || + (riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_FCFI)) + return -EINVAL; + + return 0; +} + +static int riscv_cfiss_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (!IS_ENABLED(CONFIG_RISCV_USER_CFI) || + (riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_BCFI)) + return -EINVAL; + + return 0; +} + static const unsigned int riscv_a_exts[] = { RISCV_ISA_EXT_ZAAMO, RISCV_ISA_EXT_ZALRSC, @@ -309,6 +330,27 @@ static const unsigned int riscv_a_exts[] = { RISCV_ISA_EXT_ZKNE, \ RISCV_ISA_EXT_ZKNH +static int riscv_ext_smcdeleg_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_SSCSRIND) && + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZIHPM) && + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZICNTR)) + return 0; + + return -EPROBE_DEFER; +} + +static int riscv_ext_ssccfg_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (!riscv_ext_smcdeleg_validate(data, isa_bitmap) && + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_SMCDELEG)) + return 0; + + return -EPROBE_DEFER; +} + static const unsigned int riscv_zk_bundled_exts[] = { RISCV_ISA_EXT_ZKN, RISCV_ISA_EXT_ZKR, @@ -370,6 +412,19 @@ static const unsigned int riscv_zvbb_exts[] = { RISCV_ISA_EXT_ZVKB }; +/* + * The RISC-V ISA manual specifies that Zfh implies Zfhmin and Zvfh implies + * Zvfhmin. Report the implied subset extensions whenever the supersets are + * detected (see https://github.com/riscv/riscv-isa-manual/pull/3070). + */ +static const unsigned int riscv_zfh_exts[] = { + RISCV_ISA_EXT_ZFHMIN +}; + +static const unsigned int riscv_zvfh_exts[] = { + RISCV_ISA_EXT_ZVFHMIN +}; + #define RISCV_ISA_EXT_ZVE64F_IMPLY_LIST \ RISCV_ISA_EXT_ZVE64X, \ RISCV_ISA_EXT_ZVE32F, \ @@ -431,7 +486,8 @@ static const unsigned int riscv_c_exts[] = { /* * The canonical order of ISA extension names in the ISA string is defined in - * chapter 27 of the unprivileged specification. + * Chapter 27 of the RISC-V Instruction Set Manual Volume I Unprivileged ISA + * (Document Version 20191213). * * Ordinarily, for in-kernel data structures, this order is unimportant but * isa_ext_arr defines the order of the ISA string in /proc/cpuinfo. @@ -469,19 +525,27 @@ static const unsigned int riscv_c_exts[] = { * New entries to this struct should follow the ordering rules described above. */ const struct riscv_isa_ext_data riscv_isa_ext[] = { - __RISCV_ISA_EXT_DATA(i, RISCV_ISA_EXT_i), - __RISCV_ISA_EXT_DATA(m, RISCV_ISA_EXT_m), - __RISCV_ISA_EXT_SUPERSET(a, RISCV_ISA_EXT_a, riscv_a_exts), - __RISCV_ISA_EXT_DATA_VALIDATE(f, RISCV_ISA_EXT_f, riscv_ext_f_validate), - __RISCV_ISA_EXT_DATA_VALIDATE(d, RISCV_ISA_EXT_d, riscv_ext_d_validate), - __RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_q), - __RISCV_ISA_EXT_SUPERSET(c, RISCV_ISA_EXT_c, riscv_c_exts), - __RISCV_ISA_EXT_SUPERSET_VALIDATE(v, RISCV_ISA_EXT_v, riscv_v_exts, riscv_ext_vector_float_validate), - __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_h), + __RISCV_ISA_EXT_DATA(i, RISCV_ISA_EXT_I), + __RISCV_ISA_EXT_DATA(m, RISCV_ISA_EXT_M), + __RISCV_ISA_EXT_SUPERSET(a, RISCV_ISA_EXT_A, riscv_a_exts), + __RISCV_ISA_EXT_DATA_VALIDATE(f, RISCV_ISA_EXT_F, riscv_ext_f_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(d, RISCV_ISA_EXT_D, riscv_ext_d_validate), + __RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_Q), + __RISCV_ISA_EXT_SUPERSET(c, RISCV_ISA_EXT_C, riscv_c_exts), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(v, RISCV_ISA_EXT_V, riscv_v_exts, + riscv_ext_vector_float_validate), + __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_H), __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicbom, RISCV_ISA_EXT_ZICBOM, riscv_xlinuxenvcfg_exts, riscv_ext_zicbom_validate), __RISCV_ISA_EXT_DATA_VALIDATE(zicbop, RISCV_ISA_EXT_ZICBOP, riscv_ext_zicbop_validate), __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts, riscv_ext_zicboz_validate), + __RISCV_ISA_EXT_DATA(ziccamoa, RISCV_ISA_EXT_ZICCAMOA), + __RISCV_ISA_EXT_DATA(ziccif, RISCV_ISA_EXT_ZICCIF), + __RISCV_ISA_EXT_DATA(zicclsm, RISCV_ISA_EXT_ZICCLSM), __RISCV_ISA_EXT_DATA(ziccrse, RISCV_ISA_EXT_ZICCRSE), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicfilp, RISCV_ISA_EXT_ZICFILP, riscv_xlinuxenvcfg_exts, + riscv_cfilp_validate), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicfiss, RISCV_ISA_EXT_ZICFISS, riscv_xlinuxenvcfg_exts, + riscv_cfiss_validate), __RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR), __RISCV_ISA_EXT_DATA(zicond, RISCV_ISA_EXT_ZICOND), __RISCV_ISA_EXT_DATA(zicsr, RISCV_ISA_EXT_ZICSR), @@ -490,6 +554,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM), __RISCV_ISA_EXT_DATA(zimop, RISCV_ISA_EXT_ZIMOP), + __RISCV_ISA_EXT_DATA(za64rs, RISCV_ISA_EXT_ZA64RS), __RISCV_ISA_EXT_DATA(zaamo, RISCV_ISA_EXT_ZAAMO), __RISCV_ISA_EXT_DATA(zabha, RISCV_ISA_EXT_ZABHA), __RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS), @@ -498,7 +563,8 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zawrs, RISCV_ISA_EXT_ZAWRS), __RISCV_ISA_EXT_DATA_VALIDATE(zfa, RISCV_ISA_EXT_ZFA, riscv_ext_f_depends), __RISCV_ISA_EXT_DATA_VALIDATE(zfbfmin, RISCV_ISA_EXT_ZFBFMIN, riscv_ext_f_depends), - __RISCV_ISA_EXT_DATA_VALIDATE(zfh, RISCV_ISA_EXT_ZFH, riscv_ext_f_depends), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zfh, RISCV_ISA_EXT_ZFH, + riscv_zfh_exts, riscv_ext_f_depends), __RISCV_ISA_EXT_DATA_VALIDATE(zfhmin, RISCV_ISA_EXT_ZFHMIN, riscv_ext_f_depends), __RISCV_ISA_EXT_DATA(zca, RISCV_ISA_EXT_ZCA), __RISCV_ISA_EXT_DATA_VALIDATE(zcb, RISCV_ISA_EXT_ZCB, riscv_ext_zca_depends), @@ -534,7 +600,9 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64x, RISCV_ISA_EXT_ZVE64X, riscv_zve64x_exts, riscv_ext_vector_x_validate), __RISCV_ISA_EXT_DATA_VALIDATE(zvfbfmin, RISCV_ISA_EXT_ZVFBFMIN, riscv_vector_f_validate), __RISCV_ISA_EXT_DATA_VALIDATE(zvfbfwma, RISCV_ISA_EXT_ZVFBFWMA, riscv_ext_zvfbfwma_validate), - __RISCV_ISA_EXT_DATA(zvfh, RISCV_ISA_EXT_ZVFH), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zvfh, RISCV_ISA_EXT_ZVFH, + riscv_zvfh_exts, + riscv_ext_vector_float_validate), __RISCV_ISA_EXT_DATA(zvfhmin, RISCV_ISA_EXT_ZVFHMIN), __RISCV_ISA_EXT_DATA_VALIDATE(zvkb, RISCV_ISA_EXT_ZVKB, riscv_ext_vector_crypto_validate), __RISCV_ISA_EXT_DATA_VALIDATE(zvkg, RISCV_ISA_EXT_ZVKG, riscv_ext_vector_crypto_validate), @@ -551,12 +619,19 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvksg, riscv_zvksg_bundled_exts, riscv_ext_vector_crypto_validate), __RISCV_ISA_EXT_DATA_VALIDATE(zvkt, RISCV_ISA_EXT_ZVKT, riscv_ext_vector_crypto_validate), __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA), + __RISCV_ISA_EXT_DATA_VALIDATE(smcdeleg, RISCV_ISA_EXT_SMCDELEG, + riscv_ext_smcdeleg_validate), + __RISCV_ISA_EXT_DATA(smcntrpmf, RISCV_ISA_EXT_SMCNTRPMF), + __RISCV_ISA_EXT_DATA(smcsrind, RISCV_ISA_EXT_SMCSRIND), __RISCV_ISA_EXT_DATA(smmpm, RISCV_ISA_EXT_SMMPM), __RISCV_ISA_EXT_SUPERSET(smnpm, RISCV_ISA_EXT_SMNPM, riscv_xlinuxenvcfg_exts), __RISCV_ISA_EXT_DATA(smstateen, RISCV_ISA_EXT_SMSTATEEN), __RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA), + __RISCV_ISA_EXT_DATA_VALIDATE(ssccfg, RISCV_ISA_EXT_SSCCFG, riscv_ext_ssccfg_validate), __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF), + __RISCV_ISA_EXT_DATA(sscsrind, RISCV_ISA_EXT_SSCSRIND), __RISCV_ISA_EXT_SUPERSET(ssnpm, RISCV_ISA_EXT_SSNPM, riscv_xlinuxenvcfg_exts), + __RISCV_ISA_EXT_DATA(ssqosid, RISCV_ISA_EXT_SSQOSID), __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC), __RISCV_ISA_EXT_DATA(svade, RISCV_ISA_EXT_SVADE), __RISCV_ISA_EXT_DATA_VALIDATE(svadu, RISCV_ISA_EXT_SVADU, riscv_ext_svadu_validate), @@ -871,10 +946,8 @@ static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap) * CPU cores with the ratified spec will contain non-zero * marchid. */ - if (acpi_disabled && boot_vendorid == THEAD_VENDOR_ID && boot_archid == 0x0) { - this_hwcap &= ~isa2hwcap[RISCV_ISA_EXT_v]; - clear_bit(RISCV_ISA_EXT_v, source_isa); - } + if (acpi_disabled && boot_vendorid == THEAD_VENDOR_ID && boot_archid == 0x0) + clear_bit(RISCV_ISA_EXT_V, source_isa); riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap); @@ -1079,16 +1152,16 @@ early_param("riscv_isa_fallback", riscv_isa_fallback_setup); void __init riscv_fill_hwcap(void) { char print_str[NUM_ALPHA_EXTS + 1]; - unsigned long isa2hwcap[26] = {0}; + unsigned long isa2hwcap[RISCV_ISA_EXT_BASE] = {0}; int i, j; - isa2hwcap['i' - 'a'] = COMPAT_HWCAP_ISA_I; - isa2hwcap['m' - 'a'] = COMPAT_HWCAP_ISA_M; - isa2hwcap['a' - 'a'] = COMPAT_HWCAP_ISA_A; - isa2hwcap['f' - 'a'] = COMPAT_HWCAP_ISA_F; - isa2hwcap['d' - 'a'] = COMPAT_HWCAP_ISA_D; - isa2hwcap['c' - 'a'] = COMPAT_HWCAP_ISA_C; - isa2hwcap['v' - 'a'] = COMPAT_HWCAP_ISA_V; + isa2hwcap[RISCV_ISA_EXT_I] = COMPAT_HWCAP_ISA_I; + isa2hwcap[RISCV_ISA_EXT_M] = COMPAT_HWCAP_ISA_M; + isa2hwcap[RISCV_ISA_EXT_A] = COMPAT_HWCAP_ISA_A; + isa2hwcap[RISCV_ISA_EXT_F] = COMPAT_HWCAP_ISA_F; + isa2hwcap[RISCV_ISA_EXT_D] = COMPAT_HWCAP_ISA_D; + isa2hwcap[RISCV_ISA_EXT_C] = COMPAT_HWCAP_ISA_C; + isa2hwcap[RISCV_ISA_EXT_V] = COMPAT_HWCAP_ISA_V; if (!acpi_disabled) { riscv_fill_hwcap_from_isa_string(isa2hwcap); diff --git a/arch/riscv/kernel/efi.c b/arch/riscv/kernel/efi.c index b64bf1624a05..2d3cc57b4535 100644 --- a/arch/riscv/kernel/efi.c +++ b/arch/riscv/kernel/efi.c @@ -95,3 +95,8 @@ int __init efi_set_mapping_permissions(struct mm_struct *mm, md->num_pages << EFI_PAGE_SHIFT, set_permissions, md); } + +bool efi_poweroff_required(void) +{ + return efi_enabled(EFI_RUNTIME_SERVICES); +} diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index 9b9dec6893b8..d799c4e56f80 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -20,44 +20,44 @@ .section .irqentry.text, "ax" -.macro new_vmalloc_check +.macro new_valid_map_cpus_check REG_S a0, TASK_TI_A0(tp) csrr a0, CSR_CAUSE /* Exclude IRQs */ - blt a0, zero, .Lnew_vmalloc_restore_context_a0 + blt a0, zero, .Lnew_valid_map_cpus_restore_context_a0 REG_S a1, TASK_TI_A1(tp) - /* Only check new_vmalloc if we are in page/protection fault */ + /* Only check new_valid_map_cpus if we are in page/protection fault */ li a1, EXC_LOAD_PAGE_FAULT - beq a0, a1, .Lnew_vmalloc_kernel_address + beq a0, a1, .Lnew_valid_map_cpus_kernel_address li a1, EXC_STORE_PAGE_FAULT - beq a0, a1, .Lnew_vmalloc_kernel_address + beq a0, a1, .Lnew_valid_map_cpus_kernel_address li a1, EXC_INST_PAGE_FAULT - bne a0, a1, .Lnew_vmalloc_restore_context_a1 + bne a0, a1, .Lnew_valid_map_cpus_restore_context_a1 -.Lnew_vmalloc_kernel_address: +.Lnew_valid_map_cpus_kernel_address: /* Is it a kernel address? */ csrr a0, CSR_TVAL - bge a0, zero, .Lnew_vmalloc_restore_context_a1 + bge a0, zero, .Lnew_valid_map_cpus_restore_context_a1 /* Check if a new vmalloc mapping appeared that could explain the trap */ REG_S a2, TASK_TI_A2(tp) /* * Computes: - * a0 = &new_vmalloc[BIT_WORD(cpu)] + * a0 = &new_valid_map_cpus[BIT_WORD(cpu)] * a1 = BIT_MASK(cpu) */ lw a2, TASK_TI_CPU(tp) /* - * Compute the new_vmalloc element position: + * Compute the new_valid_map_cpus element position: * (cpu / 64) * 8 = (cpu >> 6) << 3 */ srli a1, a2, 6 slli a1, a1, 3 - la a0, new_vmalloc + la a0, new_valid_map_cpus add a0, a0, a1 /* - * Compute the bit position in the new_vmalloc element: + * Compute the bit position in the new_valid_map_cpus element: * bit_pos = cpu % 64 = cpu - (cpu / 64) * 64 = cpu - (cpu >> 6) << 6 * = cpu - ((cpu >> 6) << 3) << 3 */ @@ -67,16 +67,19 @@ li a2, 1 sll a1, a2, a1 - /* Check the value of new_vmalloc for this cpu */ + /* Check the value of new_valid_map_cpus for this cpu */ REG_L a2, 0(a0) and a2, a2, a1 - beq a2, zero, .Lnew_vmalloc_restore_context + beq a2, zero, .Lnew_valid_map_cpus_restore_context - /* Atomically reset the current cpu bit in new_vmalloc */ + /* Atomically reset the current cpu bit in new_valid_map_cpus */ amoxor.d a0, a1, (a0) - /* Only emit a sfence.vma if the uarch caches invalid entries */ - ALTERNATIVE("sfence.vma", "nop", 0, RISCV_ISA_EXT_SVVPTC, 1) + /* + * A sfence.vma is required here. Even if we had Svvptc, there's no + * guarantee that after returning we wouldn't just fault again. + */ + sfence.vma REG_L a0, TASK_TI_A0(tp) REG_L a1, TASK_TI_A1(tp) @@ -84,14 +87,43 @@ csrw CSR_SCRATCH, x0 sret -.Lnew_vmalloc_restore_context: +.Lnew_valid_map_cpus_restore_context: REG_L a2, TASK_TI_A2(tp) -.Lnew_vmalloc_restore_context_a1: +.Lnew_valid_map_cpus_restore_context_a1: REG_L a1, TASK_TI_A1(tp) -.Lnew_vmalloc_restore_context_a0: +.Lnew_valid_map_cpus_restore_context_a0: REG_L a0, TASK_TI_A0(tp) .endm +/* + * If previous mode was U, capture shadow stack pointer and save it away + * Zero CSR_SSP at the same time for sanitization. + */ +.macro save_userssp tmp, status + ALTERNATIVE("nops(4)", + __stringify( \ + andi \tmp, \status, SR_SPP; \ + bnez \tmp, skip_ssp_save; \ + csrrw \tmp, CSR_SSP, x0; \ + REG_S \tmp, TASK_TI_USER_SSP(tp); \ + skip_ssp_save:), + 0, + RISCV_ISA_EXT_ZICFISS, + CONFIG_RISCV_USER_CFI) +.endm + +.macro restore_userssp tmp, status + ALTERNATIVE("nops(4)", + __stringify( \ + andi \tmp, \status, SR_SPP; \ + bnez \tmp, skip_ssp_restore; \ + REG_L \tmp, TASK_TI_USER_SSP(tp); \ + csrw CSR_SSP, \tmp; \ + skip_ssp_restore:), + 0, + RISCV_ISA_EXT_ZICFISS, + CONFIG_RISCV_USER_CFI) +.endm SYM_CODE_START(handle_exception) /* @@ -105,17 +137,19 @@ SYM_CODE_START(handle_exception) .Lrestore_kernel_tpsp: csrr tp, CSR_SCRATCH -#ifdef CONFIG_64BIT +#if defined(CONFIG_64BIT) && defined(CONFIG_MMU) /* - * The RISC-V kernel does not eagerly emit a sfence.vma after each - * new vmalloc mapping, which may result in exceptions: + * The RISC-V kernel does not flush TLBs on all CPUS after each new + * vmalloc mapping or kfence_unprotect(), which may result in + * exceptions: + * * - if the uarch caches invalid entries, the new mapping would not be * observed by the page table walker and an invalidation is needed. * - if the uarch does not cache invalid entries, a reordered access * could "miss" the new mapping and traps: in that case, we only need * to retry the access, no sfence.vma is required. */ - new_vmalloc_check + new_valid_map_cpus_check #endif REG_S sp, TASK_TI_KERNEL_SP(tp) @@ -145,9 +179,14 @@ SYM_CODE_START(handle_exception) * or vector in kernel space. */ li t0, SR_SUM | SR_FS_VS +#ifdef CONFIG_64BIT + li t1, SR_ELP + or t0, t0, t1 +#endif REG_L s0, TASK_TI_USER_SP(tp) csrrc s1, CSR_STATUS, t0 + save_userssp s2, s1 csrr s2, CSR_EPC csrr s3, CSR_TVAL csrr s4, CSR_CAUSE @@ -243,6 +282,7 @@ SYM_CODE_START_NOALIGN(ret_from_exception) call riscv_v_context_nesting_end #endif REG_L a0, PT_STATUS(sp) + restore_userssp s3, a0 /* * The current load reservation is effectively part of the processor's * state, in the sense that load reservations cannot be shared between @@ -346,8 +386,8 @@ SYM_CODE_END(ret_from_fork_user_asm) SYM_FUNC_START(call_on_irq_stack) /* Create a frame record to save ra and s0 (fp) */ addi sp, sp, -STACKFRAME_SIZE_ON_STACK - REG_S ra, STACKFRAME_RA(sp) - REG_S s0, STACKFRAME_FP(sp) + REG_S ra, (STACKFRAME_SIZE_ON_STACK + STACKFRAME_RA)(sp) + REG_S s0, (STACKFRAME_SIZE_ON_STACK + STACKFRAME_FP)(sp) addi s0, sp, STACKFRAME_SIZE_ON_STACK /* Switch to the per-CPU shadow call stack */ @@ -365,8 +405,8 @@ SYM_FUNC_START(call_on_irq_stack) /* Switch back to the thread stack and restore ra and s0 */ addi sp, s0, -STACKFRAME_SIZE_ON_STACK - REG_L ra, STACKFRAME_RA(sp) - REG_L s0, STACKFRAME_FP(sp) + REG_L ra, (STACKFRAME_SIZE_ON_STACK + STACKFRAME_RA)(sp) + REG_L s0, (STACKFRAME_SIZE_ON_STACK + STACKFRAME_FP)(sp) addi sp, sp, STACKFRAME_SIZE_ON_STACK ret @@ -460,6 +500,10 @@ SYM_DATA_START_LOCAL(excp_vect_table) RISCV_PTR do_page_fault /* load page fault */ RISCV_PTR do_trap_unknown RISCV_PTR do_page_fault /* store page fault */ + RISCV_PTR do_trap_unknown /* cause=16 */ + RISCV_PTR do_trap_unknown /* cause=17 */ + RISCV_PTR do_trap_software_check /* cause=18 is sw check exception */ + RISCV_PTR do_trap_hardware_error /* hardware error (19) */ SYM_DATA_END_LABEL(excp_vect_table, SYM_L_LOCAL, excp_vect_table_end) #ifndef CONFIG_MMU diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c index 8d18d6727f0f..be8b68514417 100644 --- a/arch/riscv/kernel/ftrace.c +++ b/arch/riscv/kernel/ftrace.c @@ -12,6 +12,7 @@ #include <linux/stop_machine.h> #include <asm/cacheflush.h> #include <asm/text-patching.h> +#include <asm/insn.h> #ifdef CONFIG_DYNAMIC_FTRACE void ftrace_arch_code_modify_prepare(void) @@ -63,7 +64,9 @@ static int __ftrace_modify_call(unsigned long source, unsigned long target, bool if (copy_from_kernel_nofault(replaced, (void *)source, 2 * MCOUNT_INSN_SIZE)) return -EFAULT; - if (replaced[0] != call[0]) { + /* Bypass the check if the auipc insn is a kprobe breakpoint */ + if (replaced[0] != call[0] && + !(riscv_insn_is_ebreak(replaced[0]) || riscv_insn_is_c_ebreak(replaced[0]))) { pr_err("%p: expected (%08x) but got (%08x)\n", (void *)source, call[0], replaced[0]); return -EINVAL; @@ -148,7 +151,7 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long ad /* * This is called early on, and isn't wrapped by - * ftrace_arch_code_modify_{prepare,post_process}() and therefor doesn't hold + * ftrace_arch_code_modify_{prepare,post_process}() and therefore doesn't hold * text_mutex, which triggers a lockdep failure. SMP isn't running so we could * just directly poke the text, but it's simpler to just take the lock * ourselves. diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index bdf3352acf4c..f6a8ca49e627 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -14,7 +14,7 @@ #include <asm/hwcap.h> #include <asm/image.h> #include <asm/scs.h> -#include <asm/xip_fixup.h> +#include <asm/usercfi.h> #include "efi-header.S" __HEAD @@ -74,13 +74,12 @@ pe_head_start: relocate_enable_mmu: /* Relocate return address */ la a1, kernel_map - XIP_FIXUP_OFFSET a1 REG_L a1, KERNEL_MAP_VIRT_ADDR(a1) la a2, _start sub a1, a1, a2 add ra, ra, a1 - /* Point stvec to virtual address of intruction after satp write */ + /* Point stvec to virtual address of instruction after satp write */ la a2, 1f add a2, a2, a1 csrw CSR_TVEC, a2 @@ -88,7 +87,6 @@ relocate_enable_mmu: /* Compute satp for kernel page tables, but don't load it yet */ srl a2, a0, PAGE_SHIFT la a1, satp_mode - XIP_FIXUP_OFFSET a1 REG_L a1, 0(a1) or a2, a2, a1 @@ -99,7 +97,6 @@ relocate_enable_mmu: * to ensure the new translations are in use. */ la a0, trampoline_pg_dir - XIP_FIXUP_OFFSET a0 srl a0, a0, PAGE_SHIFT or a0, a0, a1 sfence.vma @@ -153,11 +150,9 @@ secondary_start_sbi: /* a0 contains the hartid & a1 contains boot data */ li a2, SBI_HART_BOOT_TASK_PTR_OFFSET - XIP_FIXUP_OFFSET a2 add a2, a2, a1 REG_L tp, (a2) li a3, SBI_HART_BOOT_STACK_PTR_OFFSET - XIP_FIXUP_OFFSET a3 add a3, a3, a1 REG_L sp, (a3) @@ -166,10 +161,22 @@ secondary_start_sbi: #ifdef CONFIG_MMU /* Enable virtual memory and relocate to virtual address */ la a0, swapper_pg_dir - XIP_FIXUP_OFFSET a0 call relocate_enable_mmu #endif call .Lsetup_trap_vector +#if defined(CONFIG_RISCV_SBI) && defined(CONFIG_RISCV_USER_CFI) + li a7, SBI_EXT_FWFT + li a6, SBI_EXT_FWFT_SET + li a0, SBI_FWFT_SHADOW_STACK + li a1, 1 /* enable supervisor to access shadow stack access */ + li a2, SBI_FWFT_SET_FLAG_LOCK + ecall + beqz a0, 1f + la a1, riscv_nousercfi + li a0, CMDLINE_DISABLE_RISCV_USERCFI_BCFI + REG_S a0, (a1) +1: +#endif scs_load_current call smp_callin #endif /* CONFIG_SMP */ @@ -255,40 +262,13 @@ SYM_CODE_START(_start_kernel) .Lgood_cores: /* The lottery system is only required for spinwait booting method */ -#ifndef CONFIG_XIP_KERNEL /* Pick one hart to run the main boot sequence */ la a3, hart_lottery li a2, 1 amoadd.w a3, a2, (a3) bnez a3, .Lsecondary_start - -#else - /* hart_lottery in flash contains a magic number */ - la a3, hart_lottery - mv a2, a3 - XIP_FIXUP_OFFSET a2 - XIP_FIXUP_FLASH_OFFSET a3 - lw t1, (a3) - amoswap.w t0, t1, (a2) - /* first time here if hart_lottery in RAM is not set */ - beq t0, t1, .Lsecondary_start - -#endif /* CONFIG_XIP */ #endif /* CONFIG_RISCV_BOOT_SPINWAIT */ -#ifdef CONFIG_XIP_KERNEL - la sp, _end + THREAD_SIZE - XIP_FIXUP_OFFSET sp - mv s0, a0 - mv s1, a1 - call __copy_data - - /* Restore a0 & a1 copy */ - mv a0, s0 - mv a1, s1 -#endif - -#ifndef CONFIG_XIP_KERNEL /* Clear BSS for flat non-ELF images */ la a3, __bss_start la a4, __bss_stop @@ -298,20 +278,16 @@ SYM_CODE_START(_start_kernel) add a3, a3, RISCV_SZPTR blt a3, a4, .Lclear_bss .Lclear_bss_done: -#endif la a2, boot_cpu_hartid - XIP_FIXUP_OFFSET a2 REG_S a0, (a2) /* Initialize page tables and relocate to virtual addresses */ la tp, init_task la sp, init_thread_union + THREAD_SIZE - XIP_FIXUP_OFFSET sp addi sp, sp, -PT_SIZE_ON_STACK scs_load_init_stack #ifdef CONFIG_BUILTIN_DTB la a0, __dtb_start - XIP_FIXUP_OFFSET a0 #else mv a0, a1 #endif /* CONFIG_BUILTIN_DTB */ @@ -321,7 +297,6 @@ SYM_CODE_START(_start_kernel) call setup_vm #ifdef CONFIG_MMU la a0, early_pg_dir - XIP_FIXUP_OFFSET a0 call relocate_enable_mmu #endif /* CONFIG_MMU */ @@ -330,6 +305,19 @@ SYM_CODE_START(_start_kernel) la tp, init_task la sp, init_thread_union + THREAD_SIZE addi sp, sp, -PT_SIZE_ON_STACK +#if defined(CONFIG_RISCV_SBI) && defined(CONFIG_RISCV_USER_CFI) + li a7, SBI_EXT_FWFT + li a6, SBI_EXT_FWFT_SET + li a0, SBI_FWFT_SHADOW_STACK + li a1, 1 /* enable supervisor to access shadow stack access */ + li a2, SBI_FWFT_SET_FLAG_LOCK + ecall + beqz a0, 1f + la a1, riscv_nousercfi + li a0, CMDLINE_DISABLE_RISCV_USERCFI_BCFI + REG_S a0, (a1) +1: +#endif scs_load_current #ifdef CONFIG_KASAN @@ -347,9 +335,7 @@ SYM_CODE_START(_start_kernel) slli a3, a0, LGREG la a1, __cpu_spinwait_stack_pointer - XIP_FIXUP_OFFSET a1 la a2, __cpu_spinwait_task_pointer - XIP_FIXUP_OFFSET a2 add a1, a3, a1 add a2, a3, a2 diff --git a/arch/riscv/kernel/head.h b/arch/riscv/kernel/head.h index a556fdaafed9..05a04bef442b 100644 --- a/arch/riscv/kernel/head.h +++ b/arch/riscv/kernel/head.h @@ -11,9 +11,6 @@ extern atomic_t hart_lottery; asmlinkage void __init setup_vm(uintptr_t dtb_pa); -#ifdef CONFIG_XIP_KERNEL -asmlinkage void __init __copy_data(void); -#endif #ifdef CONFIG_RISCV_BOOT_SPINWAIT extern void *__cpu_spinwait_stack_pointer[]; diff --git a/arch/riscv/kernel/hibernate.c b/arch/riscv/kernel/hibernate.c index 671b686c0158..982843828adb 100644 --- a/arch/riscv/kernel/hibernate.c +++ b/arch/riscv/kernel/hibernate.c @@ -415,7 +415,7 @@ int hibernate_resume_nonboot_cpu_disable(void) static int __init riscv_hibernate_init(void) { - hibernate_cpu_context = kzalloc(sizeof(*hibernate_cpu_context), GFP_KERNEL); + hibernate_cpu_context = kzalloc_obj(*hibernate_cpu_context); if (WARN_ON(!hibernate_cpu_context)) return -ENOMEM; diff --git a/arch/riscv/kernel/image-vars.h b/arch/riscv/kernel/image-vars.h index 3df30dd1c458..7b44b94f1283 100644 --- a/arch/riscv/kernel/image-vars.h +++ b/arch/riscv/kernel/image-vars.h @@ -29,9 +29,18 @@ __efistub__end = _end; __efistub__edata = _edata; __efistub___init_text_end = __init_text_end; #if defined(CONFIG_EFI_EARLYCON) || defined(CONFIG_SYSFB) -__efistub_screen_info = screen_info; +__efistub_sysfb_primary_display = sysfb_primary_display; #endif +/* + * These double-word integer shifts are used by the library code, and + * the first two of them are required to link EFI stub. Note __ashrdi3() + * is not actually used by the stub but this may change in the future. + */ +PROVIDE(__efistub___lshrdi3 = __lshrdi3); +PROVIDE(__efistub___ashldi3 = __ashldi3); +PROVIDE(__efistub___ashrdi3 = __ashrdi3); + #endif #endif /* __RISCV_KERNEL_IMAGE_VARS_H */ diff --git a/arch/riscv/kernel/irq.c b/arch/riscv/kernel/irq.c index b6af20bc300f..017d42e778be 100644 --- a/arch/riscv/kernel/irq.c +++ b/arch/riscv/kernel/irq.c @@ -75,28 +75,34 @@ DECLARE_PER_CPU(ulong *, irq_shadow_call_stack_ptr); DEFINE_PER_CPU(ulong *, irq_shadow_call_stack_ptr); #endif -static void init_irq_scs(void) +static void __init init_irq_scs(void) { int cpu; + void *s; if (!scs_is_enabled()) return; - for_each_possible_cpu(cpu) - per_cpu(irq_shadow_call_stack_ptr, cpu) = - scs_alloc(cpu_to_node(cpu)); + for_each_possible_cpu(cpu) { + s = scs_alloc(cpu_to_node(cpu)); + if (!s) + panic("Failed to allocate IRQ shadow call stack resources\n"); + per_cpu(irq_shadow_call_stack_ptr, cpu) = s; + } } DEFINE_PER_CPU(ulong *, irq_stack_ptr); #ifdef CONFIG_VMAP_STACK -static void init_irq_stacks(void) +static void __init init_irq_stacks(void) { int cpu; ulong *p; for_each_possible_cpu(cpu) { p = arch_alloc_vmap_stack(IRQ_STACK_SIZE, cpu_to_node(cpu)); + if (!p) + panic("Failed to allocate IRQ stack resources\n"); per_cpu(irq_stack_ptr, cpu) = p; } } @@ -104,7 +110,7 @@ static void init_irq_stacks(void) /* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */ DEFINE_PER_CPU_ALIGNED(ulong [IRQ_STACK_SIZE/sizeof(ulong)], irq_stack); -static void init_irq_stacks(void) +static void __init init_irq_stacks(void) { int cpu; @@ -129,8 +135,8 @@ void do_softirq_own_stack(void) #endif /* CONFIG_SOFTIRQ_ON_OWN_STACK */ #else -static void init_irq_scs(void) {} -static void init_irq_stacks(void) {} +static void __init init_irq_scs(void) {} +static void __init init_irq_stacks(void) {} #endif /* CONFIG_IRQ_STACKS */ int arch_show_interrupts(struct seq_file *p, int prec) diff --git a/arch/riscv/kernel/kernel_mode_vector.c b/arch/riscv/kernel/kernel_mode_vector.c index 99972a48e86b..77e98b504485 100644 --- a/arch/riscv/kernel/kernel_mode_vector.c +++ b/arch/riscv/kernel/kernel_mode_vector.c @@ -10,18 +10,34 @@ #include <linux/percpu.h> #include <linux/preempt.h> #include <linux/types.h> +#include <linux/kvm_types.h> #include <asm/vector.h> +#include <asm/kvm_vcpu_vector.h> #include <asm/switch_to.h> #include <asm/simd.h> #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE #include <asm/asm-prototypes.h> #endif -static inline void riscv_v_flags_set(u32 flags) +static void (* __rcu kvm_flush_vector_ctx_callback)(void); + +void kvm_riscv_register_vctx_callback(void (*func)(void)) +{ + if (WARN_ON_ONCE(rcu_access_pointer(kvm_flush_vector_ctx_callback))) + return; + + rcu_assign_pointer(kvm_flush_vector_ctx_callback, func); +} +EXPORT_SYMBOL_GPL(kvm_riscv_register_vctx_callback); + +void kvm_riscv_unregister_vctx_callback(void) { - WRITE_ONCE(current->thread.riscv_v_flags, flags); + rcu_assign_pointer(kvm_flush_vector_ctx_callback, NULL); + synchronize_rcu(); } +EXPORT_SYMBOL_GPL(kvm_riscv_unregister_vctx_callback); + static inline void riscv_v_start(u32 flags) { @@ -55,13 +71,16 @@ void get_cpu_vector_context(void) * disable softirqs so it is impossible for softirqs to nest * get_cpu_vector_context() when kernel is actively using Vector. */ - if (!IS_ENABLED(CONFIG_PREEMPT_RT)) - local_bh_disable(); - else + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) { + if (!irqs_disabled()) + local_bh_disable(); + } else { preempt_disable(); + } riscv_v_start(RISCV_KERNEL_MODE_V); } +EXPORT_SYMBOL_FOR_KVM(get_cpu_vector_context); /* * Release the CPU vector context. @@ -74,10 +93,29 @@ void put_cpu_vector_context(void) { riscv_v_stop(RISCV_KERNEL_MODE_V); - if (!IS_ENABLED(CONFIG_PREEMPT_RT)) - local_bh_enable(); - else + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) { + if (!irqs_disabled()) + local_bh_enable(); + } else { preempt_enable(); + } +} +EXPORT_SYMBOL_FOR_KVM(put_cpu_vector_context); + +static void __riscv_flush_vector_context(void) +{ + void (*vcpu_flush_v_callback)(void); + + if (riscv_v_flags() & RISCV_V_VCPU_CTX) { + rcu_read_lock(); + vcpu_flush_v_callback = rcu_dereference(kvm_flush_vector_ctx_callback); + vcpu_flush_v_callback(); + rcu_read_unlock(); + return; + } + + riscv_v_vstate_save(¤t->thread.vstate, task_pt_regs(current)); + riscv_v_vstate_set_restore(current, task_pt_regs(current)); } #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE @@ -121,9 +159,9 @@ static int riscv_v_stop_kernel_context(void) return 0; } -static int riscv_v_start_kernel_context(bool *is_nested) +static int riscv_v_start_kernel_context(void) { - struct __riscv_v_ext_state *kvstate, *uvstate; + struct __riscv_v_ext_state *kvstate; kvstate = ¤t->thread.kernel_vstate; if (!kvstate->datap) @@ -131,7 +169,6 @@ static int riscv_v_start_kernel_context(bool *is_nested) if (riscv_preempt_v_started(current)) { WARN_ON(riscv_v_ctx_get_depth() == 0); - *is_nested = true; get_cpu_vector_context(); if (riscv_preempt_v_dirty(current)) { __riscv_v_vstate_save(kvstate, kvstate->datap); @@ -142,12 +179,18 @@ static int riscv_v_start_kernel_context(bool *is_nested) } /* Transfer the ownership of V from user to kernel, then save */ - riscv_v_start(RISCV_PREEMPT_V | RISCV_PREEMPT_V_DIRTY); - if (__riscv_v_vstate_check(task_pt_regs(current)->status, DIRTY)) { - uvstate = ¤t->thread.vstate; - __riscv_v_vstate_save(uvstate, uvstate->datap); - } - riscv_preempt_v_clear_dirty(current); + get_cpu_vector_context(); + __riscv_flush_vector_context(); + put_cpu_vector_context(); + /* + * A voluntary context switch caused by put_cpu_vector_context() can + * raise the NEED_RESTORE flag if preempt_v starts too early due to a + * failed risv_v_is_on() check. + * + * This causes the next context_nesting_end pollute the v-reg from + * the stale context memory in kernel-mode vector. + */ + riscv_v_start(RISCV_PREEMPT_V); return 0; } @@ -187,7 +230,7 @@ asmlinkage void riscv_v_context_nesting_end(struct pt_regs *regs) } } #else -#define riscv_v_start_kernel_context(nested) (-ENOENT) +#define riscv_v_start_kernel_context() (-ENOENT) #define riscv_v_stop_kernel_context() (-ENOENT) #endif /* CONFIG_RISCV_ISA_V_PREEMPTIVE */ @@ -206,21 +249,16 @@ asmlinkage void riscv_v_context_nesting_end(struct pt_regs *regs) */ void kernel_vector_begin(void) { - bool nested = false; - if (WARN_ON(!(has_vector() || has_xtheadvector()))) return; BUG_ON(!may_use_simd()); - if (riscv_v_start_kernel_context(&nested)) { + if (riscv_v_start_kernel_context()) { get_cpu_vector_context(); - riscv_v_vstate_save(¤t->thread.vstate, task_pt_regs(current)); + __riscv_flush_vector_context(); } - if (!nested) - riscv_v_vstate_set_restore(current, task_pt_regs(current)); - riscv_v_enable(); } EXPORT_SYMBOL_GPL(kernel_vector_begin); diff --git a/arch/riscv/kernel/kexec_elf.c b/arch/riscv/kernel/kexec_elf.c index 531d348db84d..3e9a32acb8f2 100644 --- a/arch/riscv/kernel/kexec_elf.c +++ b/arch/riscv/kernel/kexec_elf.c @@ -10,8 +10,6 @@ * for kernel. */ -#define pr_fmt(fmt) "kexec_image: " fmt - #include <linux/elf.h> #include <linux/kexec.h> #include <linux/slab.h> @@ -19,6 +17,7 @@ #include <linux/libfdt.h> #include <linux/types.h> #include <linux/memblock.h> +#include <linux/minmax.h> #include <asm/setup.h> static int riscv_kexec_elf_load(struct kimage *image, struct elfhdr *ehdr, @@ -27,7 +26,6 @@ static int riscv_kexec_elf_load(struct kimage *image, struct elfhdr *ehdr, { int i; int ret = 0; - size_t size; struct kexec_buf kbuf = {}; const struct elf_phdr *phdr; @@ -38,12 +36,8 @@ static int riscv_kexec_elf_load(struct kimage *image, struct elfhdr *ehdr, if (phdr->p_type != PT_LOAD) continue; - size = phdr->p_filesz; - if (size > phdr->p_memsz) - size = phdr->p_memsz; - kbuf.buffer = (void *) elf_info->buffer + phdr->p_offset; - kbuf.bufsz = size; + kbuf.bufsz = min(phdr->p_filesz, phdr->p_memsz); kbuf.buf_align = phdr->p_align; kbuf.mem = phdr->p_paddr - old_pbase + new_pbase; kbuf.memsz = phdr->p_memsz; diff --git a/arch/riscv/kernel/kgdb.c b/arch/riscv/kernel/kgdb.c index 15fec5d1e6de..0bf629204c76 100644 --- a/arch/riscv/kernel/kgdb.c +++ b/arch/riscv/kernel/kgdb.c @@ -175,7 +175,7 @@ struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] = { {DBG_REG_T1, GDB_SIZEOF_REG, offsetof(struct pt_regs, t1)}, {DBG_REG_T2, GDB_SIZEOF_REG, offsetof(struct pt_regs, t2)}, {DBG_REG_FP, GDB_SIZEOF_REG, offsetof(struct pt_regs, s0)}, - {DBG_REG_S1, GDB_SIZEOF_REG, offsetof(struct pt_regs, a1)}, + {DBG_REG_S1, GDB_SIZEOF_REG, offsetof(struct pt_regs, s1)}, {DBG_REG_A0, GDB_SIZEOF_REG, offsetof(struct pt_regs, a0)}, {DBG_REG_A1, GDB_SIZEOF_REG, offsetof(struct pt_regs, a1)}, {DBG_REG_A2, GDB_SIZEOF_REG, offsetof(struct pt_regs, a2)}, @@ -244,8 +244,9 @@ sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *task) gdb_regs[DBG_REG_S6_OFF] = task->thread.s[6]; gdb_regs[DBG_REG_S7_OFF] = task->thread.s[7]; gdb_regs[DBG_REG_S8_OFF] = task->thread.s[8]; - gdb_regs[DBG_REG_S9_OFF] = task->thread.s[10]; - gdb_regs[DBG_REG_S10_OFF] = task->thread.s[11]; + gdb_regs[DBG_REG_S9_OFF] = task->thread.s[9]; + gdb_regs[DBG_REG_S10_OFF] = task->thread.s[10]; + gdb_regs[DBG_REG_S11_OFF] = task->thread.s[11]; gdb_regs[DBG_REG_EPC_OFF] = task->thread.ra; } diff --git a/arch/riscv/kernel/machine_kexec.c b/arch/riscv/kernel/machine_kexec.c index 2306ce3e5f22..738df176ff6f 100644 --- a/arch/riscv/kernel/machine_kexec.c +++ b/arch/riscv/kernel/machine_kexec.c @@ -41,6 +41,9 @@ machine_kexec_prepare(struct kimage *image) if (image->segment[i].memsz <= sizeof(fdt)) continue; + if (!image->segment[i].buf) + continue; + if (image->file_mode) memcpy(&fdt, image->segment[i].buf, sizeof(fdt)); else if (copy_from_user(&fdt, image->segment[i].buf, sizeof(fdt))) diff --git a/arch/riscv/kernel/machine_kexec_file.c b/arch/riscv/kernel/machine_kexec_file.c index dd9d92a96517..26cd2a8bd0cd 100644 --- a/arch/riscv/kernel/machine_kexec_file.c +++ b/arch/riscv/kernel/machine_kexec_file.c @@ -13,6 +13,7 @@ #include <linux/libfdt.h> #include <linux/types.h> #include <linux/memblock.h> +#include <linux/pfn.h> #include <linux/vmalloc.h> #include <asm/setup.h> #include <asm/insn.h> @@ -44,6 +45,15 @@ static int get_nr_ram_ranges_callback(struct resource *res, void *arg) return 0; } +unsigned int arch_get_system_nr_ranges(void) +{ + unsigned int nr_ranges = 2 + crashk_cma_cnt; /* For exclusion of crashkernel region */ + + walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback); + + return nr_ranges; +} + static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg) { struct crash_mem *cmem = arg; @@ -55,33 +65,9 @@ static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg) return 0; } -static int prepare_elf_headers(void **addr, unsigned long *sz) +int arch_crash_populate_cmem(struct crash_mem *cmem) { - struct crash_mem *cmem; - unsigned int nr_ranges; - int ret; - - nr_ranges = 1; /* For exclusion of crashkernel region */ - walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback); - - cmem = kmalloc(struct_size(cmem, ranges, nr_ranges), GFP_KERNEL); - if (!cmem) - return -ENOMEM; - - cmem->max_nr_ranges = nr_ranges; - cmem->nr_ranges = 0; - ret = walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback); - if (ret) - goto out; - - /* Exclude crashkernel region */ - ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end); - if (!ret) - ret = crash_prepare_elf64_headers(cmem, true, addr, sz); - -out: - kfree(cmem); - return ret; + return walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback); } static char *setup_kdump_cmdline(struct kimage *image, char *cmdline, @@ -266,14 +252,14 @@ int load_extra_segments(struct kimage *image, unsigned long kernel_start, kbuf.image = image; kbuf.buf_min = kernel_start + kernel_len; - kbuf.buf_max = ULONG_MAX; + kbuf.buf_max = PFN_PHYS(max_low_pfn); #ifdef CONFIG_CRASH_DUMP /* Add elfcorehdr */ if (image->type == KEXEC_TYPE_CRASH) { void *headers; unsigned long headers_sz; - ret = prepare_elf_headers(&headers, &headers_sz); + ret = crash_prepare_headers(true, &headers, &headers_sz, NULL); if (ret) { pr_err("Preparing elf core header failed\n"); goto out; diff --git a/arch/riscv/kernel/mcount-dyn.S b/arch/riscv/kernel/mcount-dyn.S index 48f6c4f7dca0..082fe0b0e3c0 100644 --- a/arch/riscv/kernel/mcount-dyn.S +++ b/arch/riscv/kernel/mcount-dyn.S @@ -66,7 +66,7 @@ * 8(sp) stores the function return address (i.e. parent IP) that * can be accessed by &(fregs)->ra in tracing function. * -* The other regs are saved at the respective localtion and accessed +* The other regs are saved at the respective location and accessed * by the respective ftrace_regs member. * * Here is the layout of stack for your reference. diff --git a/arch/riscv/kernel/module-sections.c b/arch/riscv/kernel/module-sections.c index 1675cbad8619..b3b11b7f7ed9 100644 --- a/arch/riscv/kernel/module-sections.c +++ b/arch/riscv/kernel/module-sections.c @@ -56,17 +56,15 @@ unsigned long module_emit_plt_entry(struct module *mod, unsigned long val) return (unsigned long)&plt[i]; } -#define cmp_3way(a, b) ((a) < (b) ? -1 : (a) > (b)) - static int cmp_rela(const void *a, const void *b) { const Elf_Rela *x = a, *y = b; int i; /* sort by type, symbol index and addend */ - i = cmp_3way(x->r_info, y->r_info); + i = cmp_int(x->r_info, y->r_info); if (i == 0) - i = cmp_3way(x->r_addend, y->r_addend); + i = cmp_int(x->r_addend, y->r_addend); return i; } @@ -148,7 +146,7 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, return -ENOEXEC; } - /* Calculate the maxinum number of entries */ + /* Calculate the maximum number of entries */ for (i = 0; i < ehdr->e_shnum; i++) { size_t num_relas = sechdrs[i].sh_size / sizeof(Elf_Rela); Elf_Rela *relas = (void *)ehdr + sechdrs[i].sh_offset; diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c index 7f6147c18033..1961135689db 100644 --- a/arch/riscv/kernel/module.c +++ b/arch/riscv/kernel/module.c @@ -663,7 +663,7 @@ static int add_relocation_to_accumulate(struct module *me, int type, struct used_bucket *bucket; unsigned long hash; - entry = kmalloc(sizeof(*entry), GFP_KERNEL); + entry = kmalloc_obj(*entry); if (!entry) return -ENOMEM; @@ -697,7 +697,7 @@ static int add_relocation_to_accumulate(struct module *me, int type, * relocation_entry. */ if (!found) { - rel_head = kmalloc(sizeof(*rel_head), GFP_KERNEL); + rel_head = kmalloc_obj(*rel_head); if (!rel_head) { kfree(entry); @@ -709,7 +709,7 @@ static int add_relocation_to_accumulate(struct module *me, int type, INIT_HLIST_NODE(&rel_head->node); if (!current_head->first) { bucket = - kmalloc(sizeof(struct used_bucket), GFP_KERNEL); + kmalloc_obj(struct used_bucket); if (!bucket) { kfree(entry); @@ -753,9 +753,8 @@ initialize_relocation_hashtable(unsigned int num_relocations, hashtable_size <<= should_double_size; /* Number of relocations may be large, so kvmalloc it */ - *relocation_hashtable = kvmalloc_array(hashtable_size, - sizeof(**relocation_hashtable), - GFP_KERNEL); + *relocation_hashtable = kvmalloc_objs(**relocation_hashtable, + hashtable_size); if (!*relocation_hashtable) return 0; diff --git a/arch/riscv/kernel/paravirt.c b/arch/riscv/kernel/paravirt.c index fa6b0339a65d..5f56be79cd06 100644 --- a/arch/riscv/kernel/paravirt.c +++ b/arch/riscv/kernel/paravirt.c @@ -16,22 +16,13 @@ #include <linux/printk.h> #include <linux/static_call.h> #include <linux/types.h> +#include <linux/sched/cputime.h> #include <asm/barrier.h> #include <asm/page.h> #include <asm/paravirt.h> #include <asm/sbi.h> -struct static_key paravirt_steal_enabled; -struct static_key paravirt_steal_rq_enabled; - -static u64 native_steal_clock(int cpu) -{ - return 0; -} - -DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); - static bool steal_acc = true; static int __init parse_no_stealacc(char *arg) { diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c index db13c9ddf9e3..2239c28981bc 100644 --- a/arch/riscv/kernel/patch.c +++ b/arch/riscv/kernel/patch.c @@ -42,19 +42,22 @@ static inline bool is_kernel_exittext(uintptr_t addr) static __always_inline void *patch_map(void *addr, const unsigned int fixmap) { uintptr_t uintaddr = (uintptr_t) addr; - struct page *page; + phys_addr_t phys; - if (core_kernel_text(uintaddr) || is_kernel_exittext(uintaddr)) - page = phys_to_page(__pa_symbol(addr)); - else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) - page = vmalloc_to_page(addr); - else - return addr; + if (core_kernel_text(uintaddr) || is_kernel_exittext(uintaddr)) { + if (!IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) + return addr; + phys = __pa_symbol(addr); + } else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) { + struct page *page = vmalloc_to_page(addr); - BUG_ON(!page); + BUG_ON(!page); + phys = page_to_phys(page) + offset_in_page(addr); + } else { + return addr; + } - return (void *)set_fixmap_offset(fixmap, page_to_phys(page) + - offset_in_page(addr)); + return (void *)set_fixmap_offset(fixmap, phys); } static void patch_unmap(int fixmap) diff --git a/arch/riscv/kernel/pi/cmdline_early.c b/arch/riscv/kernel/pi/cmdline_early.c index 389d086a0718..0afbe4077cb8 100644 --- a/arch/riscv/kernel/pi/cmdline_early.c +++ b/arch/riscv/kernel/pi/cmdline_early.c @@ -12,8 +12,8 @@ static char early_cmdline[COMMAND_LINE_SIZE]; static char *get_early_cmdline(uintptr_t dtb_pa) { - const char *fdt_cmdline = NULL; - unsigned int fdt_cmdline_size = 0; + const char *fdt_cmdline; + ssize_t fdt_cmdline_size = 0; int chosen_node; if (!IS_ENABLED(CONFIG_CMDLINE_FORCE)) { @@ -22,18 +22,18 @@ static char *get_early_cmdline(uintptr_t dtb_pa) fdt_cmdline = fdt_getprop((void *)dtb_pa, chosen_node, "bootargs", NULL); if (fdt_cmdline) { - fdt_cmdline_size = strlen(fdt_cmdline); - strscpy(early_cmdline, fdt_cmdline, - COMMAND_LINE_SIZE); + fdt_cmdline_size = strscpy(early_cmdline, fdt_cmdline); + if (fdt_cmdline_size < 0) + return early_cmdline; } } } if (IS_ENABLED(CONFIG_CMDLINE_EXTEND) || IS_ENABLED(CONFIG_CMDLINE_FORCE) || - fdt_cmdline_size == 0 /* CONFIG_CMDLINE_FALLBACK */) { - strlcat(early_cmdline, CONFIG_CMDLINE, COMMAND_LINE_SIZE); - } + fdt_cmdline_size == 0 /* CONFIG_CMDLINE_FALLBACK */) + strscpy(early_cmdline + fdt_cmdline_size, CONFIG_CMDLINE, + COMMAND_LINE_SIZE - fdt_cmdline_size); return early_cmdline; } diff --git a/arch/riscv/kernel/probes/decode-insn.c b/arch/riscv/kernel/probes/decode-insn.c index 65d9590bfb9f..433d9035b888 100644 --- a/arch/riscv/kernel/probes/decode-insn.c +++ b/arch/riscv/kernel/probes/decode-insn.c @@ -29,12 +29,12 @@ riscv_probe_decode_insn(probe_opcode_t *addr, struct arch_probe_insn *api) * TODO: the REJECTED ones below need to be implemented */ #ifdef CONFIG_RISCV_ISA_C - RISCV_INSN_REJECTED(c_jal, insn); RISCV_INSN_REJECTED(c_ebreak, insn); RISCV_INSN_SET_SIMULATE(c_j, insn); RISCV_INSN_SET_SIMULATE(c_jr, insn); RISCV_INSN_SET_SIMULATE(c_jalr, insn); + RISCV_INSN_SET_SIMULATE(c_jal, insn); RISCV_INSN_SET_SIMULATE(c_beqz, insn); RISCV_INSN_SET_SIMULATE(c_bnez, insn); #endif diff --git a/arch/riscv/kernel/probes/kprobes.c b/arch/riscv/kernel/probes/kprobes.c index 8723390c7cad..9e2afabf94e8 100644 --- a/arch/riscv/kernel/probes/kprobes.c +++ b/arch/riscv/kernel/probes/kprobes.c @@ -149,7 +149,7 @@ static void __kprobes set_current_kprobe(struct kprobe *p) /* * Interrupts need to be disabled before single-step mode is set, and not - * reenabled until after single-step mode ends. + * re-enabled until after single-step mode ends. * Without disabling interrupt on local CPU, there is a chance of * interrupt occurrence in the period of exception return and start of * out-of-line single-step, that result in wrongly single stepping diff --git a/arch/riscv/kernel/probes/rethook_trampoline.S b/arch/riscv/kernel/probes/rethook_trampoline.S index f2cd83d9b0f0..c3aa8d8cf5af 100644 --- a/arch/riscv/kernel/probes/rethook_trampoline.S +++ b/arch/riscv/kernel/probes/rethook_trampoline.S @@ -41,6 +41,9 @@ REG_S x29, PT_T4(sp) REG_S x30, PT_T5(sp) REG_S x31, PT_T6(sp) + /* save original sp */ + addi a0, sp, PT_SIZE_ON_STACK + REG_S a0, PT_SP(sp) .endm .macro restore_all_base_regs diff --git a/arch/riscv/kernel/probes/simulate-insn.c b/arch/riscv/kernel/probes/simulate-insn.c index fa581590c1f8..f8a2f6857877 100644 --- a/arch/riscv/kernel/probes/simulate-insn.c +++ b/arch/riscv/kernel/probes/simulate-insn.c @@ -163,6 +163,13 @@ bool __kprobes simulate_c_j(u32 opcode, unsigned long addr, struct pt_regs *regs return true; } +bool __kprobes simulate_c_jal(u32 opcode, unsigned long addr, struct pt_regs *regs) +{ + regs->ra = addr + 2; + + return simulate_c_j(opcode, addr, regs); +} + static bool __kprobes simulate_c_jr_jalr(u32 opcode, unsigned long addr, struct pt_regs *regs, bool is_jalr) { diff --git a/arch/riscv/kernel/probes/simulate-insn.h b/arch/riscv/kernel/probes/simulate-insn.h index 44ebbc444db9..b89e1bb01842 100644 --- a/arch/riscv/kernel/probes/simulate-insn.h +++ b/arch/riscv/kernel/probes/simulate-insn.h @@ -25,6 +25,7 @@ bool simulate_branch(u32 opcode, unsigned long addr, struct pt_regs *regs); bool simulate_jal(u32 opcode, unsigned long addr, struct pt_regs *regs); bool simulate_jalr(u32 opcode, unsigned long addr, struct pt_regs *regs); bool simulate_c_j(u32 opcode, unsigned long addr, struct pt_regs *regs); +bool simulate_c_jal(u32 opcode, unsigned long addr, struct pt_regs *regs); bool simulate_c_jr(u32 opcode, unsigned long addr, struct pt_regs *regs); bool simulate_c_jalr(u32 opcode, unsigned long addr, struct pt_regs *regs); bool simulate_c_bnez(u32 opcode, unsigned long addr, struct pt_regs *regs); diff --git a/arch/riscv/kernel/probes/uprobes.c b/arch/riscv/kernel/probes/uprobes.c index cc15f7ca6cc1..eb177d0ce8ab 100644 --- a/arch/riscv/kernel/probes/uprobes.c +++ b/arch/riscv/kernel/probes/uprobes.c @@ -111,7 +111,7 @@ void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) current->thread.bad_cause = utask->autask.saved_cause; /* - * Task has received a fatal signal, so reset back to probbed + * Task has received a fatal signal, so reset back to probed * address. */ instruction_pointer_set(regs, utask->vaddr); @@ -165,7 +165,7 @@ void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, void *src, unsigned long len) { /* Initialize the slot */ - void *kaddr = kmap_atomic(page); + void *kaddr = kmap_local_page(page); void *dst = kaddr + (vaddr & ~PAGE_MASK); unsigned long start = (unsigned long)dst; @@ -178,5 +178,5 @@ void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, } flush_icache_range(start, start + len); - kunmap_atomic(kaddr); + kunmap_local(kaddr); } diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index 31a392993cb4..7cc5a6a5c020 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -31,6 +31,7 @@ #include <asm/vector.h> #include <asm/cpufeature.h> #include <asm/exec.h> +#include <asm/usercfi.h> #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK) #include <linux/stackprotector.h> @@ -92,8 +93,8 @@ void __show_regs(struct pt_regs *regs) regs->s8, regs->s9, regs->s10); pr_cont(" s11: " REG_FMT " t3 : " REG_FMT " t4 : " REG_FMT "\n", regs->s11, regs->t3, regs->t4); - pr_cont(" t5 : " REG_FMT " t6 : " REG_FMT "\n", - regs->t5, regs->t6); + pr_cont(" t5 : " REG_FMT " t6 : " REG_FMT " ssp : " REG_FMT "\n", + regs->t5, regs->t6, get_active_shstk(current)); pr_cont("status: " REG_FMT " badaddr: " REG_FMT " cause: " REG_FMT "\n", regs->status, regs->badaddr, regs->cause); @@ -155,6 +156,21 @@ void start_thread(struct pt_regs *regs, unsigned long pc, regs->epc = pc; regs->sp = sp; + /* + * clear shadow stack state on exec. + * libc will set it later via prctl. + */ + set_shstk_lock(current, false); + set_shstk_status(current, false); + set_shstk_base(current, 0, 0); + set_active_shstk(current, 0); + /* + * disable indirect branch tracking on exec. + * libc will enable it later via prctl. + */ + set_indir_lp_lock(current, false); + set_indir_lp_status(current, false); + #ifdef CONFIG_64BIT regs->status &= ~SR_UXL; @@ -226,6 +242,7 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) u64 clone_flags = args->flags; unsigned long usp = args->stack; unsigned long tls = args->tls; + unsigned long ssp = 0; struct pt_regs *childregs = task_pt_regs(p); /* Ensure all threads in this mm have the same pointer masking mode. */ @@ -245,11 +262,19 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) p->thread.s[1] = (unsigned long)args->fn_arg; p->thread.ra = (unsigned long)ret_from_fork_kernel_asm; } else { + /* allocate new shadow stack if needed. In case of CLONE_VM we have to */ + ssp = shstk_alloc_thread_stack(p, args); + if (IS_ERR_VALUE(ssp)) + return PTR_ERR((void *)ssp); + *childregs = *(current_pt_regs()); /* Turn off status.VS */ riscv_v_vstate_off(childregs); if (usp) /* User fork */ childregs->sp = usp; + /* if needed, set new ssp */ + if (ssp) + set_active_shstk(p, ssp); if (clone_flags & CLONE_SETTLS) childregs->tp = tls; childregs->a0 = 0; /* Return value of fork() */ diff --git a/arch/riscv/kernel/ptrace.c b/arch/riscv/kernel/ptrace.c index e6272d74572f..f336a183667e 100644 --- a/arch/riscv/kernel/ptrace.c +++ b/arch/riscv/kernel/ptrace.c @@ -19,6 +19,7 @@ #include <linux/regset.h> #include <linux/sched.h> #include <linux/sched/task_stack.h> +#include <asm/usercfi.h> enum riscv_regset { REGSET_X, @@ -31,6 +32,9 @@ enum riscv_regset { #ifdef CONFIG_RISCV_ISA_SUPM REGSET_TAGGED_ADDR_CTRL, #endif +#ifdef CONFIG_RISCV_USER_CFI + REGSET_CFI, +#endif }; static int riscv_gpr_get(struct task_struct *target, @@ -95,9 +99,12 @@ static int riscv_vr_get(struct task_struct *target, struct __riscv_v_ext_state *vstate = &target->thread.vstate; struct __riscv_v_regset_state ptrace_vstate; - if (!riscv_v_vstate_query(task_pt_regs(target))) + if (!(has_vector() || has_xtheadvector())) return -EINVAL; + if (!riscv_v_vstate_query(task_pt_regs(target))) + return -ENODATA; + /* * Ensure the vector registers have been saved to the memory before * copying them to membuf. @@ -121,6 +128,92 @@ static int riscv_vr_get(struct task_struct *target, return membuf_write(&to, vstate->datap, riscv_v_vsize); } +static int invalid_ptrace_v_csr(struct __riscv_v_ext_state *vstate, + struct __riscv_v_regset_state *ptrace) +{ + unsigned long vsew, vlmul, vfrac, vl; + unsigned long elen, vlen; + unsigned long sew, lmul; + unsigned long reserved; + + vlen = vstate->vlenb * 8; + if (vstate->vlenb != ptrace->vlenb) + return 1; + + /* do not allow to set vcsr/vxrm/vxsat reserved bits */ + reserved = ~(CSR_VXSAT_MASK | (CSR_VXRM_MASK << CSR_VXRM_SHIFT)); + if (ptrace->vcsr & reserved) + return 1; + + if (has_vector()) { + /* do not allow to set vtype reserved bits and vill bit */ + reserved = ~(VTYPE_VSEW | VTYPE_VLMUL | VTYPE_VMA | VTYPE_VTA); + if (ptrace->vtype & reserved) + return 1; + + elen = riscv_has_extension_unlikely(RISCV_ISA_EXT_ZVE64X) ? 64 : 32; + vsew = (ptrace->vtype & VTYPE_VSEW) >> VTYPE_VSEW_SHIFT; + sew = 8 << vsew; + + if (sew > elen) + return 1; + + vfrac = (ptrace->vtype & VTYPE_VLMUL_FRAC); + vlmul = (ptrace->vtype & VTYPE_VLMUL); + + /* RVV 1.0 spec 3.4.2: VLMUL(0x4) reserved */ + if (vlmul == 4) + return 1; + + /* RVV 1.0 spec 3.4.2: (LMUL < SEW_min / ELEN) reserved */ + if (vlmul == 5 && elen == 32) + return 1; + + /* for zero vl verify that at least one element is possible */ + vl = ptrace->vl ? ptrace->vl : 1; + + if (vfrac) { + /* integer 1/LMUL: VL =< VLMAX = VLEN / SEW / LMUL */ + lmul = 2 << (3 - (vlmul - vfrac)); + if (vlen < vl * sew * lmul) + return 1; + } else { + /* integer LMUL: VL =< VLMAX = LMUL * VLEN / SEW */ + lmul = 1 << vlmul; + if (vl * sew > lmul * vlen) + return 1; + } + } + + if (has_xtheadvector()) { + /* do not allow to set vtype reserved bits and vill bit */ + reserved = ~(VTYPE_VSEW_THEAD | VTYPE_VLMUL_THEAD | VTYPE_VEDIV_THEAD); + if (ptrace->vtype & reserved) + return 1; + + /* + * THead ISA Extension spec chapter 16: + * divided element extension ('Zvediv') is not part of XTheadVector + */ + if (ptrace->vtype & VTYPE_VEDIV_THEAD) + return 1; + + vsew = (ptrace->vtype & VTYPE_VSEW_THEAD) >> VTYPE_VSEW_THEAD_SHIFT; + sew = 8 << vsew; + + vlmul = (ptrace->vtype & VTYPE_VLMUL_THEAD); + lmul = 1 << vlmul; + + /* for zero vl verify that at least one element is possible */ + vl = ptrace->vl ? ptrace->vl : 1; + + if (vl * sew > lmul * vlen) + return 1; + } + + return 0; +} + static int riscv_vr_set(struct task_struct *target, const struct user_regset *regset, unsigned int pos, unsigned int count, @@ -130,16 +223,19 @@ static int riscv_vr_set(struct task_struct *target, struct __riscv_v_ext_state *vstate = &target->thread.vstate; struct __riscv_v_regset_state ptrace_vstate; - if (!riscv_v_vstate_query(task_pt_regs(target))) + if (!(has_vector() || has_xtheadvector())) return -EINVAL; + if (!riscv_v_vstate_query(task_pt_regs(target))) + return -ENODATA; + /* Copy rest of the vstate except datap */ ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ptrace_vstate, 0, sizeof(struct __riscv_v_regset_state)); if (unlikely(ret)) return ret; - if (vstate->vlenb != ptrace_vstate.vlenb) + if (invalid_ptrace_v_csr(vstate, &ptrace_vstate)) return -EINVAL; vstate->vstart = ptrace_vstate.vstart; @@ -195,6 +291,87 @@ static int tagged_addr_ctrl_set(struct task_struct *target, } #endif +#ifdef CONFIG_RISCV_USER_CFI +static int riscv_cfi_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + struct user_cfi_state user_cfi; + struct pt_regs *regs; + + memset(&user_cfi, 0, sizeof(user_cfi)); + regs = task_pt_regs(target); + + if (is_indir_lp_enabled(target)) { + user_cfi.cfi_status.cfi_state |= PTRACE_CFI_BRANCH_LANDING_PAD_EN_STATE; + user_cfi.cfi_status.cfi_state |= is_indir_lp_locked(target) ? + PTRACE_CFI_BRANCH_LANDING_PAD_LOCK_STATE : 0; + user_cfi.cfi_status.cfi_state |= (regs->status & SR_ELP) ? + PTRACE_CFI_BRANCH_EXPECTED_LANDING_PAD_STATE : 0; + } + + if (is_shstk_enabled(target)) { + user_cfi.cfi_status.cfi_state |= (PTRACE_CFI_SHADOW_STACK_EN_STATE | + PTRACE_CFI_SHADOW_STACK_PTR_STATE); + user_cfi.cfi_status.cfi_state |= is_shstk_locked(target) ? + PTRACE_CFI_SHADOW_STACK_LOCK_STATE : 0; + user_cfi.shstk_ptr = get_active_shstk(target); + } + + return membuf_write(&to, &user_cfi, sizeof(user_cfi)); +} + +/* + * Does it make sense to allow enable / disable of cfi via ptrace? + * We don't allow enable / disable / locking control via ptrace for now. + * Setting the shadow stack pointer is allowed. GDB might use it to unwind or + * some other fixup. Similarly gdb might want to suppress elp and may want + * to reset elp state. + */ +static int riscv_cfi_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + int ret; + struct user_cfi_state user_cfi; + struct pt_regs *regs; + + regs = task_pt_regs(target); + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &user_cfi, 0, -1); + if (ret) + return ret; + + /* + * Not allowing enabling or locking shadow stack or landing pad + * There is no disabling of shadow stack or landing pad via ptrace + * rsvd field should be set to zero so that if those fields are needed in future + */ + if ((user_cfi.cfi_status.cfi_state & + (PTRACE_CFI_BRANCH_LANDING_PAD_EN_STATE | PTRACE_CFI_BRANCH_LANDING_PAD_LOCK_STATE | + PTRACE_CFI_SHADOW_STACK_EN_STATE | PTRACE_CFI_SHADOW_STACK_LOCK_STATE)) || + (user_cfi.cfi_status.cfi_state & PTRACE_CFI_STATE_INVALID_MASK)) + return -EINVAL; + + /* If lpad is enabled on target and ptrace requests to set / clear elp, do that */ + if (is_indir_lp_enabled(target)) { + if (user_cfi.cfi_status.cfi_state & + PTRACE_CFI_BRANCH_EXPECTED_LANDING_PAD_STATE) /* set elp state */ + regs->status |= SR_ELP; + else + regs->status &= ~SR_ELP; /* clear elp state */ + } + + /* If shadow stack enabled on target, set new shadow stack pointer */ + if (is_shstk_enabled(target) && + (user_cfi.cfi_status.cfi_state & PTRACE_CFI_SHADOW_STACK_PTR_STATE)) + set_active_shstk(target, user_cfi.shstk_ptr); + + return 0; +} +#endif + static struct user_regset riscv_user_regset[] __ro_after_init = { [REGSET_X] = { USER_REGSET_NOTE_TYPE(PRSTATUS), @@ -234,6 +411,16 @@ static struct user_regset riscv_user_regset[] __ro_after_init = { .set = tagged_addr_ctrl_set, }, #endif +#ifdef CONFIG_RISCV_USER_CFI + [REGSET_CFI] = { + USER_REGSET_NOTE_TYPE(RISCV_USER_CFI), + .align = sizeof(__u64), + .n = sizeof(struct user_cfi_state) / sizeof(__u64), + .size = sizeof(__u64), + .regset_get = riscv_cfi_get, + .set = riscv_cfi_set, + }, +#endif }; static const struct user_regset_view riscv_user_native_view = { @@ -390,8 +577,8 @@ static int compat_riscv_gpr_set(struct task_struct *target, struct compat_user_regs_struct cregs; ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &cregs, 0, -1); - - cregs_to_regs(&cregs, task_pt_regs(target)); + if (!ret) + cregs_to_regs(&cregs, task_pt_regs(target)); return ret; } diff --git a/arch/riscv/kernel/qos.c b/arch/riscv/kernel/qos.c new file mode 100644 index 000000000000..99925c0e93dd --- /dev/null +++ b/arch/riscv/kernel/qos.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include <linux/cpu.h> +#include <linux/cpu_pm.h> +#include <linux/cpuhotplug.h> +#include <linux/notifier.h> +#include <linux/percpu-defs.h> +#include <linux/types.h> + +#include <asm/cpufeature-macros.h> +#include <asm/hwcap.h> +#include <asm/qos.h> + +/* + * Cached value of srmcfg csr for each cpu. Seeded to U32_MAX so the next + * __switch_to_srmcfg() unconditionally writes the CSR. The encoding + * MCID << 16 | RCID with both fields well under 16 bits can never + * produce this sentinel. This covers early-boot context switches that + * happen before riscv_srmcfg_init() runs as an arch_initcall. + */ +DEFINE_PER_CPU(u32, cpu_srmcfg) = U32_MAX; + +/* default srmcfg value for each cpu, set via resctrl cpu assignment */ +DEFINE_PER_CPU(u32, cpu_srmcfg_default); + +/* + * Invalidate the per-CPU srmcfg cache. Used as both the cpuhp startup + * and teardown callback. U32_MAX is not a valid srmcfg value + * (MCID << 16 | RCID, both fields under 16 bits), so the next + * __switch_to_srmcfg() always writes the CSR. + * + * Ssqosid leaves the CSR implementation-defined across hart stop/start, + * so the cached value cannot be trusted after online. The startup + * callback runs at CPUHP_AP_ONLINE_DYN, before CPUHP_AP_ACTIVE makes + * the CPU schedulable, so the cache is invalidated before any normal + * task runs and the CSR is written on that task's first switch. + * The teardown callback is not relied on. Idle and per-CPU kthreads keep + * switching as the CPU goes down and overwrite the sentinel with the CPU + * default, so it does not survive the offline period. + */ +static int riscv_srmcfg_reset_cache(unsigned int cpu) +{ + per_cpu(cpu_srmcfg, cpu) = U32_MAX; + return 0; +} + +/* + * CPU PM notifier: invalidate the cached srmcfg on resume from a deep + * idle / suspend. Ssqosid leaves CSR_SRMCFG state across low-power + * transitions implementation-defined, and the boot CPU never goes + * through the cpuhp online callback during system suspend, so without + * this hook __switch_to_srmcfg() would skip the CSR write when the + * outgoing task happens to share its srmcfg with the pre-suspend cache. + */ +static int riscv_srmcfg_pm_notify(struct notifier_block *nb, + unsigned long action, void *unused) +{ + switch (action) { + case CPU_PM_EXIT: + case CPU_PM_ENTER_FAILED: + /* + * The CSR is implementation-defined across the low-power + * transition. Invalidate the cache and eagerly rewrite the + * CSR for the current task so it does not run mis-tagged + * until the next context switch. + */ + __this_cpu_write(cpu_srmcfg, U32_MAX); + __switch_to_srmcfg(current); + break; + } + return NOTIFY_OK; +} + +static struct notifier_block riscv_srmcfg_pm_nb = { + .notifier_call = riscv_srmcfg_pm_notify, +}; + +static int __init riscv_srmcfg_init(void) +{ + int err; + + if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_SSQOSID)) + return 0; + + /* + * cpuhp_setup_state() invokes the startup callback locally on every + * already-online CPU, so no separate seed loop is needed here. + */ + err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "riscv/srmcfg:online", + riscv_srmcfg_reset_cache, riscv_srmcfg_reset_cache); + if (err < 0) + pr_warn("srmcfg: cpuhp setup failed (%d), cache not invalidated on CPU online\n", + err); + + /* + * Register the PM notifier even if the cpuhp setup failed. It is + * independent of the cpuhp state and guards suspend/resume. + */ + cpu_pm_register_notifier(&riscv_srmcfg_pm_nb); + return 0; +} +arch_initcall(riscv_srmcfg_init); diff --git a/arch/riscv/kernel/reset.c b/arch/riscv/kernel/reset.c index 912288572226..14eb08a6db85 100644 --- a/arch/riscv/kernel/reset.c +++ b/arch/riscv/kernel/reset.c @@ -3,10 +3,11 @@ * Copyright (C) 2012 Regents of the University of California */ +#include <linux/efi.h> #include <linux/reboot.h> #include <linux/pm.h> -static void default_power_off(void) +static void __noreturn default_power_off(void) { while (1) wait_for_interrupt(); @@ -17,6 +18,12 @@ EXPORT_SYMBOL(pm_power_off); void machine_restart(char *cmd) { + /* + * UpdateCapsule() depends on the system being reset via ResetSystem(). + */ + if (efi_enabled(EFI_RUNTIME_SERVICES)) + efi_reboot(reboot_mode, NULL); + do_kernel_restart(cmd); while (1); } diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index b5bc5fc65cea..a32344bb220d 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -46,11 +46,7 @@ * This is used before the kernel initializes the BSS so it can't be in the * BSS. */ -atomic_t hart_lottery __section(".sdata") -#ifdef CONFIG_XIP_KERNEL -= ATOMIC_INIT(0xC001BEEF) -#endif -; +atomic_t hart_lottery __section(".sdata"); unsigned long boot_cpu_hartid; EXPORT_SYMBOL_GPL(boot_cpu_hartid); @@ -75,16 +71,13 @@ static struct resource *standard_resources; static int __init add_resource(struct resource *parent, struct resource *res) { - int ret = 0; + int ret; ret = insert_resource(parent, res); - if (ret < 0) { - pr_err("Failed to add a %s resource at %llx\n", - res->name, (unsigned long long) res->start); - return ret; - } + if (ret < 0) + pr_err("Failed to add resource %s %pR\n", res->name, res); - return 1; + return ret; } static int __init add_kernel_resources(void) @@ -328,6 +321,8 @@ void __init setup_arch(char **cmdline_p) efi_init(); paging_init(); + acpi_table_upgrade(); + /* Parse the ACPI tables for possible boot-time configuration */ acpi_boot_table_init(); diff --git a/arch/riscv/kernel/signal.c b/arch/riscv/kernel/signal.c index 5a956108b1ea..59784dc117e4 100644 --- a/arch/riscv/kernel/signal.c +++ b/arch/riscv/kernel/signal.c @@ -22,11 +22,13 @@ #include <asm/vector.h> #include <asm/csr.h> #include <asm/cacheflush.h> +#include <asm/usercfi.h> unsigned long signal_minsigstksz __ro_after_init; extern u32 __user_rt_sigreturn[2]; static size_t riscv_v_sc_size __ro_after_init; +static size_t riscv_zicfiss_sc_size __ro_after_init; #define DEBUG_SIG 0 @@ -140,19 +142,79 @@ static long __restore_v_state(struct pt_regs *regs, void __user *sc_vec) return copy_from_user(current->thread.vstate.datap, datap, riscv_v_vsize); } +static long save_cfiss_state(struct pt_regs *regs, void __user *sc_cfi) +{ + struct __sc_riscv_cfi_state __user *state = sc_cfi; + unsigned long ss_ptr = 0; + long err = 0; + + if (!is_shstk_enabled(current)) + return 0; + + /* + * Save a pointer to the shadow stack itself on shadow stack as a form of token. + * A token on the shadow stack gives the following properties: + * - Safe save and restore for shadow stack switching. Any save of a shadow stack + * must have saved a token on the shadow stack. Similarly any restore of shadow + * stack must check the token before restore. Since writing to the shadow stack with + * address of the shadow stack itself is not easily allowed, a restore without a save + * is quite difficult for an attacker to perform. + * - A natural break. A token in shadow stack provides a natural break in shadow stack + * So a single linear range can be bucketed into different shadow stack segments. Any + * sspopchk will detect the condition and fault to kernel as a sw check exception. + */ + err |= save_user_shstk(current, &ss_ptr); + err |= __put_user(ss_ptr, &state->ss_ptr); + if (unlikely(err)) + return -EFAULT; + + return riscv_zicfiss_sc_size; +} + +static long __restore_cfiss_state(struct pt_regs *regs, void __user *sc_cfi) +{ + struct __sc_riscv_cfi_state __user *state = sc_cfi; + unsigned long ss_ptr = 0; + long err; + + /* + * Restore shadow stack as a form of token stored on the shadow stack itself as a safe + * way to restore. + * A token on the shadow stack gives the following properties: + * - Safe save and restore for shadow stack switching. Any save of shadow stack + * must have saved a token on shadow stack. Similarly any restore of shadow + * stack must check the token before restore. Since writing to a shadow stack with + * the address of shadow stack itself is not easily allowed, a restore without a save + * is quite difficult for an attacker to perform. + * - A natural break. A token in the shadow stack provides a natural break in shadow stack + * So a single linear range can be bucketed into different shadow stack segments. + * sspopchk will detect the condition and fault to kernel as a sw check exception. + */ + err = __copy_from_user(&ss_ptr, &state->ss_ptr, sizeof(unsigned long)); + + if (unlikely(err)) + return err; + + return restore_user_shstk(current, ss_ptr); +} + struct arch_ext_priv { __u32 magic; long (*save)(struct pt_regs *regs, void __user *sc_vec); }; -struct arch_ext_priv arch_ext_list[] = { +static struct arch_ext_priv arch_ext_list[] = { { .magic = RISCV_V_MAGIC, .save = &save_v_state, }, + { + .magic = RISCV_ZICFISS_MAGIC, + .save = &save_cfiss_state, + }, }; -const size_t nr_arch_exts = ARRAY_SIZE(arch_ext_list); +static const size_t nr_arch_exts = ARRAY_SIZE(arch_ext_list); static long restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc) @@ -202,6 +264,12 @@ static long restore_sigcontext(struct pt_regs *regs, err = __restore_v_state(regs, sc_ext_ptr); break; + case RISCV_ZICFISS_MAGIC: + if (!is_shstk_enabled(current) || size != riscv_zicfiss_sc_size) + return -EINVAL; + + err = __restore_cfiss_state(regs, sc_ext_ptr); + break; default: return -EINVAL; } @@ -223,6 +291,16 @@ static size_t get_rt_frame_size(bool cal_all) total_context_size += riscv_v_sc_size; } + if (is_shstk_enabled(current)) + total_context_size += riscv_zicfiss_sc_size; + + /* + * Preserved a __riscv_ctx_hdr for END signal context header if an + * extension uses __riscv_extra_ext_header + */ + if (total_context_size) + total_context_size += sizeof(struct __riscv_ctx_hdr); + frame_size += total_context_size; frame_size = round_up(frame_size, 16); @@ -297,7 +375,7 @@ static long setup_sigcontext(struct rt_sigframe __user *frame, } else { err |= __put_user(arch_ext->magic, &sc_ext_ptr->magic); err |= __put_user(ext_size, &sc_ext_ptr->size); - sc_ext_ptr = (void *)sc_ext_ptr + ext_size; + sc_ext_ptr = (void __user *)sc_ext_ptr + ext_size; } } /* Write zero to fp-reserved space and check it on restore_sigcontext */ @@ -359,6 +437,11 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, #ifdef CONFIG_MMU regs->ra = (unsigned long)VDSO_SYMBOL( current->mm->context.vdso, rt_sigreturn); + + /* if bcfi is enabled x1 (ra) and x5 (t0) must match. not sure if we need this? */ + if (is_shstk_enabled(current)) + regs->t0 = regs->ra; + #else /* * For the nommu case we don't have a VDSO. Instead we push two @@ -487,6 +570,9 @@ void __init init_rt_signal_env(void) { riscv_v_sc_size = sizeof(struct __riscv_ctx_hdr) + sizeof(struct __sc_riscv_v_state) + riscv_v_vsize; + + riscv_zicfiss_sc_size = sizeof(struct __riscv_ctx_hdr) + + sizeof(struct __sc_riscv_cfi_state); /* * Determine the stack space required for guaranteed signal delivery. * The signal_minsigstksz will be populated into the AT_MINSIGSTKSZ entry diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index 5ed5095320e6..fa66f9c97d74 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -226,8 +226,7 @@ void show_ipi_stats(struct seq_file *p, int prec) unsigned int cpu, i; for (i = 0; i < IPI_MAX; i++) { - seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, - prec >= 4 ? " " : ""); + seq_printf(p, "%*s%u:", prec - 1, "IPI", i); for_each_online_cpu(cpu) seq_printf(p, "%10u ", irq_desc_kstat_cpu(ipi_desc[i], cpu)); seq_printf(p, " %s\n", ipi_names[i]); diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index d85916a3660c..f6ef57930b50 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -189,13 +189,12 @@ int arch_cpuhp_kick_ap_alive(unsigned int cpu, struct task_struct *tidle) #else int __cpu_up(unsigned int cpu, struct task_struct *tidle) { - int ret = 0; + int ret; tidle->thread_info.cpu = cpu; ret = start_secondary_cpu(cpu, tidle); if (!ret) { - wait_for_completion_timeout(&cpu_running, - msecs_to_jiffies(1000)); + wait_for_completion_timeout(&cpu_running, secs_to_jiffies(1)); if (!cpu_online(cpu)) { pr_crit("CPU%u: failed to come online\n", cpu); @@ -251,18 +250,14 @@ asmlinkage __visible void smp_callin(void) set_cpu_online(curr_cpuid, true); /* - * Remote cache and TLB flushes are ignored while the CPU is offline, - * so flush them both right now just in case. + * Remote instruction cache and TLB flushes are ignored while the CPU + * is offline, so flush them both right now just in case. */ local_flush_icache_all(); local_flush_tlb_all(); #ifndef CONFIG_HOTPLUG_PARALLEL complete(&cpu_running); #endif - /* - * Disable preemption before enabling interrupts, so we don't try to - * schedule a CPU that hasn't actually started yet. - */ local_irq_enable(); cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); } diff --git a/arch/riscv/kernel/soc.c b/arch/riscv/kernel/soc.c index a0516172a33c..7d69e3b6bab4 100644 --- a/arch/riscv/kernel/soc.c +++ b/arch/riscv/kernel/soc.c @@ -8,7 +8,7 @@ #include <asm/soc.h> /* - * This is called extremly early, before parse_dtb(), to allow initializing + * This is called extremely early, before parse_dtb(), to allow initializing * SoC hardware before memory or any device driver initialization. */ void __init soc_early_init(void) diff --git a/arch/riscv/kernel/stacktrace.c b/arch/riscv/kernel/stacktrace.c index b41b6255751c..c7555447149b 100644 --- a/arch/riscv/kernel/stacktrace.c +++ b/arch/riscv/kernel/stacktrace.c @@ -87,7 +87,7 @@ void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, } else { fp = READ_ONCE_TASK_STACK(task, frame->fp); pc = READ_ONCE_TASK_STACK(task, frame->ra); - pc = ftrace_graph_ret_addr(current, &graph_idx, pc, + pc = ftrace_graph_ret_addr(task, &graph_idx, pc, &frame->ra); if (pc >= (unsigned long)handle_exception && pc < (unsigned long)&ret_from_exception_end) { @@ -129,7 +129,7 @@ void notrace walk_stackframe(struct task_struct *task, while (!kstack_end(ksp)) { if (__kernel_text_address(pc) && unlikely(!fn(arg, pc))) break; - pc = READ_ONCE_NOCHECK(*ksp++) - 0x4; + pc = READ_ONCE_NOCHECK(*ksp++); } } diff --git a/arch/riscv/kernel/suspend.c b/arch/riscv/kernel/suspend.c index aff93090c4ef..3efbf7874f3b 100644 --- a/arch/riscv/kernel/suspend.c +++ b/arch/riscv/kernel/suspend.c @@ -78,7 +78,7 @@ int cpu_suspend(unsigned long arg, suspend_save_csrs(&context); /* - * Function graph tracer state gets incosistent when the kernel + * Function graph tracer state gets inconsistent when the kernel * calls functions that never return (aka finishers) hence disable * graph tracing during their execution. */ diff --git a/arch/riscv/kernel/suspend_entry.S b/arch/riscv/kernel/suspend_entry.S index 2d54f309c140..d71b55fd6259 100644 --- a/arch/riscv/kernel/suspend_entry.S +++ b/arch/riscv/kernel/suspend_entry.S @@ -10,7 +10,6 @@ #include <asm/asm-offsets.h> #include <asm/assembler.h> #include <asm/csr.h> -#include <asm/xip_fixup.h> .text .altmacro @@ -70,7 +69,6 @@ SYM_TYPED_FUNC_START(__cpu_resume_enter) /* Enable MMU */ la a0, swapper_pg_dir - XIP_FIXUP_OFFSET a0 call relocate_enable_mmu /* Restore A0 and A1 */ diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c index e6787ba7f2fc..7818e1d32622 100644 --- a/arch/riscv/kernel/sys_hwprobe.c +++ b/arch/riscv/kernel/sys_hwprobe.c @@ -5,9 +5,6 @@ * more details. */ #include <linux/syscalls.h> -#include <linux/completion.h> -#include <linux/atomic.h> -#include <linux/once.h> #include <asm/cacheflush.h> #include <asm/cpufeature.h> #include <asm/hwprobe.h> @@ -24,6 +21,14 @@ #include <vdso/vsyscall.h> +#define EXT_KEY(isa_arg, ext, pv, missing) \ + do { \ + if (__riscv_isa_extension_available(isa_arg, RISCV_ISA_EXT_##ext)) \ + pv |= RISCV_HWPROBE_EXT_##ext; \ + else \ + missing |= RISCV_HWPROBE_EXT_##ext; \ + } while (false) + static void hwprobe_arch_id(struct riscv_hwprobe *pair, const struct cpumask *cpus) { @@ -80,10 +85,10 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, if (has_fpu()) pair->value |= RISCV_HWPROBE_IMA_FD; - if (riscv_isa_extension_available(NULL, c)) + if (riscv_isa_extension_available(NULL, C)) pair->value |= RISCV_HWPROBE_IMA_C; - if (has_vector() && riscv_isa_extension_available(NULL, v)) + if (has_vector() && riscv_isa_extension_available(NULL, V)) pair->value |= RISCV_HWPROBE_IMA_V; /* @@ -93,90 +98,115 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, for_each_cpu(cpu, cpus) { struct riscv_isainfo *isainfo = &hart_isa[cpu]; -#define EXT_KEY(ext) \ - do { \ - if (__riscv_isa_extension_available(isainfo->isa, RISCV_ISA_EXT_##ext)) \ - pair->value |= RISCV_HWPROBE_EXT_##ext; \ - else \ - missing |= RISCV_HWPROBE_EXT_##ext; \ - } while (false) - /* * Only use EXT_KEY() for extensions which can be exposed to userspace, * regardless of the kernel's configuration, as no other checks, besides * presence in the hart_isa bitmap, are made. */ - EXT_KEY(ZAAMO); - EXT_KEY(ZABHA); - EXT_KEY(ZACAS); - EXT_KEY(ZALASR); - EXT_KEY(ZALRSC); - EXT_KEY(ZAWRS); - EXT_KEY(ZBA); - EXT_KEY(ZBB); - EXT_KEY(ZBC); - EXT_KEY(ZBKB); - EXT_KEY(ZBKC); - EXT_KEY(ZBKX); - EXT_KEY(ZBS); - EXT_KEY(ZCA); - EXT_KEY(ZCB); - EXT_KEY(ZCLSD); - EXT_KEY(ZCMOP); - EXT_KEY(ZICBOM); - EXT_KEY(ZICBOP); - EXT_KEY(ZICBOZ); - EXT_KEY(ZICNTR); - EXT_KEY(ZICOND); - EXT_KEY(ZIHINTNTL); - EXT_KEY(ZIHINTPAUSE); - EXT_KEY(ZIHPM); - EXT_KEY(ZILSD); - EXT_KEY(ZIMOP); - EXT_KEY(ZKND); - EXT_KEY(ZKNE); - EXT_KEY(ZKNH); - EXT_KEY(ZKSED); - EXT_KEY(ZKSH); - EXT_KEY(ZKT); - EXT_KEY(ZTSO); + EXT_KEY(isainfo->isa, ZAAMO, pair->value, missing); + EXT_KEY(isainfo->isa, ZABHA, pair->value, missing); + EXT_KEY(isainfo->isa, ZACAS, pair->value, missing); + EXT_KEY(isainfo->isa, ZALASR, pair->value, missing); + EXT_KEY(isainfo->isa, ZALRSC, pair->value, missing); + EXT_KEY(isainfo->isa, ZAWRS, pair->value, missing); + EXT_KEY(isainfo->isa, ZBA, pair->value, missing); + EXT_KEY(isainfo->isa, ZBB, pair->value, missing); + EXT_KEY(isainfo->isa, ZBC, pair->value, missing); + EXT_KEY(isainfo->isa, ZBKB, pair->value, missing); + EXT_KEY(isainfo->isa, ZBKC, pair->value, missing); + EXT_KEY(isainfo->isa, ZBKX, pair->value, missing); + EXT_KEY(isainfo->isa, ZBS, pair->value, missing); + EXT_KEY(isainfo->isa, ZCA, pair->value, missing); + EXT_KEY(isainfo->isa, ZCB, pair->value, missing); + EXT_KEY(isainfo->isa, ZCLSD, pair->value, missing); + EXT_KEY(isainfo->isa, ZCMOP, pair->value, missing); + EXT_KEY(isainfo->isa, ZICBOM, pair->value, missing); + EXT_KEY(isainfo->isa, ZICBOP, pair->value, missing); + EXT_KEY(isainfo->isa, ZICBOZ, pair->value, missing); + EXT_KEY(isainfo->isa, ZICFILP, pair->value, missing); + EXT_KEY(isainfo->isa, ZICNTR, pair->value, missing); + EXT_KEY(isainfo->isa, ZICOND, pair->value, missing); + EXT_KEY(isainfo->isa, ZIHINTNTL, pair->value, missing); + EXT_KEY(isainfo->isa, ZIHINTPAUSE, pair->value, missing); + EXT_KEY(isainfo->isa, ZIHPM, pair->value, missing); + EXT_KEY(isainfo->isa, ZILSD, pair->value, missing); + EXT_KEY(isainfo->isa, ZIMOP, pair->value, missing); + EXT_KEY(isainfo->isa, ZKND, pair->value, missing); + EXT_KEY(isainfo->isa, ZKNE, pair->value, missing); + EXT_KEY(isainfo->isa, ZKNH, pair->value, missing); + EXT_KEY(isainfo->isa, ZKSED, pair->value, missing); + EXT_KEY(isainfo->isa, ZKSH, pair->value, missing); + EXT_KEY(isainfo->isa, ZKT, pair->value, missing); + EXT_KEY(isainfo->isa, ZTSO, pair->value, missing); /* * All the following extensions must depend on the kernel * support of V. */ if (has_vector()) { - EXT_KEY(ZVBB); - EXT_KEY(ZVBC); - EXT_KEY(ZVE32F); - EXT_KEY(ZVE32X); - EXT_KEY(ZVE64D); - EXT_KEY(ZVE64F); - EXT_KEY(ZVE64X); - EXT_KEY(ZVFBFMIN); - EXT_KEY(ZVFBFWMA); - EXT_KEY(ZVFH); - EXT_KEY(ZVFHMIN); - EXT_KEY(ZVKB); - EXT_KEY(ZVKG); - EXT_KEY(ZVKNED); - EXT_KEY(ZVKNHA); - EXT_KEY(ZVKNHB); - EXT_KEY(ZVKSED); - EXT_KEY(ZVKSH); - EXT_KEY(ZVKT); + EXT_KEY(isainfo->isa, ZVBB, pair->value, missing); + EXT_KEY(isainfo->isa, ZVBC, pair->value, missing); + EXT_KEY(isainfo->isa, ZVE32F, pair->value, missing); + EXT_KEY(isainfo->isa, ZVE32X, pair->value, missing); + EXT_KEY(isainfo->isa, ZVE64D, pair->value, missing); + EXT_KEY(isainfo->isa, ZVE64F, pair->value, missing); + EXT_KEY(isainfo->isa, ZVE64X, pair->value, missing); + EXT_KEY(isainfo->isa, ZVFBFMIN, pair->value, missing); + EXT_KEY(isainfo->isa, ZVFBFWMA, pair->value, missing); + EXT_KEY(isainfo->isa, ZVFH, pair->value, missing); + EXT_KEY(isainfo->isa, ZVFHMIN, pair->value, missing); + EXT_KEY(isainfo->isa, ZVKB, pair->value, missing); + EXT_KEY(isainfo->isa, ZVKG, pair->value, missing); + EXT_KEY(isainfo->isa, ZVKNED, pair->value, missing); + EXT_KEY(isainfo->isa, ZVKNHA, pair->value, missing); + EXT_KEY(isainfo->isa, ZVKNHB, pair->value, missing); + EXT_KEY(isainfo->isa, ZVKSED, pair->value, missing); + EXT_KEY(isainfo->isa, ZVKSH, pair->value, missing); + EXT_KEY(isainfo->isa, ZVKT, pair->value, missing); } - EXT_KEY(ZCD); - EXT_KEY(ZCF); - EXT_KEY(ZFA); - EXT_KEY(ZFBFMIN); - EXT_KEY(ZFH); - EXT_KEY(ZFHMIN); + EXT_KEY(isainfo->isa, ZCD, pair->value, missing); + EXT_KEY(isainfo->isa, ZCF, pair->value, missing); + EXT_KEY(isainfo->isa, ZFA, pair->value, missing); + EXT_KEY(isainfo->isa, ZFBFMIN, pair->value, missing); + EXT_KEY(isainfo->isa, ZFH, pair->value, missing); + EXT_KEY(isainfo->isa, ZFHMIN, pair->value, missing); if (IS_ENABLED(CONFIG_RISCV_ISA_SUPM)) - EXT_KEY(SUPM); -#undef EXT_KEY + EXT_KEY(isainfo->isa, SUPM, pair->value, missing); + } + + /* Now turn off reporting features if any CPU is missing it. */ + pair->value &= ~missing; +} + +static void hwprobe_isa_ext1(struct riscv_hwprobe *pair, + const struct cpumask *cpus) +{ + int cpu; + u64 missing = 0; + + pair->value = 0; + + /* + * Loop through and record extensions that 1) anyone has, and 2) anyone + * doesn't have. + */ + for_each_cpu(cpu, cpus) { + struct riscv_isainfo *isainfo = &hart_isa[cpu]; + + /* + * Only use EXT_KEY() for extensions which can be + * exposed to userspace, regardless of the kernel's + * configuration, as no other checks, besides presence + * in the hart_isa bitmap, are made. + */ + EXT_KEY(isainfo->isa, ZICFISS, pair->value, missing); + EXT_KEY(isainfo->isa, ZICCLSM, pair->value, missing); + EXT_KEY(isainfo->isa, ZICCAMOA, pair->value, missing); + EXT_KEY(isainfo->isa, ZICCIF, pair->value, missing); + EXT_KEY(isainfo->isa, ZICCRSE, pair->value, missing); + EXT_KEY(isainfo->isa, ZA64RS, pair->value, missing); } /* Now turn off reporting features if any CPU is missing it. */ @@ -267,6 +297,8 @@ static u64 hwprobe_vec_misaligned(const struct cpumask *cpus) static void hwprobe_one_pair(struct riscv_hwprobe *pair, const struct cpumask *cpus) { + pair->value = 0; + switch (pair->key) { case RISCV_HWPROBE_KEY_MVENDORID: case RISCV_HWPROBE_KEY_MARCHID: @@ -287,6 +319,10 @@ static void hwprobe_one_pair(struct riscv_hwprobe *pair, hwprobe_isa_ext0(pair, cpus); break; + case RISCV_HWPROBE_KEY_IMA_EXT_1: + hwprobe_isa_ext1(pair, cpus); + break; + case RISCV_HWPROBE_KEY_CPUPERF_0: case RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF: pair->value = hwprobe_misaligned(cpus); @@ -297,17 +333,14 @@ static void hwprobe_one_pair(struct riscv_hwprobe *pair, break; case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE: - pair->value = 0; if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ)) pair->value = riscv_cboz_block_size; break; case RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE: - pair->value = 0; if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOM)) pair->value = riscv_cbom_block_size; break; case RISCV_HWPROBE_KEY_ZICBOP_BLOCK_SIZE: - pair->value = 0; if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOP)) pair->value = riscv_cbop_block_size; break; @@ -418,6 +451,7 @@ static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs, if (cpusetsize > cpumask_size()) cpusetsize = cpumask_size(); + cpumask_clear(&cpus); ret = copy_from_user(&cpus, cpus_user, cpusetsize); if (ret) return -EFAULT; @@ -472,32 +506,28 @@ static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs, return 0; } -#ifdef CONFIG_MMU - -static DECLARE_COMPLETION(boot_probes_done); -static atomic_t pending_boot_probes = ATOMIC_INIT(1); - -void riscv_hwprobe_register_async_probe(void) +static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs, + size_t pair_count, size_t cpusetsize, + unsigned long __user *cpus_user, + unsigned int flags) { - atomic_inc(&pending_boot_probes); -} + if (flags & RISCV_HWPROBE_WHICH_CPUS) + return hwprobe_get_cpus(pairs, pair_count, cpusetsize, + cpus_user, flags); -void riscv_hwprobe_complete_async_probe(void) -{ - if (atomic_dec_and_test(&pending_boot_probes)) - complete(&boot_probes_done); + return hwprobe_get_values(pairs, pair_count, cpusetsize, + cpus_user, flags); } -static int complete_hwprobe_vdso_data(void) +#ifdef CONFIG_MMU + +static int __init init_hwprobe_vdso_data(void) { struct vdso_arch_data *avd = vdso_k_arch_data; u64 id_bitsmash = 0; struct riscv_hwprobe pair; int key; - if (unlikely(!atomic_dec_and_test(&pending_boot_probes))) - wait_for_completion(&boot_probes_done); - /* * Initialize vDSO data with the answers for the "all CPUs" case, to * save a syscall in the common case. @@ -525,52 +555,13 @@ static int complete_hwprobe_vdso_data(void) * vDSO should defer to the kernel for exotic cpu masks. */ avd->homogeneous_cpus = id_bitsmash != 0 && id_bitsmash != -1; - - /* - * Make sure all the VDSO values are visible before we look at them. - * This pairs with the implicit "no speculativly visible accesses" - * barrier in the VDSO hwprobe code. - */ - smp_wmb(); - avd->ready = true; - return 0; -} - -static int __init init_hwprobe_vdso_data(void) -{ - struct vdso_arch_data *avd = vdso_k_arch_data; - - /* - * Prevent the vDSO cached values from being used, as they're not ready - * yet. - */ - avd->ready = false; return 0; } arch_initcall_sync(init_hwprobe_vdso_data); -#else - -static int complete_hwprobe_vdso_data(void) { return 0; } - #endif /* CONFIG_MMU */ -static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs, - size_t pair_count, size_t cpusetsize, - unsigned long __user *cpus_user, - unsigned int flags) -{ - DO_ONCE_SLEEPABLE(complete_hwprobe_vdso_data); - - if (flags & RISCV_HWPROBE_WHICH_CPUS) - return hwprobe_get_cpus(pairs, pair_count, cpusetsize, - cpus_user, flags); - - return hwprobe_get_values(pairs, pair_count, cpusetsize, - cpus_user, flags); -} - SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs, size_t, pair_count, size_t, cpusetsize, unsigned long __user *, cpus, unsigned int, flags) diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c index 795b2e815ac9..22fc9b3268be 100644 --- a/arch/riscv/kernel/sys_riscv.c +++ b/arch/riscv/kernel/sys_riscv.c @@ -7,6 +7,7 @@ #include <linux/syscalls.h> #include <asm/cacheflush.h> +#include <asm-generic/mman-common.h> static long riscv_sys_mmap(unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, @@ -16,6 +17,15 @@ static long riscv_sys_mmap(unsigned long addr, unsigned long len, if (unlikely(offset & (~PAGE_MASK >> page_shift_offset))) return -EINVAL; + /* + * If PROT_WRITE is specified then extend that to PROT_READ + * protection_map[VM_WRITE] is now going to select shadow stack encodings. + * So specifying PROT_WRITE actually should select protection_map [VM_WRITE | VM_READ] + * If user wants to create shadow stack then they should use `map_shadow_stack` syscall. + */ + if (unlikely((prot & PROT_WRITE) && !(prot & PROT_READ))) + prot |= PROT_READ; + return ksys_mmap_pgoff(addr, len, prot, flags, fd, offset >> (PAGE_SHIFT - page_shift_offset)); } diff --git a/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S b/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S index f16deee9e091..15e50934bbd7 100644 --- a/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S +++ b/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S @@ -179,6 +179,23 @@ test_kprobes_c_bnez_addr3: ret SYM_FUNC_END(test_kprobes_c_bnez) +#ifdef CONFIG_32BIT +SYM_FUNC_START(test_kprobes_c_jal) + li a0, 0 + mv a1, ra +test_kprobes_c_jal_addr1: + c.jal 2f + ret +1: li a0, KPROBE_TEST_MAGIC_UPPER + ret +test_kprobes_c_jal_addr2: +2: c.jal 1b + li a2, KPROBE_TEST_MAGIC_LOWER + add a0, a0, a2 + jr a1 +SYM_FUNC_END(test_kprobes_c_jal) +#endif + #endif /* CONFIG_RISCV_ISA_C */ .section .rodata @@ -209,6 +226,10 @@ SYM_DATA_START(test_kprobes_addresses) RISCV_PTR test_kprobes_c_bnez_addr1 RISCV_PTR test_kprobes_c_bnez_addr2 RISCV_PTR test_kprobes_c_bnez_addr3 +#ifdef CONFIG_32BIT + RISCV_PTR test_kprobes_c_jal_addr1 + RISCV_PTR test_kprobes_c_jal_addr2 +#endif #endif /* CONFIG_RISCV_ISA_C */ RISCV_PTR 0 SYM_DATA_END(test_kprobes_addresses) @@ -226,6 +247,9 @@ SYM_DATA_START(test_kprobes_functions) RISCV_PTR test_kprobes_c_jalr RISCV_PTR test_kprobes_c_beqz RISCV_PTR test_kprobes_c_bnez +#ifdef CONFIG_32BIT + RISCV_PTR test_kprobes_c_jal +#endif #endif /* CONFIG_RISCV_ISA_C */ RISCV_PTR 0 SYM_DATA_END(test_kprobes_functions) diff --git a/arch/riscv/kernel/tests/kprobes/test-kprobes.c b/arch/riscv/kernel/tests/kprobes/test-kprobes.c index 664535ca0a98..027424a3ff7b 100644 --- a/arch/riscv/kernel/tests/kprobes/test-kprobes.c +++ b/arch/riscv/kernel/tests/kprobes/test-kprobes.c @@ -20,7 +20,7 @@ static void test_kprobe_riscv(struct kunit *test) while (test_kprobes_addresses[num_kprobe]) num_kprobe++; - kp = kcalloc(num_kprobe, sizeof(*kp), GFP_KERNEL); + kp = kzalloc_objs(*kp, num_kprobe); KUNIT_EXPECT_TRUE(test, kp); if (!kp) return; diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index 47afea4ff1a8..f8292adda099 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -7,7 +7,6 @@ #include <linux/kernel.h> #include <linux/init.h> #include <linux/irqflags.h> -#include <linux/randomize_kstack.h> #include <linux/sched.h> #include <linux/sched/debug.h> #include <linux/sched/signal.h> @@ -142,11 +141,7 @@ static void do_trap_error(struct pt_regs *regs, int signo, int code, } } -#if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_RISCV_ALTERNATIVE) -#define __trap_section __noinstr_section(".xip.traps") -#else #define __trap_section noinstr -#endif #define DO_ERROR_INFO(name, signo, code, str) \ asmlinkage __visible __trap_section void name(struct pt_regs *regs) \ { \ @@ -165,6 +160,8 @@ asmlinkage __visible __trap_section void name(struct pt_regs *regs) \ DO_ERROR_INFO(do_trap_unknown, SIGILL, ILL_ILLTRP, "unknown exception"); +DO_ERROR_INFO(do_trap_hardware_error, + SIGBUS, BUS_MCEERR_AR, "hardware error"); DO_ERROR_INFO(do_trap_insn_misaligned, SIGBUS, BUS_ADRALN, "instruction address misaligned"); DO_ERROR_INFO(do_trap_insn_fault, @@ -272,6 +269,7 @@ static bool probe_single_step_handler(struct pt_regs *regs) return user ? uprobe_single_step_handler(regs) : kprobe_single_step_handler(regs); } +NOKPROBE_SYMBOL(probe_single_step_handler); static bool probe_breakpoint_handler(struct pt_regs *regs) { @@ -279,6 +277,7 @@ static bool probe_breakpoint_handler(struct pt_regs *regs) return user ? uprobe_breakpoint_handler(regs) : kprobe_breakpoint_handler(regs); } +NOKPROBE_SYMBOL(probe_breakpoint_handler); void handle_break(struct pt_regs *regs) { @@ -303,6 +302,7 @@ void handle_break(struct pt_regs *regs) else die(regs, "Kernel BUG"); } +NOKPROBE_SYMBOL(handle_break); asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs) { @@ -335,27 +335,12 @@ void do_trap_ecall_u(struct pt_regs *regs) riscv_v_vstate_discard(regs); - syscall = syscall_enter_from_user_mode(regs, syscall); - - add_random_kstack_offset(); - - if (syscall >= 0 && syscall < NR_syscalls) { - syscall = array_index_nospec(syscall, NR_syscalls); - syscall_handler(regs, syscall); + if (likely(syscall_enter_from_user_mode_randomize_stack(regs, &syscall))) { + if (syscall >= 0 && syscall < NR_syscalls) { + syscall = array_index_nospec(syscall, NR_syscalls); + syscall_handler(regs, syscall); + } } - - /* - * Ultimately, this value will get limited by KSTACK_OFFSET_MAX(), - * so the maximum stack offset is 1k bytes (10 bits). - * - * The actual entropy will be further reduced by the compiler when - * applying stack alignment constraints: 16-byte (i.e. 4-bit) aligned - * for RV32I or RV64I. - * - * The resulting 6 bits of entropy is seen in SP[9:4]. - */ - choose_random_kstack_offset(get_random_u16()); - syscall_exit_to_user_mode(regs); } else { irqentry_state_t state = irqentry_nmi_enter(regs); @@ -368,6 +353,60 @@ void do_trap_ecall_u(struct pt_regs *regs) } +#define CFI_TVAL_FCFI_CODE 2 +#define CFI_TVAL_BCFI_CODE 3 +/* handle cfi violations */ +bool handle_user_cfi_violation(struct pt_regs *regs) +{ + unsigned long tval = csr_read(CSR_TVAL); + bool is_fcfi = (tval == CFI_TVAL_FCFI_CODE && cpu_supports_indirect_br_lp_instr()); + bool is_bcfi = (tval == CFI_TVAL_BCFI_CODE && cpu_supports_shadow_stack()); + + /* + * Handle uprobe event first. The probe point can be a valid target + * of indirect jumps or calls, in this case, forward cfi violation + * will be triggered instead of breakpoint exception. Clear ELP flag + * on sstatus image as well to avoid recurring fault. + */ + if (is_fcfi && probe_breakpoint_handler(regs)) { + regs->status &= ~SR_ELP; + return true; + } + + if (is_fcfi || is_bcfi) { + do_trap_error(regs, SIGSEGV, SEGV_CPERR, regs->epc, + "Oops - control flow violation"); + return true; + } + + return false; +} + +/* + * software check exception is defined with risc-v cfi spec. Software check + * exception is raised when: + * a) An indirect branch doesn't land on 4 byte aligned PC or `lpad` + * instruction or `label` value programmed in `lpad` instr doesn't + * match with value setup in `x7`. reported code in `xtval` is 2. + * b) `sspopchk` instruction finds a mismatch between top of shadow stack (ssp) + * and x1/x5. reported code in `xtval` is 3. + */ +asmlinkage __visible __trap_section void do_trap_software_check(struct pt_regs *regs) +{ + if (user_mode(regs)) { + irqentry_enter_from_user_mode(regs); + + /* not a cfi violation, then merge into flow of unknown trap handler */ + if (!handle_user_cfi_violation(regs)) + do_trap_unknown(regs); + + irqentry_exit_to_user_mode(regs); + } else { + /* sw check exception coming from kernel is a bug in kernel */ + die(regs, "Kernel BUG"); + } +} + #ifdef CONFIG_MMU asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs) { diff --git a/arch/riscv/kernel/traps_misaligned.c b/arch/riscv/kernel/traps_misaligned.c index 2a27d3ff4ac6..6e8ae6c66322 100644 --- a/arch/riscv/kernel/traps_misaligned.c +++ b/arch/riscv/kernel/traps_misaligned.c @@ -522,10 +522,10 @@ static bool unaligned_ctl __read_mostly; static void check_unaligned_access_emulated(void *arg __always_unused) { int cpu = smp_processor_id(); - long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu); unsigned long tmp_var, tmp_val; - *mas_ptr = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; + if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) + return; __asm__ __volatile__ ( " "REG_L" %[tmp], 1(%[ptr])\n" @@ -584,7 +584,7 @@ static int cpu_online_check_unaligned_access_emulated(unsigned int cpu) static bool misaligned_traps_delegated; -#ifdef CONFIG_RISCV_SBI +#if defined(CONFIG_RISCV_SBI_FWFT_DELEGATE_MISALIGNED) static int cpu_online_sbi_unaligned_setup(unsigned int cpu) { diff --git a/arch/riscv/kernel/unaligned_access_speed.c b/arch/riscv/kernel/unaligned_access_speed.c index 70b5e6927620..67f1a0371659 100644 --- a/arch/riscv/kernel/unaligned_access_speed.c +++ b/arch/riscv/kernel/unaligned_access_speed.c @@ -6,7 +6,6 @@ #include <linux/cpu.h> #include <linux/cpumask.h> #include <linux/jump_label.h> -#include <linux/kthread.h> #include <linux/mm.h> #include <linux/smp.h> #include <linux/types.h> @@ -16,7 +15,7 @@ #include "copy-unaligned.h" -#define MISALIGNED_ACCESS_JIFFIES_LG2 1 +#define MISALIGNED_ACCESS_NS 8000000 #define MISALIGNED_BUFFER_SIZE 0x4000 #define MISALIGNED_BUFFER_ORDER get_order(MISALIGNED_BUFFER_SIZE) #define MISALIGNED_COPY_SIZE ((MISALIGNED_BUFFER_SIZE / 2) - 0x80) @@ -27,111 +26,122 @@ DEFINE_PER_CPU(long, vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR static long unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; static long unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; -static cpumask_t fast_misaligned_access; - -#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS -static int check_unaligned_access(void *param) +static u64 __maybe_unused +measure_cycles(void (*func)(void *dst, const void *src, size_t len), + void *dst, void *src, size_t len) { - int cpu = smp_processor_id(); - u64 start_cycles, end_cycles; - u64 word_cycles; - u64 byte_cycles; - int ratio; - unsigned long start_jiffies, now; - struct page *page = param; - void *dst; - void *src; - long speed = RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW; - - if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) - return 0; + u64 start_cycles, end_cycles, cycles = -1ULL; + u64 start_ns; - /* Make an unaligned destination buffer. */ - dst = (void *)((unsigned long)page_address(page) | 0x1); - /* Unalign src as well, but differently (off by 1 + 2 = 3). */ - src = dst + (MISALIGNED_BUFFER_SIZE / 2); - src += 2; - word_cycles = -1ULL; /* Do a warmup. */ - __riscv_copy_words_unaligned(dst, src, MISALIGNED_COPY_SIZE); + func(dst, src, len); + preempt_disable(); - start_jiffies = jiffies; - while ((now = jiffies) == start_jiffies) - cpu_relax(); /* * For a fixed amount of time, repeatedly try the function, and take * the best time in cycles as the measurement. */ - while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) { + start_ns = ktime_get_mono_fast_ns(); + while (ktime_get_mono_fast_ns() < start_ns + MISALIGNED_ACCESS_NS) { start_cycles = get_cycles64(); /* Ensure the CSR read can't reorder WRT to the copy. */ mb(); - __riscv_copy_words_unaligned(dst, src, MISALIGNED_COPY_SIZE); + func(dst, src, len); /* Ensure the copy ends before the end time is snapped. */ mb(); end_cycles = get_cycles64(); - if ((end_cycles - start_cycles) < word_cycles) - word_cycles = end_cycles - start_cycles; + if ((end_cycles - start_cycles) < cycles) + cycles = end_cycles - start_cycles; } - byte_cycles = -1ULL; - __riscv_copy_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE); - start_jiffies = jiffies; - while ((now = jiffies) == start_jiffies) - cpu_relax(); + preempt_enable(); - while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) { - start_cycles = get_cycles64(); - mb(); - __riscv_copy_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE); - mb(); - end_cycles = get_cycles64(); - if ((end_cycles - start_cycles) < byte_cycles) - byte_cycles = end_cycles - start_cycles; - } + return cycles; +} - preempt_enable(); +/* + * Return: + * 1 if unaligned accesses are fast + * 0 if unaligned accesses are slow + * -1 if check cannot be done + */ +static int __maybe_unused +compare_unaligned_access(void (*word_copy)(void *dst, const void *src, size_t len), + void (*byte_copy)(void *dst, const void *src, size_t len), + void *buf, const char *type) +{ + int cpu = smp_processor_id(); + u64 word_cycles; + u64 byte_cycles; + void *dst, *src; + bool fast; + int ratio; + + /* Make an unaligned destination buffer. */ + dst = (void *)((unsigned long)buf | 0x1); + /* Unalign src as well, but differently (off by 1 + 2 = 3). */ + src = dst + (MISALIGNED_BUFFER_SIZE / 2); + src += 2; + + word_cycles = measure_cycles(word_copy, dst, src, MISALIGNED_COPY_SIZE); + byte_cycles = measure_cycles(byte_copy, dst, src, MISALIGNED_COPY_SIZE); /* Don't divide by zero. */ if (!word_cycles || !byte_cycles) { - pr_warn("cpu%d: rdtime lacks granularity needed to measure unaligned access speed\n", - cpu); + pr_warn("cpu%d: rdtime lacks granularity needed to measure %s unaligned access speed\n", + cpu, type); - return 0; + return -1; } - if (word_cycles < byte_cycles) - speed = RISCV_HWPROBE_MISALIGNED_SCALAR_FAST; + fast = word_cycles < byte_cycles; ratio = div_u64((byte_cycles * 100), word_cycles); - pr_info("cpu%d: Ratio of byte access time to unaligned word access is %d.%02d, unaligned accesses are %s\n", + pr_info("cpu%d: %s unaligned word access speed is %d.%02dx byte access speed (%s)\n", cpu, + type, ratio / 100, ratio % 100, - (speed == RISCV_HWPROBE_MISALIGNED_SCALAR_FAST) ? "fast" : "slow"); + fast ? "fast" : "slow"); - per_cpu(misaligned_access_speed, cpu) = speed; + return fast; +} + +#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS +static int check_unaligned_access(struct page *page) +{ + void *buf = page_address(page); + int cpu = smp_processor_id(); + int ret; + + if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) + return 0; + + ret = compare_unaligned_access(__riscv_copy_words_unaligned, + __riscv_copy_bytes_unaligned, + buf, "scalar"); + if (ret < 0) + return 0; /* * Set the value of fast_misaligned_access of a CPU. These operations * are atomic to avoid race conditions. */ - if (speed == RISCV_HWPROBE_MISALIGNED_SCALAR_FAST) - cpumask_set_cpu(cpu, &fast_misaligned_access); + if (ret) + per_cpu(misaligned_access_speed, cpu) = RISCV_HWPROBE_MISALIGNED_SCALAR_FAST; else - cpumask_clear_cpu(cpu, &fast_misaligned_access); + per_cpu(misaligned_access_speed, cpu) = RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW; return 0; } -static void __init check_unaligned_access_nonboot_cpu(void *param) +static void __init _check_unaligned_access(void *param) { unsigned int cpu = smp_processor_id(); struct page **pages = param; - if (smp_processor_id() != 0) - check_unaligned_access(pages[cpu]); + check_unaligned_access(pages[cpu]); } /* Measure unaligned access speed on all CPUs present at boot in parallel. */ @@ -139,7 +149,7 @@ static void __init check_unaligned_access_speed_all_cpus(void) { unsigned int cpu; unsigned int cpu_count = num_possible_cpus(); - struct page **bufs = kcalloc(cpu_count, sizeof(*bufs), GFP_KERNEL); + struct page **bufs = kzalloc_objs(*bufs, cpu_count); if (!bufs) { pr_warn("Allocation failure, not measuring misaligned performance\n"); @@ -158,11 +168,7 @@ static void __init check_unaligned_access_speed_all_cpus(void) } } - /* Check everybody except 0, who stays behind to tend jiffies. */ - on_each_cpu(check_unaligned_access_nonboot_cpu, bufs, 1); - - /* Check core 0. */ - smp_call_on_cpu(0, check_unaligned_access, bufs[0], true); + on_each_cpu(_check_unaligned_access, bufs, 1); out: for_each_cpu(cpu, cpu_online_mask) { @@ -180,60 +186,24 @@ static void __init check_unaligned_access_speed_all_cpus(void) DEFINE_STATIC_KEY_FALSE(fast_unaligned_access_speed_key); -static void modify_unaligned_access_branches(cpumask_t *mask, int weight) +static void modify_unaligned_access_branches(const cpumask_t *mask) { - if (cpumask_weight(mask) == weight) + bool fast = true; + int cpu; + + for_each_cpu(cpu, mask) { + if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_FAST) { + fast = false; + break; + } + } + + if (fast) static_branch_enable_cpuslocked(&fast_unaligned_access_speed_key); else static_branch_disable_cpuslocked(&fast_unaligned_access_speed_key); } -static void set_unaligned_access_static_branches_except_cpu(int cpu) -{ - /* - * Same as set_unaligned_access_static_branches, except excludes the - * given CPU from the result. When a CPU is hotplugged into an offline - * state, this function is called before the CPU is set to offline in - * the cpumask, and thus the CPU needs to be explicitly excluded. - */ - - cpumask_t fast_except_me; - - cpumask_and(&fast_except_me, &fast_misaligned_access, cpu_online_mask); - cpumask_clear_cpu(cpu, &fast_except_me); - - modify_unaligned_access_branches(&fast_except_me, num_online_cpus() - 1); -} - -static void set_unaligned_access_static_branches(void) -{ - /* - * This will be called after check_unaligned_access_all_cpus so the - * result of unaligned access speed for all CPUs will be available. - * - * To avoid the number of online cpus changing between reading - * cpu_online_mask and calling num_online_cpus, cpus_read_lock must be - * held before calling this function. - */ - - cpumask_t fast_and_online; - - cpumask_and(&fast_and_online, &fast_misaligned_access, cpu_online_mask); - - modify_unaligned_access_branches(&fast_and_online, num_online_cpus()); -} - -static int __init lock_and_set_unaligned_access_static_branch(void) -{ - cpus_read_lock(); - set_unaligned_access_static_branches(); - cpus_read_unlock(); - - return 0; -} - -arch_initcall_sync(lock_and_set_unaligned_access_static_branch); - static int riscv_online_cpu(unsigned int cpu) { int ret = cpu_online_unaligned_access_init(cpu); @@ -265,14 +235,19 @@ static int riscv_online_cpu(unsigned int cpu) #endif exit: - set_unaligned_access_static_branches(); + modify_unaligned_access_branches(cpu_online_mask); return 0; } static int riscv_offline_cpu(unsigned int cpu) { - set_unaligned_access_static_branches_except_cpu(cpu); + cpumask_t mask; + + cpumask_copy(&mask, cpu_online_mask); + cpumask_clear_cpu(cpu, &mask); + + modify_unaligned_access_branches(&mask); return 0; } @@ -281,15 +256,8 @@ static int riscv_offline_cpu(unsigned int cpu) static void check_vector_unaligned_access(struct work_struct *work __always_unused) { int cpu = smp_processor_id(); - u64 start_cycles, end_cycles; - u64 word_cycles; - u64 byte_cycles; - int ratio; - unsigned long start_jiffies, now; struct page *page; - void *dst; - void *src; - long speed = RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW; + int ret; if (per_cpu(vector_misaligned_access, cpu) != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) return; @@ -300,93 +268,28 @@ static void check_vector_unaligned_access(struct work_struct *work __always_unus return; } - /* Make an unaligned destination buffer. */ - dst = (void *)((unsigned long)page_address(page) | 0x1); - /* Unalign src as well, but differently (off by 1 + 2 = 3). */ - src = dst + (MISALIGNED_BUFFER_SIZE / 2); - src += 2; - word_cycles = -1ULL; - - /* Do a warmup. */ kernel_vector_begin(); - __riscv_copy_vec_words_unaligned(dst, src, MISALIGNED_COPY_SIZE); - - start_jiffies = jiffies; - while ((now = jiffies) == start_jiffies) - cpu_relax(); - - /* - * For a fixed amount of time, repeatedly try the function, and take - * the best time in cycles as the measurement. - */ - while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) { - start_cycles = get_cycles64(); - /* Ensure the CSR read can't reorder WRT to the copy. */ - mb(); - __riscv_copy_vec_words_unaligned(dst, src, MISALIGNED_COPY_SIZE); - /* Ensure the copy ends before the end time is snapped. */ - mb(); - end_cycles = get_cycles64(); - if ((end_cycles - start_cycles) < word_cycles) - word_cycles = end_cycles - start_cycles; - } - - byte_cycles = -1ULL; - __riscv_copy_vec_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE); - start_jiffies = jiffies; - while ((now = jiffies) == start_jiffies) - cpu_relax(); - - while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) { - start_cycles = get_cycles64(); - /* Ensure the CSR read can't reorder WRT to the copy. */ - mb(); - __riscv_copy_vec_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE); - /* Ensure the copy ends before the end time is snapped. */ - mb(); - end_cycles = get_cycles64(); - if ((end_cycles - start_cycles) < byte_cycles) - byte_cycles = end_cycles - start_cycles; - } + ret = compare_unaligned_access(__riscv_copy_vec_words_unaligned, + __riscv_copy_vec_bytes_unaligned, + page_address(page), "vector"); kernel_vector_end(); - /* Don't divide by zero. */ - if (!word_cycles || !byte_cycles) { - pr_warn("cpu%d: rdtime lacks granularity needed to measure unaligned vector access speed\n", - cpu); - + if (ret < 0) goto free; - } - - if (word_cycles < byte_cycles) - speed = RISCV_HWPROBE_MISALIGNED_VECTOR_FAST; - ratio = div_u64((byte_cycles * 100), word_cycles); - pr_info("cpu%d: Ratio of vector byte access time to vector unaligned word access is %d.%02d, unaligned accesses are %s\n", - cpu, - ratio / 100, - ratio % 100, - (speed == RISCV_HWPROBE_MISALIGNED_VECTOR_FAST) ? "fast" : "slow"); - - per_cpu(vector_misaligned_access, cpu) = speed; + if (ret) + per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_FAST; + else + per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW; free: __free_pages(page, MISALIGNED_BUFFER_ORDER); } -/* Measure unaligned access speed on all CPUs present at boot in parallel. */ -static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) -{ - schedule_on_each_cpu(check_vector_unaligned_access); - riscv_hwprobe_complete_async_probe(); - - return 0; -} #else /* CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS */ -static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) +static void check_vector_unaligned_access(struct work_struct *work __always_unused) { - return 0; } #endif @@ -474,12 +377,7 @@ static int __init check_unaligned_access_all_cpus(void) per_cpu(vector_misaligned_access, cpu) = unaligned_vector_speed_param; } else if (!check_vector_unaligned_access_emulated_all_cpus() && IS_ENABLED(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS)) { - riscv_hwprobe_register_async_probe(); - if (IS_ERR(kthread_run(vec_check_unaligned_access_speed_all_cpus, - NULL, "vec_check_unaligned_access_speed_all_cpus"))) { - pr_warn("Failed to create vec_unalign_check kthread\n"); - riscv_hwprobe_complete_async_probe(); - } + schedule_on_each_cpu(check_vector_unaligned_access); } /* @@ -491,7 +389,17 @@ static int __init check_unaligned_access_all_cpus(void) cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online", riscv_online_cpu_vec, NULL); + cpus_read_lock(); + modify_unaligned_access_branches(cpu_online_mask); + cpus_read_unlock(); + return 0; } -arch_initcall(check_unaligned_access_all_cpus); +/* + * Run after clocksource_done_booting() so measure_cycles() uses a stable + * clocksource, but before rootfs_initcall() enables usermode helpers. Those + * helpers can reach hwprobe and populate the vDSO cache, so async hwprobe + * probes must be registered first. + */ +fs_initcall_sync(check_unaligned_access_all_cpus); diff --git a/arch/riscv/kernel/usercfi.c b/arch/riscv/kernel/usercfi.c new file mode 100644 index 000000000000..dec0ba5eff5e --- /dev/null +++ b/arch/riscv/kernel/usercfi.c @@ -0,0 +1,534 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2024 Rivos, Inc. + * Deepak Gupta <debug@rivosinc.com> + */ + +#include <linux/sched.h> +#include <linux/bitops.h> +#include <linux/types.h> +#include <linux/mm.h> +#include <linux/mman.h> +#include <linux/uaccess.h> +#include <linux/sizes.h> +#include <linux/user.h> +#include <linux/syscalls.h> +#include <linux/prctl.h> +#include <asm/csr.h> +#include <asm/usercfi.h> + +unsigned long riscv_nousercfi __read_mostly; + +#define SHSTK_ENTRY_SIZE sizeof(void *) + +bool is_shstk_enabled(struct task_struct *task) +{ + return task->thread_info.user_cfi_state.ubcfi_en; +} + +bool is_shstk_allocated(struct task_struct *task) +{ + return task->thread_info.user_cfi_state.shdw_stk_base; +} + +bool is_shstk_locked(struct task_struct *task) +{ + return task->thread_info.user_cfi_state.ubcfi_locked; +} + +void set_shstk_base(struct task_struct *task, unsigned long shstk_addr, unsigned long size) +{ + task->thread_info.user_cfi_state.shdw_stk_base = shstk_addr; + task->thread_info.user_cfi_state.shdw_stk_size = size; +} + +unsigned long get_shstk_base(struct task_struct *task, unsigned long *size) +{ + if (size) + *size = task->thread_info.user_cfi_state.shdw_stk_size; + return task->thread_info.user_cfi_state.shdw_stk_base; +} + +void set_active_shstk(struct task_struct *task, unsigned long shstk_addr) +{ + task->thread_info.user_cfi_state.user_shdw_stk = shstk_addr; +} + +unsigned long get_active_shstk(struct task_struct *task) +{ + return task->thread_info.user_cfi_state.user_shdw_stk; +} + +void set_shstk_status(struct task_struct *task, bool enable) +{ + if (!is_user_shstk_enabled()) + return; + + task->thread_info.user_cfi_state.ubcfi_en = enable ? 1 : 0; + + if (enable) + task->thread.envcfg |= ENVCFG_SSE; + else + task->thread.envcfg &= ~ENVCFG_SSE; + + csr_write(CSR_ENVCFG, task->thread.envcfg); +} + +void set_shstk_lock(struct task_struct *task, bool lock) +{ + task->thread_info.user_cfi_state.ubcfi_locked = lock; +} + +bool is_indir_lp_enabled(struct task_struct *task) +{ + return task->thread_info.user_cfi_state.ufcfi_en; +} + +bool is_indir_lp_locked(struct task_struct *task) +{ + return task->thread_info.user_cfi_state.ufcfi_locked; +} + +void set_indir_lp_status(struct task_struct *task, bool enable) +{ + if (!is_user_lpad_enabled()) + return; + + task->thread_info.user_cfi_state.ufcfi_en = enable ? 1 : 0; + + if (enable) + task->thread.envcfg |= ENVCFG_LPE; + else + task->thread.envcfg &= ~ENVCFG_LPE; + + csr_write(CSR_ENVCFG, task->thread.envcfg); +} + +void set_indir_lp_lock(struct task_struct *task, bool lock) +{ + task->thread_info.user_cfi_state.ufcfi_locked = lock; +} +/* + * The shadow stack only stores the return address and not any variables + * 512M should be more than sufficient for most applications. + * Else PAGE_ALIGN it and return back + */ +static unsigned long calc_shstk_size(unsigned long size) +{ + if (size) + return PAGE_ALIGN(size); + + return PAGE_ALIGN(min(rlimit(RLIMIT_STACK) / 8, SZ_512M)); +} + +/* + * Writes on shadow stack can either be `sspush` or `ssamoswap`. `sspush` can happen + * implicitly on current shadow stack pointed to by CSR_SSP. `ssamoswap` takes pointer to + * shadow stack. To keep it simple, we plan to use `ssamoswap` to perform writes on shadow + * stack. + */ +static noinline unsigned long amo_user_shstk(unsigned long __user *addr, unsigned long val) +{ + /* + * Never expect -1 on shadow stack. Expect return addresses and zero + */ + unsigned long swap = -1; + + __enable_user_access(); + asm goto(".option push\n" + ".option arch, +zicfiss\n" + "1: ssamoswap.d %[swap], %[val], %[addr]\n" + _ASM_EXTABLE(1b, %l[fault]) + ".option pop\n" + : [swap] "=r" (swap), [addr] "+A" (*(__force unsigned long *)addr) + : [val] "r" (val) + : "memory" + : fault + ); + __disable_user_access(); + return swap; +fault: + __disable_user_access(); + return -1; +} + +/* + * Create a restore token on the shadow stack. A token is always XLEN wide + * and aligned to XLEN. + */ +static int create_rstor_token(unsigned long ssp, unsigned long *token_addr) +{ + unsigned long addr; + + /* Token must be aligned */ + if (!IS_ALIGNED(ssp, SHSTK_ENTRY_SIZE)) + return -EINVAL; + + /* On RISC-V we're constructing token to be function of address itself */ + addr = ssp - SHSTK_ENTRY_SIZE; + + if (amo_user_shstk((unsigned long __user *)addr, (unsigned long)ssp) == -1) + return -EFAULT; + + if (token_addr) + *token_addr = addr; + + return 0; +} + +/* + * Save user shadow stack pointer on the shadow stack itself and return a pointer to saved location. + * Returns -EFAULT if unsuccessful. + */ +int save_user_shstk(struct task_struct *tsk, unsigned long *saved_shstk_ptr) +{ + unsigned long ss_ptr = 0; + unsigned long token_loc = 0; + int ret = 0; + + if (!saved_shstk_ptr) + return -EINVAL; + + ss_ptr = get_active_shstk(tsk); + ret = create_rstor_token(ss_ptr, &token_loc); + + if (!ret) { + *saved_shstk_ptr = token_loc; + set_active_shstk(tsk, token_loc); + } + + return ret; +} + +/* + * Restores the user shadow stack pointer from the token on the shadow stack for task 'tsk'. + * Returns -EFAULT if unsuccessful. + */ +int restore_user_shstk(struct task_struct *tsk, unsigned long shstk_ptr) +{ + unsigned long token = 0; + + token = amo_user_shstk((unsigned long __user *)shstk_ptr, 0); + + if (token == -1) + return -EFAULT; + + /* invalid token, return EINVAL */ + if ((token - shstk_ptr) != SHSTK_ENTRY_SIZE) { + pr_info_ratelimited("%s[%d]: bad restore token in %s: pc=%p sp=%p, token=%p, shstk_ptr=%p\n", + tsk->comm, task_pid_nr(tsk), __func__, + (void *)(task_pt_regs(tsk)->epc), + (void *)(task_pt_regs(tsk)->sp), + (void *)token, (void *)shstk_ptr); + return -EINVAL; + } + + /* all checks passed, set active shstk and return success */ + set_active_shstk(tsk, token); + return 0; +} + +static unsigned long allocate_shadow_stack(unsigned long addr, unsigned long size, + unsigned long token_offset, bool set_tok) +{ + addr = vm_mmap_shadow_stack(addr, size, 0); + + if (!set_tok || IS_ERR_VALUE(addr)) + goto out; + + if (create_rstor_token(addr + token_offset, NULL)) { + vm_munmap(addr, size); + return -EINVAL; + } + +out: + return addr; +} + +SYSCALL_DEFINE3(map_shadow_stack, unsigned long, addr, unsigned long, size, unsigned int, flags) +{ + bool set_tok = flags & SHADOW_STACK_SET_TOKEN; + unsigned long aligned_size = 0; + + if (!is_user_shstk_enabled()) + return -EOPNOTSUPP; + + /* Anything other than set token should result in invalid param */ + if (flags & ~SHADOW_STACK_SET_TOKEN) + return -EINVAL; + + /* + * Unlike other architectures, on RISC-V, SSP pointer is held in CSR_SSP and is an available + * CSR in all modes. CSR accesses are performed using 12bit index programmed in instruction + * itself. This provides static property on register programming and writes to CSR can't + * be unintentional from programmer's perspective. As long as programmer has guarded areas + * which perform writes to CSR_SSP properly, shadow stack pivoting is not possible. Since + * CSR_SSP is writable by user mode, it itself can setup a shadow stack token subsequent + * to allocation. Although in order to provide portablity with other architectures (because + * `map_shadow_stack` is arch agnostic syscall), RISC-V will follow expectation of a token + * flag in flags and if provided in flags, will setup a token at the base. + */ + + /* If there isn't space for a token */ + if (set_tok && size < SHSTK_ENTRY_SIZE) + return -ENOSPC; + + if (addr && (addr & (PAGE_SIZE - 1))) + return -EINVAL; + + aligned_size = PAGE_ALIGN(size); + if (aligned_size < size) + return -EOVERFLOW; + + return allocate_shadow_stack(addr, aligned_size, size, set_tok); +} + +/* + * This gets called during clone/clone3/fork. And is needed to allocate a shadow stack for + * cases where CLONE_VM is specified and thus a different stack is specified by user. We + * thus need a separate shadow stack too. How a separate shadow stack is specified by + * user is still being debated. Once that's settled, remove this part of the comment. + * This function simply returns 0 if shadow stacks are not supported or if separate shadow + * stack allocation is not needed (like in case of !CLONE_VM) + */ +unsigned long shstk_alloc_thread_stack(struct task_struct *tsk, + const struct kernel_clone_args *args) +{ + unsigned long addr, size; + + /* If shadow stack is not supported, return 0 */ + if (!is_user_shstk_enabled()) + return 0; + + /* + * If shadow stack is not enabled on the new thread, skip any + * switch to a new shadow stack. + */ + if (!is_shstk_enabled(tsk)) + return 0; + + /* + * For CLONE_VFORK the child will share the parents shadow stack. + * Set base = 0 and size = 0, this is special means to track this state + * so the freeing logic run for child knows to leave it alone. + */ + if (args->flags & CLONE_VFORK) { + set_shstk_base(tsk, 0, 0); + return 0; + } + + /* + * For !CLONE_VM the child will use a copy of the parents shadow + * stack. + */ + if (!(args->flags & CLONE_VM)) + return 0; + + /* + * reaching here means, CLONE_VM was specified and thus a separate shadow + * stack is needed for new cloned thread. Note: below allocation is happening + * using current mm. + */ + size = calc_shstk_size(args->stack_size); + addr = allocate_shadow_stack(0, size, 0, false); + if (IS_ERR_VALUE(addr)) + return addr; + + set_shstk_base(tsk, addr, size); + + return addr + size; +} + +void shstk_release(struct task_struct *tsk) +{ + unsigned long base = 0, size = 0; + /* If shadow stack is not supported or not enabled, nothing to release */ + if (!is_user_shstk_enabled() || !is_shstk_enabled(tsk)) + return; + + /* + * When fork() with CLONE_VM fails, the child (tsk) already has a + * shadow stack allocated, and exit_thread() calls this function to + * free it. In this case the parent (current) and the child share + * the same mm struct. Move forward only when they're same. + */ + if (!tsk->mm || tsk->mm != current->mm) + return; + + /* + * We know shadow stack is enabled but if base is NULL, then + * this task is not managing its own shadow stack (CLONE_VFORK). So + * skip freeing it. + */ + base = get_shstk_base(tsk, &size); + if (!base) + return; + + vm_munmap(base, size); + set_shstk_base(tsk, 0, 0); +} + +int arch_get_shadow_stack_status(struct task_struct *t, unsigned long __user *status) +{ + unsigned long bcfi_status = 0; + + if (!is_user_shstk_enabled()) + return -EINVAL; + + /* this means shadow stack is enabled on the task */ + bcfi_status |= (is_shstk_enabled(t) ? PR_SHADOW_STACK_ENABLE : 0); + + return copy_to_user(status, &bcfi_status, sizeof(bcfi_status)) ? -EFAULT : 0; +} + +int arch_set_shadow_stack_status(struct task_struct *t, unsigned long status) +{ + unsigned long size = 0, addr = 0; + bool enable_shstk = false; + + if (!is_user_shstk_enabled()) + return -EINVAL; + + /* Reject unknown flags */ + if (status & ~PR_SHADOW_STACK_SUPPORTED_STATUS_MASK) + return -EINVAL; + + /* bcfi status is locked and further can't be modified by user */ + if (is_shstk_locked(t)) + return -EINVAL; + + enable_shstk = status & PR_SHADOW_STACK_ENABLE; + /* Request is to enable shadow stack and shadow stack is not enabled already */ + if (enable_shstk && !is_shstk_enabled(t)) { + /* shadow stack was allocated and enable request again + * no need to support such usecase and return EINVAL. + */ + if (is_shstk_allocated(t)) + return -EINVAL; + + size = calc_shstk_size(0); + addr = allocate_shadow_stack(0, size, 0, false); + if (IS_ERR_VALUE(addr)) + return -ENOMEM; + set_shstk_base(t, addr, size); + set_active_shstk(t, addr + size); + } + + /* + * If a request to disable shadow stack happens, let's go ahead and release it + * Although, if CLONE_VFORKed child did this, then in that case we will end up + * not releasing the shadow stack (because it might be needed in parent). Although + * we will disable it for VFORKed child. And if VFORKed child tries to enable again + * then in that case, it'll get entirely new shadow stack because following condition + * are true + * - shadow stack was not enabled for vforked child + * - shadow stack base was anyways pointing to 0 + * This shouldn't be a big issue because we want parent to have availability of shadow + * stack whenever VFORKed child releases resources via exit or exec but at the same + * time we want VFORKed child to break away and establish new shadow stack if it desires + * + */ + if (!enable_shstk) + shstk_release(t); + + set_shstk_status(t, enable_shstk); + return 0; +} + +int arch_lock_shadow_stack_status(struct task_struct *task, + unsigned long arg) +{ + /* If shtstk not supported or not enabled on task, nothing to lock here */ + if (!is_user_shstk_enabled() || + !is_shstk_enabled(task) || arg != 0) + return -EINVAL; + + set_shstk_lock(task, true); + + return 0; +} + +int arch_prctl_get_branch_landing_pad_state(struct task_struct *t, + unsigned long __user *state) +{ + unsigned long fcfi_status = 0; + + if (!is_user_lpad_enabled()) + return -EINVAL; + + fcfi_status = (is_indir_lp_enabled(t) ? PR_CFI_ENABLE : PR_CFI_DISABLE); + fcfi_status |= (is_indir_lp_locked(t) ? PR_CFI_LOCK : 0); + + return copy_to_user(state, &fcfi_status, sizeof(fcfi_status)) ? -EFAULT : 0; +} + +int arch_prctl_set_branch_landing_pad_state(struct task_struct *t, unsigned long state) +{ + if (!is_user_lpad_enabled()) + return -EINVAL; + + if (state & ~PR_CFI_SUPPORTED_STATUS_MASK) + return -EINVAL; + + /* indirect branch tracking is locked and further can't be modified by user */ + if (is_indir_lp_locked(t)) + return -EINVAL; + + if (!(state & (PR_CFI_ENABLE | PR_CFI_DISABLE))) + return -EINVAL; + + if (state & PR_CFI_ENABLE && state & PR_CFI_DISABLE) + return -EINVAL; + + set_indir_lp_status(t, !!(state & PR_CFI_ENABLE)); + + return 0; +} + +int arch_prctl_lock_branch_landing_pad_state(struct task_struct *task) +{ + /* + * If indirect branch tracking is not supported or not enabled on task, + * nothing to lock here + */ + if (!is_user_lpad_enabled() || + !is_indir_lp_enabled(task)) + return -EINVAL; + + set_indir_lp_lock(task, true); + + return 0; +} + +bool is_user_shstk_enabled(void) +{ + return (cpu_supports_shadow_stack() && + !(riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_BCFI)); +} + +bool is_user_lpad_enabled(void) +{ + return (cpu_supports_indirect_br_lp_instr() && + !(riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_FCFI)); +} + +static int __init setup_global_riscv_enable(char *str) +{ + if (strcmp(str, "all") == 0) + riscv_nousercfi = CMDLINE_DISABLE_RISCV_USERCFI; + + if (strcmp(str, "fcfi") == 0) + riscv_nousercfi |= CMDLINE_DISABLE_RISCV_USERCFI_FCFI; + + if (strcmp(str, "bcfi") == 0) + riscv_nousercfi |= CMDLINE_DISABLE_RISCV_USERCFI_BCFI; + + if (riscv_nousercfi) + pr_info("RISC-V user CFI disabled via cmdline - shadow stack status : %s, landing pad status : %s\n", + str_disabled_enabled(riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_BCFI), + str_disabled_enabled(riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_FCFI)); + + return 1; +} + +__setup("riscv_nousercfi=", setup_global_riscv_enable); diff --git a/arch/riscv/kernel/vdso.c b/arch/riscv/kernel/vdso.c index 3a8e038b10a2..9c2f5e442338 100644 --- a/arch/riscv/kernel/vdso.c +++ b/arch/riscv/kernel/vdso.c @@ -55,9 +55,7 @@ static void __init __vdso_init(struct __vdso_info *vdso_info) vdso_info->vdso_code_start) >> PAGE_SHIFT; - vdso_pagelist = kcalloc(vdso_info->vdso_pages, - sizeof(struct page *), - GFP_KERNEL); + vdso_pagelist = kzalloc_objs(struct page *, vdso_info->vdso_pages); if (vdso_pagelist == NULL) panic("vDSO kcalloc failed!\n"); @@ -98,6 +96,13 @@ static struct __vdso_info compat_vdso_info __ro_after_init = { static int __init vdso_init(void) { + /* Hart implements zimop, expose cfi compiled vdso */ + if (IS_ENABLED(CONFIG_RISCV_USER_CFI) && + riscv_has_extension_unlikely(RISCV_ISA_EXT_ZIMOP)) { + vdso_info.vdso_code_start = vdso_cfi_start; + vdso_info.vdso_code_end = vdso_cfi_end; + } + __vdso_init(&vdso_info); #ifdef CONFIG_COMPAT __vdso_init(&compat_vdso_info); diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index 9ebb5e590f93..8dbf2532a573 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -17,6 +17,15 @@ ifdef CONFIG_VDSO_GETRANDOM vdso-syms += getrandom endif +ifdef VDSO_CFI_BUILD +CFI_MARCH = _zicfilp_zicfiss +CFI_FULL = -fcf-protection=full +CFI_SUFFIX = -cfi +OFFSET_SUFFIX = _cfi +ccflags-y += -DVDSO_CFI=1 +asflags-y += -DVDSO_CFI=1 +endif + # Files to link into the vdso obj-vdso = $(patsubst %, %.o, $(vdso-syms)) note.o @@ -27,6 +36,12 @@ endif ccflags-y := -fno-stack-protector ccflags-y += -DDISABLE_BRANCH_PROFILING ccflags-y += -fno-builtin +ccflags-y += $(KBUILD_BASE_ISA)$(CFI_MARCH) +ccflags-y += $(CFI_FULL) +asflags-y += $(KBUILD_BASE_ISA)$(CFI_MARCH) +asflags-y += $(CFI_FULL) + +ccflags-remove-y += $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_LTO) ifneq ($(c-gettimeofday-y),) CFLAGS_vgettimeofday.o += -fPIC -include $(c-gettimeofday-y) @@ -39,27 +54,29 @@ endif CFLAGS_hwprobe.o += -fPIC # Build rules -targets := $(obj-vdso) vdso.so vdso.so.dbg vdso.lds +vdso_offsets := vdso$(if $(VDSO_CFI_BUILD),$(CFI_SUFFIX),)-offsets.h +vdso_o := vdso$(if $(VDSO_CFI_BUILD),$(CFI_SUFFIX),).o +vdso_so := vdso$(if $(VDSO_CFI_BUILD),$(CFI_SUFFIX),).so +vdso_so_dbg := vdso$(if $(VDSO_CFI_BUILD),$(CFI_SUFFIX),).so.dbg +vdso_lds := vdso.lds + +targets := $(obj-vdso) $(vdso_so) $(vdso_so_dbg) $(vdso_lds) + obj-vdso := $(addprefix $(obj)/, $(obj-vdso)) -obj-y += vdso.o -CPPFLAGS_vdso.lds += -P -C -U$(ARCH) +obj-y += vdso$(if $(VDSO_CFI_BUILD),$(CFI_SUFFIX),).o +CPPFLAGS_$(vdso_lds) += -P -C -U$(ARCH) ifneq ($(filter vgettimeofday, $(vdso-syms)),) -CPPFLAGS_vdso.lds += -DHAS_VGETTIMEOFDAY +CPPFLAGS_$(vdso_lds) += -DHAS_VGETTIMEOFDAY endif -# Disable -pg to prevent insert call site -CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) -CFLAGS_REMOVE_getrandom.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) -CFLAGS_REMOVE_hwprobe.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) - # Force dependency -$(obj)/vdso.o: $(obj)/vdso.so +$(obj)/$(vdso_o): $(obj)/$(vdso_so) # link rule for the .so file, .lds has to be first -$(obj)/vdso.so.dbg: $(obj)/vdso.lds $(obj-vdso) FORCE +$(obj)/$(vdso_so_dbg): $(obj)/$(vdso_lds) $(obj-vdso) FORCE $(call if_changed,vdsold_and_check) -LDFLAGS_vdso.so.dbg = -shared -soname=linux-vdso.so.1 \ +LDFLAGS_$(vdso_so_dbg) = -shared -soname=linux-vdso.so.1 \ --build-id=sha1 --eh-frame-hdr # strip rule for the .so file @@ -70,16 +87,16 @@ $(obj)/%.so: $(obj)/%.so.dbg FORCE # Generate VDSO offsets using helper script gen-vdsosym := $(src)/gen_vdso_offsets.sh quiet_cmd_vdsosym = VDSOSYM $@ - cmd_vdsosym = $(NM) $< | $(gen-vdsosym) | LC_ALL=C sort > $@ + cmd_vdsosym = $(NM) $< | $(gen-vdsosym) $(OFFSET_SUFFIX) | LC_ALL=C sort > $@ -include/generated/vdso-offsets.h: $(obj)/vdso.so.dbg FORCE +include/generated/$(vdso_offsets): $(obj)/$(vdso_so_dbg) FORCE $(call if_changed,vdsosym) # actual build commands # The DSO images are built using a special linker script # Make sure only to export the intended __vdso_xxx symbol offsets. quiet_cmd_vdsold_and_check = VDSOLD $@ - cmd_vdsold_and_check = $(LD) $(ld_flags) -T $(filter-out FORCE,$^) -o $@.tmp && \ + cmd_vdsold_and_check = $(LD) $(CFI_FULL) $(ld_flags) -T $(filter-out FORCE,$^) -o $@.tmp && \ $(OBJCOPY) $(patsubst %, -G __vdso_%, $(vdso-syms)) $@.tmp $@ && \ rm $@.tmp && \ $(cmd_vdso_check) diff --git a/arch/riscv/kernel/vdso/flush_icache.S b/arch/riscv/kernel/vdso/flush_icache.S index 8f884227e8bc..e4c56970905e 100644 --- a/arch/riscv/kernel/vdso/flush_icache.S +++ b/arch/riscv/kernel/vdso/flush_icache.S @@ -5,11 +5,13 @@ #include <linux/linkage.h> #include <asm/unistd.h> +#include <asm/assembler.h> .text /* int __vdso_flush_icache(void *start, void *end, unsigned long flags); */ SYM_FUNC_START(__vdso_flush_icache) .cfi_startproc + vdso_lpad #ifdef CONFIG_SMP li a7, __NR_riscv_flush_icache ecall @@ -20,3 +22,5 @@ SYM_FUNC_START(__vdso_flush_icache) ret .cfi_endproc SYM_FUNC_END(__vdso_flush_icache) + +emit_riscv_feature_1_and diff --git a/arch/riscv/kernel/vdso/gen_vdso_offsets.sh b/arch/riscv/kernel/vdso/gen_vdso_offsets.sh index c2e5613f3495..bd5d5afaaa14 100755 --- a/arch/riscv/kernel/vdso/gen_vdso_offsets.sh +++ b/arch/riscv/kernel/vdso/gen_vdso_offsets.sh @@ -2,4 +2,6 @@ # SPDX-License-Identifier: GPL-2.0 LC_ALL=C -sed -n -e 's/^[0]\+\(0[0-9a-fA-F]*\) . \(__vdso_[a-zA-Z0-9_]*\)$/\#define \2_offset\t0x\1/p' +SUFFIX=${1:-""} +sed -n -e \ +'s/^[0]\+\(0[0-9a-fA-F]*\) . \(__vdso_[a-zA-Z0-9_]*\)$/\#define \2'$SUFFIX'_offset\t0x\1/p' diff --git a/arch/riscv/kernel/vdso/getcpu.S b/arch/riscv/kernel/vdso/getcpu.S index 9c1bd531907f..5c1ecc4e1465 100644 --- a/arch/riscv/kernel/vdso/getcpu.S +++ b/arch/riscv/kernel/vdso/getcpu.S @@ -5,14 +5,18 @@ #include <linux/linkage.h> #include <asm/unistd.h> +#include <asm/assembler.h> .text /* int __vdso_getcpu(unsigned *cpu, unsigned *node, void *unused); */ SYM_FUNC_START(__vdso_getcpu) .cfi_startproc + vdso_lpad /* For now, just do the syscall. */ li a7, __NR_getcpu ecall ret .cfi_endproc SYM_FUNC_END(__vdso_getcpu) + +emit_riscv_feature_1_and diff --git a/arch/riscv/kernel/vdso/hwprobe.c b/arch/riscv/kernel/vdso/hwprobe.c index 8f45500d0a6e..2ddeba6c68dd 100644 --- a/arch/riscv/kernel/vdso/hwprobe.c +++ b/arch/riscv/kernel/vdso/hwprobe.c @@ -27,7 +27,7 @@ static int riscv_vdso_get_values(struct riscv_hwprobe *pairs, size_t pair_count, * homogeneous, then this function can handle requests for arbitrary * masks. */ - if (flags != 0 || (!all_cpus && !avd->homogeneous_cpus) || unlikely(!avd->ready)) + if ((flags != 0) || (!all_cpus && !avd->homogeneous_cpus)) return riscv_hwprobe(pairs, pair_count, cpusetsize, cpus, flags); /* This is something we can handle, fill out the pairs. */ diff --git a/arch/riscv/kernel/vdso/note.S b/arch/riscv/kernel/vdso/note.S index 2a956c942211..69bfe48be037 100644 --- a/arch/riscv/kernel/vdso/note.S +++ b/arch/riscv/kernel/vdso/note.S @@ -4,9 +4,15 @@ * Here we can supply some information useful to userland. */ +#include <linux/build-salt.h> #include <linux/elfnote.h> #include <linux/version.h> +#include <asm/assembler.h> ELFNOTE_START(Linux, 0, "a") .long LINUX_VERSION_CODE ELFNOTE_END + +BUILD_SALT + +emit_riscv_feature_1_and diff --git a/arch/riscv/kernel/vdso/rt_sigreturn.S b/arch/riscv/kernel/vdso/rt_sigreturn.S index 3dc022aa8931..d6f96b1abe40 100644 --- a/arch/riscv/kernel/vdso/rt_sigreturn.S +++ b/arch/riscv/kernel/vdso/rt_sigreturn.S @@ -5,7 +5,17 @@ #include <linux/linkage.h> #include <asm/unistd.h> +#include <asm/assembler.h> +/* + * WARNING: Do NOT add a CFI landing pad at the start of this function. + * Unwinders such as libgcc identify the sigreturn trampoline by matching the + * instruction sequence. Adding a landing pad here would break unwinding from + * signal handlers. + * + * This trampoline is used only for signal return and not via an indirect + * call/jump from userspace, so adding CFI landing pad is unnecessary. + */ .text SYM_FUNC_START(__vdso_rt_sigreturn) .cfi_startproc @@ -14,3 +24,5 @@ SYM_FUNC_START(__vdso_rt_sigreturn) ecall .cfi_endproc SYM_FUNC_END(__vdso_rt_sigreturn) + +emit_riscv_feature_1_and diff --git a/arch/riscv/kernel/vdso/sys_hwprobe.S b/arch/riscv/kernel/vdso/sys_hwprobe.S index 77e57f830521..f1694451a60c 100644 --- a/arch/riscv/kernel/vdso/sys_hwprobe.S +++ b/arch/riscv/kernel/vdso/sys_hwprobe.S @@ -3,13 +3,17 @@ #include <linux/linkage.h> #include <asm/unistd.h> +#include <asm/assembler.h> .text SYM_FUNC_START(riscv_hwprobe) .cfi_startproc + vdso_lpad li a7, __NR_riscv_hwprobe ecall ret .cfi_endproc SYM_FUNC_END(riscv_hwprobe) + +emit_riscv_feature_1_and diff --git a/arch/riscv/kernel/vdso/vdso.lds.S b/arch/riscv/kernel/vdso/vdso.lds.S index c29ef12a63bb..5e9424df932b 100644 --- a/arch/riscv/kernel/vdso/vdso.lds.S +++ b/arch/riscv/kernel/vdso/vdso.lds.S @@ -45,9 +45,11 @@ SECTIONS .text : { *(.text .text.*) } :text . = ALIGN(4); + __vdso_alternatives_start = .; .alternative : { *(.alternative) } + __vdso_alternatives_end = .; } /* diff --git a/arch/riscv/kernel/vdso/vgetrandom-chacha.S b/arch/riscv/kernel/vdso/vgetrandom-chacha.S index 5f0dad8f2373..916ab30a88f7 100644 --- a/arch/riscv/kernel/vdso/vgetrandom-chacha.S +++ b/arch/riscv/kernel/vdso/vgetrandom-chacha.S @@ -7,6 +7,7 @@ #include <asm/asm.h> #include <linux/linkage.h> +#include <asm/assembler.h> .text @@ -74,7 +75,7 @@ SYM_FUNC_START(__arch_chacha20_blocks_nostack) #define _20 20, 20, 20, 20 #define _24 24, 24, 24, 24 #define _25 25, 25, 25, 25 - + vdso_lpad /* * The ABI requires s0-s9 saved. * This does not violate the stack-less requirement: no sensitive data @@ -247,3 +248,5 @@ SYM_FUNC_START(__arch_chacha20_blocks_nostack) ret SYM_FUNC_END(__arch_chacha20_blocks_nostack) + +emit_riscv_feature_1_and diff --git a/arch/riscv/kernel/vdso_cfi/.gitignore b/arch/riscv/kernel/vdso_cfi/.gitignore new file mode 100644 index 000000000000..220b6ebece4f --- /dev/null +++ b/arch/riscv/kernel/vdso_cfi/.gitignore @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copied source files from the main vdso directory +*.c +*.S +!vdso-cfi.S +vdso.lds +*.tmp +vdso-syms.S diff --git a/arch/riscv/kernel/vdso_cfi/Makefile b/arch/riscv/kernel/vdso_cfi/Makefile new file mode 100644 index 000000000000..10292498b765 --- /dev/null +++ b/arch/riscv/kernel/vdso_cfi/Makefile @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: GPL-2.0-only +# RISC-V VDSO CFI Makefile +# This Makefile builds the VDSO with CFI support when CONFIG_RISCV_USER_CFI is enabled + +# setting VDSO_CFI_BUILD triggers build for vdso differently +VDSO_CFI_BUILD := 1 + +# Set the source directory to the main vdso directory +src := $(srctree)/arch/riscv/kernel/vdso + +# Copy all .S and .c files from vdso directory to vdso_cfi object build directory +vdso_c_sources := $(wildcard $(src)/*.c) +vdso_S_sources := $(wildcard $(src)/*.S) +vdso_c_objects := $(addprefix $(obj)/, $(notdir $(vdso_c_sources))) +vdso_S_objects := $(addprefix $(obj)/, $(notdir $(vdso_S_sources))) + +$(vdso_S_objects): $(obj)/%.S: $(src)/%.S + $(Q)cp $< $@ + +$(vdso_c_objects): $(obj)/%.c: $(src)/%.c + $(Q)cp $< $@ + +# Include the main VDSO Makefile which contains all the build rules and sources +# The VDSO_CFI_BUILD variable will be passed to it to enable CFI compilation +include $(src)/Makefile + +# Clean rules - remove the copied source files +clean-files += $(notdir $(vdso_c_sources)) $(notdir $(vdso_S_sources)) diff --git a/arch/riscv/kernel/vdso_cfi/vdso-cfi.S b/arch/riscv/kernel/vdso_cfi/vdso-cfi.S new file mode 100644 index 000000000000..d426f6accb35 --- /dev/null +++ b/arch/riscv/kernel/vdso_cfi/vdso-cfi.S @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2025 Rivos, Inc + */ + +#define vdso_start vdso_cfi_start +#define vdso_end vdso_cfi_end + +#define __VDSO_PATH "arch/riscv/kernel/vdso_cfi/vdso-cfi.so" + +#include "../vdso/vdso.S" diff --git a/arch/riscv/kernel/vec-copy-unaligned.S b/arch/riscv/kernel/vec-copy-unaligned.S index 7ce4de6f6e69..361039f7b944 100644 --- a/arch/riscv/kernel/vec-copy-unaligned.S +++ b/arch/riscv/kernel/vec-copy-unaligned.S @@ -2,6 +2,7 @@ /* Copyright (C) 2024 Rivos Inc. */ #include <linux/args.h> +#include <linux/cfi_types.h> #include <linux/linkage.h> #include <asm/asm.h> @@ -16,7 +17,7 @@ /* void __riscv_copy_vec_words_unaligned(void *, const void *, size_t) */ /* Performs a memcpy without aligning buffers, using word loads and stores. */ /* Note: The size is truncated to a multiple of WORD_EEW */ -SYM_FUNC_START(__riscv_copy_vec_words_unaligned) +SYM_TYPED_FUNC_START(__riscv_copy_vec_words_unaligned) andi a4, a2, ~(WORD_EEW-1) beqz a4, 2f add a3, a1, a4 @@ -38,7 +39,7 @@ SYM_FUNC_END(__riscv_copy_vec_words_unaligned) /* void __riscv_copy_vec_bytes_unaligned(void *, const void *, size_t) */ /* Performs a memcpy without aligning buffers, using only byte accesses. */ /* Note: The size is truncated to a multiple of 8 */ -SYM_FUNC_START(__riscv_copy_vec_bytes_unaligned) +SYM_TYPED_FUNC_START(__riscv_copy_vec_bytes_unaligned) andi a4, a2, ~(8-1) beqz a4, 2f add a3, a1, a4 diff --git a/arch/riscv/kernel/vector.c b/arch/riscv/kernel/vector.c index 3ed071dab9d8..b112166d51e9 100644 --- a/arch/riscv/kernel/vector.c +++ b/arch/riscv/kernel/vector.c @@ -111,8 +111,8 @@ bool insn_is_vector(u32 insn_buf) return false; } -static int riscv_v_thread_zalloc(struct kmem_cache *cache, - struct __riscv_v_ext_state *ctx) +static int riscv_v_thread_ctx_alloc(struct kmem_cache *cache, + struct __riscv_v_ext_state *ctx) { void *datap; @@ -122,13 +122,15 @@ static int riscv_v_thread_zalloc(struct kmem_cache *cache, ctx->datap = datap; memset(ctx, 0, offsetof(struct __riscv_v_ext_state, datap)); + ctx->vlenb = riscv_v_vsize / 32; + return 0; } void riscv_v_thread_alloc(struct task_struct *tsk) { #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE - riscv_v_thread_zalloc(riscv_v_kernel_cachep, &tsk->thread.kernel_vstate); + riscv_v_thread_ctx_alloc(riscv_v_kernel_cachep, &tsk->thread.kernel_vstate); #endif } @@ -214,12 +216,14 @@ bool riscv_v_first_use_handler(struct pt_regs *regs) * context where VS has been off. So, try to allocate the user's V * context and resume execution. */ - if (riscv_v_thread_zalloc(riscv_v_user_cachep, ¤t->thread.vstate)) { + if (riscv_v_thread_ctx_alloc(riscv_v_user_cachep, ¤t->thread.vstate)) { force_sig(SIGBUS); return true; } + riscv_v_vstate_on(regs); riscv_v_vstate_set_restore(current, regs); + return true; } diff --git a/arch/riscv/kernel/vmcore_info.c b/arch/riscv/kernel/vmcore_info.c index d5e448aa90e7..c27efceec3cc 100644 --- a/arch/riscv/kernel/vmcore_info.c +++ b/arch/riscv/kernel/vmcore_info.c @@ -3,6 +3,11 @@ #include <linux/vmcore_info.h> #include <linux/pagemap.h> +static inline u64 get_satp_value(void) +{ + return csr_read(CSR_SATP); +} + void arch_crash_save_vmcoreinfo(void) { VMCOREINFO_NUMBER(phys_ram_base); @@ -19,13 +24,8 @@ void arch_crash_save_vmcoreinfo(void) #endif #endif vmcoreinfo_append_str("NUMBER(KERNEL_LINK_ADDR)=0x%lx\n", KERNEL_LINK_ADDR); -#ifdef CONFIG_XIP_KERNEL - /* TODO: Communicate with crash-utility developers on the information to - * export. The XIP case is more complicated, because the virtual-physical - * address offset depends on whether the address is in ROM or in RAM. - */ -#else vmcoreinfo_append_str("NUMBER(va_kernel_pa_offset)=0x%lx\n", kernel_map.va_kernel_pa_offset); -#endif + vmcoreinfo_append_str("KERNELOFFSET=%lx\n", kaslr_offset()); + vmcoreinfo_append_str("NUMBER(satp)=0x%llx\n", get_satp_value()); } diff --git a/arch/riscv/kernel/vmlinux-xip.lds.S b/arch/riscv/kernel/vmlinux-xip.lds.S deleted file mode 100644 index a7611789bad5..000000000000 --- a/arch/riscv/kernel/vmlinux-xip.lds.S +++ /dev/null @@ -1,143 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 Regents of the University of California - * Copyright (C) 2017 SiFive - * Copyright (C) 2020 Vitaly Wool, Konsulko AB - */ - -#include <asm/pgtable.h> -#define LOAD_OFFSET KERNEL_LINK_ADDR -/* No __ro_after_init data in the .rodata section - which will always be ro */ -#define RO_AFTER_INIT_DATA - -#include <asm/vmlinux.lds.h> -#include <asm/page.h> -#include <asm/cache.h> -#include <asm/thread_info.h> -#include <asm/set_memory.h> - -OUTPUT_ARCH(riscv) -ENTRY(_start) - -jiffies = jiffies_64; - -SECTIONS -{ - /* Beginning of code and text segment */ - . = LOAD_OFFSET; - _xiprom = .; - _start = .; - HEAD_TEXT_SECTION - INIT_TEXT_SECTION(PAGE_SIZE) - /* we have to discard exit text and such at runtime, not link time */ - __exittext_begin = .; - .exit.text : - { - EXIT_TEXT - } - __exittext_end = .; - - .text : { - _text = .; - _stext = .; - TEXT_TEXT - SCHED_TEXT - LOCK_TEXT - KPROBES_TEXT - ENTRY_TEXT - IRQENTRY_TEXT - SOFTIRQENTRY_TEXT - _etext = .; - } - RO_DATA(L1_CACHE_BYTES) - .srodata : { - *(.srodata*) - } - .init.rodata : { - INIT_SETUP(16) - INIT_CALLS - CON_INITCALL - INIT_RAM_FS - } - _exiprom = .; /* End of XIP ROM area */ - - -/* - * From this point, stuff is considered writable and will be copied to RAM - */ - __data_loc = ALIGN(PAGE_SIZE); /* location in file */ - . = ALIGN(SECTION_ALIGN); /* location in memory */ - -#undef LOAD_OFFSET -#define LOAD_OFFSET (KERNEL_LINK_ADDR + _sdata - __data_loc) - - _sdata = .; /* Start of data section */ - _data = .; - RW_DATA(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE) - _edata = .; - __start_ro_after_init = .; - .data.ro_after_init : AT(ADDR(.data.ro_after_init) - LOAD_OFFSET) { - *(.data..ro_after_init) - } - __end_ro_after_init = .; - - . = ALIGN(PAGE_SIZE); - __init_begin = .; - .init.data : { - INIT_DATA - } - .exit.data : { - EXIT_DATA - } - . = ALIGN(8); - __soc_early_init_table : { - __soc_early_init_table_start = .; - KEEP(*(__soc_early_init_table)) - __soc_early_init_table_end = .; - } - __soc_builtin_dtb_table : { - __soc_builtin_dtb_table_start = .; - KEEP(*(__soc_builtin_dtb_table)) - __soc_builtin_dtb_table_end = .; - } - - __init_end = .; - - . = ALIGN(16); - .xip.traps : { - __xip_traps_start = .; - *(.xip.traps) - __xip_traps_end = .; - } - - . = ALIGN(PAGE_SIZE); - .sdata : { - __global_pointer$ = . + 0x800; - *(.sdata*) - *(.sbss*) - } - - BSS_SECTION(PAGE_SIZE, PAGE_SIZE, 0) - - PERCPU_SECTION(L1_CACHE_BYTES) - - .rel.dyn : AT(ADDR(.rel.dyn) - LOAD_OFFSET) { - *(.rel.dyn*) - } - - /* - * End of copied data. We need a dummy section to get its LMA. - * Also located before final ALIGN() as trailing padding is not stored - * in the resulting binary file and useless to copy. - */ - .data.endmark : AT(ADDR(.data.endmark) - LOAD_OFFSET) { } - _edata_loc = LOADADDR(.data.endmark); - - . = ALIGN(PAGE_SIZE); - _end = .; - - STABS_DEBUG - DWARF_DEBUG - - DISCARDS -} diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 61bd5ba6680a..1f4f8496941a 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -7,10 +7,6 @@ #define RO_EXCEPTION_TABLE_ALIGN 4 #define RUNTIME_DISCARD_EXIT -#ifdef CONFIG_XIP_KERNEL -#include "vmlinux-xip.lds.S" -#else - #include <asm/pgtable.h> #define LOAD_OFFSET KERNEL_LINK_ADDR @@ -170,9 +166,9 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS .riscv.attributes 0 : { *(.riscv.attributes) } DISCARDS } -#endif /* CONFIG_XIP_KERNEL */ diff --git a/arch/riscv/kvm/Kconfig b/arch/riscv/kvm/Kconfig index 77379f77840a..ec2cee0a39e0 100644 --- a/arch/riscv/kvm/Kconfig +++ b/arch/riscv/kvm/Kconfig @@ -30,7 +30,6 @@ config KVM select KVM_GENERIC_HARDWARE_ENABLING select KVM_MMIO select VIRT_XFER_TO_GUEST_WORK - select KVM_GENERIC_MMU_NOTIFIER select SCHED_INFO select GUEST_PERF_EVENTS if PERF_EVENTS help diff --git a/arch/riscv/kvm/Makefile b/arch/riscv/kvm/Makefile index 3b8afb038b35..296c2ba05089 100644 --- a/arch/riscv/kvm/Makefile +++ b/arch/riscv/kvm/Makefile @@ -15,11 +15,13 @@ kvm-y += aia_aplic.o kvm-y += aia_device.o kvm-y += aia_imsic.o kvm-y += gstage.o +kvm-y += isa.o kvm-y += main.o kvm-y += mmu.o kvm-y += nacl.o kvm-y += tlb.o kvm-y += vcpu.o +kvm-y += vcpu_config.o kvm-y += vcpu_exit.o kvm-y += vcpu_fp.o kvm-y += vcpu_insn.o diff --git a/arch/riscv/kvm/aia.c b/arch/riscv/kvm/aia.c index dad318185660..b3b466e4db33 100644 --- a/arch/riscv/kvm/aia.c +++ b/arch/riscv/kvm/aia.c @@ -13,6 +13,7 @@ #include <linux/irqchip/riscv-imsic.h> #include <linux/irqdomain.h> #include <linux/kvm_host.h> +#include <linux/nospec.h> #include <linux/percpu.h> #include <linux/spinlock.h> #include <asm/cpufeature.h> @@ -20,13 +21,16 @@ struct aia_hgei_control { raw_spinlock_t lock; + bool free_bitmap_initialized; unsigned long free_bitmap; struct kvm_vcpu *owners[BITS_PER_LONG]; + unsigned int nr_hgei; + unsigned long saved_hgeie; }; static DEFINE_PER_CPU(struct aia_hgei_control, aia_hgei); static int hgei_parent_irq; -unsigned int kvm_riscv_aia_nr_hgei; +atomic_t kvm_riscv_aia_nr_hgei; unsigned int kvm_riscv_aia_max_ids; DEFINE_STATIC_KEY_FALSE(kvm_riscv_aia_available); @@ -50,12 +54,15 @@ void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu) struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; unsigned long mask, val; + lockdep_assert_held(&vcpu->arch.irqs_pending_lock); + if (!kvm_riscv_aia_available()) return; - if (READ_ONCE(vcpu->arch.irqs_pending_mask[1])) { - mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[1], 0); - val = READ_ONCE(vcpu->arch.irqs_pending[1]) & mask; + mask = vcpu->arch.irqs_pending_mask[1]; + if (mask) { + vcpu->arch.irqs_pending_mask[1] = 0; + val = vcpu->arch.irqs_pending[1] & mask; csr->hviph &= ~mask; csr->hviph |= val; @@ -66,6 +73,8 @@ void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu) { struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; + lockdep_assert_held(&vcpu->arch.irqs_pending_lock); + if (kvm_riscv_aia_available()) csr->vsieh = ncsr_read(CSR_VSIEH); } @@ -74,13 +83,22 @@ void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu) bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) { unsigned long seip; +#ifdef CONFIG_32BIT + unsigned long flags; + bool pending; +#endif if (!kvm_riscv_aia_available()) return false; #ifdef CONFIG_32BIT - if (READ_ONCE(vcpu->arch.irqs_pending[1]) & - (vcpu->arch.aia_context.guest_csr.vsieh & upper_32_bits(mask))) + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + pending = vcpu->arch.irqs_pending[1] & + (vcpu->arch.aia_context.guest_csr.vsieh & + upper_32_bits(mask)); + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); + + if (pending) return true; #endif @@ -182,9 +200,14 @@ int kvm_riscv_vcpu_aia_get_csr(struct kvm_vcpu *vcpu, unsigned long *out_val) { struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; + unsigned long regs_max = sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); - if (reg_num >= sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long)) + if (!riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) return -ENOENT; + if (reg_num >= regs_max) + return -ENOENT; + + reg_num = array_index_nospec(reg_num, regs_max); *out_val = 0; if (kvm_riscv_aia_available()) @@ -198,16 +221,28 @@ int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu, unsigned long val) { struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; + unsigned long regs_max = sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); +#ifdef CONFIG_32BIT + unsigned long flags; +#endif - if (reg_num >= sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long)) + if (!riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) return -ENOENT; + if (reg_num >= regs_max) + return -ENOENT; + + reg_num = array_index_nospec(reg_num, regs_max); if (kvm_riscv_aia_available()) { ((unsigned long *)csr)[reg_num] = val; #ifdef CONFIG_32BIT - if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph)) - WRITE_ONCE(vcpu->arch.irqs_pending_mask[1], 0); + if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph)) { + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + vcpu->arch.irqs_pending_mask[1] = 0; + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, + flags); + } #endif } @@ -441,7 +476,7 @@ void kvm_riscv_aia_free_hgei(int cpu, int hgei) raw_spin_lock_irqsave(&hgctrl->lock, flags); - if (hgei > 0 && hgei <= kvm_riscv_aia_nr_hgei) { + if (hgei > 0 && hgei <= hgctrl->nr_hgei) { if (!(hgctrl->free_bitmap & BIT(hgei))) { hgctrl->free_bitmap |= BIT(hgei); hgctrl->owners[hgei] = NULL; @@ -475,26 +510,18 @@ static irqreturn_t hgei_interrupt(int irq, void *dev_id) static int aia_hgei_init(void) { - int cpu, rc; - struct irq_domain *domain; struct aia_hgei_control *hgctrl; + struct irq_domain *domain; + int cpu, rc; /* Initialize per-CPU guest external interrupt line management */ for_each_possible_cpu(cpu) { hgctrl = per_cpu_ptr(&aia_hgei, cpu); raw_spin_lock_init(&hgctrl->lock); - if (kvm_riscv_aia_nr_hgei) { - hgctrl->free_bitmap = - BIT(kvm_riscv_aia_nr_hgei + 1) - 1; - hgctrl->free_bitmap &= ~BIT(0); - } else - hgctrl->free_bitmap = 0; + hgctrl->free_bitmap_initialized = false; + hgctrl->free_bitmap = 0; } - /* Skip SGEI interrupt setup for zero guest external interrupts */ - if (!kvm_riscv_aia_nr_hgei) - goto skip_sgei_interrupt; - /* Find INTC irq domain */ domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), DOMAIN_BUS_ANY); @@ -518,25 +545,101 @@ static int aia_hgei_init(void) return rc; } -skip_sgei_interrupt: return 0; } static void aia_hgei_exit(void) { - /* Do nothing for zero guest external interrupts */ - if (!kvm_riscv_aia_nr_hgei) - return; - /* Free per-CPU SGEI interrupt */ free_percpu_irq(hgei_parent_irq, &aia_hgei); } +void kvm_riscv_aia_pm_exit(void) +{ + struct aia_hgei_control *hgctrl; + + if (!kvm_riscv_aia_available()) + return; + + hgctrl = this_cpu_ptr(&aia_hgei); + csr_write(CSR_HGEIE, hgctrl->saved_hgeie); + + csr_write(CSR_HVICTL, aia_hvictl_value(false)); + csr_write(CSR_HVIPRIO1, 0x0); + csr_write(CSR_HVIPRIO2, 0x0); +#ifdef CONFIG_32BIT + csr_write(CSR_HVIPH, 0x0); + csr_write(CSR_HIDELEGH, 0x0); + csr_write(CSR_HVIPRIO1H, 0x0); + csr_write(CSR_HVIPRIO2H, 0x0); +#endif + csr_set(CSR_HIE, BIT(IRQ_S_GEXT)); + /* Enable IRQ filtering for overflow interrupt only if sscofpmf is present */ + if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF)) + csr_set(CSR_HVIEN, BIT(IRQ_PMU_OVF)); +} + +void kvm_riscv_aia_pm_enter(void) +{ + struct aia_hgei_control *hgctrl; + + if (!kvm_riscv_aia_available()) + return; + + if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF)) + csr_clear(CSR_HVIEN, BIT(IRQ_PMU_OVF)); + + csr_write(CSR_HVICTL, aia_hvictl_value(false)); + + hgctrl = this_cpu_ptr(&aia_hgei); + hgctrl->saved_hgeie = csr_read(CSR_HGEIE); +} + void kvm_riscv_aia_enable(void) { + const struct imsic_global_config *gc; + const struct imsic_local_config *lc; + struct aia_hgei_control *hgctrl; + unsigned long flags; + int aia_nr_hgei; + if (!kvm_riscv_aia_available()) return; + gc = imsic_get_global_config(); + lc = (gc) ? this_cpu_ptr(gc->local) : NULL; + hgctrl = this_cpu_ptr(&aia_hgei); + + /* Figure-out number of bits in HGEIE */ + csr_write(CSR_HGEIE, -1UL); + hgctrl->nr_hgei = fls_long(csr_read(CSR_HGEIE)); + csr_write(CSR_HGEIE, 0); + if (hgctrl->nr_hgei) + hgctrl->nr_hgei--; + + /* + * Number of usable per-HART HGEI lines should be minimum of + * per-HART IMSIC guest files and number of bits in HGEIE. + */ + if (lc) + hgctrl->nr_hgei = min((ulong)hgctrl->nr_hgei, lc->nr_guest_files); + else + hgctrl->nr_hgei = 0; + + /* Update the number of IMSIC guest files across all HARTs */ + aia_nr_hgei = atomic_read(&kvm_riscv_aia_nr_hgei); + do { + if (aia_nr_hgei <= hgctrl->nr_hgei) + break; + } while (!atomic_try_cmpxchg(&kvm_riscv_aia_nr_hgei, &aia_nr_hgei, hgctrl->nr_hgei)); + + raw_spin_lock_irqsave(&hgctrl->lock, flags); + if (!hgctrl->free_bitmap_initialized) { + hgctrl->free_bitmap = (hgctrl->nr_hgei) ? GENMASK_ULL(hgctrl->nr_hgei, 1) : 0; + hgctrl->free_bitmap_initialized = true; + } + raw_spin_unlock_irqrestore(&hgctrl->lock, flags); + csr_write(CSR_HVICTL, aia_hvictl_value(false)); csr_write(CSR_HVIPRIO1, 0x0); csr_write(CSR_HVIPRIO2, 0x0); @@ -577,7 +680,7 @@ void kvm_riscv_aia_disable(void) raw_spin_lock_irqsave(&hgctrl->lock, flags); - for (i = 0; i <= kvm_riscv_aia_nr_hgei; i++) { + for (i = 0; i <= hgctrl->nr_hgei; i++) { vcpu = hgctrl->owners[i]; if (!vcpu) continue; @@ -617,26 +720,15 @@ int kvm_riscv_aia_init(void) return -ENODEV; gc = imsic_get_global_config(); - /* Figure-out number of bits in HGEIE */ - csr_write(CSR_HGEIE, -1UL); - kvm_riscv_aia_nr_hgei = fls_long(csr_read(CSR_HGEIE)); - csr_write(CSR_HGEIE, 0); - if (kvm_riscv_aia_nr_hgei) - kvm_riscv_aia_nr_hgei--; - - /* - * Number of usable HGEI lines should be minimum of per-HART - * IMSIC guest files and number of bits in HGEIE - */ + /* Set initial value of IMSIC guest files across all HARTs */ if (gc) - kvm_riscv_aia_nr_hgei = min((ulong)kvm_riscv_aia_nr_hgei, - BIT(gc->guest_index_bits) - 1); + atomic_set(&kvm_riscv_aia_nr_hgei, gc->nr_guest_files); else - kvm_riscv_aia_nr_hgei = 0; + atomic_set(&kvm_riscv_aia_nr_hgei, 0); /* Find number of guest MSI IDs */ kvm_riscv_aia_max_ids = IMSIC_MAX_ID; - if (gc && kvm_riscv_aia_nr_hgei) + if (gc) kvm_riscv_aia_max_ids = gc->nr_guest_ids + 1; /* Initialize guest external interrupt line management */ diff --git a/arch/riscv/kvm/aia_aplic.c b/arch/riscv/kvm/aia_aplic.c index f59d1c0c8c43..be902ea6480b 100644 --- a/arch/riscv/kvm/aia_aplic.c +++ b/arch/riscv/kvm/aia_aplic.c @@ -10,6 +10,7 @@ #include <linux/irqchip/riscv-aplic.h> #include <linux/kvm_host.h> #include <linux/math.h> +#include <linux/nospec.h> #include <linux/spinlock.h> #include <linux/swab.h> #include <kvm/iodev.h> @@ -34,7 +35,7 @@ struct aplic { u32 nr_irqs; u32 nr_words; - struct aplic_irq *irqs; + struct aplic_irq irqs[] __counted_by(nr_irqs); }; static u32 aplic_read_sourcecfg(struct aplic *aplic, u32 irq) @@ -45,7 +46,7 @@ static u32 aplic_read_sourcecfg(struct aplic *aplic, u32 irq) if (!irq || aplic->nr_irqs <= irq) return 0; - irqd = &aplic->irqs[irq]; + irqd = &aplic->irqs[array_index_nospec(irq, aplic->nr_irqs)]; raw_spin_lock_irqsave(&irqd->lock, flags); ret = irqd->sourcecfg; @@ -61,7 +62,7 @@ static void aplic_write_sourcecfg(struct aplic *aplic, u32 irq, u32 val) if (!irq || aplic->nr_irqs <= irq) return; - irqd = &aplic->irqs[irq]; + irqd = &aplic->irqs[array_index_nospec(irq, aplic->nr_irqs)]; if (val & APLIC_SOURCECFG_D) val = 0; @@ -81,7 +82,7 @@ static u32 aplic_read_target(struct aplic *aplic, u32 irq) if (!irq || aplic->nr_irqs <= irq) return 0; - irqd = &aplic->irqs[irq]; + irqd = &aplic->irqs[array_index_nospec(irq, aplic->nr_irqs)]; raw_spin_lock_irqsave(&irqd->lock, flags); ret = irqd->target; @@ -97,7 +98,7 @@ static void aplic_write_target(struct aplic *aplic, u32 irq, u32 val) if (!irq || aplic->nr_irqs <= irq) return; - irqd = &aplic->irqs[irq]; + irqd = &aplic->irqs[array_index_nospec(irq, aplic->nr_irqs)]; val &= APLIC_TARGET_EIID_MASK | (APLIC_TARGET_HART_IDX_MASK << APLIC_TARGET_HART_IDX_SHIFT) | @@ -116,7 +117,7 @@ static bool aplic_read_pending(struct aplic *aplic, u32 irq) if (!irq || aplic->nr_irqs <= irq) return false; - irqd = &aplic->irqs[irq]; + irqd = &aplic->irqs[array_index_nospec(irq, aplic->nr_irqs)]; raw_spin_lock_irqsave(&irqd->lock, flags); ret = (irqd->state & APLIC_IRQ_STATE_PENDING) ? true : false; @@ -132,7 +133,7 @@ static void aplic_write_pending(struct aplic *aplic, u32 irq, bool pending) if (!irq || aplic->nr_irqs <= irq) return; - irqd = &aplic->irqs[irq]; + irqd = &aplic->irqs[array_index_nospec(irq, aplic->nr_irqs)]; raw_spin_lock_irqsave(&irqd->lock, flags); @@ -170,7 +171,7 @@ static bool aplic_read_enabled(struct aplic *aplic, u32 irq) if (!irq || aplic->nr_irqs <= irq) return false; - irqd = &aplic->irqs[irq]; + irqd = &aplic->irqs[array_index_nospec(irq, aplic->nr_irqs)]; raw_spin_lock_irqsave(&irqd->lock, flags); ret = (irqd->state & APLIC_IRQ_STATE_ENABLED) ? true : false; @@ -186,7 +187,7 @@ static void aplic_write_enabled(struct aplic *aplic, u32 irq, bool enabled) if (!irq || aplic->nr_irqs <= irq) return; - irqd = &aplic->irqs[irq]; + irqd = &aplic->irqs[array_index_nospec(irq, aplic->nr_irqs)]; raw_spin_lock_irqsave(&irqd->lock, flags); if (enabled) @@ -205,7 +206,7 @@ static bool aplic_read_input(struct aplic *aplic, u32 irq) if (!irq || aplic->nr_irqs <= irq) return false; - irqd = &aplic->irqs[irq]; + irqd = &aplic->irqs[array_index_nospec(irq, aplic->nr_irqs)]; raw_spin_lock_irqsave(&irqd->lock, flags); @@ -254,7 +255,7 @@ static void aplic_update_irq_range(struct kvm *kvm, u32 first, u32 last) for (irq = first; irq <= last; irq++) { if (!irq || aplic->nr_irqs <= irq) continue; - irqd = &aplic->irqs[irq]; + irqd = &aplic->irqs[array_index_nospec(irq, aplic->nr_irqs)]; raw_spin_lock_irqsave(&irqd->lock, flags); @@ -283,7 +284,7 @@ int kvm_riscv_aia_aplic_inject(struct kvm *kvm, u32 source, bool level) if (!aplic || !source || (aplic->nr_irqs <= source)) return -ENODEV; - irqd = &aplic->irqs[source]; + irqd = &aplic->irqs[array_index_nospec(source, aplic->nr_irqs)]; ie = (aplic->domaincfg & APLIC_DOMAINCFG_IE) ? true : false; raw_spin_lock_irqsave(&irqd->lock, flags); @@ -580,7 +581,8 @@ int kvm_riscv_aia_aplic_init(struct kvm *kvm) return 0; /* Allocate APLIC global state */ - aplic = kzalloc(sizeof(*aplic), GFP_KERNEL); + aplic = kzalloc_flex(*aplic, irqs, kvm->arch.aia.nr_sources + 1, + GFP_KERNEL_ACCOUNT); if (!aplic) return -ENOMEM; kvm->arch.aia.aplic_state = aplic; @@ -588,12 +590,6 @@ int kvm_riscv_aia_aplic_init(struct kvm *kvm) /* Setup APLIC IRQs */ aplic->nr_irqs = kvm->arch.aia.nr_sources + 1; aplic->nr_words = DIV_ROUND_UP(aplic->nr_irqs, 32); - aplic->irqs = kcalloc(aplic->nr_irqs, - sizeof(*aplic->irqs), GFP_KERNEL); - if (!aplic->irqs) { - ret = -ENOMEM; - goto fail_free_aplic; - } for (i = 0; i < aplic->nr_irqs; i++) raw_spin_lock_init(&aplic->irqs[i].lock); @@ -606,7 +602,7 @@ int kvm_riscv_aia_aplic_init(struct kvm *kvm) &aplic->iodev); mutex_unlock(&kvm->slots_lock); if (ret) - goto fail_free_aplic_irqs; + goto fail_free_aplic; /* Setup default IRQ routing */ ret = kvm_riscv_setup_default_irq_routing(kvm, aplic->nr_irqs); @@ -619,8 +615,6 @@ fail_unreg_iodev: mutex_lock(&kvm->slots_lock); kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &aplic->iodev); mutex_unlock(&kvm->slots_lock); -fail_free_aplic_irqs: - kfree(aplic->irqs); fail_free_aplic: kvm->arch.aia.aplic_state = NULL; kfree(aplic); @@ -638,8 +632,6 @@ void kvm_riscv_aia_aplic_cleanup(struct kvm *kvm) kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &aplic->iodev); mutex_unlock(&kvm->slots_lock); - kfree(aplic->irqs); - kvm->arch.aia.aplic_state = NULL; kfree(aplic); } diff --git a/arch/riscv/kvm/aia_device.c b/arch/riscv/kvm/aia_device.c index b195a93add1c..efc7c0bcfba9 100644 --- a/arch/riscv/kvm/aia_device.c +++ b/arch/riscv/kvm/aia_device.c @@ -11,6 +11,7 @@ #include <linux/irqchip/riscv-imsic.h> #include <linux/kvm_host.h> #include <linux/uaccess.h> +#include <asm/kvm_isa.h> static int aia_create(struct kvm_device *dev, u32 type) { @@ -22,6 +23,9 @@ static int aia_create(struct kvm_device *dev, u32 type) if (irqchip_in_kernel(kvm)) return -EEXIST; + if (kvm_riscv_isa_check_host(SSAIA)) + return -ENODEV; + ret = -EBUSY; if (kvm_trylock_all_vcpus(kvm)) return ret; @@ -67,7 +71,7 @@ static int aia_config(struct kvm *kvm, unsigned long type, * external interrupts (i.e. non-zero * VS-level IMSIC pages). */ - if (!kvm_riscv_aia_nr_hgei) + if (!atomic_read(&kvm_riscv_aia_nr_hgei)) return -EINVAL; break; default: @@ -244,6 +248,12 @@ static int aia_init(struct kvm *kvm) if (aia->nr_sources && aia->aplic_addr == KVM_RISCV_AIA_UNDEF_ADDR) return -EINVAL; + /* Group index bits must not overlap guest and HART index bits. */ + if (aia->nr_group_bits && + aia->nr_group_shift < (IMSIC_MMIO_PAGE_SHIFT + + aia->nr_guest_bits + aia->nr_hart_bits)) + return -EINVAL; + /* Initialize APLIC */ ret = kvm_riscv_aia_aplic_init(kvm); if (ret) @@ -437,7 +447,7 @@ static int aia_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr) static int aia_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr) { - int nr_vcpus; + int nr_vcpus, r = -ENXIO; switch (attr->group) { case KVM_DEV_RISCV_AIA_GRP_CONFIG: @@ -466,12 +476,18 @@ static int aia_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr) } break; case KVM_DEV_RISCV_AIA_GRP_APLIC: - return kvm_riscv_aia_aplic_has_attr(dev->kvm, attr->attr); + mutex_lock(&dev->kvm->lock); + r = kvm_riscv_aia_aplic_has_attr(dev->kvm, attr->attr); + mutex_unlock(&dev->kvm->lock); + break; case KVM_DEV_RISCV_AIA_GRP_IMSIC: - return kvm_riscv_aia_imsic_has_attr(dev->kvm, attr->attr); + mutex_lock(&dev->kvm->lock); + r = kvm_riscv_aia_imsic_has_attr(dev->kvm, attr->attr); + mutex_unlock(&dev->kvm->lock); + break; } - return -ENXIO; + return r; } struct kvm_device_ops kvm_riscv_aia_device_ops = { @@ -618,7 +634,7 @@ void kvm_riscv_aia_init_vm(struct kvm *kvm) */ /* Initialize default values in AIA global context */ - aia->mode = (kvm_riscv_aia_nr_hgei) ? + aia->mode = (atomic_read(&kvm_riscv_aia_nr_hgei)) ? KVM_DEV_RISCV_AIA_MODE_AUTO : KVM_DEV_RISCV_AIA_MODE_EMUL; aia->nr_ids = kvm_riscv_aia_max_ids - 1; aia->nr_sources = 0; diff --git a/arch/riscv/kvm/aia_imsic.c b/arch/riscv/kvm/aia_imsic.c index e597e86491c3..f17dc559a709 100644 --- a/arch/riscv/kvm/aia_imsic.c +++ b/arch/riscv/kvm/aia_imsic.c @@ -220,15 +220,11 @@ static unsigned long imsic_mrif_atomic_rmw(struct imsic_mrif *mrif, { unsigned long old_val = 0, tmp = 0; - __asm__ __volatile__ ( - "0: lr.w.aq %1, %0\n" - " and %2, %1, %3\n" - " or %2, %2, %4\n" - " sc.w.rl %2, %2, %0\n" - " bnez %2, 0b" - : "+A" (*ptr), "+r" (old_val), "+r" (tmp) - : "r" (~wr_mask), "r" (new_val & wr_mask) - : "memory"); + new_val &= wr_mask; + old_val = READ_ONCE(*ptr); + do { + tmp = (old_val & ~wr_mask) | new_val; + } while (!try_cmpxchg(ptr, &old_val, tmp)); return old_val; } @@ -683,6 +679,9 @@ bool kvm_riscv_vcpu_aia_imsic_has_interrupt(struct kvm_vcpu *vcpu) unsigned long flags; bool ret = false; + if (!imsic) + return false; + /* * The IMSIC SW-file directly injects interrupt via hvip so * only check for interrupt when IMSIC VS-file is being used. @@ -722,6 +721,9 @@ void kvm_riscv_vcpu_aia_imsic_put(struct kvm_vcpu *vcpu) struct imsic *imsic = vcpu->arch.aia_context.imsic_state; unsigned long flags; + if (!imsic) + return; + if (!kvm_vcpu_is_blocking(vcpu)) return; @@ -738,6 +740,9 @@ void kvm_riscv_vcpu_aia_imsic_release(struct kvm_vcpu *vcpu) int old_vsfile_hgei, old_vsfile_cpu; struct imsic *imsic = vcpu->arch.aia_context.imsic_state; + if (!imsic) + return; + /* Read and clear IMSIC VS-file details */ write_lock_irqsave(&imsic->vsfile_lock, flags); old_vsfile_hgei = imsic->vsfile_hgei; @@ -797,6 +802,10 @@ int kvm_riscv_vcpu_aia_imsic_update(struct kvm_vcpu *vcpu) if (kvm->arch.aia.mode == KVM_DEV_RISCV_AIA_MODE_EMUL) return 1; + /* IMSIC vCPU state may not be initialized yet */ + if (!imsic) + return 1; + /* Read old IMSIC VS-file details */ read_lock_irqsave(&imsic->vsfile_lock, flags); old_vsfile_hgei = imsic->vsfile_hgei; @@ -904,6 +913,10 @@ int kvm_riscv_vcpu_aia_imsic_rmw(struct kvm_vcpu *vcpu, unsigned long isel, int r, rc = KVM_INSN_CONTINUE_NEXT_SEPC; struct imsic *imsic = vcpu->arch.aia_context.imsic_state; + /* If IMSIC vCPU state not initialized then forward to user space */ + if (!imsic) + return KVM_INSN_EXIT_TO_USER_SPACE; + if (isel == KVM_RISCV_AIA_IMSIC_TOPEI) { /* Read pending and enabled interrupt with highest priority */ topei = imsic_mrif_topei(imsic->swfile, imsic->nr_eix, @@ -952,8 +965,15 @@ int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type, if (!vcpu) return -ENODEV; - isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type); + if (mutex_lock_killable(&vcpu->mutex)) + return -EINTR; + imsic = vcpu->arch.aia_context.imsic_state; + if (!imsic) { + rc = -ENODEV; + goto out_unlock; + } + isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type); read_lock_irqsave(&imsic->vsfile_lock, flags); @@ -976,6 +996,8 @@ int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type, rc = imsic_vsfile_rw(vsfile_hgei, vsfile_cpu, imsic->nr_eix, isel, write, val); +out_unlock: + mutex_unlock(&vcpu->mutex); return rc; } @@ -993,8 +1015,11 @@ int kvm_riscv_aia_imsic_has_attr(struct kvm *kvm, unsigned long type) if (!vcpu) return -ENODEV; - isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type); imsic = vcpu->arch.aia_context.imsic_state; + if (!imsic) + return -ENODEV; + + isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type); return imsic_mrif_isel_check(imsic->nr_eix, isel); } @@ -1086,7 +1111,7 @@ int kvm_riscv_vcpu_aia_imsic_init(struct kvm_vcpu *vcpu) return -EINVAL; /* Allocate IMSIC context */ - imsic = kzalloc(sizeof(*imsic), GFP_KERNEL); + imsic = kzalloc_obj(*imsic, GFP_KERNEL_ACCOUNT); if (!imsic) return -ENOMEM; vcpu->arch.aia_context.imsic_state = imsic; @@ -1099,7 +1124,7 @@ int kvm_riscv_vcpu_aia_imsic_init(struct kvm_vcpu *vcpu) imsic->vsfile_hgei = imsic->vsfile_cpu = -1; /* Setup IMSIC SW-file */ - swfile_page = alloc_pages(GFP_KERNEL | __GFP_ZERO, + swfile_page = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO, get_order(sizeof(*imsic->swfile))); if (!swfile_page) { ret = -ENOMEM; diff --git a/arch/riscv/kvm/gstage.c b/arch/riscv/kvm/gstage.c index b67d60d722c2..e5002cb9cbef 100644 --- a/arch/riscv/kvm/gstage.c +++ b/arch/riscv/kvm/gstage.c @@ -5,29 +5,30 @@ */ #include <linux/bitops.h> +#include <linux/cpufeature.h> #include <linux/errno.h> #include <linux/kvm_host.h> #include <linux/module.h> #include <linux/pgtable.h> #include <asm/kvm_gstage.h> +#include <asm/hwcap.h> #ifdef CONFIG_64BIT -unsigned long kvm_riscv_gstage_mode __ro_after_init = HGATP_MODE_SV39X4; -unsigned long kvm_riscv_gstage_pgd_levels __ro_after_init = 3; +unsigned long kvm_riscv_gstage_max_pgd_levels __ro_after_init = 3; #else -unsigned long kvm_riscv_gstage_mode __ro_after_init = HGATP_MODE_SV32X4; -unsigned long kvm_riscv_gstage_pgd_levels __ro_after_init = 2; +unsigned long kvm_riscv_gstage_max_pgd_levels __ro_after_init = 2; #endif #define gstage_pte_leaf(__ptep) \ (pte_val(*(__ptep)) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC)) -static inline unsigned long gstage_pte_index(gpa_t addr, u32 level) +static inline unsigned long gstage_pte_index(struct kvm_gstage *gstage, + gpa_t addr, u32 level) { unsigned long mask; unsigned long shift = HGATP_PAGE_SHIFT + (kvm_riscv_gstage_index_bits * level); - if (level == (kvm_riscv_gstage_pgd_levels - 1)) + if (level == gstage->pgd_levels - 1) mask = (PTRS_PER_PTE * (1UL << kvm_riscv_gstage_pgd_xbits)) - 1; else mask = PTRS_PER_PTE - 1; @@ -40,12 +41,13 @@ static inline unsigned long gstage_pte_page_vaddr(pte_t pte) return (unsigned long)pfn_to_virt(__page_val_to_pfn(pte_val(pte))); } -static int gstage_page_size_to_level(unsigned long page_size, u32 *out_level) +static int gstage_page_size_to_level(struct kvm_gstage *gstage, unsigned long page_size, + u32 *out_level) { u32 i; unsigned long psz = 1UL << 12; - for (i = 0; i < kvm_riscv_gstage_pgd_levels; i++) { + for (i = 0; i < gstage->pgd_levels; i++) { if (page_size == (psz << (i * kvm_riscv_gstage_index_bits))) { *out_level = i; return 0; @@ -55,21 +57,23 @@ static int gstage_page_size_to_level(unsigned long page_size, u32 *out_level) return -EINVAL; } -static int gstage_level_to_page_order(u32 level, unsigned long *out_pgorder) +static int gstage_level_to_page_order(struct kvm_gstage *gstage, u32 level, + unsigned long *out_pgorder) { - if (kvm_riscv_gstage_pgd_levels < level) + if (gstage->pgd_levels < level) return -EINVAL; *out_pgorder = 12 + (level * kvm_riscv_gstage_index_bits); return 0; } -static int gstage_level_to_page_size(u32 level, unsigned long *out_pgsize) +static int gstage_level_to_page_size(struct kvm_gstage *gstage, u32 level, + unsigned long *out_pgsize) { int rc; unsigned long page_order = PAGE_SHIFT; - rc = gstage_level_to_page_order(level, &page_order); + rc = gstage_level_to_page_order(gstage, level, &page_order); if (rc) return rc; @@ -81,11 +85,11 @@ bool kvm_riscv_gstage_get_leaf(struct kvm_gstage *gstage, gpa_t addr, pte_t **ptepp, u32 *ptep_level) { pte_t *ptep; - u32 current_level = kvm_riscv_gstage_pgd_levels - 1; + u32 current_level = gstage->pgd_levels - 1; *ptep_level = current_level; ptep = (pte_t *)gstage->pgd; - ptep = &ptep[gstage_pte_index(addr, current_level)]; + ptep = &ptep[gstage_pte_index(gstage, addr, current_level)]; while (ptep && pte_val(ptep_get(ptep))) { if (gstage_pte_leaf(ptep)) { *ptep_level = current_level; @@ -97,7 +101,7 @@ bool kvm_riscv_gstage_get_leaf(struct kvm_gstage *gstage, gpa_t addr, current_level--; *ptep_level = current_level; ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep)); - ptep = &ptep[gstage_pte_index(addr, current_level)]; + ptep = &ptep[gstage_pte_index(gstage, addr, current_level)]; } else { ptep = NULL; } @@ -110,7 +114,7 @@ static void gstage_tlb_flush(struct kvm_gstage *gstage, u32 level, gpa_t addr) { unsigned long order = PAGE_SHIFT; - if (gstage_level_to_page_order(level, &order)) + if (gstage_level_to_page_order(gstage, level, &order)) return; addr &= ~(BIT(order) - 1); @@ -121,13 +125,27 @@ static void gstage_tlb_flush(struct kvm_gstage *gstage, u32 level, gpa_t addr) gstage->vmid); } +bool kvm_riscv_gstage_try_update_pte(struct kvm_gstage *gstage, u32 level, + gpa_t addr, pte_t *ptep, + pte_t old_pte, pte_t new_pte) +{ + if (cmpxchg(&ptep->pte, pte_val(old_pte), pte_val(new_pte)) != + pte_val(old_pte)) + return false; + + if (pte_val(old_pte) != pte_val(new_pte)) + gstage_tlb_flush(gstage, level, addr); + + return true; +} + int kvm_riscv_gstage_set_pte(struct kvm_gstage *gstage, struct kvm_mmu_memory_cache *pcache, const struct kvm_gstage_mapping *map) { - u32 current_level = kvm_riscv_gstage_pgd_levels - 1; + u32 current_level = gstage->pgd_levels - 1; pte_t *next_ptep = (pte_t *)gstage->pgd; - pte_t *ptep = &next_ptep[gstage_pte_index(map->addr, current_level)]; + pte_t *ptep = &next_ptep[gstage_pte_index(gstage, map->addr, current_level)]; if (current_level < map->level) return -EINVAL; @@ -151,31 +169,57 @@ int kvm_riscv_gstage_set_pte(struct kvm_gstage *gstage, } current_level--; - ptep = &next_ptep[gstage_pte_index(map->addr, current_level)]; + ptep = &next_ptep[gstage_pte_index(gstage, map->addr, current_level)]; } if (pte_val(*ptep) != pte_val(map->pte)) { + bool was_invalid = !pte_val(*ptep); set_pte(ptep, map->pte); - if (gstage_pte_leaf(ptep)) + if (gstage_pte_leaf(ptep) && + !(was_invalid && riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC))) gstage_tlb_flush(gstage, current_level, map->addr); } return 0; } +static void kvm_riscv_gstage_update_pte_prot(struct kvm_gstage *gstage, u32 level, + gpa_t addr, pte_t *ptep, pgprot_t prot) +{ + pte_t old_pte, new_pte; + + for (;;) { + old_pte = ptep_get(ptep); + if (pgprot_val(pte_pgprot(old_pte)) == pgprot_val(prot)) + return; + + new_pte = pfn_pte(pte_pfn(old_pte), prot); + new_pte = pte_mkdirty(new_pte); + + if (kvm_riscv_gstage_try_update_pte(gstage, level, addr, ptep, + old_pte, new_pte)) + return; + + cpu_relax(); + } +} + int kvm_riscv_gstage_map_page(struct kvm_gstage *gstage, struct kvm_mmu_memory_cache *pcache, gpa_t gpa, phys_addr_t hpa, unsigned long page_size, bool page_rdonly, bool page_exec, struct kvm_gstage_mapping *out_map) { + bool found_leaf; + u32 ptep_level; pgprot_t prot; + pte_t *ptep; int ret; out_map->addr = gpa; out_map->level = 0; - ret = gstage_page_size_to_level(page_size, &out_map->level); + ret = gstage_page_size_to_level(gstage, page_size, &out_map->level); if (ret) return ret; @@ -203,41 +247,158 @@ int kvm_riscv_gstage_map_page(struct kvm_gstage *gstage, else prot = PAGE_WRITE; } + + found_leaf = kvm_riscv_gstage_get_leaf(gstage, gpa, &ptep, &ptep_level); + if (found_leaf) { + /* + * ptep_level is the current gstage mapping level of addr, out_map->level + * is the required mapping level during fault handling. + * + * 1) ptep_level > out_map->level + * This happens when dirty logging is enabled and huge pages are used. + * KVM must track the pages at 4K level, and split the huge mapping + * into 4K mappings. + * + * 2) ptep_level < out_map->level + * This happens when dirty logging is disabled and huge pages are used. + * The gstage is split into 4K mappings, but the out_map level is now + * back to the huge page level. Ignore the out_map level this time, and + * just update the pte prot here. Otherwise, we would fall back to mapping + * the gstage at huge page level in `kvm_riscv_gstage_set_pte`, with the + * overhead of freeing the page tables(not support now), which would slow + * down the vCPUs' performance. + * + * It is better to recover the huge page mapping in the ioctl context when + * disabling dirty logging. + * + * 3) ptep_level == out_map->level + * We already have the ptep, just update the pte prot if the pfn not change. + * There is no need to invoke `kvm_riscv_gstage_set_pte` again. + */ + if (ptep_level > out_map->level) { + kvm_riscv_gstage_split_huge(gstage, pcache, gpa, + out_map->level, true); + } else if (ALIGN_DOWN(PFN_PHYS(pte_pfn(ptep_get(ptep))), page_size) == hpa) { + kvm_riscv_gstage_update_pte_prot(gstage, ptep_level, gpa, ptep, prot); + return 0; + } + } + out_map->pte = pfn_pte(PFN_DOWN(hpa), prot); out_map->pte = pte_mkdirty(out_map->pte); return kvm_riscv_gstage_set_pte(gstage, pcache, out_map); } -void kvm_riscv_gstage_op_pte(struct kvm_gstage *gstage, gpa_t addr, +static inline unsigned long make_child_pte(unsigned long huge_pte, int index, + unsigned long child_page_size) +{ + unsigned long child_pte = huge_pte; + unsigned long child_pfn_offset; + + /* + * The child_pte already has the base address of the huge page being + * split. So we just have to OR in the offset to the page at the next + * lower level for the given index. + */ + child_pfn_offset = index * (child_page_size / PAGE_SIZE); + child_pte |= pte_val(pfn_pte(child_pfn_offset, __pgprot(0))); + + return child_pte; +} + +bool kvm_riscv_gstage_split_huge(struct kvm_gstage *gstage, + struct kvm_mmu_memory_cache *pcache, + gpa_t addr, u32 target_level, bool flush) +{ + u32 current_level = gstage->pgd_levels - 1; + pte_t *next_ptep = (pte_t *)gstage->pgd; + unsigned long huge_pte, child_pte; + unsigned long child_page_size; + bool need_flush = false; + pte_t *ptep; + int i, ret; + + if (!pcache) + return false; + + while(current_level > target_level) { + ptep = (pte_t *)&next_ptep[gstage_pte_index(gstage, addr, current_level)]; + + if (!pte_val(ptep_get(ptep))) + break; + + if (!gstage_pte_leaf(ptep)) { + next_ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep)); + current_level--; + continue; + } + + huge_pte = pte_val(ptep_get(ptep)); + + ret = gstage_level_to_page_size(gstage, current_level - 1, &child_page_size); + if (ret) + return need_flush; + + next_ptep = kvm_mmu_memory_cache_alloc(pcache); + if (!next_ptep) + return need_flush; + + for (i = 0; i < PTRS_PER_PTE; i++) { + child_pte = make_child_pte(huge_pte, i, child_page_size); + set_pte((pte_t *)&next_ptep[i], __pte(child_pte)); + } + + /* + * Ensure the writes to the child PTEs are visible before + * linking the new page table to the parent PTE. + */ + smp_wmb(); + + set_pte(ptep, pfn_pte(PFN_DOWN(__pa(next_ptep)), + __pgprot(_PAGE_TABLE))); + + if (flush) + gstage_tlb_flush(gstage, current_level, addr); + else + need_flush = true; + + current_level--; + } + + return need_flush; +} + +bool kvm_riscv_gstage_op_pte(struct kvm_gstage *gstage, gpa_t addr, pte_t *ptep, u32 ptep_level, enum kvm_riscv_gstage_op op) { int i, ret; pte_t old_pte, *next_ptep; u32 next_ptep_level; unsigned long next_page_size, page_size; + bool flush = false; - ret = gstage_level_to_page_size(ptep_level, &page_size); + ret = gstage_level_to_page_size(gstage, ptep_level, &page_size); if (ret) - return; + return false; WARN_ON(addr & (page_size - 1)); if (!pte_val(ptep_get(ptep))) - return; + return false; if (ptep_level && !gstage_pte_leaf(ptep)) { next_ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep)); next_ptep_level = ptep_level - 1; - ret = gstage_level_to_page_size(next_ptep_level, &next_page_size); + ret = gstage_level_to_page_size(gstage, next_ptep_level, &next_page_size); if (ret) - return; + return false; if (op == GSTAGE_OP_CLEAR) set_pte(ptep, __pte(0)); for (i = 0; i < PTRS_PER_PTE; i++) - kvm_riscv_gstage_op_pte(gstage, addr + i * next_page_size, - &next_ptep[i], next_ptep_level, op); + flush |= kvm_riscv_gstage_op_pte(gstage, addr + i * next_page_size, + &next_ptep[i], next_ptep_level, op); if (op == GSTAGE_OP_CLEAR) put_page(virt_to_page(next_ptep)); } else { @@ -247,11 +408,13 @@ void kvm_riscv_gstage_op_pte(struct kvm_gstage *gstage, gpa_t addr, else if (op == GSTAGE_OP_WP) set_pte(ptep, __pte(pte_val(ptep_get(ptep)) & ~_PAGE_WRITE)); if (pte_val(*ptep) != pte_val(old_pte)) - gstage_tlb_flush(gstage, ptep_level, addr); + flush = true; } + + return flush; } -void kvm_riscv_gstage_unmap_range(struct kvm_gstage *gstage, +bool kvm_riscv_gstage_unmap_range(struct kvm_gstage *gstage, gpa_t start, gpa_t size, bool may_block) { int ret; @@ -260,33 +423,40 @@ void kvm_riscv_gstage_unmap_range(struct kvm_gstage *gstage, bool found_leaf; unsigned long page_size; gpa_t addr = start, end = start + size; + bool flush = false; while (addr < end) { found_leaf = kvm_riscv_gstage_get_leaf(gstage, addr, &ptep, &ptep_level); - ret = gstage_level_to_page_size(ptep_level, &page_size); + ret = gstage_level_to_page_size(gstage, ptep_level, &page_size); if (ret) break; - if (!found_leaf) - goto next; - - if (!(addr & (page_size - 1)) && ((end - addr) >= page_size)) - kvm_riscv_gstage_op_pte(gstage, addr, ptep, - ptep_level, GSTAGE_OP_CLEAR); - -next: - addr += page_size; + if (!found_leaf) { + addr = ALIGN(addr + 1, page_size); + } else { + if (!(addr & (page_size - 1)) && ((end - addr) >= page_size)) + flush |= kvm_riscv_gstage_op_pte(gstage, addr, ptep, + ptep_level, GSTAGE_OP_CLEAR); + else { + WARN_ONCE(1, "Skip unmap range addr: %#llx, end: %#llx, page_size: %#lx\n", + addr, end, page_size); + } + + addr += page_size; + } /* * If the range is too large, release the kvm->mmu_lock * to prevent starvation and lockup detector warnings. */ if (!(gstage->flags & KVM_GSTAGE_FLAGS_LOCAL) && may_block && addr < end) - cond_resched_lock(&gstage->kvm->mmu_lock); + cond_resched_rwlock_write(&gstage->kvm->mmu_lock); } + + return flush; } -void kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end) +bool kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end) { int ret; pte_t *ptep; @@ -294,23 +464,84 @@ void kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end bool found_leaf; gpa_t addr = start; unsigned long page_size; + bool flush = false; while (addr < end) { found_leaf = kvm_riscv_gstage_get_leaf(gstage, addr, &ptep, &ptep_level); - ret = gstage_level_to_page_size(ptep_level, &page_size); + ret = gstage_level_to_page_size(gstage, ptep_level, &page_size); if (ret) break; - if (!found_leaf) - goto next; + if (!found_leaf) { + addr = ALIGN(addr + 1, page_size); + } else { + addr = ALIGN_DOWN(addr, page_size); + flush |= kvm_riscv_gstage_op_pte(gstage, addr, ptep, + ptep_level, GSTAGE_OP_WP); + addr += page_size; + } + } - if (!(addr & (page_size - 1)) && ((end - addr) >= page_size)) - kvm_riscv_gstage_op_pte(gstage, addr, ptep, - ptep_level, GSTAGE_OP_WP); + return flush; +} + +static inline void clear_huge_mask(unsigned long *mask, unsigned long page_size, + gfn_t base_gfn, gpa_t addr) +{ + unsigned long start_index = 0; + unsigned long end_index = BITS_PER_LONG - 1; + unsigned long end_gfn = base_gfn + end_index; + unsigned long aligned_start_gfn = addr >> PAGE_SHIFT; + unsigned long aligned_end_gfn = aligned_start_gfn + (page_size >> PAGE_SHIFT) - 1; + unsigned int nbits = 0; + + if (aligned_start_gfn > base_gfn) + start_index = aligned_start_gfn - base_gfn; + + if (aligned_end_gfn < end_gfn) + end_index = aligned_end_gfn - base_gfn; + + nbits = end_index - start_index + 1; + bitmap_clear(mask, start_index, nbits); +} + +bool kvm_riscv_gstage_wp_pt_masked(struct kvm_gstage *gstage, gfn_t base_gfn, + unsigned long mask) +{ + unsigned long page_size; + bool flush = false; + bool found_leaf; + u32 ptep_level; + pte_t *ptep; + gpa_t addr = 0; + int ret; + + while (mask) { + addr = (base_gfn + __ffs(mask)) << PAGE_SHIFT; + + found_leaf = kvm_riscv_gstage_get_leaf(gstage, addr, &ptep, &ptep_level); + ret = gstage_level_to_page_size(gstage, ptep_level, &page_size); + if (ret) + break; + + if (found_leaf) { + if (ptep_level) { + addr = ALIGN_DOWN(addr, page_size); + clear_huge_mask(&mask, page_size, base_gfn, addr); + } + + flush |= kvm_riscv_gstage_op_pte(gstage, addr, ptep, + ptep_level, GSTAGE_OP_WP); -next: - addr += page_size; + if (ptep_level) + continue; + } + + /* clear the first set bit*/ + mask &= mask - 1; } + + return flush; } void __init kvm_riscv_gstage_mode_detect(void) @@ -319,39 +550,34 @@ void __init kvm_riscv_gstage_mode_detect(void) /* Try Sv57x4 G-stage mode */ csr_write(CSR_HGATP, HGATP_MODE_SV57X4 << HGATP_MODE_SHIFT); if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV57X4) { - kvm_riscv_gstage_mode = HGATP_MODE_SV57X4; - kvm_riscv_gstage_pgd_levels = 5; + kvm_riscv_gstage_max_pgd_levels = 5; goto done; } /* Try Sv48x4 G-stage mode */ csr_write(CSR_HGATP, HGATP_MODE_SV48X4 << HGATP_MODE_SHIFT); if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV48X4) { - kvm_riscv_gstage_mode = HGATP_MODE_SV48X4; - kvm_riscv_gstage_pgd_levels = 4; + kvm_riscv_gstage_max_pgd_levels = 4; goto done; } /* Try Sv39x4 G-stage mode */ csr_write(CSR_HGATP, HGATP_MODE_SV39X4 << HGATP_MODE_SHIFT); if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV39X4) { - kvm_riscv_gstage_mode = HGATP_MODE_SV39X4; - kvm_riscv_gstage_pgd_levels = 3; + kvm_riscv_gstage_max_pgd_levels = 3; goto done; } #else /* CONFIG_32BIT */ /* Try Sv32x4 G-stage mode */ csr_write(CSR_HGATP, HGATP_MODE_SV32X4 << HGATP_MODE_SHIFT); if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV32X4) { - kvm_riscv_gstage_mode = HGATP_MODE_SV32X4; - kvm_riscv_gstage_pgd_levels = 2; + kvm_riscv_gstage_max_pgd_levels = 2; goto done; } #endif /* KVM depends on !HGATP_MODE_OFF */ - kvm_riscv_gstage_mode = HGATP_MODE_OFF; - kvm_riscv_gstage_pgd_levels = 0; + kvm_riscv_gstage_max_pgd_levels = 0; done: csr_write(CSR_HGATP, 0); diff --git a/arch/riscv/kvm/isa.c b/arch/riscv/kvm/isa.c new file mode 100644 index 000000000000..971f44655100 --- /dev/null +++ b/arch/riscv/kvm/isa.c @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Qualcomm Technologies, Inc. + */ + +#include <linux/cpufeature.h> +#include <linux/errno.h> +#include <linux/kvm_host.h> +#include <linux/nospec.h> +#include <linux/pgtable.h> +#include <asm/kvm_isa.h> +#include <asm/vector.h> + +#define KVM_ISA_EXT_ARR(ext) \ +[KVM_RISCV_ISA_EXT_##ext] = RISCV_ISA_EXT_##ext + +/* Mapping between KVM ISA Extension ID & guest ISA extension ID */ +static const unsigned long kvm_isa_ext_arr[] = { + /* Single letter extensions (alphabetically sorted) */ + KVM_ISA_EXT_ARR(A), + KVM_ISA_EXT_ARR(C), + KVM_ISA_EXT_ARR(D), + KVM_ISA_EXT_ARR(F), + KVM_ISA_EXT_ARR(H), + KVM_ISA_EXT_ARR(I), + KVM_ISA_EXT_ARR(M), + KVM_ISA_EXT_ARR(V), + /* Multi letter extensions (alphabetically sorted) */ + KVM_ISA_EXT_ARR(SMNPM), + KVM_ISA_EXT_ARR(SMSTATEEN), + KVM_ISA_EXT_ARR(SSAIA), + KVM_ISA_EXT_ARR(SSCOFPMF), + KVM_ISA_EXT_ARR(SSNPM), + KVM_ISA_EXT_ARR(SSTC), + KVM_ISA_EXT_ARR(SVADE), + KVM_ISA_EXT_ARR(SVADU), + KVM_ISA_EXT_ARR(SVINVAL), + KVM_ISA_EXT_ARR(SVNAPOT), + KVM_ISA_EXT_ARR(SVPBMT), + KVM_ISA_EXT_ARR(SVVPTC), + KVM_ISA_EXT_ARR(ZAAMO), + KVM_ISA_EXT_ARR(ZABHA), + KVM_ISA_EXT_ARR(ZACAS), + KVM_ISA_EXT_ARR(ZALASR), + KVM_ISA_EXT_ARR(ZALRSC), + KVM_ISA_EXT_ARR(ZAWRS), + KVM_ISA_EXT_ARR(ZBA), + KVM_ISA_EXT_ARR(ZBB), + KVM_ISA_EXT_ARR(ZBC), + KVM_ISA_EXT_ARR(ZBKB), + KVM_ISA_EXT_ARR(ZBKC), + KVM_ISA_EXT_ARR(ZBKX), + KVM_ISA_EXT_ARR(ZBS), + KVM_ISA_EXT_ARR(ZCA), + KVM_ISA_EXT_ARR(ZCB), + KVM_ISA_EXT_ARR(ZCD), + KVM_ISA_EXT_ARR(ZCF), + KVM_ISA_EXT_ARR(ZCLSD), + KVM_ISA_EXT_ARR(ZCMOP), + KVM_ISA_EXT_ARR(ZFA), + KVM_ISA_EXT_ARR(ZFBFMIN), + KVM_ISA_EXT_ARR(ZFH), + KVM_ISA_EXT_ARR(ZFHMIN), + KVM_ISA_EXT_ARR(ZICBOM), + KVM_ISA_EXT_ARR(ZICBOP), + KVM_ISA_EXT_ARR(ZICBOZ), + KVM_ISA_EXT_ARR(ZICCRSE), + KVM_ISA_EXT_ARR(ZICFILP), + KVM_ISA_EXT_ARR(ZICFISS), + KVM_ISA_EXT_ARR(ZICNTR), + KVM_ISA_EXT_ARR(ZICOND), + KVM_ISA_EXT_ARR(ZICSR), + KVM_ISA_EXT_ARR(ZIFENCEI), + KVM_ISA_EXT_ARR(ZIHINTNTL), + KVM_ISA_EXT_ARR(ZIHINTPAUSE), + KVM_ISA_EXT_ARR(ZIHPM), + KVM_ISA_EXT_ARR(ZILSD), + KVM_ISA_EXT_ARR(ZIMOP), + KVM_ISA_EXT_ARR(ZKND), + KVM_ISA_EXT_ARR(ZKNE), + KVM_ISA_EXT_ARR(ZKNH), + KVM_ISA_EXT_ARR(ZKR), + KVM_ISA_EXT_ARR(ZKSED), + KVM_ISA_EXT_ARR(ZKSH), + KVM_ISA_EXT_ARR(ZKT), + KVM_ISA_EXT_ARR(ZTSO), + KVM_ISA_EXT_ARR(ZVBB), + KVM_ISA_EXT_ARR(ZVBC), + KVM_ISA_EXT_ARR(ZVFBFMIN), + KVM_ISA_EXT_ARR(ZVFBFWMA), + KVM_ISA_EXT_ARR(ZVFH), + KVM_ISA_EXT_ARR(ZVFHMIN), + KVM_ISA_EXT_ARR(ZVKB), + KVM_ISA_EXT_ARR(ZVKG), + KVM_ISA_EXT_ARR(ZVKNED), + KVM_ISA_EXT_ARR(ZVKNHA), + KVM_ISA_EXT_ARR(ZVKNHB), + KVM_ISA_EXT_ARR(ZVKSED), + KVM_ISA_EXT_ARR(ZVKSH), + KVM_ISA_EXT_ARR(ZVKT), +}; + +unsigned long kvm_riscv_base2isa_ext(unsigned long base_ext) +{ + unsigned long i; + + for (i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) { + if (kvm_isa_ext_arr[i] == base_ext) + return i; + } + + return KVM_RISCV_ISA_EXT_MAX; +} + +int __kvm_riscv_isa_check_host(unsigned long kvm_ext, unsigned long *base_ext) +{ + unsigned long host_ext; + + if (kvm_ext >= KVM_RISCV_ISA_EXT_MAX || + kvm_ext >= ARRAY_SIZE(kvm_isa_ext_arr)) + return -ENOENT; + + kvm_ext = array_index_nospec(kvm_ext, ARRAY_SIZE(kvm_isa_ext_arr)); + switch (kvm_isa_ext_arr[kvm_ext]) { + case RISCV_ISA_EXT_SMNPM: + /* + * Pointer masking effective in (H)S-mode is provided by the + * Smnpm extension, so that extension is reported to the guest, + * even though the CSR bits for configuring VS-mode pointer + * masking on the host side are part of the Ssnpm extension. + */ + host_ext = RISCV_ISA_EXT_SSNPM; + break; + default: + host_ext = kvm_isa_ext_arr[kvm_ext]; + break; + } + + if (!__riscv_isa_extension_available(NULL, host_ext)) + return -ENOENT; + + if (base_ext) + *base_ext = kvm_isa_ext_arr[kvm_ext]; + + return 0; +} + +bool kvm_riscv_isa_enable_allowed(unsigned long ext) +{ + switch (ext) { + case KVM_RISCV_ISA_EXT_H: + return false; + case KVM_RISCV_ISA_EXT_SSCOFPMF: + /* Sscofpmf depends on interrupt filtering defined in ssaia */ + return !kvm_riscv_isa_check_host(SSAIA); + case KVM_RISCV_ISA_EXT_SVADU: + /* + * The henvcfg.ADUE is read-only zero if menvcfg.ADUE is zero. + * Guest OS can use Svadu only when host OS enable Svadu. + */ + return arch_has_hw_pte_young(); + case KVM_RISCV_ISA_EXT_V: + return riscv_v_vstate_ctrl_user_allowed(); + default: + break; + } + + return true; +} + +bool kvm_riscv_isa_disable_allowed(unsigned long ext) +{ + switch (ext) { + /* Extensions which don't have any mechanism to disable */ + case KVM_RISCV_ISA_EXT_A: + case KVM_RISCV_ISA_EXT_C: + case KVM_RISCV_ISA_EXT_I: + case KVM_RISCV_ISA_EXT_M: + /* There is not architectural config bit to disable sscofpmf completely */ + case KVM_RISCV_ISA_EXT_SSCOFPMF: + case KVM_RISCV_ISA_EXT_SSNPM: + case KVM_RISCV_ISA_EXT_SSTC: + case KVM_RISCV_ISA_EXT_SVINVAL: + case KVM_RISCV_ISA_EXT_SVNAPOT: + case KVM_RISCV_ISA_EXT_SVVPTC: + case KVM_RISCV_ISA_EXT_ZAAMO: + case KVM_RISCV_ISA_EXT_ZABHA: + case KVM_RISCV_ISA_EXT_ZACAS: + case KVM_RISCV_ISA_EXT_ZALASR: + case KVM_RISCV_ISA_EXT_ZALRSC: + case KVM_RISCV_ISA_EXT_ZAWRS: + case KVM_RISCV_ISA_EXT_ZBA: + case KVM_RISCV_ISA_EXT_ZBB: + case KVM_RISCV_ISA_EXT_ZBC: + case KVM_RISCV_ISA_EXT_ZBKB: + case KVM_RISCV_ISA_EXT_ZBKC: + case KVM_RISCV_ISA_EXT_ZBKX: + case KVM_RISCV_ISA_EXT_ZBS: + case KVM_RISCV_ISA_EXT_ZCA: + case KVM_RISCV_ISA_EXT_ZCB: + case KVM_RISCV_ISA_EXT_ZCD: + case KVM_RISCV_ISA_EXT_ZCF: + case KVM_RISCV_ISA_EXT_ZCMOP: + case KVM_RISCV_ISA_EXT_ZFA: + case KVM_RISCV_ISA_EXT_ZFBFMIN: + case KVM_RISCV_ISA_EXT_ZFH: + case KVM_RISCV_ISA_EXT_ZFHMIN: + case KVM_RISCV_ISA_EXT_ZICBOP: + case KVM_RISCV_ISA_EXT_ZICCRSE: + case KVM_RISCV_ISA_EXT_ZICNTR: + case KVM_RISCV_ISA_EXT_ZICOND: + case KVM_RISCV_ISA_EXT_ZICSR: + case KVM_RISCV_ISA_EXT_ZIFENCEI: + case KVM_RISCV_ISA_EXT_ZIHINTNTL: + case KVM_RISCV_ISA_EXT_ZIHINTPAUSE: + case KVM_RISCV_ISA_EXT_ZIHPM: + case KVM_RISCV_ISA_EXT_ZIMOP: + case KVM_RISCV_ISA_EXT_ZKND: + case KVM_RISCV_ISA_EXT_ZKNE: + case KVM_RISCV_ISA_EXT_ZKNH: + case KVM_RISCV_ISA_EXT_ZKR: + case KVM_RISCV_ISA_EXT_ZKSED: + case KVM_RISCV_ISA_EXT_ZKSH: + case KVM_RISCV_ISA_EXT_ZKT: + case KVM_RISCV_ISA_EXT_ZTSO: + case KVM_RISCV_ISA_EXT_ZVBB: + case KVM_RISCV_ISA_EXT_ZVBC: + case KVM_RISCV_ISA_EXT_ZVFBFMIN: + case KVM_RISCV_ISA_EXT_ZVFBFWMA: + case KVM_RISCV_ISA_EXT_ZVFH: + case KVM_RISCV_ISA_EXT_ZVFHMIN: + case KVM_RISCV_ISA_EXT_ZVKB: + case KVM_RISCV_ISA_EXT_ZVKG: + case KVM_RISCV_ISA_EXT_ZVKNED: + case KVM_RISCV_ISA_EXT_ZVKNHA: + case KVM_RISCV_ISA_EXT_ZVKNHB: + case KVM_RISCV_ISA_EXT_ZVKSED: + case KVM_RISCV_ISA_EXT_ZVKSH: + case KVM_RISCV_ISA_EXT_ZVKT: + return false; + /* Extensions which can be disabled using Smstateen */ + case KVM_RISCV_ISA_EXT_SSAIA: + return riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN); + case KVM_RISCV_ISA_EXT_SVADE: + /* + * The henvcfg.ADUE is read-only zero if menvcfg.ADUE is zero. + * Svade can't be disabled unless we support Svadu. + */ + return arch_has_hw_pte_young(); + default: + break; + } + + return true; +} diff --git a/arch/riscv/kvm/main.c b/arch/riscv/kvm/main.c index 45536af521f0..89568ccce01d 100644 --- a/arch/riscv/kvm/main.c +++ b/arch/riscv/kvm/main.c @@ -10,10 +10,14 @@ #include <linux/err.h> #include <linux/module.h> #include <linux/kvm_host.h> +#include <linux/cpu_pm.h> #include <asm/cpufeature.h> #include <asm/kvm_mmu.h> #include <asm/kvm_nacl.h> #include <asm/sbi.h> +#include <asm/kvm_vcpu_vector.h> + +static DEFINE_PER_CPU(bool, kvm_riscv_virtualization_enabled); DEFINE_STATIC_KEY_FALSE(kvm_riscv_vsstage_tlb_no_gpa); @@ -33,31 +37,21 @@ long kvm_arch_dev_ioctl(struct file *filp, return -EINVAL; } -int kvm_arch_enable_virtualization_cpu(void) +/* Initialize hypervisor CSRs - called during CPU online and non-retention idle resume */ +static void kvm_riscv_csr_init(void) { - int rc; - - rc = kvm_riscv_nacl_enable(); - if (rc) - return rc; - - csr_write(CSR_HEDELEG, KVM_HEDELEG_DEFAULT); - csr_write(CSR_HIDELEG, KVM_HIDELEG_DEFAULT); + csr_write(CSR_HEDELEG, 0); + csr_write(CSR_HIDELEG, 0); /* VS should access only the time counter directly. Everything else should trap */ csr_write(CSR_HCOUNTEREN, 0x02); csr_write(CSR_HVIP, 0); - - kvm_riscv_aia_enable(); - - return 0; } -void kvm_arch_disable_virtualization_cpu(void) +/* Clear hypervisor CSRs - called during CPU offline and non-retention idle entry */ +static void kvm_riscv_csr_cleanup(void) { - kvm_riscv_aia_disable(); - /* * After clearing the hideleg CSR, the host kernel will receive * spurious interrupts if hvip CSR has pending interrupts and the @@ -69,13 +63,75 @@ void kvm_arch_disable_virtualization_cpu(void) csr_write(CSR_HEDELEG, 0); csr_write(CSR_HIDELEG, 0); + kvm_riscv_clear_former_vcpu(); +} + +int kvm_arch_enable_virtualization_cpu(void) +{ + int rc; + + rc = kvm_riscv_nacl_enable(); + if (rc) + return rc; + + kvm_riscv_csr_init(); + kvm_riscv_aia_enable(); + + __this_cpu_write(kvm_riscv_virtualization_enabled, true); + + return 0; +} + +void kvm_arch_disable_virtualization_cpu(void) +{ + kvm_riscv_aia_disable(); + kvm_riscv_csr_cleanup(); kvm_riscv_nacl_disable(); + + __this_cpu_write(kvm_riscv_virtualization_enabled, false); +} + +static int kvm_riscv_cpu_pm_notifier(struct notifier_block *self, unsigned long cmd, void *v) +{ + switch (cmd) { + case CPU_PM_EXIT: + case CPU_PM_ENTER_FAILED: + /* + * Only restore hypervisor state if KVM virtualization is + * enabled on this CPU. This prevents unintentional re-enabling + * of virtualization after it has been explicitly disabled. + */ + if (__this_cpu_read(kvm_riscv_virtualization_enabled)) { + kvm_riscv_csr_init(); + kvm_riscv_aia_pm_exit(); + } + return NOTIFY_OK; + case CPU_PM_ENTER: + /* + * Only save and clear hypervisor state if KVM virtualization + * is enabled on this CPU. + */ + if (__this_cpu_read(kvm_riscv_virtualization_enabled)) { + kvm_riscv_aia_pm_enter(); + kvm_riscv_csr_cleanup(); + } + return NOTIFY_OK; + default: + break; + } + + return NOTIFY_DONE; } +static struct notifier_block kvm_riscv_cpu_pm_nb = { + .notifier_call = kvm_riscv_cpu_pm_notifier, +}; + static void kvm_riscv_teardown(void) { kvm_riscv_aia_exit(); kvm_riscv_nacl_exit(); + kvm_riscv_v_exit(); kvm_unregister_perf_callbacks(); } @@ -85,7 +141,7 @@ static int __init riscv_kvm_init(void) char slist[64]; const char *str; - if (!riscv_isa_extension_available(NULL, h)) { + if (!riscv_isa_extension_available(NULL, H)) { kvm_info("hypervisor extension not available\n"); return -ENODEV; } @@ -105,17 +161,17 @@ static int __init riscv_kvm_init(void) return rc; kvm_riscv_gstage_mode_detect(); - switch (kvm_riscv_gstage_mode) { - case HGATP_MODE_SV32X4: + switch (kvm_riscv_gstage_max_pgd_levels) { + case 2: str = "Sv32x4"; break; - case HGATP_MODE_SV39X4: + case 3: str = "Sv39x4"; break; - case HGATP_MODE_SV48X4: + case 4: str = "Sv48x4"; break; - case HGATP_MODE_SV57X4: + case 5: str = "Sv57x4"; break; default: @@ -164,25 +220,41 @@ static int __init riscv_kvm_init(void) (rc) ? slist : "no features"); } - kvm_info("using %s G-stage page table format\n", str); + kvm_info("highest G-stage page table mode is %s\n", str); kvm_info("VMID %ld bits available\n", kvm_riscv_gstage_vmid_bits()); - if (kvm_riscv_aia_available()) - kvm_info("AIA available with %d guest external interrupts\n", - kvm_riscv_aia_nr_hgei); - kvm_riscv_setup_vendor_features(); - kvm_register_perf_callbacks(NULL); + kvm_riscv_v_init(); - rc = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE); - if (rc) { - kvm_riscv_teardown(); - return rc; + kvm_register_perf_callbacks(); + + /* Register CPU PM notifier for CPU idle non-retention states */ + if (IS_ENABLED(CONFIG_CPU_PM)) { + rc = cpu_pm_register_notifier(&kvm_riscv_cpu_pm_nb); + if (rc) { + kvm_err("Failed to register CPU PM notifier: %d\n", rc); + goto err_teardown; + } } + rc = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE); + if (rc) + goto err_unregister_cpu_pm; + + if (kvm_riscv_aia_available()) + kvm_info("AIA available with %d guest external interrupts\n", + atomic_read(&kvm_riscv_aia_nr_hgei)); + return 0; + +err_unregister_cpu_pm: + if (IS_ENABLED(CONFIG_CPU_PM)) + cpu_pm_unregister_notifier(&kvm_riscv_cpu_pm_nb); +err_teardown: + kvm_riscv_teardown(); + return rc; } module_init(riscv_kvm_init); @@ -190,6 +262,10 @@ static void __exit riscv_kvm_exit(void) { kvm_exit(); + /* Unregister CPU PM notifier */ + if (IS_ENABLED(CONFIG_CPU_PM)) + cpu_pm_unregister_notifier(&kvm_riscv_cpu_pm_nb); + kvm_riscv_teardown(); } module_exit(riscv_kvm_exit); diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index 4ab06697bfc0..3e955d808743 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -16,6 +16,9 @@ #include <asm/kvm_mmu.h> #include <asm/kvm_nacl.h> +static bool __read_mostly eager_page_split = true; +module_param(eager_page_split, bool, 0644); + static void mmu_wp_memory_region(struct kvm *kvm, int slot) { struct kvm_memslots *slots = kvm_memslots(kvm); @@ -23,16 +26,15 @@ static void mmu_wp_memory_region(struct kvm *kvm, int slot) phys_addr_t start = memslot->base_gfn << PAGE_SHIFT; phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT; struct kvm_gstage gstage; + bool flush; - gstage.kvm = kvm; - gstage.flags = 0; - gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); - gstage.pgd = kvm->arch.pgd; + kvm_riscv_gstage_init(&gstage, kvm); - spin_lock(&kvm->mmu_lock); - kvm_riscv_gstage_wp_range(&gstage, start, end); - spin_unlock(&kvm->mmu_lock); - kvm_flush_remote_tlbs_memslot(kvm, memslot); + write_lock(&kvm->mmu_lock); + flush = kvm_riscv_gstage_wp_range(&gstage, start, end); + write_unlock(&kvm->mmu_lock); + if (flush) + kvm_flush_remote_tlbs_memslot(kvm, memslot); } int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa, @@ -42,6 +44,7 @@ int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa, pgprot_t prot; unsigned long pfn; phys_addr_t addr, end; + unsigned long pgd_levels = kvm->arch.pgd_levels; struct kvm_mmu_memory_cache pcache = { .gfp_custom = (in_atomic) ? GFP_ATOMIC | __GFP_ACCOUNT : 0, .gfp_zero = __GFP_ZERO, @@ -49,10 +52,7 @@ int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa, struct kvm_gstage_mapping map; struct kvm_gstage gstage; - gstage.kvm = kvm; - gstage.flags = 0; - gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); - gstage.pgd = kvm->arch.pgd; + kvm_riscv_gstage_init(&gstage, kvm); end = (gpa + size + PAGE_SIZE - 1) & PAGE_MASK; pfn = __phys_to_pfn(hpa); @@ -67,13 +67,13 @@ int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa, if (!writable) map.pte = pte_wrprotect(map.pte); - ret = kvm_mmu_topup_memory_cache(&pcache, kvm_riscv_gstage_pgd_levels); + ret = __kvm_mmu_topup_memory_cache(&pcache, pgd_levels, pgd_levels); if (ret) goto out; - spin_lock(&kvm->mmu_lock); + write_lock(&kvm->mmu_lock); ret = kvm_riscv_gstage_set_pte(&gstage, &pcache, &map); - spin_unlock(&kvm->mmu_lock); + write_unlock(&kvm->mmu_lock); if (ret) goto out; @@ -88,15 +88,73 @@ out: void kvm_riscv_mmu_iounmap(struct kvm *kvm, gpa_t gpa, unsigned long size) { struct kvm_gstage gstage; + bool flush; + + kvm_riscv_gstage_init(&gstage, kvm); + + write_lock(&kvm->mmu_lock); + flush = kvm_riscv_gstage_unmap_range(&gstage, gpa, size, false); + write_unlock(&kvm->mmu_lock); + + if (flush) + kvm_flush_remote_tlbs_range(kvm, gpa >> PAGE_SHIFT, + size >> PAGE_SHIFT); +} + +static bool need_topup_split_caches_or_resched(struct kvm *kvm, int count) +{ + struct kvm_mmu_memory_cache *cache; + + if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) + return true; + + cache = &kvm->arch.pgd_split_page_cache; + return kvm_mmu_memory_cache_nr_free_objects(cache) < count; +} + +static bool mmu_split_huge_pages(struct kvm_gstage *gstage, + phys_addr_t start, phys_addr_t end) +{ + struct kvm *kvm = gstage->kvm; + struct kvm_mmu_memory_cache *pcache = &kvm->arch.pgd_split_page_cache; + phys_addr_t addr = ALIGN_DOWN(start, PMD_SIZE); + phys_addr_t last_flush_gfn = addr >> PAGE_SHIFT; + int count = gstage->pgd_levels; + bool flush = false; + int ret; + + lockdep_assert_held_write(&kvm->mmu_lock); + + while (addr < end) { + if (need_topup_split_caches_or_resched(kvm, count)) { + if (flush) { + kvm_flush_remote_tlbs_range(kvm, last_flush_gfn, + (addr >> PAGE_SHIFT) - last_flush_gfn); + last_flush_gfn = addr >> PAGE_SHIFT; + flush = false; + } + + write_unlock(&kvm->mmu_lock); + cond_resched(); - gstage.kvm = kvm; - gstage.flags = 0; - gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); - gstage.pgd = kvm->arch.pgd; + ret = kvm_mmu_topup_memory_cache(pcache, count); + if (ret) { + kvm_err("Failed to toup split page cache\n"); + write_lock(&kvm->mmu_lock); + return flush; + } + write_lock(&kvm->mmu_lock); + } + + if (!kvm->arch.pgd) + return flush; - spin_lock(&kvm->mmu_lock); - kvm_riscv_gstage_unmap_range(&gstage, gpa, size, false); - spin_unlock(&kvm->mmu_lock); + flush |= kvm_riscv_gstage_split_huge(gstage, pcache, addr, 0, false); + + addr += PMD_SIZE; + } + + return flush; } void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, @@ -109,12 +167,19 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT; struct kvm_gstage gstage; - gstage.kvm = kvm; - gstage.flags = 0; - gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); - gstage.pgd = kvm->arch.pgd; + kvm_riscv_gstage_init(&gstage, kvm); + + kvm_riscv_gstage_wp_pt_masked(&gstage, base_gfn, mask); + + if (kvm_dirty_log_manual_protect_and_init_set(kvm)) { + if (READ_ONCE(eager_page_split)) + mmu_split_huge_pages(&gstage, start, end); + } - kvm_riscv_gstage_wp_range(&gstage, start, end); + /* + * Remote TLB flush is not needed here since callers of + * kvm_arch_mmu_enable_log_dirty_pt_masked() already do it. + */ } void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) @@ -140,15 +205,35 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm, gpa_t gpa = slot->base_gfn << PAGE_SHIFT; phys_addr_t size = slot->npages << PAGE_SHIFT; struct kvm_gstage gstage; + bool flush; + + kvm_riscv_gstage_init(&gstage, kvm); + + write_lock(&kvm->mmu_lock); + flush = kvm_riscv_gstage_unmap_range(&gstage, gpa, size, false); + write_unlock(&kvm->mmu_lock); + if (flush) + kvm_flush_remote_tlbs_range(kvm, gpa >> PAGE_SHIFT, + size >> PAGE_SHIFT); +} + +static void mmu_split_memory_region(struct kvm *kvm, int slot) +{ + struct kvm_memslots *slots = kvm_memslots(kvm); + struct kvm_memory_slot *memslot = id_to_memslot(slots, slot); + phys_addr_t start = memslot->base_gfn << PAGE_SHIFT; + phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT; + struct kvm_gstage gstage; + bool flush; + + kvm_riscv_gstage_init(&gstage, kvm); - gstage.kvm = kvm; - gstage.flags = 0; - gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); - gstage.pgd = kvm->arch.pgd; + write_lock(&kvm->mmu_lock); + flush = mmu_split_huge_pages(&gstage, start, end); + write_unlock(&kvm->mmu_lock); - spin_lock(&kvm->mmu_lock); - kvm_riscv_gstage_unmap_range(&gstage, gpa, size, false); - spin_unlock(&kvm->mmu_lock); + if (flush) + kvm_flush_remote_tlbs_memslot(kvm, memslot); } void kvm_arch_commit_memory_region(struct kvm *kvm, @@ -157,14 +242,16 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, enum kvm_mr_change change) { /* - * At this point memslot has been committed and there is an - * allocated dirty_bitmap[], dirty pages will be tracked while - * the memory slot is write protected. + * At this point memslot has been committed and dirty pages will be + * tracked while the memory slot is write protected. */ if (change != KVM_MR_DELETE && new->flags & KVM_MEM_LOG_DIRTY_PAGES) { if (kvm_dirty_log_manual_protect_and_init_set(kvm)) return; mmu_wp_memory_region(kvm, new->id); + + if (READ_ONCE(eager_page_split)) + mmu_split_memory_region(kvm, new->id); } } @@ -185,8 +272,8 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, * Prevent userspace from creating a memory region outside of the GPA * space addressable by the KVM guest GPA space. */ - if ((new->base_gfn + new->npages) >= - (kvm_riscv_gstage_gpa_size >> PAGE_SHIFT)) + if ((new->base_gfn + new->npages) > + kvm_riscv_gstage_gpa_size(kvm->arch.pgd_levels) >> PAGE_SHIFT) return -EFAULT; hva = new->userspace_addr; @@ -245,17 +332,20 @@ out: bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) { struct kvm_gstage gstage; + bool flush; if (!kvm->arch.pgd) return false; - gstage.kvm = kvm; - gstage.flags = 0; - gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); - gstage.pgd = kvm->arch.pgd; - kvm_riscv_gstage_unmap_range(&gstage, range->start << PAGE_SHIFT, - (range->end - range->start) << PAGE_SHIFT, - range->may_block); + lockdep_assert_held_write(&kvm->mmu_lock); + + kvm_riscv_gstage_init(&gstage, kvm); + flush = kvm_riscv_gstage_unmap_range(&gstage, range->start << PAGE_SHIFT, + (range->end - range->start) << PAGE_SHIFT, + range->may_block); + if (flush) + kvm_flush_remote_tlbs_range(kvm, range->start, + range->end - range->start); return false; } @@ -271,10 +361,7 @@ bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE); - gstage.kvm = kvm; - gstage.flags = 0; - gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); - gstage.pgd = kvm->arch.pgd; + kvm_riscv_gstage_init(&gstage, kvm); if (!kvm_riscv_gstage_get_leaf(&gstage, range->start << PAGE_SHIFT, &ptep, &ptep_level)) return false; @@ -294,10 +381,7 @@ bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE); - gstage.kvm = kvm; - gstage.flags = 0; - gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); - gstage.pgd = kvm->arch.pgd; + kvm_riscv_gstage_init(&gstage, kvm); if (!kvm_riscv_gstage_get_leaf(&gstage, range->start << PAGE_SHIFT, &ptep, &ptep_level)) return false; @@ -305,34 +389,268 @@ bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) return pte_young(ptep_get(ptep)); } +static bool fault_supports_gstage_huge_mapping(struct kvm_memory_slot *memslot, + unsigned long hva, + unsigned long map_size) +{ + hva_t uaddr_start, uaddr_end; + gpa_t gpa_start; + size_t size; + + size = memslot->npages * PAGE_SIZE; + uaddr_start = memslot->userspace_addr; + uaddr_end = uaddr_start + size; + + gpa_start = memslot->base_gfn << PAGE_SHIFT; + + /* + * Pages belonging to memslots that don't have the same alignment + * within a huge page for userspace and GPA cannot be mapped with + * g-stage block entries, because we'll end up mapping the wrong pages. + * + * Consider a layout like the following: + * + * memslot->userspace_addr: + * +-----+--------------------+--------------------+---+ + * |abcde|fgh vs-stage block | vs-stage block tv|xyz| + * +-----+--------------------+--------------------+---+ + * + * memslot->base_gfn << PAGE_SHIFT: + * +---+--------------------+--------------------+-----+ + * |abc|def g-stage block | g-stage block |tvxyz| + * +---+--------------------+--------------------+-----+ + * + * If we create those g-stage blocks, we'll end up with this incorrect + * mapping: + * d -> f + * e -> g + * f -> h + */ + if ((gpa_start & (map_size - 1)) != (uaddr_start & (map_size - 1))) + return false; + + /* + * Next, let's make sure we're not trying to map anything not covered + * by the memslot. This means we have to prohibit block size mappings + * for the beginning and end of a non-block aligned and non-block sized + * memory slot (illustrated by the head and tail parts of the + * userspace view above containing pages 'abcde' and 'xyz', + * respectively). + * + * Note that it doesn't matter if we do the check using the + * userspace_addr or the base_gfn, as both are equally aligned (per + * the check above) and equally sized. + */ + return (hva >= ALIGN(uaddr_start, map_size)) && + (hva < ALIGN_DOWN(uaddr_end, map_size)); +} + +static int get_hva_mapping_size(struct kvm *kvm, + unsigned long hva) +{ + int size = PAGE_SIZE; + unsigned long flags; + pgd_t pgd; + p4d_t p4d; + pud_t pud; + pmd_t pmd; + + /* + * Disable IRQs to prevent concurrent tear down of host page tables, + * e.g. if the primary MMU promotes a P*D to a huge page and then frees + * the original page table. + */ + local_irq_save(flags); + + /* + * Read each entry once. As above, a non-leaf entry can be promoted to + * a huge page _during_ this walk. Re-reading the entry could send the + * walk into the weeks, e.g. p*d_leaf() returns false (sees the old + * value) and then p*d_offset() walks into the target huge page instead + * of the old page table (sees the new value). + */ + pgd = pgdp_get(pgd_offset(kvm->mm, hva)); + if (pgd_none(pgd)) + goto out; + + p4d = p4dp_get(p4d_offset(&pgd, hva)); + if (p4d_none(p4d) || !p4d_present(p4d)) + goto out; + + pud = pudp_get(pud_offset(&p4d, hva)); + if (pud_none(pud) || !pud_present(pud)) + goto out; + + if (pud_leaf(pud)) { + size = PUD_SIZE; + goto out; + } + + pmd = pmdp_get(pmd_offset(&pud, hva)); + if (pmd_none(pmd) || !pmd_present(pmd)) + goto out; + + if (pmd_leaf(pmd)) + size = PMD_SIZE; + +out: + local_irq_restore(flags); + return size; +} + +static unsigned long transparent_hugepage_adjust(struct kvm *kvm, + struct kvm_memory_slot *memslot, + unsigned long hva, + kvm_pfn_t *hfnp, gpa_t *gpa) +{ + kvm_pfn_t hfn = *hfnp; + + /* + * Make sure the adjustment is done only for THP pages. Also make + * sure that the HVA and GPA are sufficiently aligned and that the + * block map is contained within the memslot. + */ + if (fault_supports_gstage_huge_mapping(memslot, hva, PMD_SIZE)) { + int sz; + + sz = get_hva_mapping_size(kvm, hva); + if (sz < PMD_SIZE) + return sz; + + *gpa &= PMD_MASK; + hfn &= ~(PTRS_PER_PMD - 1); + *hfnp = hfn; + + return PMD_SIZE; + } + + return PAGE_SIZE; +} + +static unsigned long hugetlb_mapping_size(struct kvm_memory_slot *memslot, + unsigned long hva, + unsigned long map_size) +{ + switch (map_size) { +#ifndef CONFIG_32BIT + case PUD_SIZE: + if (fault_supports_gstage_huge_mapping(memslot, hva, PUD_SIZE)) + return PUD_SIZE; + fallthrough; +#endif + case PMD_SIZE: + if (fault_supports_gstage_huge_mapping(memslot, hva, PMD_SIZE)) + return PMD_SIZE; + fallthrough; + case PAGE_SIZE: + return PAGE_SIZE; + default: + return map_size; + } +} + +static bool kvm_riscv_mmu_dirty_log_write_fault_fast(struct kvm *kvm, + struct kvm_memory_slot *memslot, + gpa_t gpa, + struct kvm_gstage_mapping *out_map) +{ + struct kvm_gstage gstage; + unsigned long mmu_seq; + pte_t old_pte, new_pte; + pte_t *ptep; + gfn_t gfn = gpa >> PAGE_SHIFT; + u32 ptep_level; + bool dirty_marked = false; + bool ret; + + kvm_riscv_gstage_init(&gstage, kvm); + mmu_seq = kvm->mmu_invalidate_seq; + + read_lock(&kvm->mmu_lock); + + if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) { + ret = false; + goto out_unlock; + } + + if (!kvm_riscv_gstage_get_leaf(&gstage, gpa, &ptep, &ptep_level) || + ptep_level) { + ret = false; + goto out_unlock; + } + + for (;;) { + old_pte = ptep_get(ptep); + if (!(pte_val(old_pte) & _PAGE_LEAF)) { + ret = false; + break; + } + + if (!dirty_marked) { + mark_page_dirty_in_slot(kvm, memslot, gfn); + dirty_marked = true; + } + + if ((pte_val(old_pte) & (_PAGE_WRITE | _PAGE_DIRTY)) == + (_PAGE_WRITE | _PAGE_DIRTY)) { + new_pte = old_pte; + ret = true; + break; + } + + new_pte = pte_mkdirty(pte_mkwrite_novma(old_pte)); + + if (kvm_riscv_gstage_try_update_pte(&gstage, ptep_level, gpa, + ptep, old_pte, new_pte)) { + ret = true; + break; + } + cpu_relax(); + } + +out_unlock: + read_unlock(&kvm->mmu_lock); + + if (ret) { + out_map->addr = gpa & PAGE_MASK; + out_map->level = 0; + out_map->pte = new_pte; + } + + return ret; +} + int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, gpa_t gpa, unsigned long hva, bool is_write, struct kvm_gstage_mapping *out_map) { int ret; kvm_pfn_t hfn; + bool is_hugetlb; + bool unused = false; bool writable; - short vma_pageshift; + unsigned int vma_pageshift; gfn_t gfn = gpa >> PAGE_SHIFT; struct vm_area_struct *vma; struct kvm *kvm = vcpu->kvm; struct kvm_mmu_memory_cache *pcache = &vcpu->arch.mmu_page_cache; - bool logging = (memslot->dirty_bitmap && - !(memslot->flags & KVM_MEM_READONLY)) ? true : false; + bool logging = kvm_slot_dirty_track_enabled(memslot) && + !(memslot->flags & KVM_MEM_READONLY); unsigned long vma_pagesize, mmu_seq; struct kvm_gstage gstage; struct page *page; - gstage.kvm = kvm; - gstage.flags = 0; - gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); - gstage.pgd = kvm->arch.pgd; + kvm_riscv_gstage_init(&gstage, kvm); /* Setup initial state of output mapping */ memset(out_map, 0, sizeof(*out_map)); + if (is_write && logging && + kvm_riscv_mmu_dirty_log_write_fault_fast(kvm, memslot, gpa, out_map)) + return 0; + /* We need minimum second+third level pages */ - ret = kvm_mmu_topup_memory_cache(pcache, kvm_riscv_gstage_pgd_levels); + ret = kvm_mmu_topup_memory_cache(pcache, kvm->arch.pgd_levels); if (ret) { kvm_err("Failed to topup G-stage cache\n"); return ret; @@ -347,16 +665,23 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, return -EFAULT; } - if (is_vm_hugetlb_page(vma)) + is_hugetlb = is_vm_hugetlb_page(vma); + if (is_hugetlb) vma_pageshift = huge_page_shift(hstate_vma(vma)); else vma_pageshift = PAGE_SHIFT; vma_pagesize = 1ULL << vma_pageshift; if (logging || (vma->vm_flags & VM_PFNMAP)) vma_pagesize = PAGE_SIZE; + else if (is_hugetlb) + vma_pagesize = hugetlb_mapping_size(memslot, hva, vma_pagesize); + /* + * For hugetlb mappings, vma_pagesize might have been reduced from the + * VMA size to a smaller safe mapping size. + */ if (vma_pagesize == PMD_SIZE || vma_pagesize == PUD_SIZE) - gfn = (gpa & huge_page_mask(hstate_vma(vma))) >> PAGE_SHIFT; + gfn = ALIGN_DOWN(gpa, vma_pagesize) >> PAGE_SHIFT; /* * Read mmu_invalidate_seq so that KVM can detect if the results of @@ -383,6 +708,8 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, vma_pageshift, current); return 0; } + if (is_sigpending_pfn(hfn)) + return -EINTR; if (is_error_noslot_pfn(hfn)) return -EFAULT; @@ -393,10 +720,20 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, if (logging && !is_write) writable = false; - spin_lock(&kvm->mmu_lock); + write_lock(&kvm->mmu_lock); - if (mmu_invalidate_retry(kvm, mmu_seq)) + if (mmu_invalidate_retry(kvm, mmu_seq)) { + unused = true; goto out_unlock; + } + + /* + * Check if we are backed by a THP and thus use block mapping if + * possible. Hugetlb mappings already selected their target size above, + * so do not promote them through the THP helper. + */ + if (!logging && !is_hugetlb && vma_pagesize == PAGE_SIZE) + vma_pagesize = transparent_hugepage_adjust(kvm, memslot, hva, &hfn, &gpa); if (writable) { mark_page_dirty_in_slot(kvm, memslot, gfn); @@ -411,8 +748,9 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, kvm_err("Failed to map in G-stage\n"); out_unlock: - kvm_release_faultin_page(kvm, page, ret && ret != -EEXIST, writable); - spin_unlock(&kvm->mmu_lock); + kvm_release_faultin_page(kvm, page, + unused || (ret && ret != -EEXIST), writable); + write_unlock(&kvm->mmu_lock); return ret; } @@ -425,12 +763,14 @@ int kvm_riscv_mmu_alloc_pgd(struct kvm *kvm) return -EINVAL; } - pgd_page = alloc_pages(GFP_KERNEL | __GFP_ZERO, - get_order(kvm_riscv_gstage_pgd_size)); + pgd_page = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO, + get_order(kvm_riscv_gstage_pgd_size)); if (!pgd_page) return -ENOMEM; kvm->arch.pgd = page_to_virt(pgd_page); kvm->arch.pgd_phys = page_to_phys(pgd_page); + kvm->arch.pgd_levels = kvm_riscv_gstage_max_pgd_levels; + kvm->arch.pgd_split_page_cache.gfp_zero = __GFP_ZERO; return 0; } @@ -439,31 +779,37 @@ void kvm_riscv_mmu_free_pgd(struct kvm *kvm) { struct kvm_gstage gstage; void *pgd = NULL; + bool flush = false; - spin_lock(&kvm->mmu_lock); + write_lock(&kvm->mmu_lock); if (kvm->arch.pgd) { - gstage.kvm = kvm; - gstage.flags = 0; - gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid); - gstage.pgd = kvm->arch.pgd; - kvm_riscv_gstage_unmap_range(&gstage, 0UL, kvm_riscv_gstage_gpa_size, false); + kvm_riscv_gstage_init(&gstage, kvm); + flush = kvm_riscv_gstage_unmap_range(&gstage, 0UL, + kvm_riscv_gstage_gpa_size(kvm->arch.pgd_levels), false); pgd = READ_ONCE(kvm->arch.pgd); kvm->arch.pgd = NULL; kvm->arch.pgd_phys = 0; + kvm->arch.pgd_levels = 0; } - spin_unlock(&kvm->mmu_lock); + write_unlock(&kvm->mmu_lock); + + if (flush) + kvm_flush_remote_tlbs(kvm); if (pgd) free_pages((unsigned long)pgd, get_order(kvm_riscv_gstage_pgd_size)); + + kvm_mmu_free_memory_cache(&kvm->arch.pgd_split_page_cache); } void kvm_riscv_mmu_update_hgatp(struct kvm_vcpu *vcpu) { - unsigned long hgatp = kvm_riscv_gstage_mode << HGATP_MODE_SHIFT; - struct kvm_arch *k = &vcpu->kvm->arch; + struct kvm_arch *ka = &vcpu->kvm->arch; + unsigned long hgatp = kvm_riscv_gstage_mode(ka->pgd_levels) + << HGATP_MODE_SHIFT; - hgatp |= (READ_ONCE(k->vmid.vmid) << HGATP_VMID_SHIFT) & HGATP_VMID; - hgatp |= (k->pgd_phys >> PAGE_SHIFT) & HGATP_PPN; + hgatp |= (READ_ONCE(ka->vmid.vmid) << HGATP_VMID_SHIFT) & HGATP_VMID; + hgatp |= (ka->pgd_phys >> PAGE_SHIFT) & HGATP_PPN; ncsr_write(CSR_HGATP, hgatp); diff --git a/arch/riscv/kvm/nacl.c b/arch/riscv/kvm/nacl.c index 08a95ad9ada2..a5cda9a65156 100644 --- a/arch/riscv/kvm/nacl.c +++ b/arch/riscv/kvm/nacl.c @@ -20,7 +20,7 @@ void __kvm_riscv_nacl_hfence(void *shmem, unsigned long page_count) { int i, ent = -1, try_count = 5; - unsigned long *entp; + __lelong *entp; again: for (i = 0; i < SBI_NACL_SHMEM_HFENCE_ENTRY_MAX; i++) { @@ -33,7 +33,7 @@ again: } if (ent < 0) { - if (try_count) { + if (try_count--) { nacl_sync_hfence(-1UL); goto again; } else { @@ -42,12 +42,24 @@ again: } } - entp = shmem + SBI_NACL_SHMEM_HFENCE_ENTRY_CONFIG(i); - *entp = cpu_to_lelong(control); + /* + * Per SBI v3.0 section 15.1.2, the Page_Number and Page_Count + * words must be updated before the Config word with its Pending + * bit set. WRITE_ONCE() stops the compiler from reordering the + * stores and smp_wmb() makes the parameter words globally + * visible to the SBI implementation (or NACL hardware) before + * the Pending bit is set. + */ entp = shmem + SBI_NACL_SHMEM_HFENCE_ENTRY_PNUM(i); - *entp = cpu_to_lelong(page_num); + WRITE_ONCE(*entp, cpu_to_lelong(page_num)); entp = shmem + SBI_NACL_SHMEM_HFENCE_ENTRY_PCOUNT(i); - *entp = cpu_to_lelong(page_count); + WRITE_ONCE(*entp, cpu_to_lelong(page_count)); + + /* Ensure the parameter words are visible before the Pending bit */ + smp_wmb(); + + entp = shmem + SBI_NACL_SHMEM_HFENCE_ENTRY_CONFIG(i); + WRITE_ONCE(*entp, cpu_to_lelong(control)); } int kvm_riscv_nacl_enable(void) diff --git a/arch/riscv/kvm/tlb.c b/arch/riscv/kvm/tlb.c index ff1aeac4eb8e..2ae34632cdcb 100644 --- a/arch/riscv/kvm/tlb.c +++ b/arch/riscv/kvm/tlb.c @@ -8,6 +8,7 @@ #include <linux/errno.h> #include <linux/err.h> #include <linux/module.h> +#include <linux/overflow.h> #include <linux/smp.h> #include <linux/kvm_host.h> #include <asm/cacheflush.h> @@ -24,7 +25,12 @@ void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid, gpa_t gpa, gpa_t gpsz, unsigned long order) { - gpa_t pos; + gpa_t end, pos, step = BIT(order); + + if (check_add_overflow(gpa, gpsz, &end)) { + kvm_riscv_local_hfence_gvma_vmid_all(vmid); + return; + } if (PTRS_PER_PTE < (gpsz >> order)) { kvm_riscv_local_hfence_gvma_vmid_all(vmid); @@ -33,14 +39,20 @@ void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid, if (has_svinval()) { asm volatile (SFENCE_W_INVAL() ::: "memory"); - for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) + for (pos = gpa; pos < end; pos += step) { asm volatile (HINVAL_GVMA(%0, %1) : : "r" (pos >> 2), "r" (vmid) : "memory"); + if (end - pos <= step) + break; + } asm volatile (SFENCE_INVAL_IR() ::: "memory"); } else { - for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) + for (pos = gpa; pos < end; pos += step) { asm volatile (HFENCE_GVMA(%0, %1) : : "r" (pos >> 2), "r" (vmid) : "memory"); + if (end - pos <= step) + break; + } } } @@ -52,7 +64,12 @@ void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid) void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz, unsigned long order) { - gpa_t pos; + gpa_t end, pos, step = BIT(order); + + if (check_add_overflow(gpa, gpsz, &end)) { + kvm_riscv_local_hfence_gvma_all(); + return; + } if (PTRS_PER_PTE < (gpsz >> order)) { kvm_riscv_local_hfence_gvma_all(); @@ -61,14 +78,20 @@ void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz, if (has_svinval()) { asm volatile (SFENCE_W_INVAL() ::: "memory"); - for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) + for (pos = gpa; pos < end; pos += step) { asm volatile(HINVAL_GVMA(%0, zero) : : "r" (pos >> 2) : "memory"); + if (end - pos <= step) + break; + } asm volatile (SFENCE_INVAL_IR() ::: "memory"); } else { - for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) + for (pos = gpa; pos < end; pos += step) { asm volatile(HFENCE_GVMA(%0, zero) : : "r" (pos >> 2) : "memory"); + if (end - pos <= step) + break; + } } } @@ -83,7 +106,13 @@ void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid, unsigned long gvsz, unsigned long order) { - unsigned long pos, hgatp; + unsigned long end, pos, step = BIT(order); + unsigned long hgatp; + + if (check_add_overflow(gva, gvsz, &end)) { + kvm_riscv_local_hfence_vvma_asid_all(vmid, asid); + return; + } if (PTRS_PER_PTE < (gvsz >> order)) { kvm_riscv_local_hfence_vvma_asid_all(vmid, asid); @@ -94,14 +123,20 @@ void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid, if (has_svinval()) { asm volatile (SFENCE_W_INVAL() ::: "memory"); - for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) + for (pos = gva; pos < end; pos += step) { asm volatile(HINVAL_VVMA(%0, %1) : : "r" (pos), "r" (asid) : "memory"); + if (end - pos <= step) + break; + } asm volatile (SFENCE_INVAL_IR() ::: "memory"); } else { - for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) + for (pos = gva; pos < end; pos += step) { asm volatile(HFENCE_VVMA(%0, %1) : : "r" (pos), "r" (asid) : "memory"); + if (end - pos <= step) + break; + } } csr_write(CSR_HGATP, hgatp); @@ -123,7 +158,13 @@ void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid, unsigned long gva, unsigned long gvsz, unsigned long order) { - unsigned long pos, hgatp; + unsigned long end, pos, step = BIT(order); + unsigned long hgatp; + + if (check_add_overflow(gva, gvsz, &end)) { + kvm_riscv_local_hfence_vvma_all(vmid); + return; + } if (PTRS_PER_PTE < (gvsz >> order)) { kvm_riscv_local_hfence_vvma_all(vmid); @@ -134,14 +175,20 @@ void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid, if (has_svinval()) { asm volatile (SFENCE_W_INVAL() ::: "memory"); - for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) + for (pos = gva; pos < end; pos += step) { asm volatile(HINVAL_VVMA(%0, zero) : : "r" (pos) : "memory"); + if (end - pos <= step) + break; + } asm volatile (SFENCE_INVAL_IR() ::: "memory"); } else { - for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) + for (pos = gva; pos < end; pos += step) { asm volatile(HFENCE_VVMA(%0, zero) : : "r" (pos) : "memory"); + if (end - pos <= step) + break; + } } csr_write(CSR_HGATP, hgatp); @@ -182,7 +229,7 @@ void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu) /* * Flush VS-stage TLB entries for implementation where VS-stage - * TLB does not cahce guest physical address and VMID. + * TLB does not cache guest physical address and VMID. */ if (static_branch_unlikely(&kvm_riscv_vsstage_tlb_no_gpa)) kvm_riscv_local_hfence_vvma_all(vmid); @@ -325,29 +372,52 @@ void kvm_riscv_hfence_process(struct kvm_vcpu *vcpu) } } -static void make_xfence_request(struct kvm *kvm, - unsigned long hbase, unsigned long hmask, - unsigned int req, unsigned int fallback_req, - const struct kvm_riscv_hfence *data) +static void make_xfence_request_nodata(struct kvm *kvm, unsigned long hbase, + unsigned long hmask, unsigned int req) { unsigned long i; struct kvm_vcpu *vcpu; - unsigned int actual_req = req; DECLARE_BITMAP(vcpu_mask, KVM_MAX_VCPUS); bitmap_zero(vcpu_mask, KVM_MAX_VCPUS); kvm_for_each_vcpu(i, vcpu, kvm) { if (hbase != -1UL) { - if (vcpu->vcpu_id < hbase) + if (vcpu->vcpu_id < hbase || + vcpu->vcpu_id >= hbase + BITS_PER_LONG) continue; if (!(hmask & (1UL << (vcpu->vcpu_id - hbase)))) continue; } bitmap_set(vcpu_mask, i, 1); + } + + kvm_make_vcpus_request_mask(kvm, req, vcpu_mask); +} + +static void make_xfence_request(struct kvm *kvm, + unsigned long hbase, unsigned long hmask, + unsigned int req, unsigned int fallback_req, + const struct kvm_riscv_hfence *data) +{ + unsigned long i; + struct kvm_vcpu *vcpu; + DECLARE_BITMAP(req_vcpu_mask, KVM_MAX_VCPUS); + DECLARE_BITMAP(fallback_req_vcpu_mask, KVM_MAX_VCPUS); + + if (!data || !data->type) + return; - if (!data || !data->type) - continue; + bitmap_zero(req_vcpu_mask, KVM_MAX_VCPUS); + bitmap_zero(fallback_req_vcpu_mask, KVM_MAX_VCPUS); + kvm_for_each_vcpu(i, vcpu, kvm) { + if (hbase != -1UL) { + if (vcpu->vcpu_id < hbase || + vcpu->vcpu_id >= hbase + BITS_PER_LONG) + continue; + if (!(hmask & (1UL << (vcpu->vcpu_id - hbase)))) + continue; + } /* * Enqueue hfence data to VCPU hfence queue. If we don't @@ -355,17 +425,19 @@ static void make_xfence_request(struct kvm *kvm, * a more conservative hfence request. */ if (!vcpu_hfence_enqueue(vcpu, data)) - actual_req = fallback_req; + bitmap_set(fallback_req_vcpu_mask, i, 1); + else + bitmap_set(req_vcpu_mask, i, 1); } - kvm_make_vcpus_request_mask(kvm, actual_req, vcpu_mask); + kvm_make_vcpus_request_mask(kvm, req, req_vcpu_mask); + kvm_make_vcpus_request_mask(kvm, fallback_req, fallback_req_vcpu_mask); } void kvm_riscv_fence_i(struct kvm *kvm, unsigned long hbase, unsigned long hmask) { - make_xfence_request(kvm, hbase, hmask, KVM_REQ_FENCE_I, - KVM_REQ_FENCE_I, NULL); + make_xfence_request_nodata(kvm, hbase, hmask, KVM_REQ_FENCE_I); } void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm, diff --git a/arch/riscv/kvm/trace.h b/arch/riscv/kvm/trace.h index 3d54175d805c..23a5fd1b686e 100644 --- a/arch/riscv/kvm/trace.h +++ b/arch/riscv/kvm/trace.h @@ -25,7 +25,7 @@ TRACE_EVENT(kvm_entry, __entry->pc = vcpu->arch.guest_context.sepc; ), - TP_printk("PC: 0x016%lx", __entry->pc) + TP_printk("PC: 0x%016lx", __entry->pc) ); TRACE_EVENT(kvm_exit, @@ -56,7 +56,84 @@ TRACE_EVENT(kvm_exit, __entry->htinst) ); -#endif /* _TRACE_RSICV_KVM_H */ +TRACE_EVENT(kvm_mmio_emulate, + TP_PROTO(unsigned long vcpu_id, unsigned long sepc, unsigned long insn, + unsigned long fault_addr, bool write, int len), + TP_ARGS(vcpu_id, sepc, insn, fault_addr, write, len), + + TP_STRUCT__entry( + __field(unsigned long, vcpu_id) + __field(unsigned long, sepc) + __field(unsigned long, insn) + __field(unsigned long, fault_addr) + __field(bool, write) + __field(int, len) + ), + + TP_fast_assign( + __entry->vcpu_id = vcpu_id; + __entry->sepc = sepc; + __entry->insn = insn; + __entry->fault_addr = fault_addr; + __entry->write = write; + __entry->len = len; + ), + + TP_printk("VCPU: %lu, %s MMIO at 0x%lx, len %d, insn 0x%lx, sepc 0x%lx", + __entry->vcpu_id, __entry->write ? "Store" : "Load", + __entry->fault_addr, __entry->len, __entry->insn, + __entry->sepc) +); + +TRACE_EVENT(kvm_vcpu_exit, + TP_PROTO(unsigned long vcpu_id, unsigned long sepc, unsigned long scause, + unsigned long stval, unsigned long htval, unsigned long htinst), + TP_ARGS(vcpu_id, sepc, scause, stval, htval, htinst), + + TP_STRUCT__entry( + __field(unsigned long, vcpu_id) + __field(unsigned long, sepc) + __field(unsigned long, scause) + __field(unsigned long, stval) + __field(unsigned long, htval) + __field(unsigned long, htinst) + ), + + TP_fast_assign( + __entry->vcpu_id = vcpu_id; + __entry->sepc = sepc; + __entry->scause = scause; + __entry->stval = stval; + __entry->htval = htval; + __entry->htinst = htinst; + ), + + TP_printk("VCPU: %lu, SEPC: 0x%lx, SCAUSE: 0x%lx, STVAL: 0x%lx, HTVAL: 0x%lx, HTINST: 0x%lx", + __entry->vcpu_id, __entry->sepc, __entry->scause, + __entry->stval, __entry->htval, __entry->htinst) +); + +TRACE_EVENT(kvm_vcpu_irq, + TP_PROTO(unsigned long vcpu_id, unsigned int irq, int level), + TP_ARGS(vcpu_id, irq, level), + + TP_STRUCT__entry( + __field(unsigned long, vcpu_id) + __field(unsigned int, irq) + __field(int, level) + ), + + TP_fast_assign( + __entry->vcpu_id = vcpu_id; + __entry->irq = irq; + __entry->level = level; + ), + + TP_printk("VCPU: %lu, IRQ: %u, level: %d", + __entry->vcpu_id, __entry->irq, __entry->level) +); + +#endif /* _TRACE_KVM_H */ #undef TRACE_INCLUDE_PATH #define TRACE_INCLUDE_PATH . diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index a55a95da54d0..7f1636c44d41 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -24,7 +24,20 @@ #define CREATE_TRACE_POINTS #include "trace.h" -const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { +static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_former_vcpu); + +void kvm_riscv_clear_former_vcpu(void) +{ + /* + * Clear the per-CPU former VCPU pointer because hypervisor CSR state + * will be lost. This ensures that the next VCPU entry will properly + * restore all CSRs instead of incorrectly skipping CSR restoration + * via the fast-path optimization. + */ + __this_cpu_write(kvm_former_vcpu, NULL); +} + +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), STATS_DESC_COUNTER(VCPU, ecall_exit_stat), STATS_DESC_COUNTER(VCPU, wfi_exit_stat), @@ -61,6 +74,7 @@ static void kvm_riscv_vcpu_context_reset(struct kvm_vcpu *vcpu, memset(cntx, 0, sizeof(*cntx)); memset(csr, 0, sizeof(*csr)); memset(&vcpu->arch.smstateen_csr, 0, sizeof(vcpu->arch.smstateen_csr)); + memset(&vcpu->arch.zicfiss_csr, 0, sizeof(vcpu->arch.zicfiss_csr)); /* Restore datap as it's not a part of the guest context. */ cntx->vector.datap = vector_datap; @@ -78,6 +92,7 @@ static void kvm_riscv_vcpu_context_reset(struct kvm_vcpu *vcpu, static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu, bool kvm_sbi_reset) { + unsigned long flags; bool loaded; /** @@ -102,8 +117,10 @@ static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu, bool kvm_sbi_reset) kvm_riscv_vcpu_aia_reset(vcpu); + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); bitmap_zero(vcpu->arch.irqs_pending, KVM_RISCV_VCPU_NR_IRQS); bitmap_zero(vcpu->arch.irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS); + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); kvm_riscv_vcpu_pmu_reset(vcpu); @@ -133,10 +150,12 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) /* Mark this VCPU never ran */ vcpu->arch.ran_atleast_once = false; - vcpu->arch.cfg.hedeleg = KVM_HEDELEG_DEFAULT; vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO; bitmap_zero(vcpu->arch.isa, RISCV_ISA_EXT_MAX); + /* Setup VCPU config */ + kvm_riscv_vcpu_config_init(vcpu); + /* Setup ISA features available to VCPU */ kvm_riscv_vcpu_setup_isa(vcpu); @@ -147,6 +166,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) /* Setup VCPU hfence queue */ spin_lock_init(&vcpu->arch.hfence_lock); + raw_spin_lock_init(&vcpu->arch.irqs_pending_lock); spin_lock_init(&vcpu->arch.reset_state.lock); @@ -348,10 +368,14 @@ void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu) { struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; unsigned long mask, val; + unsigned long flags; + + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); - if (READ_ONCE(vcpu->arch.irqs_pending_mask[0])) { - mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[0], 0); - val = READ_ONCE(vcpu->arch.irqs_pending[0]) & mask; + mask = vcpu->arch.irqs_pending_mask[0]; + if (mask) { + vcpu->arch.irqs_pending_mask[0] = 0; + val = vcpu->arch.irqs_pending[0] & mask; csr->hvip &= ~mask; csr->hvip |= val; @@ -359,11 +383,14 @@ void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu) /* Flush AIA high interrupts */ kvm_riscv_vcpu_aia_flush_interrupts(vcpu); + + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); } void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu) { unsigned long hvip; + unsigned long flags; struct kvm_vcpu_arch *v = &vcpu->arch; struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; @@ -372,34 +399,41 @@ void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu) /* Sync-up HVIP.VSSIP bit changes does by Guest */ hvip = ncsr_read(CSR_HVIP); + + raw_spin_lock_irqsave(&v->irqs_pending_lock, flags); + if ((csr->hvip ^ hvip) & (1UL << IRQ_VS_SOFT)) { if (hvip & (1UL << IRQ_VS_SOFT)) { - if (!test_and_set_bit(IRQ_VS_SOFT, - v->irqs_pending_mask)) - set_bit(IRQ_VS_SOFT, v->irqs_pending); + if (!__test_and_set_bit(IRQ_VS_SOFT, + v->irqs_pending_mask)) + __set_bit(IRQ_VS_SOFT, v->irqs_pending); } else { - if (!test_and_set_bit(IRQ_VS_SOFT, - v->irqs_pending_mask)) - clear_bit(IRQ_VS_SOFT, v->irqs_pending); + if (!__test_and_set_bit(IRQ_VS_SOFT, + v->irqs_pending_mask)) + __clear_bit(IRQ_VS_SOFT, v->irqs_pending); } } /* Sync up the HVIP.LCOFIP bit changes (only clear) by the guest */ if ((csr->hvip ^ hvip) & (1UL << IRQ_PMU_OVF)) { if (!(hvip & (1UL << IRQ_PMU_OVF)) && - !test_and_set_bit(IRQ_PMU_OVF, v->irqs_pending_mask)) - clear_bit(IRQ_PMU_OVF, v->irqs_pending); + !__test_and_set_bit(IRQ_PMU_OVF, v->irqs_pending_mask)) + __clear_bit(IRQ_PMU_OVF, v->irqs_pending); } /* Sync-up AIA high interrupts */ kvm_riscv_vcpu_aia_sync_interrupts(vcpu); + raw_spin_unlock_irqrestore(&v->irqs_pending_lock, flags); + /* Sync-up timer CSRs */ kvm_riscv_vcpu_timer_sync(vcpu); } int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) { + unsigned long flags; + /* * We only allow VS-mode software, timer, and external * interrupts when irq is one of the local interrupts @@ -412,9 +446,12 @@ int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) irq != IRQ_PMU_OVF) return -EINVAL; - set_bit(irq, vcpu->arch.irqs_pending); - smp_mb__before_atomic(); - set_bit(irq, vcpu->arch.irqs_pending_mask); + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + __set_bit(irq, vcpu->arch.irqs_pending); + __set_bit(irq, vcpu->arch.irqs_pending_mask); + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); + + trace_kvm_vcpu_irq(vcpu->vcpu_id, irq, 1); kvm_vcpu_kick(vcpu); @@ -423,6 +460,8 @@ int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) { + unsigned long flags; + /* * We only allow VS-mode software, timer, counter overflow and external * interrupts when irq is one of the local interrupts @@ -435,26 +474,34 @@ int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) irq != IRQ_PMU_OVF) return -EINVAL; - clear_bit(irq, vcpu->arch.irqs_pending); - smp_mb__before_atomic(); - set_bit(irq, vcpu->arch.irqs_pending_mask); + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + __clear_bit(irq, vcpu->arch.irqs_pending); + __set_bit(irq, vcpu->arch.irqs_pending_mask); + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); + + trace_kvm_vcpu_irq(vcpu->vcpu_id, irq, 0); return 0; } bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) { + unsigned long flags; unsigned long ie; + bool ret; - ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK) - << VSIP_TO_HVIP_SHIFT) & (unsigned long)mask; + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + ie = vsip_to_hvip(vcpu->arch.guest_csr.vsie) & (unsigned long)mask; ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK & (unsigned long)mask; - if (READ_ONCE(vcpu->arch.irqs_pending[0]) & ie) - return true; + ret = vcpu->arch.irqs_pending[0] & ie; + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); /* Check AIA high interrupts */ - return kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask); + if (!ret) + ret = kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask); + + return ret; } void __kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu) @@ -529,57 +576,42 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg) { - if (dbg->control & KVM_GUESTDBG_ENABLE) { + if (dbg->control & KVM_GUESTDBG_ENABLE) vcpu->guest_debug = dbg->control; - vcpu->arch.cfg.hedeleg &= ~BIT(EXC_BREAKPOINT); - } else { + else vcpu->guest_debug = 0; - vcpu->arch.cfg.hedeleg |= BIT(EXC_BREAKPOINT); - } + kvm_riscv_vcpu_config_guest_debug(vcpu); return 0; } -static void kvm_riscv_vcpu_setup_config(struct kvm_vcpu *vcpu) -{ - const unsigned long *isa = vcpu->arch.isa; - struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; - - if (riscv_isa_extension_available(isa, SVPBMT)) - cfg->henvcfg |= ENVCFG_PBMTE; - - if (riscv_isa_extension_available(isa, SSTC)) - cfg->henvcfg |= ENVCFG_STCE; - - if (riscv_isa_extension_available(isa, ZICBOM)) - cfg->henvcfg |= (ENVCFG_CBIE | ENVCFG_CBCFE); - - if (riscv_isa_extension_available(isa, ZICBOZ)) - cfg->henvcfg |= ENVCFG_CBZE; - - if (riscv_isa_extension_available(isa, SVADU) && - !riscv_isa_extension_available(isa, SVADE)) - cfg->henvcfg |= ENVCFG_ADUE; - - if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) { - cfg->hstateen0 |= SMSTATEEN0_HSENVCFG; - if (riscv_isa_extension_available(isa, SSAIA)) - cfg->hstateen0 |= SMSTATEEN0_AIA_IMSIC | - SMSTATEEN0_AIA | - SMSTATEEN0_AIA_ISEL; - if (riscv_isa_extension_available(isa, SMSTATEEN)) - cfg->hstateen0 |= SMSTATEEN0_SSTATEEN0; - } - - if (vcpu->guest_debug) - cfg->hedeleg &= ~BIT(EXC_BREAKPOINT); -} - void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { void *nsh; struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; - struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; + + /* + * If VCPU is being reloaded on the same physical CPU and no + * other KVM VCPU has run on this CPU since it was last put, + * we can skip the expensive CSR and HGATP writes. + * + * Note: If a new CSR is added to this fast-path skip block, + * make sure that 'csr_dirty' is set to true in any + * ioctl (e.g., KVM_SET_ONE_REG) that modifies it. + */ + if (vcpu != __this_cpu_read(kvm_former_vcpu)) + __this_cpu_write(kvm_former_vcpu, vcpu); + else if (vcpu->arch.last_exit_cpu == cpu && !vcpu->arch.csr_dirty) + goto csr_restore_done; + + vcpu->arch.csr_dirty = false; + + /* + * Load VCPU config CSRs before other CSRs because + * the read/write behaviour of certain CSRs change + * based on VCPU config CSRs. + */ + kvm_riscv_vcpu_config_load(vcpu); if (kvm_riscv_nacl_sync_csr_available()) { nsh = nacl_shmem(); @@ -590,17 +622,8 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) nacl_csr_write(nsh, CSR_VSEPC, csr->vsepc); nacl_csr_write(nsh, CSR_VSCAUSE, csr->vscause); nacl_csr_write(nsh, CSR_VSTVAL, csr->vstval); - nacl_csr_write(nsh, CSR_HEDELEG, cfg->hedeleg); nacl_csr_write(nsh, CSR_HVIP, csr->hvip); nacl_csr_write(nsh, CSR_VSATP, csr->vsatp); - nacl_csr_write(nsh, CSR_HENVCFG, cfg->henvcfg); - if (IS_ENABLED(CONFIG_32BIT)) - nacl_csr_write(nsh, CSR_HENVCFGH, cfg->henvcfg >> 32); - if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) { - nacl_csr_write(nsh, CSR_HSTATEEN0, cfg->hstateen0); - if (IS_ENABLED(CONFIG_32BIT)) - nacl_csr_write(nsh, CSR_HSTATEEN0H, cfg->hstateen0 >> 32); - } } else { csr_write(CSR_VSSTATUS, csr->vsstatus); csr_write(CSR_VSIE, csr->vsie); @@ -609,31 +632,25 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) csr_write(CSR_VSEPC, csr->vsepc); csr_write(CSR_VSCAUSE, csr->vscause); csr_write(CSR_VSTVAL, csr->vstval); - csr_write(CSR_HEDELEG, cfg->hedeleg); csr_write(CSR_HVIP, csr->hvip); csr_write(CSR_VSATP, csr->vsatp); - csr_write(CSR_HENVCFG, cfg->henvcfg); - if (IS_ENABLED(CONFIG_32BIT)) - csr_write(CSR_HENVCFGH, cfg->henvcfg >> 32); - if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) { - csr_write(CSR_HSTATEEN0, cfg->hstateen0); - if (IS_ENABLED(CONFIG_32BIT)) - csr_write(CSR_HSTATEEN0H, cfg->hstateen0 >> 32); - } } kvm_riscv_mmu_update_hgatp(vcpu); + kvm_riscv_vcpu_aia_load(vcpu, cpu); + +csr_restore_done: kvm_riscv_vcpu_timer_restore(vcpu); kvm_riscv_vcpu_host_fp_save(&vcpu->arch.host_context); kvm_riscv_vcpu_guest_fp_restore(&vcpu->arch.guest_context, vcpu->arch.isa); + get_cpu_vector_context(); kvm_riscv_vcpu_host_vector_save(&vcpu->arch.host_context); kvm_riscv_vcpu_guest_vector_restore(&vcpu->arch.guest_context, vcpu->arch.isa); - - kvm_riscv_vcpu_aia_load(vcpu, cpu); + put_cpu_vector_context(); kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); @@ -654,9 +671,11 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) kvm_riscv_vcpu_host_fp_restore(&vcpu->arch.host_context); kvm_riscv_vcpu_timer_save(vcpu); + get_cpu_vector_context(); kvm_riscv_vcpu_guest_vector_save(&vcpu->arch.guest_context, vcpu->arch.isa); kvm_riscv_vcpu_host_vector_restore(&vcpu->arch.host_context); + put_cpu_vector_context(); if (kvm_riscv_nacl_available()) { nsh = nacl_shmem(); @@ -748,30 +767,30 @@ static void kvm_riscv_update_hvip(struct kvm_vcpu *vcpu) static __always_inline void kvm_riscv_vcpu_swap_in_guest_state(struct kvm_vcpu *vcpu) { + struct kvm_vcpu_zicfiss_csr *zicficsr = &vcpu->arch.zicfiss_csr; struct kvm_vcpu_smstateen_csr *smcsr = &vcpu->arch.smstateen_csr; struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; - struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; vcpu->arch.host_scounteren = csr_swap(CSR_SCOUNTEREN, csr->scounteren); vcpu->arch.host_senvcfg = csr_swap(CSR_SENVCFG, csr->senvcfg); - if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN) && - (cfg->hstateen0 & SMSTATEEN0_SSTATEEN0)) - vcpu->arch.host_sstateen0 = csr_swap(CSR_SSTATEEN0, - smcsr->sstateen0); + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) + vcpu->arch.host_sstateen0 = csr_swap(CSR_SSTATEEN0, smcsr->sstateen0); + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICFISS)) + csr_write(CSR_SSP, zicficsr->ssp); } static __always_inline void kvm_riscv_vcpu_swap_in_host_state(struct kvm_vcpu *vcpu) { + struct kvm_vcpu_zicfiss_csr *zicficsr = &vcpu->arch.zicfiss_csr; struct kvm_vcpu_smstateen_csr *smcsr = &vcpu->arch.smstateen_csr; struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; - struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; csr->scounteren = csr_swap(CSR_SCOUNTEREN, vcpu->arch.host_scounteren); csr->senvcfg = csr_swap(CSR_SENVCFG, vcpu->arch.host_senvcfg); - if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN) && - (cfg->hstateen0 & SMSTATEEN0_SSTATEEN0)) - smcsr->sstateen0 = csr_swap(CSR_SSTATEEN0, - vcpu->arch.host_sstateen0); + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) + smcsr->sstateen0 = csr_swap(CSR_SSTATEEN0, vcpu->arch.host_sstateen0); + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICFISS)) + zicficsr->ssp = csr_swap(CSR_SSP, 0); } /* @@ -799,6 +818,14 @@ static void noinstr kvm_riscv_vcpu_enter_exit(struct kvm_vcpu *vcpu, kvm_riscv_vcpu_swap_in_guest_state(vcpu); guest_state_enter_irqoff(); + /* sstatus.VS != SR_VS_OFF is guaranteed when NEED_RESTORE is set */ + if (current->thread.riscv_v_flags & RISCV_V_VCPU_NEED_RESTORE) { + current->thread.riscv_v_flags &= ~RISCV_V_VCPU_NEED_RESTORE; + current->thread.riscv_v_flags |= RISCV_V_VCPU_CTX; + __kvm_riscv_vector_restore(gcntx); + gcntx->sstatus = (gcntx->sstatus & ~SR_VS) | SR_VS_CLEAN; + } + if (kvm_riscv_nacl_sync_sret_available()) { nsh = nacl_shmem(); @@ -868,7 +895,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) struct kvm_run *run = vcpu->run; if (!vcpu->arch.ran_atleast_once) - kvm_riscv_vcpu_setup_config(vcpu); + kvm_riscv_vcpu_config_ran_once(vcpu); /* Mark this VCPU ran at least once */ vcpu->arch.ran_atleast_once = true; diff --git a/arch/riscv/kvm/vcpu_config.c b/arch/riscv/kvm/vcpu_config.c new file mode 100644 index 000000000000..b68aa830aaf5 --- /dev/null +++ b/arch/riscv/kvm/vcpu_config.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Qualcomm Technologies, Inc. + */ + +#include <linux/kvm_host.h> +#include <asm/kvm_nacl.h> + +#define KVM_HEDELEG_DEFAULT (BIT(EXC_INST_MISALIGNED) | \ + BIT(EXC_INST_ILLEGAL) | \ + BIT(EXC_BREAKPOINT) | \ + BIT(EXC_SYSCALL) | \ + BIT(EXC_INST_PAGE_FAULT) | \ + BIT(EXC_LOAD_PAGE_FAULT) | \ + BIT(EXC_STORE_PAGE_FAULT)) + +#define KVM_HIDELEG_DEFAULT (BIT(IRQ_VS_SOFT) | \ + BIT(IRQ_VS_TIMER) | \ + BIT(IRQ_VS_EXT)) + +void kvm_riscv_vcpu_config_init(struct kvm_vcpu *vcpu) +{ + vcpu->arch.cfg.hedeleg = KVM_HEDELEG_DEFAULT; + vcpu->arch.cfg.hideleg = KVM_HIDELEG_DEFAULT; +} + +void kvm_riscv_vcpu_config_guest_debug(struct kvm_vcpu *vcpu) +{ + struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; + + if (vcpu->guest_debug) + cfg->hedeleg &= ~BIT(EXC_BREAKPOINT); + else + cfg->hedeleg |= BIT(EXC_BREAKPOINT); + + vcpu->arch.csr_dirty = true; +} + +void kvm_riscv_vcpu_config_ran_once(struct kvm_vcpu *vcpu) +{ + const unsigned long *isa = vcpu->arch.isa; + struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; + + if (riscv_isa_extension_available(isa, SVPBMT)) + cfg->henvcfg |= ENVCFG_PBMTE; + + if (riscv_isa_extension_available(isa, SSTC)) + cfg->henvcfg |= ENVCFG_STCE; + + if (riscv_isa_extension_available(isa, ZICBOM)) + cfg->henvcfg |= (ENVCFG_CBIE | ENVCFG_CBCFE); + + if (riscv_isa_extension_available(isa, ZICBOZ)) + cfg->henvcfg |= ENVCFG_CBZE; + + if (riscv_isa_extension_available(isa, SVADU) && + !riscv_isa_extension_available(isa, SVADE)) + cfg->henvcfg |= ENVCFG_ADUE; + + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) { + cfg->hstateen0 |= SMSTATEEN0_HSENVCFG; + if (riscv_isa_extension_available(isa, SSAIA)) + cfg->hstateen0 |= SMSTATEEN0_AIA_IMSIC | + SMSTATEEN0_AIA | + SMSTATEEN0_AIA_ISEL; + if (riscv_isa_extension_available(isa, SMSTATEEN)) + cfg->hstateen0 |= SMSTATEEN0_SSTATEEN0; + } + + if (vcpu->guest_debug) + cfg->hedeleg &= ~BIT(EXC_BREAKPOINT); + + kvm_riscv_vcpu_sbi_validate(vcpu); +} + +void kvm_riscv_vcpu_config_load(struct kvm_vcpu *vcpu) +{ + struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; + void *nsh; + + if (kvm_riscv_nacl_sync_csr_available()) { + nsh = nacl_shmem(); + nacl_csr_write(nsh, CSR_HEDELEG, cfg->hedeleg); + nacl_csr_write(nsh, CSR_HIDELEG, cfg->hideleg); + nacl_csr_write(nsh, CSR_HENVCFG, cfg->henvcfg); + if (IS_ENABLED(CONFIG_32BIT)) + nacl_csr_write(nsh, CSR_HENVCFGH, cfg->henvcfg >> 32); + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) { + nacl_csr_write(nsh, CSR_HSTATEEN0, cfg->hstateen0); + if (IS_ENABLED(CONFIG_32BIT)) + nacl_csr_write(nsh, CSR_HSTATEEN0H, cfg->hstateen0 >> 32); + } + } else { + csr_write(CSR_HEDELEG, cfg->hedeleg); + csr_write(CSR_HIDELEG, cfg->hideleg); + csr_write(CSR_HENVCFG, cfg->henvcfg); + if (IS_ENABLED(CONFIG_32BIT)) + csr_write(CSR_HENVCFGH, cfg->henvcfg >> 32); + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) { + csr_write(CSR_HSTATEEN0, cfg->hstateen0); + if (IS_ENABLED(CONFIG_32BIT)) + csr_write(CSR_HSTATEEN0H, cfg->hstateen0 >> 32); + } + } +} diff --git a/arch/riscv/kvm/vcpu_exit.c b/arch/riscv/kvm/vcpu_exit.c index 0bb0c51e3c89..8d36eb8abce1 100644 --- a/arch/riscv/kvm/vcpu_exit.c +++ b/arch/riscv/kvm/vcpu_exit.c @@ -11,18 +11,20 @@ #include <asm/insn-def.h> #include <asm/kvm_mmu.h> #include <asm/kvm_nacl.h> +#include "trace.h" static int gstage_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run, struct kvm_cpu_trap *trap) { struct kvm_gstage_mapping host_map; struct kvm_memory_slot *memslot; - unsigned long hva, fault_addr; + unsigned long hva; + gpa_t fault_addr; bool writable; gfn_t gfn; int ret; - fault_addr = (trap->htval << 2) | (trap->stval & 0x3); + fault_addr = ((gpa_t)trap->htval << 2) | (trap->stval & 0x3); gfn = fault_addr >> PAGE_SHIFT; memslot = gfn_to_memslot(vcpu->kvm, gfn); hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable); @@ -38,6 +40,25 @@ static int gstage_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run, return kvm_riscv_vcpu_mmio_store(vcpu, run, fault_addr, trap->htinst); + case EXC_INST_GUEST_PAGE_FAULT: { + /* + * No memslot backs this GPA and an instruction fetch + * cannot be emulated as MMIO. On bare metal a fetch + * from an unbacked physical address raises an + * instruction access fault, so reflect that back to + * the guest. + */ + struct kvm_cpu_trap inst_trap = { + .sepc = trap->sepc, + .scause = EXC_INST_ACCESS, + .stval = trap->stval, + .htval = 0, + .htinst = 0, + }; + + kvm_riscv_vcpu_trap_redirect(vcpu, &inst_trap); + return 1; + } default: return -EOPNOTSUPP; }; @@ -154,6 +175,13 @@ void kvm_riscv_vcpu_trap_redirect(struct kvm_vcpu *vcpu, /* Clear Guest SSTATUS.SIE bit */ vsstatus &= ~SR_SIE; + /* Change Guest SSTATUS.SPELP bit */ + if (vcpu->arch.cfg.henvcfg & ENVCFG_LPE) { + vsstatus &= ~SR_SPELP; + vsstatus |= vcpu->arch.guest_context.sstatus & SR_SPELP; + vcpu->arch.guest_context.sstatus &= ~SR_SPELP; + } + /* Update Guest SSTATUS */ ncsr_write(CSR_VSSTATUS, vsstatus); @@ -193,6 +221,9 @@ int kvm_riscv_vcpu_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, if (trap->scause & CAUSE_IRQ_FLAG) return 1; + trace_kvm_vcpu_exit(vcpu->vcpu_id, trap->sepc, trap->scause, + trap->stval, trap->htval, trap->htinst); + /* Handle guest traps */ ret = -EFAULT; run->exit_reason = KVM_EXIT_UNKNOWN; @@ -243,12 +274,16 @@ int kvm_riscv_vcpu_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, run->exit_reason = KVM_EXIT_DEBUG; ret = 0; break; + case EXC_SOFTWARE_CHECK: + if (vcpu->arch.cfg.henvcfg & (ENVCFG_LPE | ENVCFG_SSE)) + ret = vcpu_redirect(vcpu, trap); + break; default: break; } /* Print details in-case of error */ - if (ret < 0) { + if (ret < 0 && ret != -EINTR) { kvm_err("VCPU exit error %d\n", ret); kvm_err("SEPC=0x%lx SSTATUS=0x%lx HSTATUS=0x%lx\n", vcpu->arch.guest_context.sepc, diff --git a/arch/riscv/kvm/vcpu_fp.c b/arch/riscv/kvm/vcpu_fp.c index 030904d82b58..bb11e6757d34 100644 --- a/arch/riscv/kvm/vcpu_fp.c +++ b/arch/riscv/kvm/vcpu_fp.c @@ -10,8 +10,10 @@ #include <linux/errno.h> #include <linux/err.h> #include <linux/kvm_host.h> +#include <linux/nospec.h> #include <linux/uaccess.h> #include <asm/cpufeature.h> +#include <asm/kvm_isa.h> #ifdef CONFIG_FPU void kvm_riscv_vcpu_fp_reset(struct kvm_vcpu *vcpu) @@ -19,8 +21,8 @@ void kvm_riscv_vcpu_fp_reset(struct kvm_vcpu *vcpu) struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; cntx->sstatus &= ~SR_FS; - if (riscv_isa_extension_available(vcpu->arch.isa, f) || - riscv_isa_extension_available(vcpu->arch.isa, d)) + if (riscv_isa_extension_available(vcpu->arch.isa, F) || + riscv_isa_extension_available(vcpu->arch.isa, D)) cntx->sstatus |= SR_FS_INITIAL; else cntx->sstatus |= SR_FS_OFF; @@ -36,9 +38,9 @@ void kvm_riscv_vcpu_guest_fp_save(struct kvm_cpu_context *cntx, const unsigned long *isa) { if ((cntx->sstatus & SR_FS) == SR_FS_DIRTY) { - if (riscv_isa_extension_available(isa, d)) + if (riscv_isa_extension_available(isa, D)) __kvm_riscv_fp_d_save(cntx); - else if (riscv_isa_extension_available(isa, f)) + else if (riscv_isa_extension_available(isa, F)) __kvm_riscv_fp_f_save(cntx); kvm_riscv_vcpu_fp_clean(cntx); } @@ -48,9 +50,9 @@ void kvm_riscv_vcpu_guest_fp_restore(struct kvm_cpu_context *cntx, const unsigned long *isa) { if ((cntx->sstatus & SR_FS) != SR_FS_OFF) { - if (riscv_isa_extension_available(isa, d)) + if (riscv_isa_extension_available(isa, D)) __kvm_riscv_fp_d_restore(cntx); - else if (riscv_isa_extension_available(isa, f)) + else if (riscv_isa_extension_available(isa, F)) __kvm_riscv_fp_f_restore(cntx); kvm_riscv_vcpu_fp_clean(cntx); } @@ -59,17 +61,17 @@ void kvm_riscv_vcpu_guest_fp_restore(struct kvm_cpu_context *cntx, void kvm_riscv_vcpu_host_fp_save(struct kvm_cpu_context *cntx) { /* No need to check host sstatus as it can be modified outside */ - if (riscv_isa_extension_available(NULL, d)) + if (!kvm_riscv_isa_check_host(D)) __kvm_riscv_fp_d_save(cntx); - else if (riscv_isa_extension_available(NULL, f)) + else if (!kvm_riscv_isa_check_host(F)) __kvm_riscv_fp_f_save(cntx); } void kvm_riscv_vcpu_host_fp_restore(struct kvm_cpu_context *cntx) { - if (riscv_isa_extension_available(NULL, d)) + if (!kvm_riscv_isa_check_host(D)) __kvm_riscv_fp_d_restore(cntx); - else if (riscv_isa_extension_available(NULL, f)) + else if (!kvm_riscv_isa_check_host(F)) __kvm_riscv_fp_f_restore(cntx); } #endif @@ -87,18 +89,20 @@ int kvm_riscv_vcpu_get_reg_fp(struct kvm_vcpu *vcpu, void *reg_val; if ((rtype == KVM_REG_RISCV_FP_F) && - riscv_isa_extension_available(vcpu->arch.isa, f)) { + riscv_isa_extension_available(vcpu->arch.isa, F)) { if (KVM_REG_SIZE(reg->id) != sizeof(u32)) return -EINVAL; if (reg_num == KVM_REG_RISCV_FP_F_REG(fcsr)) reg_val = &cntx->fp.f.fcsr; else if ((KVM_REG_RISCV_FP_F_REG(f[0]) <= reg_num) && - reg_num <= KVM_REG_RISCV_FP_F_REG(f[31])) + reg_num <= KVM_REG_RISCV_FP_F_REG(f[31])) { + reg_num = array_index_nospec(reg_num, + ARRAY_SIZE(cntx->fp.f.f)); reg_val = &cntx->fp.f.f[reg_num]; - else + } else return -ENOENT; } else if ((rtype == KVM_REG_RISCV_FP_D) && - riscv_isa_extension_available(vcpu->arch.isa, d)) { + riscv_isa_extension_available(vcpu->arch.isa, D)) { if (reg_num == KVM_REG_RISCV_FP_D_REG(fcsr)) { if (KVM_REG_SIZE(reg->id) != sizeof(u32)) return -EINVAL; @@ -107,6 +111,8 @@ int kvm_riscv_vcpu_get_reg_fp(struct kvm_vcpu *vcpu, reg_num <= KVM_REG_RISCV_FP_D_REG(f[31])) { if (KVM_REG_SIZE(reg->id) != sizeof(u64)) return -EINVAL; + reg_num = array_index_nospec(reg_num, + ARRAY_SIZE(cntx->fp.d.f)); reg_val = &cntx->fp.d.f[reg_num]; } else return -ENOENT; @@ -132,18 +138,20 @@ int kvm_riscv_vcpu_set_reg_fp(struct kvm_vcpu *vcpu, void *reg_val; if ((rtype == KVM_REG_RISCV_FP_F) && - riscv_isa_extension_available(vcpu->arch.isa, f)) { + riscv_isa_extension_available(vcpu->arch.isa, F)) { if (KVM_REG_SIZE(reg->id) != sizeof(u32)) return -EINVAL; if (reg_num == KVM_REG_RISCV_FP_F_REG(fcsr)) reg_val = &cntx->fp.f.fcsr; else if ((KVM_REG_RISCV_FP_F_REG(f[0]) <= reg_num) && - reg_num <= KVM_REG_RISCV_FP_F_REG(f[31])) + reg_num <= KVM_REG_RISCV_FP_F_REG(f[31])) { + reg_num = array_index_nospec(reg_num, + ARRAY_SIZE(cntx->fp.f.f)); reg_val = &cntx->fp.f.f[reg_num]; - else + } else return -ENOENT; } else if ((rtype == KVM_REG_RISCV_FP_D) && - riscv_isa_extension_available(vcpu->arch.isa, d)) { + riscv_isa_extension_available(vcpu->arch.isa, D)) { if (reg_num == KVM_REG_RISCV_FP_D_REG(fcsr)) { if (KVM_REG_SIZE(reg->id) != sizeof(u32)) return -EINVAL; @@ -152,6 +160,8 @@ int kvm_riscv_vcpu_set_reg_fp(struct kvm_vcpu *vcpu, reg_num <= KVM_REG_RISCV_FP_D_REG(f[31])) { if (KVM_REG_SIZE(reg->id) != sizeof(u64)) return -EINVAL; + reg_num = array_index_nospec(reg_num, + ARRAY_SIZE(cntx->fp.d.f)); reg_val = &cntx->fp.d.f[reg_num]; } else return -ENOENT; diff --git a/arch/riscv/kvm/vcpu_insn.c b/arch/riscv/kvm/vcpu_insn.c index 4d89b94128ae..adbbb59a0f36 100644 --- a/arch/riscv/kvm/vcpu_insn.c +++ b/arch/riscv/kvm/vcpu_insn.c @@ -9,6 +9,7 @@ #include <asm/cpufeature.h> #include <asm/insn.h> +#include "trace.h" struct insn_func { unsigned long mask; @@ -371,11 +372,11 @@ int kvm_riscv_vcpu_virtual_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, * Returns < 0 to report failure and exit run-loop */ int kvm_riscv_vcpu_mmio_load(struct kvm_vcpu *vcpu, struct kvm_run *run, - unsigned long fault_addr, + gpa_t fault_addr, unsigned long htinst) { u8 data_buf[8]; - unsigned long insn; + unsigned long insn, raw_insn; int shift = 0, len = 0, insn_len = 0; struct kvm_cpu_trap utrap = { 0 }; struct kvm_cpu_context *ct = &vcpu->arch.guest_context; @@ -405,6 +406,7 @@ int kvm_riscv_vcpu_mmio_load(struct kvm_vcpu *vcpu, struct kvm_run *run, } insn_len = INSN_LEN(insn); } + raw_insn = insn; /* Decode length of MMIO and shift */ if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) { @@ -415,7 +417,6 @@ int kvm_riscv_vcpu_mmio_load(struct kvm_vcpu *vcpu, struct kvm_run *run, shift = 8 * (sizeof(ulong) - len); } else if ((insn & INSN_MASK_LBU) == INSN_MATCH_LBU) { len = 1; - shift = 8 * (sizeof(ulong) - len); #ifdef CONFIG_64BIT } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) { len = 8; @@ -454,6 +455,9 @@ int kvm_riscv_vcpu_mmio_load(struct kvm_vcpu *vcpu, struct kvm_run *run, if (fault_addr & (len - 1)) return -EIO; + trace_kvm_mmio_emulate(vcpu->vcpu_id, ct->sepc, raw_insn, fault_addr, + false, len); + /* Save instruction decode info */ vcpu->arch.mmio_decode.insn = insn; vcpu->arch.mmio_decode.insn_len = insn_len; @@ -495,7 +499,7 @@ int kvm_riscv_vcpu_mmio_load(struct kvm_vcpu *vcpu, struct kvm_run *run, * Returns < 0 to report failure and exit run-loop */ int kvm_riscv_vcpu_mmio_store(struct kvm_vcpu *vcpu, struct kvm_run *run, - unsigned long fault_addr, + gpa_t fault_addr, unsigned long htinst) { u8 data8; @@ -503,7 +507,7 @@ int kvm_riscv_vcpu_mmio_store(struct kvm_vcpu *vcpu, struct kvm_run *run, u32 data32; u64 data64; ulong data; - unsigned long insn; + unsigned long insn, raw_insn; int len = 0, insn_len = 0; struct kvm_cpu_trap utrap = { 0 }; struct kvm_cpu_context *ct = &vcpu->arch.guest_context; @@ -533,6 +537,7 @@ int kvm_riscv_vcpu_mmio_store(struct kvm_vcpu *vcpu, struct kvm_run *run, } insn_len = INSN_LEN(insn); } + raw_insn = insn; data = GET_RS2(insn, &vcpu->arch.guest_context); data8 = data16 = data32 = data64 = data; @@ -571,6 +576,9 @@ int kvm_riscv_vcpu_mmio_store(struct kvm_vcpu *vcpu, struct kvm_run *run, if (fault_addr & (len - 1)) return -EIO; + trace_kvm_mmio_emulate(vcpu->vcpu_id, ct->sepc, raw_insn, fault_addr, + true, len); + /* Save instruction decode info */ vcpu->arch.mmio_decode.insn = insn; vcpu->arch.mmio_decode.insn_len = insn_len; @@ -649,22 +657,22 @@ int kvm_riscv_vcpu_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run) case 1: data8 = *((u8 *)run->mmio.data); SET_RD(insn, &vcpu->arch.guest_context, - (ulong)data8 << shift >> shift); + (long)((ulong)data8 << shift) >> shift); break; case 2: data16 = *((u16 *)run->mmio.data); SET_RD(insn, &vcpu->arch.guest_context, - (ulong)data16 << shift >> shift); + (long)((ulong)data16 << shift) >> shift); break; case 4: data32 = *((u32 *)run->mmio.data); SET_RD(insn, &vcpu->arch.guest_context, - (ulong)data32 << shift >> shift); + (long)((ulong)data32 << shift) >> shift); break; case 8: data64 = *((u64 *)run->mmio.data); SET_RD(insn, &vcpu->arch.guest_context, - (ulong)data64 << shift >> shift); + (long)((ulong)data64 << shift) >> shift); break; default: return -EOPNOTSUPP; diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c index 865dae903aa0..f16e25098af1 100644 --- a/arch/riscv/kvm/vcpu_onereg.c +++ b/arch/riscv/kvm/vcpu_onereg.c @@ -10,258 +10,24 @@ #include <linux/bitops.h> #include <linux/errno.h> #include <linux/err.h> +#include <linux/nospec.h> #include <linux/uaccess.h> #include <linux/kvm_host.h> #include <asm/cacheflush.h> #include <asm/cpufeature.h> +#include <asm/kvm_isa.h> #include <asm/kvm_vcpu_vector.h> -#include <asm/pgtable.h> -#include <asm/vector.h> #define KVM_RISCV_BASE_ISA_MASK GENMASK(25, 0) -#define KVM_ISA_EXT_ARR(ext) \ -[KVM_RISCV_ISA_EXT_##ext] = RISCV_ISA_EXT_##ext - -/* Mapping between KVM ISA Extension ID & guest ISA extension ID */ -static const unsigned long kvm_isa_ext_arr[] = { - /* Single letter extensions (alphabetically sorted) */ - [KVM_RISCV_ISA_EXT_A] = RISCV_ISA_EXT_a, - [KVM_RISCV_ISA_EXT_C] = RISCV_ISA_EXT_c, - [KVM_RISCV_ISA_EXT_D] = RISCV_ISA_EXT_d, - [KVM_RISCV_ISA_EXT_F] = RISCV_ISA_EXT_f, - [KVM_RISCV_ISA_EXT_H] = RISCV_ISA_EXT_h, - [KVM_RISCV_ISA_EXT_I] = RISCV_ISA_EXT_i, - [KVM_RISCV_ISA_EXT_M] = RISCV_ISA_EXT_m, - [KVM_RISCV_ISA_EXT_V] = RISCV_ISA_EXT_v, - /* Multi letter extensions (alphabetically sorted) */ - KVM_ISA_EXT_ARR(SMNPM), - KVM_ISA_EXT_ARR(SMSTATEEN), - KVM_ISA_EXT_ARR(SSAIA), - KVM_ISA_EXT_ARR(SSCOFPMF), - KVM_ISA_EXT_ARR(SSNPM), - KVM_ISA_EXT_ARR(SSTC), - KVM_ISA_EXT_ARR(SVADE), - KVM_ISA_EXT_ARR(SVADU), - KVM_ISA_EXT_ARR(SVINVAL), - KVM_ISA_EXT_ARR(SVNAPOT), - KVM_ISA_EXT_ARR(SVPBMT), - KVM_ISA_EXT_ARR(SVVPTC), - KVM_ISA_EXT_ARR(ZAAMO), - KVM_ISA_EXT_ARR(ZABHA), - KVM_ISA_EXT_ARR(ZACAS), - KVM_ISA_EXT_ARR(ZALRSC), - KVM_ISA_EXT_ARR(ZAWRS), - KVM_ISA_EXT_ARR(ZBA), - KVM_ISA_EXT_ARR(ZBB), - KVM_ISA_EXT_ARR(ZBC), - KVM_ISA_EXT_ARR(ZBKB), - KVM_ISA_EXT_ARR(ZBKC), - KVM_ISA_EXT_ARR(ZBKX), - KVM_ISA_EXT_ARR(ZBS), - KVM_ISA_EXT_ARR(ZCA), - KVM_ISA_EXT_ARR(ZCB), - KVM_ISA_EXT_ARR(ZCD), - KVM_ISA_EXT_ARR(ZCF), - KVM_ISA_EXT_ARR(ZCMOP), - KVM_ISA_EXT_ARR(ZFA), - KVM_ISA_EXT_ARR(ZFBFMIN), - KVM_ISA_EXT_ARR(ZFH), - KVM_ISA_EXT_ARR(ZFHMIN), - KVM_ISA_EXT_ARR(ZICBOM), - KVM_ISA_EXT_ARR(ZICBOP), - KVM_ISA_EXT_ARR(ZICBOZ), - KVM_ISA_EXT_ARR(ZICCRSE), - KVM_ISA_EXT_ARR(ZICNTR), - KVM_ISA_EXT_ARR(ZICOND), - KVM_ISA_EXT_ARR(ZICSR), - KVM_ISA_EXT_ARR(ZIFENCEI), - KVM_ISA_EXT_ARR(ZIHINTNTL), - KVM_ISA_EXT_ARR(ZIHINTPAUSE), - KVM_ISA_EXT_ARR(ZIHPM), - KVM_ISA_EXT_ARR(ZIMOP), - KVM_ISA_EXT_ARR(ZKND), - KVM_ISA_EXT_ARR(ZKNE), - KVM_ISA_EXT_ARR(ZKNH), - KVM_ISA_EXT_ARR(ZKR), - KVM_ISA_EXT_ARR(ZKSED), - KVM_ISA_EXT_ARR(ZKSH), - KVM_ISA_EXT_ARR(ZKT), - KVM_ISA_EXT_ARR(ZTSO), - KVM_ISA_EXT_ARR(ZVBB), - KVM_ISA_EXT_ARR(ZVBC), - KVM_ISA_EXT_ARR(ZVFBFMIN), - KVM_ISA_EXT_ARR(ZVFBFWMA), - KVM_ISA_EXT_ARR(ZVFH), - KVM_ISA_EXT_ARR(ZVFHMIN), - KVM_ISA_EXT_ARR(ZVKB), - KVM_ISA_EXT_ARR(ZVKG), - KVM_ISA_EXT_ARR(ZVKNED), - KVM_ISA_EXT_ARR(ZVKNHA), - KVM_ISA_EXT_ARR(ZVKNHB), - KVM_ISA_EXT_ARR(ZVKSED), - KVM_ISA_EXT_ARR(ZVKSH), - KVM_ISA_EXT_ARR(ZVKT), -}; - -static unsigned long kvm_riscv_vcpu_base2isa_ext(unsigned long base_ext) -{ - unsigned long i; - - for (i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) { - if (kvm_isa_ext_arr[i] == base_ext) - return i; - } - - return KVM_RISCV_ISA_EXT_MAX; -} - -static int kvm_riscv_vcpu_isa_check_host(unsigned long kvm_ext, unsigned long *guest_ext) -{ - unsigned long host_ext; - - if (kvm_ext >= KVM_RISCV_ISA_EXT_MAX || - kvm_ext >= ARRAY_SIZE(kvm_isa_ext_arr)) - return -ENOENT; - - *guest_ext = kvm_isa_ext_arr[kvm_ext]; - switch (*guest_ext) { - case RISCV_ISA_EXT_SMNPM: - /* - * Pointer masking effective in (H)S-mode is provided by the - * Smnpm extension, so that extension is reported to the guest, - * even though the CSR bits for configuring VS-mode pointer - * masking on the host side are part of the Ssnpm extension. - */ - host_ext = RISCV_ISA_EXT_SSNPM; - break; - default: - host_ext = *guest_ext; - break; - } - - if (!__riscv_isa_extension_available(NULL, host_ext)) - return -ENOENT; - - return 0; -} - -static bool kvm_riscv_vcpu_isa_enable_allowed(unsigned long ext) -{ - switch (ext) { - case KVM_RISCV_ISA_EXT_H: - return false; - case KVM_RISCV_ISA_EXT_SSCOFPMF: - /* Sscofpmf depends on interrupt filtering defined in ssaia */ - return __riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSAIA); - case KVM_RISCV_ISA_EXT_SVADU: - /* - * The henvcfg.ADUE is read-only zero if menvcfg.ADUE is zero. - * Guest OS can use Svadu only when host OS enable Svadu. - */ - return arch_has_hw_pte_young(); - case KVM_RISCV_ISA_EXT_V: - return riscv_v_vstate_ctrl_user_allowed(); - default: - break; - } - - return true; -} - -static bool kvm_riscv_vcpu_isa_disable_allowed(unsigned long ext) -{ - switch (ext) { - /* Extensions which don't have any mechanism to disable */ - case KVM_RISCV_ISA_EXT_A: - case KVM_RISCV_ISA_EXT_C: - case KVM_RISCV_ISA_EXT_I: - case KVM_RISCV_ISA_EXT_M: - /* There is not architectural config bit to disable sscofpmf completely */ - case KVM_RISCV_ISA_EXT_SSCOFPMF: - case KVM_RISCV_ISA_EXT_SSNPM: - case KVM_RISCV_ISA_EXT_SSTC: - case KVM_RISCV_ISA_EXT_SVINVAL: - case KVM_RISCV_ISA_EXT_SVNAPOT: - case KVM_RISCV_ISA_EXT_SVVPTC: - case KVM_RISCV_ISA_EXT_ZAAMO: - case KVM_RISCV_ISA_EXT_ZABHA: - case KVM_RISCV_ISA_EXT_ZACAS: - case KVM_RISCV_ISA_EXT_ZALRSC: - case KVM_RISCV_ISA_EXT_ZAWRS: - case KVM_RISCV_ISA_EXT_ZBA: - case KVM_RISCV_ISA_EXT_ZBB: - case KVM_RISCV_ISA_EXT_ZBC: - case KVM_RISCV_ISA_EXT_ZBKB: - case KVM_RISCV_ISA_EXT_ZBKC: - case KVM_RISCV_ISA_EXT_ZBKX: - case KVM_RISCV_ISA_EXT_ZBS: - case KVM_RISCV_ISA_EXT_ZCA: - case KVM_RISCV_ISA_EXT_ZCB: - case KVM_RISCV_ISA_EXT_ZCD: - case KVM_RISCV_ISA_EXT_ZCF: - case KVM_RISCV_ISA_EXT_ZCMOP: - case KVM_RISCV_ISA_EXT_ZFA: - case KVM_RISCV_ISA_EXT_ZFBFMIN: - case KVM_RISCV_ISA_EXT_ZFH: - case KVM_RISCV_ISA_EXT_ZFHMIN: - case KVM_RISCV_ISA_EXT_ZICBOP: - case KVM_RISCV_ISA_EXT_ZICCRSE: - case KVM_RISCV_ISA_EXT_ZICNTR: - case KVM_RISCV_ISA_EXT_ZICOND: - case KVM_RISCV_ISA_EXT_ZICSR: - case KVM_RISCV_ISA_EXT_ZIFENCEI: - case KVM_RISCV_ISA_EXT_ZIHINTNTL: - case KVM_RISCV_ISA_EXT_ZIHINTPAUSE: - case KVM_RISCV_ISA_EXT_ZIHPM: - case KVM_RISCV_ISA_EXT_ZIMOP: - case KVM_RISCV_ISA_EXT_ZKND: - case KVM_RISCV_ISA_EXT_ZKNE: - case KVM_RISCV_ISA_EXT_ZKNH: - case KVM_RISCV_ISA_EXT_ZKR: - case KVM_RISCV_ISA_EXT_ZKSED: - case KVM_RISCV_ISA_EXT_ZKSH: - case KVM_RISCV_ISA_EXT_ZKT: - case KVM_RISCV_ISA_EXT_ZTSO: - case KVM_RISCV_ISA_EXT_ZVBB: - case KVM_RISCV_ISA_EXT_ZVBC: - case KVM_RISCV_ISA_EXT_ZVFBFMIN: - case KVM_RISCV_ISA_EXT_ZVFBFWMA: - case KVM_RISCV_ISA_EXT_ZVFH: - case KVM_RISCV_ISA_EXT_ZVFHMIN: - case KVM_RISCV_ISA_EXT_ZVKB: - case KVM_RISCV_ISA_EXT_ZVKG: - case KVM_RISCV_ISA_EXT_ZVKNED: - case KVM_RISCV_ISA_EXT_ZVKNHA: - case KVM_RISCV_ISA_EXT_ZVKNHB: - case KVM_RISCV_ISA_EXT_ZVKSED: - case KVM_RISCV_ISA_EXT_ZVKSH: - case KVM_RISCV_ISA_EXT_ZVKT: - return false; - /* Extensions which can be disabled using Smstateen */ - case KVM_RISCV_ISA_EXT_SSAIA: - return riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN); - case KVM_RISCV_ISA_EXT_SVADE: - /* - * The henvcfg.ADUE is read-only zero if menvcfg.ADUE is zero. - * Svade can't be disabled unless we support Svadu. - */ - return arch_has_hw_pte_young(); - default: - break; - } - - return true; -} - void kvm_riscv_vcpu_setup_isa(struct kvm_vcpu *vcpu) { unsigned long guest_ext, i; - for (i = 0; i < ARRAY_SIZE(kvm_isa_ext_arr); i++) { - if (kvm_riscv_vcpu_isa_check_host(i, &guest_ext)) + for (i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) { + if (__kvm_riscv_isa_check_host(i, &guest_ext)) continue; - if (kvm_riscv_vcpu_isa_enable_allowed(i)) + if (kvm_riscv_isa_enable_allowed(i)) set_bit(guest_ext, vcpu->arch.isa); } } @@ -284,19 +50,13 @@ static int kvm_riscv_vcpu_get_reg_config(struct kvm_vcpu *vcpu, reg_val = vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK; break; case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): - if (!riscv_isa_extension_available(NULL, ZICBOM)) - return -ENOENT; - reg_val = riscv_cbom_block_size; + reg_val = (kvm_riscv_isa_check_host(ZICBOM)) ? 0 : riscv_cbom_block_size; break; case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): - if (!riscv_isa_extension_available(NULL, ZICBOZ)) - return -ENOENT; - reg_val = riscv_cboz_block_size; + reg_val = (kvm_riscv_isa_check_host(ZICBOZ)) ? 0 : riscv_cboz_block_size; break; case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size): - if (!riscv_isa_extension_available(NULL, ZICBOP)) - return -ENOENT; - reg_val = riscv_cbop_block_size; + reg_val = (kvm_riscv_isa_check_host(ZICBOP)) ? 0 : riscv_cbop_block_size; break; case KVM_REG_RISCV_CONFIG_REG(mvendorid): reg_val = vcpu->arch.mvendorid; @@ -355,15 +115,15 @@ static int kvm_riscv_vcpu_set_reg_config(struct kvm_vcpu *vcpu, if (!vcpu->arch.ran_atleast_once) { /* Ignore the enable/disable request for certain extensions */ for (i = 0; i < RISCV_ISA_EXT_BASE; i++) { - isa_ext = kvm_riscv_vcpu_base2isa_ext(i); + isa_ext = kvm_riscv_base2isa_ext(i); if (isa_ext >= KVM_RISCV_ISA_EXT_MAX) { reg_val &= ~BIT(i); continue; } - if (!kvm_riscv_vcpu_isa_enable_allowed(isa_ext)) + if (!kvm_riscv_isa_enable_allowed(isa_ext)) if (reg_val & BIT(i)) reg_val &= ~BIT(i); - if (!kvm_riscv_vcpu_isa_disable_allowed(isa_ext)) + if (!kvm_riscv_isa_disable_allowed(isa_ext)) if (!(reg_val & BIT(i))) reg_val |= BIT(i); } @@ -378,21 +138,15 @@ static int kvm_riscv_vcpu_set_reg_config(struct kvm_vcpu *vcpu, } break; case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): - if (!riscv_isa_extension_available(NULL, ZICBOM)) - return -ENOENT; - if (reg_val != riscv_cbom_block_size) + if (reg_val && reg_val != riscv_cbom_block_size) return -EINVAL; break; case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): - if (!riscv_isa_extension_available(NULL, ZICBOZ)) - return -ENOENT; - if (reg_val != riscv_cboz_block_size) + if (reg_val && reg_val != riscv_cboz_block_size) return -EINVAL; break; case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size): - if (!riscv_isa_extension_available(NULL, ZICBOP)) - return -ENOENT; - if (reg_val != riscv_cbop_block_size) + if (reg_val && reg_val != riscv_cbop_block_size) return -EINVAL; break; case KVM_REG_RISCV_CONFIG_REG(mvendorid): @@ -439,13 +193,16 @@ static int kvm_riscv_vcpu_get_reg_core(struct kvm_vcpu *vcpu, unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_RISCV_CORE); + unsigned long regs_max = sizeof(struct kvm_riscv_core) / sizeof(unsigned long); unsigned long reg_val; if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) return -EINVAL; - if (reg_num >= sizeof(struct kvm_riscv_core) / sizeof(unsigned long)) + if (reg_num >= regs_max) return -ENOENT; + reg_num = array_index_nospec(reg_num, regs_max); + if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc)) reg_val = cntx->sepc; else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num && @@ -472,13 +229,16 @@ static int kvm_riscv_vcpu_set_reg_core(struct kvm_vcpu *vcpu, unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_RISCV_CORE); + unsigned long regs_max = sizeof(struct kvm_riscv_core) / sizeof(unsigned long); unsigned long reg_val; if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) return -EINVAL; - if (reg_num >= sizeof(struct kvm_riscv_core) / sizeof(unsigned long)) + if (reg_num >= regs_max) return -ENOENT; + reg_num = array_index_nospec(reg_num, regs_max); + if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) return -EFAULT; @@ -503,13 +263,16 @@ static int kvm_riscv_vcpu_general_get_csr(struct kvm_vcpu *vcpu, unsigned long *out_val) { struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; + unsigned long regs_max = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); - if (reg_num >= sizeof(struct kvm_riscv_csr) / sizeof(unsigned long)) + if (reg_num >= regs_max) return -ENOENT; + reg_num = array_index_nospec(reg_num, regs_max); + if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { kvm_riscv_vcpu_flush_interrupts(vcpu); - *out_val = (csr->hvip >> VSIP_TO_HVIP_SHIFT) & VSIP_VALID_MASK; + *out_val = hvip_to_vsip(csr->hvip); *out_val |= csr->hvip & ~IRQ_LOCAL_MASK; } else *out_val = ((unsigned long *)csr)[reg_num]; @@ -522,19 +285,24 @@ static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu, unsigned long reg_val) { struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; + unsigned long regs_max = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); + unsigned long flags; - if (reg_num >= sizeof(struct kvm_riscv_csr) / sizeof(unsigned long)) + if (reg_num >= regs_max) return -ENOENT; - if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { - reg_val &= VSIP_VALID_MASK; - reg_val <<= VSIP_TO_HVIP_SHIFT; - } + reg_num = array_index_nospec(reg_num, regs_max); + + if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) + reg_val = vsip_to_hvip(reg_val); ((unsigned long *)csr)[reg_num] = reg_val; - if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) - WRITE_ONCE(vcpu->arch.irqs_pending_mask[0], 0); + if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + vcpu->arch.irqs_pending_mask[0] = 0; + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); + } return 0; } @@ -544,10 +312,15 @@ static inline int kvm_riscv_vcpu_smstateen_set_csr(struct kvm_vcpu *vcpu, unsigned long reg_val) { struct kvm_vcpu_smstateen_csr *csr = &vcpu->arch.smstateen_csr; + unsigned long regs_max = sizeof(struct kvm_riscv_smstateen_csr) / + sizeof(unsigned long); - if (reg_num >= sizeof(struct kvm_riscv_smstateen_csr) / - sizeof(unsigned long)) - return -EINVAL; + if (!riscv_isa_extension_available(vcpu->arch.isa, SMSTATEEN)) + return -ENOENT; + if (reg_num >= regs_max) + return -ENOENT; + + reg_num = array_index_nospec(reg_num, regs_max); ((unsigned long *)csr)[reg_num] = reg_val; return 0; @@ -558,10 +331,53 @@ static int kvm_riscv_vcpu_smstateen_get_csr(struct kvm_vcpu *vcpu, unsigned long *out_val) { struct kvm_vcpu_smstateen_csr *csr = &vcpu->arch.smstateen_csr; + unsigned long regs_max = sizeof(struct kvm_riscv_smstateen_csr) / + sizeof(unsigned long); - if (reg_num >= sizeof(struct kvm_riscv_smstateen_csr) / - sizeof(unsigned long)) - return -EINVAL; + if (!riscv_isa_extension_available(vcpu->arch.isa, SMSTATEEN)) + return -ENOENT; + if (reg_num >= regs_max) + return -ENOENT; + + reg_num = array_index_nospec(reg_num, regs_max); + + *out_val = ((unsigned long *)csr)[reg_num]; + return 0; +} + +static inline int kvm_riscv_vcpu_zicfiss_set_csr(struct kvm_vcpu *vcpu, + unsigned long reg_num, + unsigned long reg_val) +{ + struct kvm_vcpu_zicfiss_csr *csr = &vcpu->arch.zicfiss_csr; + unsigned long regs_max = sizeof(struct kvm_vcpu_zicfiss_csr) / + sizeof(unsigned long); + + if (!riscv_isa_extension_available(vcpu->arch.isa, ZICFISS)) + return -ENOENT; + if (reg_num >= regs_max) + return -ENOENT; + + reg_num = array_index_nospec(reg_num, regs_max); + + ((unsigned long *)csr)[reg_num] = reg_val; + return 0; +} + +static int kvm_riscv_vcpu_zicfiss_get_csr(struct kvm_vcpu *vcpu, + unsigned long reg_num, + unsigned long *out_val) +{ + struct kvm_vcpu_zicfiss_csr *csr = &vcpu->arch.zicfiss_csr; + unsigned long regs_max = sizeof(struct kvm_vcpu_zicfiss_csr) / + sizeof(unsigned long); + + if (!riscv_isa_extension_available(vcpu->arch.isa, ZICFISS)) + return -ENOENT; + if (reg_num >= regs_max) + return -ENOENT; + + reg_num = array_index_nospec(reg_num, regs_max); *out_val = ((unsigned long *)csr)[reg_num]; return 0; @@ -591,10 +407,10 @@ static int kvm_riscv_vcpu_get_reg_csr(struct kvm_vcpu *vcpu, rc = kvm_riscv_vcpu_aia_get_csr(vcpu, reg_num, ®_val); break; case KVM_REG_RISCV_CSR_SMSTATEEN: - rc = -EINVAL; - if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) - rc = kvm_riscv_vcpu_smstateen_get_csr(vcpu, reg_num, - ®_val); + rc = kvm_riscv_vcpu_smstateen_get_csr(vcpu, reg_num, ®_val); + break; + case KVM_REG_RISCV_CSR_ZICFISS: + rc = kvm_riscv_vcpu_zicfiss_get_csr(vcpu, reg_num, ®_val); break; default: rc = -ENOENT; @@ -636,10 +452,10 @@ static int kvm_riscv_vcpu_set_reg_csr(struct kvm_vcpu *vcpu, rc = kvm_riscv_vcpu_aia_set_csr(vcpu, reg_num, reg_val); break; case KVM_REG_RISCV_CSR_SMSTATEEN: - rc = -EINVAL; - if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) - rc = kvm_riscv_vcpu_smstateen_set_csr(vcpu, reg_num, - reg_val); + rc = kvm_riscv_vcpu_smstateen_set_csr(vcpu, reg_num, reg_val); + break; + case KVM_REG_RISCV_CSR_ZICFISS: + rc = kvm_riscv_vcpu_zicfiss_set_csr(vcpu, reg_num, reg_val); break; default: rc = -ENOENT; @@ -648,6 +464,8 @@ static int kvm_riscv_vcpu_set_reg_csr(struct kvm_vcpu *vcpu, if (rc) return rc; + vcpu->arch.csr_dirty = true; + return 0; } @@ -658,7 +476,7 @@ static int riscv_vcpu_get_isa_ext_single(struct kvm_vcpu *vcpu, unsigned long guest_ext; int ret; - ret = kvm_riscv_vcpu_isa_check_host(reg_num, &guest_ext); + ret = __kvm_riscv_isa_check_host(reg_num, &guest_ext); if (ret) return ret; @@ -676,7 +494,7 @@ static int riscv_vcpu_set_isa_ext_single(struct kvm_vcpu *vcpu, unsigned long guest_ext; int ret; - ret = kvm_riscv_vcpu_isa_check_host(reg_num, &guest_ext); + ret = __kvm_riscv_isa_check_host(reg_num, &guest_ext); if (ret) return ret; @@ -689,10 +507,10 @@ static int riscv_vcpu_set_isa_ext_single(struct kvm_vcpu *vcpu, * extension can be disabled */ if (reg_val == 1 && - kvm_riscv_vcpu_isa_enable_allowed(reg_num)) + kvm_riscv_isa_enable_allowed(reg_num)) set_bit(guest_ext, vcpu->arch.isa); else if (!reg_val && - kvm_riscv_vcpu_isa_disable_allowed(reg_num)) + kvm_riscv_isa_disable_allowed(reg_num)) clear_bit(guest_ext, vcpu->arch.isa); else return -EINVAL; @@ -830,20 +648,6 @@ static int copy_config_reg_indices(const struct kvm_vcpu *vcpu, u64 size; u64 reg; - /* - * Avoid reporting config reg if the corresponding extension - * was not available. - */ - if (i == KVM_REG_RISCV_CONFIG_REG(zicbom_block_size) && - !riscv_isa_extension_available(NULL, ZICBOM)) - continue; - else if (i == KVM_REG_RISCV_CONFIG_REG(zicboz_block_size) && - !riscv_isa_extension_available(NULL, ZICBOZ)) - continue; - else if (i == KVM_REG_RISCV_CONFIG_REG(zicbop_block_size) && - !riscv_isa_extension_available(NULL, ZICBOP)) - continue; - size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CONFIG | i; @@ -896,6 +700,8 @@ static inline unsigned long num_csr_regs(const struct kvm_vcpu *vcpu) n += sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); if (riscv_isa_extension_available(vcpu->arch.isa, SMSTATEEN)) n += sizeof(struct kvm_riscv_smstateen_csr) / sizeof(unsigned long); + if (riscv_isa_extension_available(vcpu->arch.isa, ZICFISS)) + n += sizeof(struct kvm_riscv_zicfiss_csr) / sizeof(unsigned long); return n; } @@ -904,7 +710,7 @@ static int copy_csr_reg_indices(const struct kvm_vcpu *vcpu, u64 __user *uindices) { int n1 = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); - int n2 = 0, n3 = 0; + int n2 = 0, n3 = 0, n4 = 0; /* copy general csr regs */ for (int i = 0; i < n1; i++) { @@ -956,7 +762,25 @@ static int copy_csr_reg_indices(const struct kvm_vcpu *vcpu, } } - return n1 + n2 + n3; + /* copy Zicfiss csr regs */ + if (riscv_isa_extension_available(vcpu->arch.isa, ZICFISS)) { + n4 = sizeof(struct kvm_riscv_zicfiss_csr) / sizeof(unsigned long); + + for (int i = 0; i < n4; i++) { + u64 size = IS_ENABLED(CONFIG_32BIT) ? + KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; + u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CSR | + KVM_REG_RISCV_CSR_ZICFISS | i; + + if (uindices) { + if (put_user(reg, uindices)) + return -EFAULT; + uindices++; + } + } + } + + return n1 + n2 + n3 + n4; } static inline unsigned long num_timer_regs(void) @@ -986,7 +810,7 @@ static inline unsigned long num_fp_f_regs(const struct kvm_vcpu *vcpu) { const struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; - if (riscv_isa_extension_available(vcpu->arch.isa, f)) + if (riscv_isa_extension_available(vcpu->arch.isa, F)) return sizeof(cntx->fp.f) / sizeof(u32); else return 0; @@ -1015,7 +839,7 @@ static inline unsigned long num_fp_d_regs(const struct kvm_vcpu *vcpu) { const struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; - if (riscv_isa_extension_available(vcpu->arch.isa, d)) + if (riscv_isa_extension_available(vcpu->arch.isa, D)) return sizeof(cntx->fp.d.f) / sizeof(u64) + 1; else return 0; @@ -1062,7 +886,7 @@ static int copy_isa_ext_reg_indices(const struct kvm_vcpu *vcpu, KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_ISA_EXT | i; - if (kvm_riscv_vcpu_isa_check_host(i, &guest_ext)) + if (__kvm_riscv_isa_check_host(i, &guest_ext)) continue; if (uindices) { @@ -1094,7 +918,7 @@ static inline unsigned long num_sbi_regs(struct kvm_vcpu *vcpu) static inline unsigned long num_vector_regs(const struct kvm_vcpu *vcpu) { - if (!riscv_isa_extension_available(vcpu->arch.isa, v)) + if (!riscv_isa_extension_available(vcpu->arch.isa, V)) return 0; /* vstart, vl, vtype, vcsr, vlenb and 32 vector regs */ diff --git a/arch/riscv/kvm/vcpu_pmu.c b/arch/riscv/kvm/vcpu_pmu.c index a2fae70ee174..e095d1ff439f 100644 --- a/arch/riscv/kvm/vcpu_pmu.c +++ b/arch/riscv/kvm/vcpu_pmu.c @@ -7,15 +7,19 @@ */ #define pr_fmt(fmt) "riscv-kvm-pmu: " fmt +#include <linux/bitops.h> #include <linux/errno.h> #include <linux/err.h> #include <linux/kvm_host.h> +#include <linux/nospec.h> +#include <linux/overflow.h> #include <linux/perf/riscv_pmu.h> +#include <linux/slab.h> #include <asm/csr.h> +#include <asm/kvm_isa.h> #include <asm/kvm_vcpu_sbi.h> #include <asm/kvm_vcpu_pmu.h> #include <asm/sbi.h> -#include <linux/bitops.h> #define kvm_pmu_num_counters(pmu) ((pmu)->num_hw_ctrs + (pmu)->num_fw_ctrs) #define get_event_type(x) (((x) & SBI_PMU_EVENT_IDX_TYPE_MASK) >> 16) @@ -87,7 +91,8 @@ static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc) static u64 kvm_pmu_get_perf_event_hw_config(u32 sbi_event_code) { - return hw_event_perf_map[sbi_event_code]; + return hw_event_perf_map[array_index_nospec(sbi_event_code, + SBI_PMU_HW_GENERAL_MAX)]; } static u64 kvm_pmu_get_perf_event_cache_config(u32 sbi_event_code) @@ -218,12 +223,20 @@ static int pmu_fw_ctr_read_hi(struct kvm_vcpu *vcpu, unsigned long cidx, return -EINVAL; } + cidx = array_index_nospec(cidx, RISCV_KVM_MAX_COUNTERS); pmc = &kvpmu->pmc[cidx]; if (pmc->cinfo.type != SBI_PMU_CTR_TYPE_FW) return -EINVAL; + if (pmc->event_idx == SBI_PMU_EVENT_IDX_INVALID) + return -EINVAL; + fevent_code = get_event_code(pmc->event_idx); + if (WARN_ONCE(fevent_code >= SBI_PMU_FW_MAX, + "Invalid firmware event code: %d\n", fevent_code)) + return -EINVAL; + pmc->counter_val = kvpmu->fw_event[fevent_code].value; *out_val = pmc->counter_val >> 32; @@ -244,17 +257,26 @@ static int pmu_ctr_read(struct kvm_vcpu *vcpu, unsigned long cidx, return -EINVAL; } + cidx = array_index_nospec(cidx, RISCV_KVM_MAX_COUNTERS); pmc = &kvpmu->pmc[cidx]; if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) { + if (pmc->event_idx == SBI_PMU_EVENT_IDX_INVALID) + return -EINVAL; + fevent_code = get_event_code(pmc->event_idx); + if (WARN_ONCE(fevent_code >= SBI_PMU_FW_MAX, + "Invalid firmware event code: %d\n", fevent_code)) + return -EINVAL; + pmc->counter_val = kvpmu->fw_event[fevent_code].value; + *out_val = pmc->counter_val; } else if (pmc->perf_event) { - pmc->counter_val += perf_event_read_value(pmc->perf_event, &enabled, &running); + *out_val = pmc->counter_val + + perf_event_read_value(pmc->perf_event, &enabled, &running); } else { return -EINVAL; } - *out_val = pmc->counter_val; return 0; } @@ -262,8 +284,10 @@ static int pmu_ctr_read(struct kvm_vcpu *vcpu, unsigned long cidx, static int kvm_pmu_validate_counter_mask(struct kvm_pmu *kvpmu, unsigned long ctr_base, unsigned long ctr_mask) { - /* Make sure the we have a valid counter mask requested from the caller */ - if (!ctr_mask || (ctr_base + __fls(ctr_mask) >= kvm_pmu_num_counters(kvpmu))) + unsigned long num_ctrs = kvm_pmu_num_counters(kvpmu); + + /* Make sure we have a valid counter mask requested from the caller */ + if (!ctr_mask || ctr_base >= num_ctrs || (ctr_base + __fls(ctr_mask) >= num_ctrs)) return -EINVAL; return 0; @@ -423,20 +447,32 @@ int kvm_riscv_vcpu_pmu_snapshot_set_shmem(struct kvm_vcpu *vcpu, unsigned long s saddr = saddr_low; if (saddr_high != 0) { - if (IS_ENABLED(CONFIG_32BIT)) + if (IS_ENABLED(CONFIG_32BIT)) { saddr |= ((gpa_t)saddr_high << 32); - else + } else { sbiret = SBI_ERR_INVALID_ADDRESS; - goto out; + goto out; + } } - kvpmu->sdata = kzalloc(snapshot_area_size, GFP_ATOMIC); - if (!kvpmu->sdata) - return -ENOMEM; + /* + * Clear any previously installed snapshot area to avoid leaking + * the old sdata and to keep sdata/snapshot_addr consistent if + * the re-install fails below. + */ + if (kvpmu->snapshot_addr != INVALID_GPA) + kvm_pmu_clear_snapshot_area(vcpu); + + kvpmu->sdata = kzalloc(snapshot_area_size, GFP_ATOMIC | __GFP_ACCOUNT); + if (!kvpmu->sdata) { + sbiret = SBI_ERR_FAILURE; + goto out; + } /* No need to check writable slot explicitly as kvm_vcpu_write_guest does it internally */ if (kvm_vcpu_write_guest(vcpu, saddr, kvpmu->sdata, snapshot_area_size)) { kfree(kvpmu->sdata); + kvpmu->sdata = NULL; sbiret = SBI_ERR_INVALID_ADDRESS; goto out; } @@ -454,13 +490,14 @@ int kvm_riscv_vcpu_pmu_event_info(struct kvm_vcpu *vcpu, unsigned long saddr_low unsigned long flags, struct kvm_vcpu_sbi_return *retdata) { struct riscv_pmu_event_info *einfo = NULL; - int shmem_size = num_events * sizeof(*einfo); + size_t shmem_size; gpa_t shmem; u32 eidx, etype; u64 econfig; int ret; - if (flags != 0 || (saddr_low & (SZ_16 - 1) || num_events == 0)) { + if (flags != 0 || (saddr_low & (SZ_16 - 1)) || num_events == 0 || + check_mul_overflow(num_events, sizeof(*einfo), &shmem_size)) { ret = SBI_ERR_INVALID_PARAM; goto out; } @@ -475,9 +512,12 @@ int kvm_riscv_vcpu_pmu_event_info(struct kvm_vcpu *vcpu, unsigned long saddr_low } } - einfo = kzalloc(shmem_size, GFP_KERNEL); - if (!einfo) - return -ENOMEM; + einfo = kvzalloc_objs(*einfo, num_events, + GFP_KERNEL_ACCOUNT | __GFP_NOWARN); + if (!einfo) { + ret = SBI_ERR_FAILURE; + goto out; + } ret = kvm_vcpu_read_guest(vcpu, shmem, einfo, shmem_size); if (ret) { @@ -485,7 +525,7 @@ int kvm_riscv_vcpu_pmu_event_info(struct kvm_vcpu *vcpu, unsigned long saddr_low goto free_mem; } - for (int i = 0; i < num_events; i++) { + for (unsigned long i = 0; i < num_events; i++) { eidx = einfo[i].event_idx; etype = kvm_pmu_get_perf_event_type(eidx); econfig = kvm_pmu_get_perf_event_config(eidx, einfo[i].event_data); @@ -494,14 +534,11 @@ int kvm_riscv_vcpu_pmu_event_info(struct kvm_vcpu *vcpu, unsigned long saddr_low } ret = kvm_vcpu_write_guest(vcpu, shmem, einfo, shmem_size); - if (ret) { + if (ret) ret = SBI_ERR_INVALID_ADDRESS; - goto free_mem; - } - ret = 0; free_mem: - kfree(einfo); + kvfree(einfo); out: retdata->err_val = ret; @@ -523,11 +560,12 @@ int kvm_riscv_vcpu_pmu_ctr_info(struct kvm_vcpu *vcpu, unsigned long cidx, { struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); - if (cidx > RISCV_KVM_MAX_COUNTERS || cidx == 1) { + if (cidx >= RISCV_KVM_MAX_COUNTERS || cidx == 1) { retdata->err_val = SBI_ERR_INVALID_PARAM; return 0; } + cidx = array_index_nospec(cidx, RISCV_KVM_MAX_COUNTERS); retdata->out_val = kvpmu->pmc[cidx].cinfo.value; return 0; @@ -561,8 +599,9 @@ int kvm_riscv_vcpu_pmu_ctr_start(struct kvm_vcpu *vcpu, unsigned long ctr_base, } } /* Start the counters that have been configured and requested by the guest */ - for_each_set_bit(i, &ctr_mask, RISCV_MAX_COUNTERS) { - pmc_index = i + ctr_base; + for_each_set_bit(i, &ctr_mask, BITS_PER_LONG) { + pmc_index = array_index_nospec(i + ctr_base, + RISCV_KVM_MAX_COUNTERS); if (!test_bit(pmc_index, kvpmu->pmc_in_use)) continue; /* The guest started the counter again. Reset the overflow status */ @@ -615,7 +654,6 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, { struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); int i, pmc_index, sbiret = 0; - u64 enabled, running; struct kvm_pmc *pmc; int fevent_code; bool snap_flag_set = flags & SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT; @@ -632,8 +670,9 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, } /* Stop the counters that have been configured and requested by the guest */ - for_each_set_bit(i, &ctr_mask, RISCV_MAX_COUNTERS) { - pmc_index = i + ctr_base; + for_each_set_bit(i, &ctr_mask, BITS_PER_LONG) { + pmc_index = array_index_nospec(i + ctr_base, + RISCV_KVM_MAX_COUNTERS); if (!test_bit(pmc_index, kvpmu->pmc_in_use)) continue; pmc = &kvpmu->pmc[pmc_index]; @@ -644,14 +683,19 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, goto out; } - if (!kvpmu->fw_event[fevent_code].started) + if (!kvpmu->fw_event[fevent_code].started) { sbiret = SBI_ERR_ALREADY_STOPPED; - - kvpmu->fw_event[fevent_code].started = false; + } else { + kvpmu->fw_event[fevent_code].started = false; + pmc->counter_val = kvpmu->fw_event[fevent_code].value; + } } else if (pmc->perf_event) { if (pmc->started) { - /* Stop counting the counter */ - perf_event_disable(pmc->perf_event); + /* + * Stop the counter and fold the live count into counter_val. + * Reset the event value to avoid redundant accumulation. + */ + pmc->counter_val += perf_event_pause(pmc->perf_event, true); pmc->started = false; } else { sbiret = SBI_ERR_ALREADY_STOPPED; @@ -665,13 +709,8 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, } if (snap_flag_set && !sbiret) { - if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) - pmc->counter_val = kvpmu->fw_event[fevent_code].value; - else if (pmc->perf_event) - pmc->counter_val += perf_event_read_value(pmc->perf_event, - &enabled, &running); /* - * The counter and overflow indicies in the snapshot region are w.r.to + * The counter and overflow indices in the snapshot region are w.r.to * cbase. Modify the set bit in the counter mask instead of the pmc_index * which indicates the absolute counter index. */ @@ -696,9 +735,10 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, } } - if (shmem_needs_update) - kvm_vcpu_write_guest(vcpu, kvpmu->snapshot_addr, kvpmu->sdata, - sizeof(struct riscv_pmu_snapshot_data)); + if (shmem_needs_update && + kvm_vcpu_write_guest(vcpu, kvpmu->snapshot_addr, kvpmu->sdata, + sizeof(struct riscv_pmu_snapshot_data))) + sbiret = SBI_ERR_FAILURE; out: retdata->err_val = sbiret; @@ -764,6 +804,7 @@ int kvm_riscv_vcpu_pmu_ctr_cfg_match(struct kvm_vcpu *vcpu, unsigned long ctr_ba } } + ctr_idx = array_index_nospec(ctr_idx, RISCV_KVM_MAX_COUNTERS); pmc = &kvpmu->pmc[ctr_idx]; pmc->idx = ctr_idx; @@ -822,7 +863,7 @@ void kvm_riscv_vcpu_pmu_init(struct kvm_vcpu *vcpu) * filtering is available in the host. Otherwise, guest will always count * events while the execution is in hypervisor mode. */ - if (!riscv_isa_extension_available(NULL, SSCOFPMF)) + if (kvm_riscv_isa_check_host(SSCOFPMF)) return; ret = riscv_pmu_get_hpm_info(&hpm_width, &num_hw_ctrs); diff --git a/arch/riscv/kvm/vcpu_sbi.c b/arch/riscv/kvm/vcpu_sbi.c index 46ab7b989432..1c4f874368fb 100644 --- a/arch/riscv/kvm/vcpu_sbi.c +++ b/arch/riscv/kvm/vcpu_sbi.c @@ -221,6 +221,7 @@ static int riscv_vcpu_set_sbi_ext_single(struct kvm_vcpu *vcpu, { struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context; const struct kvm_riscv_sbi_extension_entry *sext; + const struct kvm_vcpu_sbi_extension *ext; if (reg_val != 1 && reg_val != 0) return -EINVAL; @@ -229,6 +230,12 @@ static int riscv_vcpu_set_sbi_ext_single(struct kvm_vcpu *vcpu, if (!sext || scontext->ext_status[sext->ext_idx] == KVM_RISCV_SBI_EXT_STATUS_UNAVAILABLE) return -ENOENT; + ext = sext->ext_ptr; + + if (!reg_val && scontext->ext_status[sext->ext_idx] == KVM_RISCV_SBI_EXT_STATUS_ENABLED && + ext->reset) + ext->reset(vcpu); + scontext->ext_status[sext->ext_idx] = (reg_val) ? KVM_RISCV_SBI_EXT_STATUS_ENABLED : KVM_RISCV_SBI_EXT_STATUS_DISABLED; @@ -723,3 +730,26 @@ void kvm_riscv_vcpu_sbi_reset(struct kvm_vcpu *vcpu) ext->reset(vcpu); } } + +void kvm_riscv_vcpu_sbi_validate(struct kvm_vcpu *vcpu) +{ + struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context; + const struct kvm_riscv_sbi_extension_entry *entry; + const struct kvm_vcpu_sbi_extension *ext; + int idx, i; + + for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) { + entry = &sbi_ext[i]; + ext = entry->ext_ptr; + idx = entry->ext_idx; + + if (idx < 0 || idx >= ARRAY_SIZE(scontext->ext_status)) + continue; + + if (scontext->ext_status[idx] != KVM_RISCV_SBI_EXT_STATUS_ENABLED || + !ext->validate) + continue; + + ext->validate(vcpu); + } +} diff --git a/arch/riscv/kvm/vcpu_sbi_fwft.c b/arch/riscv/kvm/vcpu_sbi_fwft.c index 62cc9c3d5759..7d0aaeff537b 100644 --- a/arch/riscv/kvm/vcpu_sbi_fwft.c +++ b/arch/riscv/kvm/vcpu_sbi_fwft.c @@ -36,6 +36,16 @@ struct kvm_sbi_fwft_feature { bool (*supported)(struct kvm_vcpu *vcpu); /** + * @init: Probe and initialize the feature on the vcpu + * + * This callback is optional. If provided, it will be called during + * vcpu initialization to probe the feature availability and perform + * any necessary initialization. Returns true if the feature is supported + * and initialized successfully, false otherwise. + */ + bool (*init)(struct kvm_vcpu *vcpu); + + /** * @reset: Reset the feature value irrespective whether feature is supported or not * * This callback is mandatory @@ -84,6 +94,44 @@ static bool kvm_fwft_is_defined_feature(enum sbi_fwft_feature_t feature) return false; } +static void kvm_sbi_fwft_envcfg_flag_reset(struct kvm_vcpu *vcpu, u64 flag) +{ + vcpu->arch.cfg.henvcfg &= ~flag; +} + +static long kvm_sbi_fwft_envcfg_flag_set(struct kvm_vcpu *vcpu, + struct kvm_sbi_fwft_config *conf, + bool one_reg_access, + unsigned long value, u64 flag) +{ + struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; + + if (value == 0) + cfg->henvcfg &= ~flag; + else if (value == 1) + cfg->henvcfg |= flag; + else + return SBI_ERR_INVALID_PARAM; + + if (!one_reg_access) { + csr_write(CSR_HENVCFG, vcpu->arch.cfg.henvcfg); + if (IS_ENABLED(CONFIG_32BIT)) + csr_write(CSR_HENVCFGH, vcpu->arch.cfg.henvcfg >> 32); + } + + return SBI_SUCCESS; +} + +static long kvm_sbi_fwft_envcfg_flag_get(struct kvm_vcpu *vcpu, + struct kvm_sbi_fwft_config *conf, + bool one_reg_access, + unsigned long *value, u64 flag) +{ + *value = (vcpu->arch.cfg.henvcfg & flag) == flag; + + return SBI_SUCCESS; +} + static bool kvm_sbi_fwft_misaligned_delegation_supported(struct kvm_vcpu *vcpu) { return misaligned_traps_can_delegate(); @@ -127,23 +175,147 @@ static long kvm_sbi_fwft_get_misaligned_delegation(struct kvm_vcpu *vcpu, return SBI_SUCCESS; } +static long kvm_sbi_fwft_set_cfi(struct kvm_vcpu *vcpu, + struct kvm_sbi_fwft_config *conf, + bool one_reg_access, unsigned long value, + u64 flag) +{ + struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; + + if (value == 0) + cfg->henvcfg &= ~flag; + else if (value == 1) + cfg->henvcfg |= flag; + else + return SBI_ERR_INVALID_PARAM; + + if (cfg->henvcfg & (ENVCFG_LPE | ENVCFG_SSE)) + cfg->hedeleg |= BIT(EXC_SOFTWARE_CHECK); + else + cfg->hedeleg &= ~BIT(EXC_SOFTWARE_CHECK); + + if (!one_reg_access) { + csr_write(CSR_HEDELEG, cfg->hedeleg); + /* + * Both Bit LPE and SSE are in the lower part, so it is safe + * to only write the henvcfg + */ + csr_write(CSR_HENVCFG, vcpu->arch.cfg.henvcfg); + } + + return SBI_SUCCESS; +} + +static bool kvm_sbi_fwft_landing_pad_supported(struct kvm_vcpu *vcpu) +{ + return riscv_isa_extension_available(vcpu->arch.isa, ZICFILP); +} + +static void kvm_sbi_fwft_reset_landing_pad(struct kvm_vcpu *vcpu) +{ + struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; + + kvm_sbi_fwft_envcfg_flag_reset(vcpu, ENVCFG_LPE); + if ((cfg->henvcfg & (ENVCFG_LPE | ENVCFG_SSE)) == 0) + cfg->hedeleg &= ~BIT(EXC_SOFTWARE_CHECK); +} + +static long kvm_sbi_fwft_set_landing_pad(struct kvm_vcpu *vcpu, + struct kvm_sbi_fwft_config *conf, + bool one_reg_access, unsigned long value) +{ + return kvm_sbi_fwft_set_cfi(vcpu, conf, one_reg_access, value, ENVCFG_LPE); +} + +static long kvm_sbi_fwft_get_landing_pad(struct kvm_vcpu *vcpu, + struct kvm_sbi_fwft_config *conf, + bool one_reg_access, unsigned long *value) +{ + return kvm_sbi_fwft_envcfg_flag_get(vcpu, conf, one_reg_access, value, ENVCFG_LPE); +} + +static bool kvm_sbi_fwft_shadow_stack_supported(struct kvm_vcpu *vcpu) +{ + return riscv_isa_extension_available(vcpu->arch.isa, ZICFISS); +} + +static void kvm_sbi_fwft_reset_shadow_stack(struct kvm_vcpu *vcpu) +{ + struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; + + kvm_sbi_fwft_envcfg_flag_reset(vcpu, ENVCFG_SSE); + if ((cfg->henvcfg & (ENVCFG_LPE | ENVCFG_SSE)) == 0) + cfg->hedeleg &= ~BIT(EXC_SOFTWARE_CHECK); +} + +static long kvm_sbi_fwft_set_shadow_stack(struct kvm_vcpu *vcpu, + struct kvm_sbi_fwft_config *conf, + bool one_reg_access, unsigned long value) +{ + return kvm_sbi_fwft_set_cfi(vcpu, conf, one_reg_access, value, ENVCFG_SSE); +} + +static long kvm_sbi_fwft_get_shadow_stack(struct kvm_vcpu *vcpu, + struct kvm_sbi_fwft_config *conf, + bool one_reg_access, unsigned long *value) +{ + return kvm_sbi_fwft_envcfg_flag_get(vcpu, conf, one_reg_access, value, ENVCFG_SSE); +} + +static bool kvm_sbi_fwft_pte_ad_hw_updating_supported(struct kvm_vcpu *vcpu) +{ + return riscv_isa_extension_available(vcpu->arch.isa, SVADU) && + riscv_isa_extension_available(vcpu->arch.isa, SVADE); +} + +static void kvm_sbi_fwft_reset_pte_ad_hw_updating(struct kvm_vcpu *vcpu) +{ + if (kvm_sbi_fwft_pte_ad_hw_updating_supported(vcpu)) + kvm_sbi_fwft_envcfg_flag_reset(vcpu, ENVCFG_ADUE); +} + +static long kvm_sbi_fwft_set_pte_ad_hw_updating(struct kvm_vcpu *vcpu, + struct kvm_sbi_fwft_config *conf, + bool one_reg_access, unsigned long value) +{ + return kvm_sbi_fwft_envcfg_flag_set(vcpu, conf, one_reg_access, value, ENVCFG_ADUE); +} + +static long kvm_sbi_fwft_get_pte_ad_hw_updating(struct kvm_vcpu *vcpu, + struct kvm_sbi_fwft_config *conf, + bool one_reg_access, unsigned long *value) +{ + return kvm_sbi_fwft_envcfg_flag_get(vcpu, conf, one_reg_access, value, ENVCFG_ADUE); +} + #ifndef CONFIG_32BIT static bool try_to_set_pmm(unsigned long value) { + unsigned long prev; + bool ret; + + prev = csr_read_clear(CSR_HENVCFG, ENVCFG_PMM); csr_set(CSR_HENVCFG, value); - return (csr_read_clear(CSR_HENVCFG, ENVCFG_PMM) & ENVCFG_PMM) == value; + ret = (csr_read_clear(CSR_HENVCFG, ENVCFG_PMM) & ENVCFG_PMM) == value; + csr_write(CSR_HENVCFG, prev); + + return ret; } static bool kvm_sbi_fwft_pointer_masking_pmlen_supported(struct kvm_vcpu *vcpu) { - struct kvm_sbi_fwft *fwft = vcpu_to_fwft(vcpu); + return riscv_isa_extension_available(vcpu->arch.isa, SMNPM); +} - if (!riscv_isa_extension_available(vcpu->arch.isa, SMNPM)) - return false; +static bool kvm_sbi_fwft_pointer_masking_pmlen_init(struct kvm_vcpu *vcpu) +{ + struct kvm_sbi_fwft *fwft = vcpu_to_fwft(vcpu); + preempt_disable(); fwft->have_vs_pmlen_7 = try_to_set_pmm(ENVCFG_PMM_PMLEN_7); fwft->have_vs_pmlen_16 = try_to_set_pmm(ENVCFG_PMM_PMLEN_16); + preempt_enable(); return fwft->have_vs_pmlen_7 || fwft->have_vs_pmlen_16; } @@ -225,12 +397,40 @@ static const struct kvm_sbi_fwft_feature features[] = { .set = kvm_sbi_fwft_set_misaligned_delegation, .get = kvm_sbi_fwft_get_misaligned_delegation, }, + { + .id = SBI_FWFT_LANDING_PAD, + .first_reg_num = offsetof(struct kvm_riscv_sbi_fwft, landing_pad.enable) / + sizeof(unsigned long), + .supported = kvm_sbi_fwft_landing_pad_supported, + .reset = kvm_sbi_fwft_reset_landing_pad, + .set = kvm_sbi_fwft_set_landing_pad, + .get = kvm_sbi_fwft_get_landing_pad, + }, + { + .id = SBI_FWFT_SHADOW_STACK, + .first_reg_num = offsetof(struct kvm_riscv_sbi_fwft, shadow_stack.enable) / + sizeof(unsigned long), + .supported = kvm_sbi_fwft_shadow_stack_supported, + .reset = kvm_sbi_fwft_reset_shadow_stack, + .set = kvm_sbi_fwft_set_shadow_stack, + .get = kvm_sbi_fwft_get_shadow_stack, + }, + { + .id = SBI_FWFT_PTE_AD_HW_UPDATING, + .first_reg_num = offsetof(struct kvm_riscv_sbi_fwft, pte_ad_hw_updating.enable) / + sizeof(unsigned long), + .supported = kvm_sbi_fwft_pte_ad_hw_updating_supported, + .reset = kvm_sbi_fwft_reset_pte_ad_hw_updating, + .set = kvm_sbi_fwft_set_pte_ad_hw_updating, + .get = kvm_sbi_fwft_get_pte_ad_hw_updating, + }, #ifndef CONFIG_32BIT { .id = SBI_FWFT_POINTER_MASKING_PMLEN, .first_reg_num = offsetof(struct kvm_riscv_sbi_fwft, pointer_masking.enable) / sizeof(unsigned long), .supported = kvm_sbi_fwft_pointer_masking_pmlen_supported, + .init = kvm_sbi_fwft_pointer_masking_pmlen_init, .reset = kvm_sbi_fwft_reset_pointer_masking_pmlen, .set = kvm_sbi_fwft_set_pointer_masking_pmlen, .get = kvm_sbi_fwft_get_pointer_masking_pmlen, @@ -281,6 +481,8 @@ static int kvm_fwft_get_feature(struct kvm_vcpu *vcpu, u32 feature, if (!tconf->supported || !tconf->enabled) return SBI_ERR_NOT_SUPPORTED; + else if (tconf->feature->supported && !tconf->feature->supported(vcpu)) + return SBI_ERR_NOT_SUPPORTED; *conf = tconf; @@ -303,9 +505,11 @@ static int kvm_sbi_fwft_set(struct kvm_vcpu *vcpu, u32 feature, if (conf->flags & SBI_FWFT_SET_FLAG_LOCK) return SBI_ERR_DENIED_LOCKED; - conf->flags = flags; + ret = conf->feature->set(vcpu, conf, false, value); + if (ret == SBI_SUCCESS) + conf->flags = flags; - return conf->feature->set(vcpu, conf, false, value); + return ret; } static int kvm_sbi_fwft_get(struct kvm_vcpu *vcpu, unsigned long feature, @@ -352,8 +556,8 @@ static int kvm_sbi_ext_fwft_init(struct kvm_vcpu *vcpu) struct kvm_sbi_fwft_config *conf; int i; - fwft->configs = kcalloc(ARRAY_SIZE(features), sizeof(struct kvm_sbi_fwft_config), - GFP_KERNEL); + fwft->configs = kzalloc_objs(struct kvm_sbi_fwft_config, + ARRAY_SIZE(features), GFP_KERNEL_ACCOUNT); if (!fwft->configs) return -ENOMEM; @@ -365,6 +569,9 @@ static int kvm_sbi_ext_fwft_init(struct kvm_vcpu *vcpu) else conf->supported = true; + if (conf->supported && feature->init) + conf->supported = feature->init(vcpu); + conf->enabled = conf->supported; conf->feature = feature; } @@ -391,6 +598,32 @@ static void kvm_sbi_ext_fwft_reset(struct kvm_vcpu *vcpu) if (feature->reset) feature->reset(vcpu); } + + vcpu->arch.csr_dirty = true; +} + +static void kvm_sbi_ext_fwft_validate(struct kvm_vcpu *vcpu) +{ + struct kvm_sbi_fwft *fwft = vcpu_to_fwft(vcpu); + const struct kvm_sbi_fwft_feature *feature; + struct kvm_sbi_fwft_config *conf; + int i; + + for (i = 0; i < ARRAY_SIZE(features); i++) { + feature = &features[i]; + conf = &fwft->configs[i]; + if (!conf->supported) + continue; + + if (!feature->supported || feature->supported(vcpu)) + continue; + + conf->enabled = false; + conf->flags = 0; + + if (feature->reset) + feature->reset(vcpu); + } } static unsigned long kvm_sbi_ext_fwft_get_reg_count(struct kvm_vcpu *vcpu) @@ -408,6 +641,8 @@ static unsigned long kvm_sbi_ext_fwft_get_reg_count(struct kvm_vcpu *vcpu) conf = kvm_sbi_fwft_get_config(vcpu, feature->id); if (!conf || !conf->supported) continue; + else if (conf->feature->supported && !conf->feature->supported(vcpu)) + continue; ret++; } @@ -430,6 +665,8 @@ static int kvm_sbi_ext_fwft_get_reg_id(struct kvm_vcpu *vcpu, int index, u64 *re conf = kvm_sbi_fwft_get_config(vcpu, feature->id); if (!conf || !conf->supported) continue; + else if (conf->feature->supported && !conf->feature->supported(vcpu)) + continue; if (index == idx) { *reg_id = KVM_REG_RISCV | @@ -465,6 +702,8 @@ static int kvm_sbi_ext_fwft_get_reg(struct kvm_vcpu *vcpu, unsigned long reg_num conf = kvm_sbi_fwft_get_config(vcpu, feature->id); if (!conf || !conf->supported) return -ENOENT; + else if (conf->feature->supported && !conf->feature->supported(vcpu)) + return -ENOENT; switch (reg_num - feature->first_reg_num) { case 0: @@ -502,6 +741,8 @@ static int kvm_sbi_ext_fwft_set_reg(struct kvm_vcpu *vcpu, unsigned long reg_num conf = kvm_sbi_fwft_get_config(vcpu, feature->id); if (!conf || !conf->supported) return -ENOENT; + else if (conf->feature->supported && !conf->feature->supported(vcpu)) + return -ENOENT; switch (reg_num - feature->first_reg_num) { case 0: @@ -521,6 +762,7 @@ static int kvm_sbi_ext_fwft_set_reg(struct kvm_vcpu *vcpu, unsigned long reg_num break; case 2: ret = conf->feature->set(vcpu, conf, true, value); + vcpu->arch.csr_dirty = true; break; default: return -ENOENT; @@ -536,6 +778,7 @@ const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_fwft = { .init = kvm_sbi_ext_fwft_init, .deinit = kvm_sbi_ext_fwft_deinit, .reset = kvm_sbi_ext_fwft_reset, + .validate = kvm_sbi_ext_fwft_validate, .state_reg_subtype = KVM_REG_RISCV_SBI_FWFT, .get_state_reg_count = kvm_sbi_ext_fwft_get_reg_count, .get_state_reg_id = kvm_sbi_ext_fwft_get_reg_id, diff --git a/arch/riscv/kvm/vcpu_sbi_hsm.c b/arch/riscv/kvm/vcpu_sbi_hsm.c index f26207f84bab..06a15629c26b 100644 --- a/arch/riscv/kvm/vcpu_sbi_hsm.c +++ b/arch/riscv/kvm/vcpu_sbi_hsm.c @@ -95,9 +95,9 @@ static int kvm_sbi_ext_hsm_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, ret = kvm_sbi_hsm_vcpu_get_status(vcpu); if (ret >= 0) { retdata->out_val = ret; - retdata->err_val = 0; + ret = 0; } - return 0; + break; case SBI_EXT_HSM_HART_SUSPEND: switch (lower_32_bits(cp->a0)) { case SBI_HSM_SUSPEND_RET_DEFAULT: diff --git a/arch/riscv/kvm/vcpu_sbi_sta.c b/arch/riscv/kvm/vcpu_sbi_sta.c index afa0545c3bcf..60e50296a008 100644 --- a/arch/riscv/kvm/vcpu_sbi_sta.c +++ b/arch/riscv/kvm/vcpu_sbi_sta.c @@ -46,7 +46,7 @@ void kvm_riscv_vcpu_record_steal_time(struct kvm_vcpu *vcpu) gfn = shmem >> PAGE_SHIFT; hva = kvm_vcpu_gfn_to_hva(vcpu, gfn); - if (WARN_ON(kvm_is_error_hva(hva))) { + if (kvm_is_error_hva(hva)) { vcpu->arch.sta.shmem = INVALID_GPA; return; } @@ -181,6 +181,7 @@ static int kvm_sbi_ext_sta_set_reg(struct kvm_vcpu *vcpu, unsigned long reg_num, unsigned long reg_size, const void *reg_val) { unsigned long value; + gpa_t new_shmem = INVALID_GPA; if (reg_size != sizeof(unsigned long)) return -EINVAL; @@ -191,18 +192,18 @@ static int kvm_sbi_ext_sta_set_reg(struct kvm_vcpu *vcpu, unsigned long reg_num, if (IS_ENABLED(CONFIG_32BIT)) { gpa_t hi = upper_32_bits(vcpu->arch.sta.shmem); - vcpu->arch.sta.shmem = value; - vcpu->arch.sta.shmem |= hi << 32; + new_shmem = value; + new_shmem |= hi << 32; } else { - vcpu->arch.sta.shmem = value; + new_shmem = value; } break; case KVM_REG_RISCV_SBI_STA_REG(shmem_hi): if (IS_ENABLED(CONFIG_32BIT)) { gpa_t lo = lower_32_bits(vcpu->arch.sta.shmem); - vcpu->arch.sta.shmem = ((gpa_t)value << 32); - vcpu->arch.sta.shmem |= lo; + new_shmem = ((gpa_t)value << 32); + new_shmem |= lo; } else if (value != 0) { return -EINVAL; } @@ -211,6 +212,11 @@ static int kvm_sbi_ext_sta_set_reg(struct kvm_vcpu *vcpu, unsigned long reg_num, return -ENOENT; } + if (new_shmem != INVALID_GPA && !IS_ALIGNED(new_shmem, 64)) + return -EINVAL; + + vcpu->arch.sta.shmem = new_shmem; + return 0; } diff --git a/arch/riscv/kvm/vcpu_sbi_system.c b/arch/riscv/kvm/vcpu_sbi_system.c index c6f7e609ac79..6f64a59e5d3c 100644 --- a/arch/riscv/kvm/vcpu_sbi_system.c +++ b/arch/riscv/kvm/vcpu_sbi_system.c @@ -35,6 +35,20 @@ static int kvm_sbi_ext_susp_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, return 0; } + /* + * Check that all other vCPUs are stopped before entering + * system suspend. + * + * There is a known TOCTOU race here: a concurrent HSM + * HART_START on another vCPU can start a vCPU after it + * has already passed this check, violating the invariant. + * + * We do not fix this because: + * 1. Triggering the race requires a pathological guest. + * 2. Only guest state is at risk, not host integrity. + * 3. Userspace can double-check vCPU states before + * proceeding with suspend. + */ kvm_for_each_vcpu(i, tmp, vcpu->kvm) { if (tmp == vcpu) continue; diff --git a/arch/riscv/kvm/vcpu_sbi_v01.c b/arch/riscv/kvm/vcpu_sbi_v01.c index 188d5ea5b3b8..de544ea3f28d 100644 --- a/arch/riscv/kvm/vcpu_sbi_v01.c +++ b/arch/riscv/kvm/vcpu_sbi_v01.c @@ -16,8 +16,9 @@ static int kvm_sbi_ext_v01_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, struct kvm_vcpu_sbi_return *retdata) { - ulong hmask; - int i, ret = 0; + ulong hbase = 0, hmask; + unsigned long i; + int ret = 0; u64 next_cycle; struct kvm_vcpu *rvcpu; struct kvm *kvm = vcpu->kvm; @@ -46,18 +47,25 @@ static int kvm_sbi_ext_v01_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, ret = kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_SOFT); break; case SBI_EXT_0_1_SEND_IPI: - if (cp->a0) + if (cp->a0) { hmask = kvm_riscv_vcpu_unpriv_read(vcpu, false, cp->a0, utrap); - else - hmask = (1UL << atomic_read(&kvm->online_vcpus)) - 1; - if (utrap->scause) - break; - - for_each_set_bit(i, &hmask, BITS_PER_LONG) { - rvcpu = kvm_get_vcpu_by_id(vcpu->kvm, i); - ret = kvm_riscv_vcpu_set_interrupt(rvcpu, IRQ_VS_SOFT); - if (ret < 0) + if (utrap->scause) break; + + for_each_set_bit(i, &hmask, BITS_PER_LONG) { + rvcpu = kvm_get_vcpu_by_id(vcpu->kvm, i); + if (!rvcpu) + continue; + ret = kvm_riscv_vcpu_set_interrupt(rvcpu, IRQ_VS_SOFT); + if (ret < 0) + break; + } + } else { + kvm_for_each_vcpu(i, rvcpu, kvm) { + ret = kvm_riscv_vcpu_set_interrupt(rvcpu, IRQ_VS_SOFT); + if (ret < 0) + break; + } } break; case SBI_EXT_0_1_SHUTDOWN: @@ -68,29 +76,31 @@ static int kvm_sbi_ext_v01_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, case SBI_EXT_0_1_REMOTE_FENCE_I: case SBI_EXT_0_1_REMOTE_SFENCE_VMA: case SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID: - if (cp->a0) + if (cp->a0) { hmask = kvm_riscv_vcpu_unpriv_read(vcpu, false, cp->a0, utrap); - else - hmask = (1UL << atomic_read(&kvm->online_vcpus)) - 1; + } else { + hbase = -1UL; + hmask = 0; + } if (utrap->scause) break; if (cp->a7 == SBI_EXT_0_1_REMOTE_FENCE_I) - kvm_riscv_fence_i(vcpu->kvm, 0, hmask); + kvm_riscv_fence_i(vcpu->kvm, hbase, hmask); else if (cp->a7 == SBI_EXT_0_1_REMOTE_SFENCE_VMA) { vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid); if (cp->a1 == 0 && cp->a2 == 0) - kvm_riscv_hfence_vvma_all(vcpu->kvm, 0, hmask, vmid); + kvm_riscv_hfence_vvma_all(vcpu->kvm, hbase, hmask, vmid); else - kvm_riscv_hfence_vvma_gva(vcpu->kvm, 0, hmask, cp->a1, + kvm_riscv_hfence_vvma_gva(vcpu->kvm, hbase, hmask, cp->a1, cp->a2, PAGE_SHIFT, vmid); } else { vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid); if (cp->a1 == 0 && cp->a2 == 0) - kvm_riscv_hfence_vvma_asid_all(vcpu->kvm, 0, hmask, + kvm_riscv_hfence_vvma_asid_all(vcpu->kvm, hbase, hmask, cp->a3, vmid); else - kvm_riscv_hfence_vvma_asid_gva(vcpu->kvm, 0, hmask, + kvm_riscv_hfence_vvma_asid_gva(vcpu->kvm, hbase, hmask, cp->a1, cp->a2, PAGE_SHIFT, cp->a3, vmid); } diff --git a/arch/riscv/kvm/vcpu_timer.c b/arch/riscv/kvm/vcpu_timer.c index f36247e4c783..a2cd277a4059 100644 --- a/arch/riscv/kvm/vcpu_timer.c +++ b/arch/riscv/kvm/vcpu_timer.c @@ -12,6 +12,7 @@ #include <linux/uaccess.h> #include <clocksource/timer-riscv.h> #include <asm/delay.h> +#include <asm/kvm_isa.h> #include <asm/kvm_nacl.h> #include <asm/kvm_vcpu_timer.h> @@ -60,10 +61,13 @@ static enum hrtimer_restart kvm_riscv_vcpu_hrtimer_expired(struct hrtimer *h) static int kvm_riscv_vcpu_timer_cancel(struct kvm_vcpu_timer *t) { - if (!t->init_done || !t->next_set) + if (!t->init_done) return -EINVAL; hrtimer_cancel(&t->hrt); + + if (!t->next_set) + return -EINVAL; t->next_set = false; return 0; @@ -230,7 +234,7 @@ int kvm_riscv_vcpu_set_reg_timer(struct kvm_vcpu *vcpu, break; case KVM_REG_RISCV_TIMER_REG(state): if (reg_val == KVM_RISCV_TIMER_STATE_ON) - ret = kvm_riscv_vcpu_timer_next_event(vcpu, reg_val); + ret = kvm_riscv_vcpu_timer_next_event(vcpu, t->next_cycles); else ret = kvm_riscv_vcpu_timer_cancel(t); break; @@ -253,7 +257,7 @@ int kvm_riscv_vcpu_timer_init(struct kvm_vcpu *vcpu) t->next_set = false; /* Enable sstc for every vcpu if available in hardware */ - if (riscv_isa_extension_available(NULL, SSTC)) { + if (!kvm_riscv_isa_check_host(SSTC)) { t->sstc_enabled = true; hrtimer_setup(&t->hrt, kvm_riscv_vcpu_vstimer_expired, CLOCK_MONOTONIC, HRTIMER_MODE_REL); diff --git a/arch/riscv/kvm/vcpu_vector.c b/arch/riscv/kvm/vcpu_vector.c index 05f3cc2d8e31..fefc2857f753 100644 --- a/arch/riscv/kvm/vcpu_vector.c +++ b/arch/riscv/kvm/vcpu_vector.c @@ -10,8 +10,10 @@ #include <linux/errno.h> #include <linux/err.h> #include <linux/kvm_host.h> +#include <linux/nospec.h> #include <linux/uaccess.h> #include <asm/cpufeature.h> +#include <asm/kvm_isa.h> #include <asm/kvm_vcpu_vector.h> #include <asm/vector.h> @@ -25,7 +27,7 @@ void kvm_riscv_vcpu_vector_reset(struct kvm_vcpu *vcpu) cntx->vector.vlenb = riscv_v_vsize / 32; - if (riscv_isa_extension_available(isa, v)) { + if (riscv_isa_extension_available(isa, V)) { cntx->sstatus |= SR_VS_INITIAL; WARN_ON(!cntx->vector.datap); memset(cntx->vector.datap, 0, riscv_v_vsize); @@ -44,7 +46,7 @@ void kvm_riscv_vcpu_guest_vector_save(struct kvm_cpu_context *cntx, unsigned long *isa) { if ((cntx->sstatus & SR_VS) == SR_VS_DIRTY) { - if (riscv_isa_extension_available(isa, v)) + if (riscv_isa_extension_available(isa, V)) __kvm_riscv_vector_save(cntx); kvm_riscv_vcpu_vector_clean(cntx); } @@ -54,34 +56,37 @@ void kvm_riscv_vcpu_guest_vector_restore(struct kvm_cpu_context *cntx, unsigned long *isa) { if ((cntx->sstatus & SR_VS) != SR_VS_OFF) { - if (riscv_isa_extension_available(isa, v)) - __kvm_riscv_vector_restore(cntx); - kvm_riscv_vcpu_vector_clean(cntx); + if (riscv_isa_extension_available(isa, V)) + riscv_v_flags_set(riscv_v_flags() | RISCV_V_VCPU_NEED_RESTORE); } } void kvm_riscv_vcpu_host_vector_save(struct kvm_cpu_context *cntx) { /* No need to check host sstatus as it can be modified outside */ - if (riscv_isa_extension_available(NULL, v)) + if (!kvm_riscv_isa_check_host(V)) __kvm_riscv_vector_save(cntx); } void kvm_riscv_vcpu_host_vector_restore(struct kvm_cpu_context *cntx) { - if (riscv_isa_extension_available(NULL, v)) + if (!kvm_riscv_isa_check_host(V)) __kvm_riscv_vector_restore(cntx); + riscv_v_flags_set(riscv_v_flags() & ~(RISCV_V_VCPU_CTX | RISCV_V_VCPU_NEED_RESTORE)); } int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu) { - vcpu->arch.guest_context.vector.datap = kzalloc(riscv_v_vsize, GFP_KERNEL); + vcpu->arch.guest_context.vector.datap = kzalloc(riscv_v_vsize, GFP_KERNEL_ACCOUNT); if (!vcpu->arch.guest_context.vector.datap) return -ENOMEM; - vcpu->arch.host_context.vector.datap = kzalloc(riscv_v_vsize, GFP_KERNEL); - if (!vcpu->arch.host_context.vector.datap) + vcpu->arch.host_context.vector.datap = kzalloc(riscv_v_vsize, GFP_KERNEL_ACCOUNT); + if (!vcpu->arch.host_context.vector.datap) { + kfree(vcpu->arch.guest_context.vector.datap); + vcpu->arch.guest_context.vector.datap = NULL; return -ENOMEM; + } return 0; } @@ -91,6 +96,24 @@ void kvm_riscv_vcpu_free_vector_context(struct kvm_vcpu *vcpu) kfree(vcpu->arch.guest_context.vector.datap); kfree(vcpu->arch.host_context.vector.datap); } + +void kvm_riscv_vcpu_flush_vector(void) +{ + struct kvm_vcpu *vcpu = *this_cpu_ptr(kvm_get_running_vcpus()); + + /* + * Only reached from __riscv_flush_vector_context() when RISCV_V_VCPU_CTX is set, which + * always have kvm_get_running_vcpus non-NULL. + */ + if (WARN_ON_ONCE(!vcpu)) + return; + + kvm_riscv_vcpu_guest_vector_save(&vcpu->arch.guest_context, vcpu->arch.isa); + + if ((vcpu->arch.guest_context.sstatus & SR_VS) != SR_VS_OFF) + riscv_v_flags_set(riscv_v_flags() | RISCV_V_VCPU_NEED_RESTORE); +} + #endif static int kvm_riscv_vcpu_vreg_addr(struct kvm_vcpu *vcpu, @@ -125,10 +148,20 @@ static int kvm_riscv_vcpu_vreg_addr(struct kvm_vcpu *vcpu, return -ENOENT; } } else if (reg_num <= KVM_REG_RISCV_VECTOR_REG(31)) { + unsigned long reg_offset; + if (reg_size != vlenb) return -EINVAL; - *reg_addr = cntx->vector.datap + - (reg_num - KVM_REG_RISCV_VECTOR_REG(0)) * vlenb; + WARN_ON(!cntx->vector.datap); + /* + * The reg_num is derived from the userspace-provided ONE_REG + * id. Sanitize it with array_index_nospec() to prevent + * speculative out-of-bounds access to the vector register + * buffer (32 vector registers: v0..v31). + */ + reg_offset = array_index_nospec( + reg_num - KVM_REG_RISCV_VECTOR_REG(0), 32); + *reg_addr = cntx->vector.datap + reg_offset * vlenb; } else { return -ENOENT; } @@ -149,7 +182,7 @@ int kvm_riscv_vcpu_get_reg_vector(struct kvm_vcpu *vcpu, void *reg_addr; int rc; - if (!riscv_isa_extension_available(isa, v)) + if (!riscv_isa_extension_available(isa, V)) return -ENOENT; rc = kvm_riscv_vcpu_vreg_addr(vcpu, reg_num, reg_size, ®_addr); @@ -175,7 +208,7 @@ int kvm_riscv_vcpu_set_reg_vector(struct kvm_vcpu *vcpu, void *reg_addr; int rc; - if (!riscv_isa_extension_available(isa, v)) + if (!riscv_isa_extension_available(isa, V)) return -ENOENT; if (reg_num == KVM_REG_RISCV_VECTOR_CSR_REG(vlenb)) { diff --git a/arch/riscv/kvm/vm.c b/arch/riscv/kvm/vm.c index 66d91ae6e9b2..a9f083feeb76 100644 --- a/arch/riscv/kvm/vm.c +++ b/arch/riscv/kvm/vm.c @@ -13,7 +13,7 @@ #include <linux/kvm_host.h> #include <asm/kvm_mmu.h> -const struct _kvm_stats_desc kvm_vm_stats_desc[] = { +const struct kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS() }; static_assert(ARRAY_SIZE(kvm_vm_stats_desc) == @@ -95,7 +95,7 @@ int kvm_riscv_setup_default_irq_routing(struct kvm *kvm, u32 lines) struct kvm_irq_routing_entry *ents; int i, rc; - ents = kcalloc(lines, sizeof(*ents), GFP_KERNEL); + ents = kzalloc_objs(*ents, lines); if (!ents) return -ENOMEM; @@ -181,7 +181,6 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) break; case KVM_CAP_IOEVENTFD: case KVM_CAP_USER_MEMORY: - case KVM_CAP_SYNC_MMU: case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: case KVM_CAP_ONE_REG: case KVM_CAP_READONLY_MEM: @@ -200,7 +199,10 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = KVM_USER_MEM_SLOTS; break; case KVM_CAP_VM_GPA_BITS: - r = kvm_riscv_gstage_gpa_bits; + if (!kvm) + r = kvm_riscv_gstage_gpa_bits(kvm_riscv_gstage_max_pgd_levels); + else + r = kvm_riscv_gstage_gpa_bits(kvm->arch.pgd_levels); break; default: r = 0; @@ -212,12 +214,52 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) { + if (cap->flags) + return -EINVAL; + switch (cap->cap) { case KVM_CAP_RISCV_MP_STATE_RESET: - if (cap->flags) - return -EINVAL; kvm->arch.mp_state_reset = true; return 0; + case KVM_CAP_VM_GPA_BITS: { + unsigned long gpa_bits = cap->args[0]; + unsigned long new_levels; + int r = 0; + + /* Decide target pgd levels from requested gpa_bits */ +#ifdef CONFIG_64BIT + if (gpa_bits <= 41) + new_levels = 3; /* Sv39x4 */ + else if (gpa_bits <= 50) + new_levels = 4; /* Sv48x4 */ + else if (gpa_bits <= 59) + new_levels = 5; /* Sv57x4 */ + else + return -EINVAL; +#else + /* 32-bit: only Sv32x4*/ + if (gpa_bits <= 34) + new_levels = 2; + else + return -EINVAL; +#endif + if (new_levels > kvm_riscv_gstage_max_pgd_levels) + return -EINVAL; + + /* Follow KVM's lock ordering: kvm->lock -> kvm->slots_lock. */ + mutex_lock(&kvm->lock); + mutex_lock(&kvm->slots_lock); + + if (kvm->created_vcpus || !kvm_are_all_memslots_empty(kvm)) + r = -EBUSY; + else + kvm->arch.pgd_levels = new_levels; + + mutex_unlock(&kvm->slots_lock); + mutex_unlock(&kvm->lock); + + return r; + } default: return -EINVAL; } diff --git a/arch/riscv/kvm/vmid.c b/arch/riscv/kvm/vmid.c index cf34d448289d..c15bdb1dd8be 100644 --- a/arch/riscv/kvm/vmid.c +++ b/arch/riscv/kvm/vmid.c @@ -26,7 +26,8 @@ static DEFINE_SPINLOCK(vmid_lock); void __init kvm_riscv_gstage_vmid_detect(void) { /* Figure-out number of VMID bits in HW */ - csr_write(CSR_HGATP, (kvm_riscv_gstage_mode << HGATP_MODE_SHIFT) | HGATP_VMID); + csr_write(CSR_HGATP, (kvm_riscv_gstage_mode(kvm_riscv_gstage_max_pgd_levels) << + HGATP_MODE_SHIFT) | HGATP_VMID); vmid_bits = csr_read(CSR_HGATP); vmid_bits = (vmid_bits & HGATP_VMID) >> HGATP_VMID_SHIFT; vmid_bits = fls_long(vmid_bits); diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile index bbc031124974..f668b98970bd 100644 --- a/arch/riscv/lib/Makefile +++ b/arch/riscv/lib/Makefile @@ -7,14 +7,17 @@ ifeq ($(CONFIG_KASAN_GENERIC)$(CONFIG_KASAN_SW_TAGS),) lib-y += strcmp.o lib-y += strlen.o lib-y += strncmp.o +lib-y += strnlen.o +lib-y += strchr.o +lib-y += strrchr.o endif lib-y += csum.o ifeq ($(CONFIG_MMU), y) lib-$(CONFIG_RISCV_ISA_V) += uaccess_vector.o endif lib-$(CONFIG_MMU) += uaccess.o +lib-$(CONFIG_32BIT) += ashldi3.o ashrdi3.o lshrdi3.o lib-$(CONFIG_64BIT) += tishift.o lib-$(CONFIG_RISCV_ISA_ZICBOZ) += clear_page.o obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o -lib-$(CONFIG_RISCV_ISA_V) += xor.o lib-$(CONFIG_RISCV_ISA_V) += riscv_v_helpers.o diff --git a/arch/riscv/lib/ashldi3.S b/arch/riscv/lib/ashldi3.S new file mode 100644 index 000000000000..c3408862e2f6 --- /dev/null +++ b/arch/riscv/lib/ashldi3.S @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/** + * Adopted for the Linux kernel from IPXE project, see + * https://github.com/ipxe/ipxe/blob/master/src/arch/riscv32/libgcc/llshift.S + */ + +#include <linux/linkage.h> +#include <asm/asm.h> + +/** + * Shift left + * + * @v a1:a0 Value to shift + * @v a2 Shift amount + * @ret a1:a0 Shifted value + */ + +SYM_FUNC_START(__ashldi3) + + /* Perform shift by 32 bits, if applicable */ + li t0, 32 + sub t1, t0, a2 + bgtz t1, 1f + mv a1, a0 + mv a0, zero +1: /* Perform shift by modulo-32 bits, if applicable */ + andi a2, a2, 0x1f + beqz a2, 2f + srl t2, a0, t1 + sll a0, a0, a2 + sll a1, a1, a2 + or a1, a1, t2 +2: ret + +SYM_FUNC_END(__ashldi3) +EXPORT_SYMBOL(__ashldi3) diff --git a/arch/riscv/lib/ashrdi3.S b/arch/riscv/lib/ashrdi3.S new file mode 100644 index 000000000000..426de0946606 --- /dev/null +++ b/arch/riscv/lib/ashrdi3.S @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/** + * Adopted for the Linux kernel from IPXE project, see + * https://github.com/ipxe/ipxe/blob/master/src/arch/riscv32/libgcc/llshift.S + */ + +#include <linux/linkage.h> +#include <asm/asm.h> + +/** + * Arithmetic shift right + * + * @v a1:a0 Value to shift + * @v a2 Shift amount + * @ret a1:a0 Shifted value + */ + +SYM_FUNC_START(__ashrdi3) + + /* Perform shift by 32 bits, if applicable */ + li t0, 32 + sub t1, t0, a2 + bgtz t1, 1f + mv a0, a1 + srai a1, a1, 31 +1: /* Perform shift by modulo-32 bits, if applicable */ + andi a2, a2, 0x1f + beqz a2, 2f + sll t2, a1, t1 + sra a1, a1, a2 + srl a0, a0, a2 + or a0, a0, t2 +2: ret + +SYM_FUNC_END(__ashrdi3) +EXPORT_SYMBOL(__ashrdi3) diff --git a/arch/riscv/lib/csum.c b/arch/riscv/lib/csum.c index 75bd0abffd63..5d4379768e5d 100644 --- a/arch/riscv/lib/csum.c +++ b/arch/riscv/lib/csum.c @@ -269,7 +269,7 @@ unsigned int do_csum(const unsigned char *buff, int len) * on machines with fast misaligned accesses. * * There is some duplicate code between the "with_alignment" and - * "no_alignment" implmentations, but the overlap is too awkward to be + * "no_alignment" implementations, but the overlap is too awkward to be * able to fit in one function without introducing multiple static * branches. The largest chunk of overlap was delegated into the * do_csum_common function. diff --git a/arch/riscv/lib/delay.c b/arch/riscv/lib/delay.c index 49d510ba75fd..bd9b3825656f 100644 --- a/arch/riscv/lib/delay.c +++ b/arch/riscv/lib/delay.c @@ -6,7 +6,6 @@ #include <linux/delay.h> #include <linux/math.h> #include <linux/param.h> -#include <linux/timex.h> #include <linux/types.h> #include <linux/export.h> @@ -109,3 +108,9 @@ void ndelay(unsigned long nsecs) __delay(ncycles >> NDELAY_SHIFT); } EXPORT_SYMBOL(ndelay); + +bool delay_read_timer(unsigned long *timer_val) +{ + *timer_val = get_cycles(); + return true; +} diff --git a/arch/riscv/lib/lshrdi3.S b/arch/riscv/lib/lshrdi3.S new file mode 100644 index 000000000000..1af03985ccb7 --- /dev/null +++ b/arch/riscv/lib/lshrdi3.S @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/** + * Adopted for the Linux kernel from IPXE project, see + * https://github.com/ipxe/ipxe/blob/master/src/arch/riscv32/libgcc/llshift.S + */ + +#include <linux/linkage.h> +#include <asm/asm.h> + +/** + * Logical shift right + * + * @v a1:a0 Value to shift + * @v a2 Shift amount + * @ret a1:a0 Shifted value + */ + +SYM_FUNC_START(__lshrdi3) + + /* Perform shift by 32 bits, if applicable */ + li t0, 32 + sub t1, t0, a2 + bgtz t1, 1f + mv a0, a1 + mv a1, zero +1: /* Perform shift by modulo-32 bits, if applicable */ + andi a2, a2, 0x1f + beqz a2, 2f + sll t2, a1, t1 + srl a1, a1, a2 + srl a0, a0, a2 + or a0, a0, t2 +2: ret + +SYM_FUNC_END(__lshrdi3) +EXPORT_SYMBOL(__lshrdi3) diff --git a/arch/riscv/lib/memmove.S b/arch/riscv/lib/memmove.S index cb3e2e7ef0ba..410e598d4677 100644 --- a/arch/riscv/lib/memmove.S +++ b/arch/riscv/lib/memmove.S @@ -40,8 +40,8 @@ SYM_FUNC_START(__memmove) * Both Copy Modes: t1 - Temporary for load-store * Both Copy Modes: t2 - Temporary for load-store * Both Copy Modes: a5 - dest to src alignment offset - * Both Copy Modes: a6 - Shift ammount - * Both Copy Modes: a7 - Inverse Shift ammount + * Both Copy Modes: a6 - Shift amount + * Both Copy Modes: a7 - Inverse Shift amount * Both Copy Modes: a2 - Alternate breakpoint for unrolled loops */ diff --git a/arch/riscv/lib/strchr.S b/arch/riscv/lib/strchr.S new file mode 100644 index 000000000000..48c3a9da53e3 --- /dev/null +++ b/arch/riscv/lib/strchr.S @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +/* + * Copyright (C) 2025 Feng Jiang <jiangfeng@kylinos.cn> + */ + +#include <linux/linkage.h> +#include <asm/asm.h> + +/* char *strchr(const char *s, int c) */ +SYM_FUNC_START(strchr) + /* + * Parameters + * a0 - The string to be searched + * a1 - The character to search for + * + * Returns + * a0 - Address of first occurrence of 'c' or 0 + * + * Clobbers + * t0 + */ + andi a1, a1, 0xff +1: + lbu t0, 0(a0) + beq t0, a1, 2f + addi a0, a0, 1 + bnez t0, 1b + li a0, 0 +2: + ret +SYM_FUNC_END(strchr) + +SYM_FUNC_ALIAS_WEAK(__pi_strchr, strchr) +EXPORT_SYMBOL(strchr) diff --git a/arch/riscv/lib/strlen.S b/arch/riscv/lib/strlen.S index eb4d2b7ed22b..e7736ccda514 100644 --- a/arch/riscv/lib/strlen.S +++ b/arch/riscv/lib/strlen.S @@ -21,13 +21,11 @@ SYM_FUNC_START(strlen) * Clobbers: * t0, t1 */ - mv t1, a0 + addi t1, a0, -1 1: - lbu t0, 0(t1) - beqz t0, 2f addi t1, t1, 1 - j 1b -2: + lbu t0, 0(t1) + bnez t0, 1b sub a0, t1, a0 ret diff --git a/arch/riscv/lib/strnlen.S b/arch/riscv/lib/strnlen.S new file mode 100644 index 000000000000..a8911605c248 --- /dev/null +++ b/arch/riscv/lib/strnlen.S @@ -0,0 +1,172 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +/* + * Base on arch/riscv/lib/strlen.S + * + * Copyright (C) Feng Jiang <jiangfeng@kylinos.cn> + */ + +#include <linux/linkage.h> +#include <asm/asm.h> +#include <asm/alternative-macros.h> +#include <asm/hwcap.h> + +/* size_t strnlen(const char *s, size_t count) */ +SYM_FUNC_START(strnlen) + + __ALTERNATIVE_CFG("nop", "j strnlen_zbb", 0, RISCV_ISA_EXT_ZBB, + IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) + + + /* + * Returns + * a0 - String length + * + * Parameters + * a0 - String to measure + * a1 - Max length of string + * + * Clobbers + * t0, t1, t2 + */ + addi t1, a0, -1 + add t2, a0, a1 +1: + addi t1, t1, 1 + beq t1, t2, 2f + lbu t0, 0(t1) + bnez t0, 1b +2: + sub a0, t1, a0 + ret + + +/* + * Variant of strnlen using the ZBB extension if available + */ +#if defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB) +strnlen_zbb: + +#ifdef CONFIG_CPU_BIG_ENDIAN +# define CZ clz +# define SHIFT sll +#else +# define CZ ctz +# define SHIFT srl +#endif + +.option push +.option arch,+zbb + + /* + * Returns + * a0 - String length + * + * Parameters + * a0 - String to measure + * a1 - Max length of string + * + * Clobbers + * t0, t1, t2, t3, t4 + */ + + /* If maxlen is 0, return 0. */ + beqz a1, 3f + + /* Number of irrelevant bytes in the first word. */ + andi t2, a0, SZREG-1 + + /* Align pointer. */ + andi t0, a0, -SZREG + + li t3, SZREG + sub t3, t3, t2 + slli t2, t2, 3 + + /* + * Aligned boundary. Use the address of the last valid byte + * (s + count - 1) to avoid loading a word past the count + * boundary in the loop below. count == 0 is handled above. + */ + add t4, a0, a1 + addi t4, t4, -1 + andi t4, t4, -SZREG + + /* Get the first word. */ + REG_L t1, 0(t0) + + /* + * Shift away the partial data we loaded to remove the irrelevant bytes + * preceding the string with the effect of adding NUL bytes at the + * end of the string's first word. + */ + SHIFT t1, t1, t2 + + /* Convert non-NUL into 0xff and NUL into 0x00. */ + orc.b t1, t1 + + /* Convert non-NUL into 0x00 and NUL into 0xff. */ + not t1, t1 + + /* + * Search for the first set bit (corresponding to a NUL byte in the + * original chunk). + */ + CZ t1, t1 + + /* + * The first chunk is special: compare against the number + * of valid bytes in this chunk. + */ + srli a0, t1, 3 + + /* Limit the result by maxlen. */ + minu a0, a0, a1 + + bgtu t3, a0, 2f + + /* All remaining bytes are in the first word, no loop needed. */ + bgeu t0, t4, 2f + + /* Prepare for the word comparison loop. */ + addi t2, t0, SZREG + li t3, -1 + + /* + * Our critical loop is 4 instructions and processes data in + * 4 byte or 8 byte chunks. + */ + .p2align 3 +1: + REG_L t1, SZREG(t0) + addi t0, t0, SZREG + orc.b t1, t1 + bgeu t0, t4, 4f + beq t1, t3, 1b +4: + not t1, t1 + CZ t1, t1 + srli t1, t1, 3 + + /* Get number of processed bytes. */ + sub t2, t0, t2 + + /* Add number of characters in the first word. */ + add a0, a0, t2 + + /* Add number of characters in the last word. */ + add a0, a0, t1 + + /* Ensure the final result does not exceed maxlen. */ + minu a0, a0, a1 +2: + ret +3: + mv a0, a1 + ret + +.option pop +#endif +SYM_FUNC_END(strnlen) +SYM_FUNC_ALIAS(__pi_strnlen, strnlen) +EXPORT_SYMBOL(strnlen) diff --git a/arch/riscv/lib/strrchr.S b/arch/riscv/lib/strrchr.S new file mode 100644 index 000000000000..ac58b20ca21d --- /dev/null +++ b/arch/riscv/lib/strrchr.S @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +/* + * Copyright (C) 2025 Feng Jiang <jiangfeng@kylinos.cn> + */ + +#include <linux/linkage.h> +#include <asm/asm.h> + +/* char *strrchr(const char *s, int c) */ +SYM_FUNC_START(strrchr) + /* + * Parameters + * a0 - The string to be searched + * a1 - The character to seaerch for + * + * Returns + * a0 - Address of last occurrence of 'c' or 0 + * + * Clobbers + * t0, t1 + */ + andi a1, a1, 0xff + mv t1, a0 + li a0, 0 +1: + lbu t0, 0(t1) + bne t0, a1, 2f + mv a0, t1 +2: + addi t1, t1, 1 + bnez t0, 1b + ret +SYM_FUNC_END(strrchr) + +SYM_FUNC_ALIAS_WEAK(__pi_strrchr, strrchr) +EXPORT_SYMBOL(strrchr) diff --git a/arch/riscv/lib/uaccess.S b/arch/riscv/lib/uaccess.S index 4efea1b3326c..cf8586a937de 100644 --- a/arch/riscv/lib/uaccess.S +++ b/arch/riscv/lib/uaccess.S @@ -76,6 +76,7 @@ SYM_FUNC_START(fallback_scalar_usercopy_sum_enabled) li a3, 9*SZREG-1 /* size must >= (word_copy stride + SZREG-1) */ bltu a2, a3, .Lbyte_copy_tail +#if !defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) /* * Copy first bytes until dst is aligned to word boundary. * a0 - start of dst @@ -103,7 +104,7 @@ SYM_FUNC_START(fallback_scalar_usercopy_sum_enabled) /* a1 - start of src */ andi a3, a1, SZREG-1 bnez a3, .Lshift_copy - +#endif .Lword_copy: /* * Both src and dst are aligned, unrolled word copy @@ -137,6 +138,7 @@ SYM_FUNC_START(fallback_scalar_usercopy_sum_enabled) addi t0, t0, 8*SZREG /* revert to original value */ j .Lbyte_copy_tail +#if !defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) .Lshift_copy: /* @@ -189,6 +191,7 @@ SYM_FUNC_START(fallback_scalar_usercopy_sum_enabled) /* Revert src to original unaligned value */ add a1, a1, a3 +#endif .Lbyte_copy_tail: /* diff --git a/arch/riscv/lib/xor.S b/arch/riscv/lib/xor.S deleted file mode 100644 index b28f2430e52f..000000000000 --- a/arch/riscv/lib/xor.S +++ /dev/null @@ -1,81 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) 2021 SiFive - */ -#include <linux/linkage.h> -#include <linux/export.h> -#include <asm/asm.h> - -SYM_FUNC_START(xor_regs_2_) - vsetvli a3, a0, e8, m8, ta, ma - vle8.v v0, (a1) - vle8.v v8, (a2) - sub a0, a0, a3 - vxor.vv v16, v0, v8 - add a2, a2, a3 - vse8.v v16, (a1) - add a1, a1, a3 - bnez a0, xor_regs_2_ - ret -SYM_FUNC_END(xor_regs_2_) -EXPORT_SYMBOL(xor_regs_2_) - -SYM_FUNC_START(xor_regs_3_) - vsetvli a4, a0, e8, m8, ta, ma - vle8.v v0, (a1) - vle8.v v8, (a2) - sub a0, a0, a4 - vxor.vv v0, v0, v8 - vle8.v v16, (a3) - add a2, a2, a4 - vxor.vv v16, v0, v16 - add a3, a3, a4 - vse8.v v16, (a1) - add a1, a1, a4 - bnez a0, xor_regs_3_ - ret -SYM_FUNC_END(xor_regs_3_) -EXPORT_SYMBOL(xor_regs_3_) - -SYM_FUNC_START(xor_regs_4_) - vsetvli a5, a0, e8, m8, ta, ma - vle8.v v0, (a1) - vle8.v v8, (a2) - sub a0, a0, a5 - vxor.vv v0, v0, v8 - vle8.v v16, (a3) - add a2, a2, a5 - vxor.vv v0, v0, v16 - vle8.v v24, (a4) - add a3, a3, a5 - vxor.vv v16, v0, v24 - add a4, a4, a5 - vse8.v v16, (a1) - add a1, a1, a5 - bnez a0, xor_regs_4_ - ret -SYM_FUNC_END(xor_regs_4_) -EXPORT_SYMBOL(xor_regs_4_) - -SYM_FUNC_START(xor_regs_5_) - vsetvli a6, a0, e8, m8, ta, ma - vle8.v v0, (a1) - vle8.v v8, (a2) - sub a0, a0, a6 - vxor.vv v0, v0, v8 - vle8.v v16, (a3) - add a2, a2, a6 - vxor.vv v0, v0, v16 - vle8.v v24, (a4) - add a3, a3, a6 - vxor.vv v0, v0, v24 - vle8.v v8, (a5) - add a4, a4, a6 - vxor.vv v16, v0, v8 - add a5, a5, a6 - vse8.v v16, (a1) - add a1, a1, a6 - bnez a0, xor_regs_5_ - ret -SYM_FUNC_END(xor_regs_5_) -EXPORT_SYMBOL(xor_regs_5_) diff --git a/arch/riscv/mm/cacheflush.c b/arch/riscv/mm/cacheflush.c index d83a612464f6..f8ead7cb7c7d 100644 --- a/arch/riscv/mm/cacheflush.c +++ b/arch/riscv/mm/cacheflush.c @@ -29,7 +29,7 @@ void flush_icache_all(void) * Make sure all previous writes to the D$ are ordered before making * the IPI. The RISC-V spec states that a hart must execute a data fence * before triggering a remote fence.i in order to make the modification - * visable for remote harts. + * visible for remote harts. * * IPIs on RISC-V are triggered by MMIO writes to either CLINT or * S-IMSIC, so the fence ensures previous data writes "happen before" diff --git a/arch/riscv/mm/dma-noncoherent.c b/arch/riscv/mm/dma-noncoherent.c index cb89d7e0ba88..a1ec2d71d1c9 100644 --- a/arch/riscv/mm/dma-noncoherent.c +++ b/arch/riscv/mm/dma-noncoherent.c @@ -140,7 +140,7 @@ void arch_setup_dma_ops(struct device *dev, bool coherent) "%s %s: device non-coherent but no non-coherent operations supported", dev_driver_string(dev), dev_name(dev)); - dev->dma_coherent = coherent; + dev_assign_dma_coherent(dev, coherent); } void riscv_noncoherent_supported(void) diff --git a/arch/riscv/mm/hugetlbpage.c b/arch/riscv/mm/hugetlbpage.c index 375dd96bb4a0..a6d217112cf4 100644 --- a/arch/riscv/mm/hugetlbpage.c +++ b/arch/riscv/mm/hugetlbpage.c @@ -447,3 +447,11 @@ static __init int gigantic_pages_init(void) } arch_initcall(gigantic_pages_init); #endif + +unsigned int __init arch_hugetlb_cma_order(void) +{ + if (IS_ENABLED(CONFIG_64BIT)) + return PUD_SHIFT - PAGE_SHIFT; + + return 0; +} diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index addb8a9305be..fb37b0b67efe 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -37,24 +37,23 @@ #include "../kernel/head.h" -u64 new_vmalloc[NR_CPUS / sizeof(u64) + 1]; +#if defined(CONFIG_64BIT) && defined(CONFIG_MMU) +DECLARE_BITMAP(new_valid_map_cpus, NR_CPUS); +#endif struct kernel_mapping kernel_map __ro_after_init; EXPORT_SYMBOL(kernel_map); -#ifdef CONFIG_XIP_KERNEL -#define kernel_map (*(struct kernel_mapping *)XIP_FIXUP(&kernel_map)) -#endif #ifdef CONFIG_64BIT -u64 satp_mode __ro_after_init = !IS_ENABLED(CONFIG_XIP_KERNEL) ? SATP_MODE_57 : SATP_MODE_39; +u64 satp_mode __ro_after_init = SATP_MODE_57; #else u64 satp_mode __ro_after_init = SATP_MODE_32; #endif EXPORT_SYMBOL(satp_mode); #ifdef CONFIG_64BIT -bool pgtable_l4_enabled __ro_after_init = !IS_ENABLED(CONFIG_XIP_KERNEL); -bool pgtable_l5_enabled __ro_after_init = !IS_ENABLED(CONFIG_XIP_KERNEL); +bool pgtable_l4_enabled __ro_after_init = true; +bool pgtable_l5_enabled __ro_after_init = true; EXPORT_SYMBOL(pgtable_l4_enabled); EXPORT_SYMBOL(pgtable_l5_enabled); #endif @@ -63,32 +62,26 @@ phys_addr_t phys_ram_base __ro_after_init; EXPORT_SYMBOL(phys_ram_base); #ifdef CONFIG_SPARSEMEM_VMEMMAP -#define VMEMMAP_ADDR_ALIGN (1ULL << SECTION_SIZE_BITS) +#define VMEMMAP_ADDR_ALIGN max(1ULL << SECTION_SIZE_BITS, \ + PFN_PHYS(MAX_FOLIO_VMEMMAP_ALIGN / \ + sizeof(struct page))) unsigned long vmemmap_start_pfn __ro_after_init; EXPORT_SYMBOL(vmemmap_start_pfn); #endif -unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] - __page_aligned_bss; -EXPORT_SYMBOL(empty_zero_page); - extern char _start[]; void *_dtb_early_va __initdata; uintptr_t _dtb_early_pa __initdata; phys_addr_t dma32_phys_limit __initdata; -static void __init zone_sizes_init(void) +void __init arch_zone_limits_init(unsigned long *max_zone_pfns) { - unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, }; - #ifdef CONFIG_ZONE_DMA32 max_zone_pfns[ZONE_DMA32] = PFN_DOWN(dma32_phys_limit); #endif max_zone_pfns[ZONE_NORMAL] = max_low_pfn; - - free_area_init(max_zone_pfns); } #if defined(CONFIG_MMU) && defined(CONFIG_DEBUG_VM) @@ -172,7 +165,9 @@ static void print_vm_layout(void) { } void __init arch_mm_preinit(void) { - bool swiotlb = max_pfn > PFN_DOWN(dma32_phys_limit); + bool swiotlb = max_pfn > PFN_DOWN(dma32_phys_limit) && + memblock_start_of_DRAM() < dma32_phys_limit; + unsigned int swiotlb_flags = SWIOTLB_VERBOSE; #ifdef CONFIG_FLATMEM BUG_ON(!mem_map); #endif /* CONFIG_FLATMEM */ @@ -180,26 +175,28 @@ void __init arch_mm_preinit(void) if (IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && !swiotlb && dma_cache_alignment != 1) { /* - * If no bouncing needed for ZONE_DMA, allocate 1MB swiotlb - * buffer per 1GB of RAM for kmalloc() bouncing on - * non-coherent platforms. + * No 32-bit DMA bouncing needed (either all DRAM is within + * the 32-bit limit, or it all starts above it), but + * kmalloc() buffers whose sizes are not cache-line-aligned + * still require bouncing for non-coherent DMA. Use + * SWIOTLB_ANY so that the buffer can be allocated from high + * memory when DRAM starts above dma32_phys_limit. Allocate + * ~1 MB per 1 GB of RAM. */ unsigned long size = DIV_ROUND_UP(memblock_phys_mem_size(), 1024); swiotlb_adjust_size(min(swiotlb_size_or_default(), size)); swiotlb = true; + swiotlb_flags |= SWIOTLB_ANY; } - swiotlb_init(swiotlb, SWIOTLB_VERBOSE); + swiotlb_init(swiotlb, swiotlb_flags); print_vm_layout(); } /* Limit the memory size via mem. */ static phys_addr_t memory_limit; -#ifdef CONFIG_XIP_KERNEL -#define memory_limit (*(phys_addr_t *)XIP_FIXUP(&memory_limit)) -#endif /* CONFIG_XIP_KERNEL */ static int __init early_mem(char *p) { @@ -223,10 +220,7 @@ static void __init setup_bootmem(void) phys_addr_t max_mapped_addr; phys_addr_t phys_ram_end, vmlinux_start; - if (IS_ENABLED(CONFIG_XIP_KERNEL)) - vmlinux_start = __pa_symbol(&_sdata); - else - vmlinux_start = __pa_symbol(&_start); + vmlinux_start = __pa_symbol(&_start); memblock_enforce_memory_limit(memory_limit); @@ -246,12 +240,10 @@ static void __init setup_bootmem(void) * Make sure we align the start of the memory on a PMD boundary so that * at worst, we map the linear mapping with PMD mappings. */ - if (!IS_ENABLED(CONFIG_XIP_KERNEL)) { - phys_ram_base = memblock_start_of_DRAM() & PMD_MASK; + phys_ram_base = memblock_start_of_DRAM() & PMD_MASK; #ifdef CONFIG_SPARSEMEM_VMEMMAP - vmemmap_start_pfn = round_down(phys_ram_base, VMEMMAP_ADDR_ALIGN) >> PAGE_SHIFT; + vmemmap_start_pfn = round_down(phys_ram_base, VMEMMAP_ADDR_ALIGN) >> PAGE_SHIFT; #endif - } /* * In 64-bit, any use of __va/__pa before this point is wrong as we @@ -313,10 +305,6 @@ static void __init setup_bootmem(void) */ if (!IS_ENABLED(CONFIG_BUILTIN_DTB)) memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va)); - - dma_contiguous_reserve(dma32_phys_limit); - if (IS_ENABLED(CONFIG_64BIT)) - hugetlb_cma_reserve(PUD_SHIFT - PAGE_SHIFT); } #ifdef CONFIG_RELOCATABLE @@ -366,17 +354,10 @@ static pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss; pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); -#ifdef CONFIG_XIP_KERNEL -#define pt_ops (*(struct pt_alloc_ops *)XIP_FIXUP(&pt_ops)) -#define trampoline_pg_dir ((pgd_t *)XIP_FIXUP(trampoline_pg_dir)) -#define fixmap_pte ((pte_t *)XIP_FIXUP(fixmap_pte)) -#define early_pg_dir ((pgd_t *)XIP_FIXUP(early_pg_dir)) -#endif /* CONFIG_XIP_KERNEL */ - static const pgprot_t protection_map[16] = { [VM_NONE] = PAGE_NONE, [VM_READ] = PAGE_READ, - [VM_WRITE] = PAGE_COPY, + [VM_WRITE] = PAGE_SHADOWSTACK, [VM_WRITE | VM_READ] = PAGE_COPY, [VM_EXEC] = PAGE_EXEC, [VM_EXEC | VM_READ] = PAGE_READ_EXEC, @@ -469,32 +450,14 @@ static pmd_t trampoline_pmd[PTRS_PER_PMD] __page_aligned_bss; static pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss; static pmd_t early_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); -#ifdef CONFIG_XIP_KERNEL -#define trampoline_pmd ((pmd_t *)XIP_FIXUP(trampoline_pmd)) -#define fixmap_pmd ((pmd_t *)XIP_FIXUP(fixmap_pmd)) -#define early_pmd ((pmd_t *)XIP_FIXUP(early_pmd)) -#endif /* CONFIG_XIP_KERNEL */ - static p4d_t trampoline_p4d[PTRS_PER_P4D] __page_aligned_bss; static p4d_t fixmap_p4d[PTRS_PER_P4D] __page_aligned_bss; static p4d_t early_p4d[PTRS_PER_P4D] __initdata __aligned(PAGE_SIZE); -#ifdef CONFIG_XIP_KERNEL -#define trampoline_p4d ((p4d_t *)XIP_FIXUP(trampoline_p4d)) -#define fixmap_p4d ((p4d_t *)XIP_FIXUP(fixmap_p4d)) -#define early_p4d ((p4d_t *)XIP_FIXUP(early_p4d)) -#endif /* CONFIG_XIP_KERNEL */ - static pud_t trampoline_pud[PTRS_PER_PUD] __page_aligned_bss; static pud_t fixmap_pud[PTRS_PER_PUD] __page_aligned_bss; static pud_t early_pud[PTRS_PER_PUD] __initdata __aligned(PAGE_SIZE); -#ifdef CONFIG_XIP_KERNEL -#define trampoline_pud ((pud_t *)XIP_FIXUP(trampoline_pud)) -#define fixmap_pud ((pud_t *)XIP_FIXUP(fixmap_pud)) -#define early_pud ((pud_t *)XIP_FIXUP(early_pud)) -#endif /* CONFIG_XIP_KERNEL */ - static pmd_t *__init get_pmd_virt_early(phys_addr_t pa) { /* Before MMU is enabled */ @@ -765,21 +728,6 @@ static uintptr_t __meminit best_map_size(phys_addr_t pa, uintptr_t va, phys_addr return PAGE_SIZE; } -#ifdef CONFIG_XIP_KERNEL -#define phys_ram_base (*(phys_addr_t *)XIP_FIXUP(&phys_ram_base)) -extern char _xiprom[], _exiprom[], __data_loc; - -/* called from head.S with MMU off */ -asmlinkage void __init __copy_data(void) -{ - void *from = (void *)(&__data_loc); - void *to = (void *)CONFIG_PHYS_RAM_BASE; - size_t sz = (size_t)((uintptr_t)(&_end) - (uintptr_t)(&_sdata)); - - memcpy(to, from, sz); -} -#endif - #ifdef CONFIG_STRICT_KERNEL_RWX static __meminit pgprot_t pgprot_from_va(uintptr_t va) { @@ -815,7 +763,7 @@ static __meminit pgprot_t pgprot_from_va(uintptr_t va) } #endif /* CONFIG_STRICT_KERNEL_RWX */ -#if defined(CONFIG_64BIT) && !defined(CONFIG_XIP_KERNEL) +#if defined(CONFIG_64BIT) u64 __pi_set_satp_mode_from_cmdline(uintptr_t dtb_pa); u64 __pi_set_satp_mode_from_fdt(uintptr_t dtb_pa); @@ -852,6 +800,27 @@ static void __init set_mmap_rnd_bits_max(void) mmap_rnd_bits_max = MMAP_VA_BITS - PAGE_SHIFT - 3; } +static bool __init is_vaddr_valid(unsigned long va) +{ + unsigned long up = 0; + + switch (satp_mode) { + case SATP_MODE_39: + up = 1UL << 38; + break; + case SATP_MODE_48: + up = 1UL << 47; + break; + case SATP_MODE_57: + up = 1UL << 56; + break; + default: + return false; + } + + return (va < up) || (va >= (ULONG_MAX - up + 1)); +} + /* * There is a simple way to determine if 4-level is supported by the * underlying hardware: establish 1:1 mapping in 4-level page table mode @@ -893,6 +862,9 @@ static __init void set_satp_mode(uintptr_t dtb_pa) set_satp_mode_pmd + PMD_SIZE, PMD_SIZE, PAGE_KERNEL_EXEC); retry: + if (!is_vaddr_valid(set_satp_mode_pmd)) + goto out; + create_pgd_mapping(early_pg_dir, set_satp_mode_pmd, pgtable_l5_enabled ? @@ -915,6 +887,7 @@ retry: disable_pgtable_l4(); } +out: memset(early_pg_dir, 0, PAGE_SIZE); memset(early_p4d, 0, PAGE_SIZE); memset(early_pud, 0, PAGE_SIZE); @@ -940,28 +913,6 @@ retry: #error "setup_vm() is called from head.S before relocate so it should not use absolute addressing." #endif -#ifdef CONFIG_XIP_KERNEL -static void __init create_kernel_page_table(pgd_t *pgdir, - __always_unused bool early) -{ - uintptr_t va, start_va, end_va; - - /* Map the flash resident part */ - end_va = kernel_map.virt_addr + kernel_map.xiprom_sz; - for (va = kernel_map.virt_addr; va < end_va; va += PMD_SIZE) - create_pgd_mapping(pgdir, va, - kernel_map.xiprom + (va - kernel_map.virt_addr), - PMD_SIZE, PAGE_KERNEL_EXEC); - - /* Map the data in RAM */ - start_va = kernel_map.virt_addr + (uintptr_t)&_sdata - (uintptr_t)&_start; - end_va = kernel_map.virt_addr + kernel_map.size; - for (va = start_va; va < end_va; va += PMD_SIZE) - create_pgd_mapping(pgdir, va, - kernel_map.phys_addr + (va - start_va), - PMD_SIZE, PAGE_KERNEL); -} -#else static void __init create_kernel_page_table(pgd_t *pgdir, bool early) { uintptr_t va, end_va; @@ -974,7 +925,6 @@ static void __init create_kernel_page_table(pgd_t *pgdir, bool early) early ? PAGE_KERNEL_EXEC : pgprot_from_va(va)); } -#endif /* * Setup a 4MB mapping that encompasses the device tree: for 64-bit kernel, @@ -1084,11 +1034,6 @@ static int __init print_nokaslr(char *p) return 0; } early_param("nokaslr", print_nokaslr); - -unsigned long kaslr_offset(void) -{ - return kernel_map.virt_offset; -} #endif asmlinkage void __init setup_vm(uintptr_t dtb_pa) @@ -1116,27 +1061,11 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) kernel_map.virt_addr = KERNEL_LINK_ADDR + kernel_map.virt_offset; -#ifdef CONFIG_XIP_KERNEL - kernel_map.xiprom = (uintptr_t)CONFIG_XIP_PHYS_ADDR; - kernel_map.xiprom_sz = (uintptr_t)(&_exiprom) - (uintptr_t)(&_xiprom); - - phys_ram_base = CONFIG_PHYS_RAM_BASE; -#ifdef CONFIG_SPARSEMEM_VMEMMAP - vmemmap_start_pfn = round_down(phys_ram_base, VMEMMAP_ADDR_ALIGN) >> PAGE_SHIFT; -#endif - kernel_map.phys_addr = (uintptr_t)CONFIG_PHYS_RAM_BASE; - kernel_map.size = (uintptr_t)(&_end) - (uintptr_t)(&_start); - - kernel_map.va_kernel_xip_text_pa_offset = kernel_map.virt_addr - kernel_map.xiprom; - kernel_map.va_kernel_xip_data_pa_offset = kernel_map.virt_addr - kernel_map.phys_addr - + (uintptr_t)&_sdata - (uintptr_t)&_start; -#else kernel_map.phys_addr = (uintptr_t)(&_start); kernel_map.size = (uintptr_t)(&_end) - kernel_map.phys_addr; kernel_map.va_kernel_pa_offset = kernel_map.virt_addr - kernel_map.phys_addr; -#endif -#if defined(CONFIG_64BIT) && !defined(CONFIG_XIP_KERNEL) +#if defined(CONFIG_64BIT) set_satp_mode(dtb_pa); set_mmap_rnd_bits_max(); #endif @@ -1209,13 +1138,8 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) if (pgtable_l4_enabled) create_pud_mapping(trampoline_pud, kernel_map.virt_addr, (uintptr_t)trampoline_pmd, PUD_SIZE, PAGE_TABLE); -#ifdef CONFIG_XIP_KERNEL - create_pmd_mapping(trampoline_pmd, kernel_map.virt_addr, - kernel_map.xiprom, PMD_SIZE, PAGE_KERNEL_EXEC); -#else create_pmd_mapping(trampoline_pmd, kernel_map.virt_addr, kernel_map.phys_addr, PMD_SIZE, PAGE_KERNEL_EXEC); -#endif #else /* Setup trampoline PGD */ create_pgd_mapping(trampoline_pg_dir, kernel_map.virt_addr, @@ -1313,8 +1237,6 @@ static void __init create_linear_mapping_page_table(void) /* Map all memory banks in the linear mapping */ for_each_mem_range(i, &start, &end) { - if (start >= end) - break; if (start <= __pa(PAGE_OFFSET) && __pa(PAGE_OFFSET) < end) start = __pa(PAGE_OFFSET); @@ -1404,7 +1326,7 @@ static inline void setup_vm_final(void) */ static void __init arch_reserve_crashkernel(void) { - unsigned long long low_size = 0; + unsigned long long low_size = 0, cma_size = 0; unsigned long long crash_base, crash_size; bool high = false; int ret; @@ -1414,11 +1336,12 @@ static void __init arch_reserve_crashkernel(void) ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(), &crash_size, &crash_base, - &low_size, NULL, &high); + &low_size, &cma_size, &high); if (ret) return; reserve_crashkernel_generic(crash_size, crash_base, low_size, high); + reserve_crashkernel_cma(cma_size); } void __init paging_init(void) @@ -1434,33 +1357,21 @@ void __init misc_mem_init(void) { early_memtest(min_low_pfn << PAGE_SHIFT, max_low_pfn << PAGE_SHIFT); arch_numa_init(); - sparse_init(); + dma_contiguous_reserve(dma32_phys_limit); #ifdef CONFIG_SPARSEMEM_VMEMMAP /* The entire VMEMMAP region has been populated. Flush TLB for this region */ local_flush_tlb_kernel_range(VMEMMAP_START, VMEMMAP_END); #endif - zone_sizes_init(); arch_reserve_crashkernel(); memblock_dump_all(); } #ifdef CONFIG_SPARSEMEM_VMEMMAP -void __meminit vmemmap_set_pmd(pmd_t *pmd, void *p, int node, - unsigned long addr, unsigned long next) -{ - pmd_set_huge(pmd, virt_to_phys(p), PAGE_KERNEL); -} - -int __meminit vmemmap_check_pmd(pmd_t *pmdp, int node, - unsigned long addr, unsigned long next) -{ - vmemmap_verify((pte_t *)pmdp, node, addr, next); - return 1; -} - int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap) { + WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END)); + /* * Note that SPARSEMEM_VMEMMAP is only selected for rv64 and that we * can't use hugepage mappings for 2-level page table because in case of @@ -1554,7 +1465,9 @@ struct execmem_info __init *execmem_arch_setup(void) [EXECMEM_KPROBES] = { .start = VMALLOC_START, .end = VMALLOC_END, - .pgprot = PAGE_KERNEL_READ_EXEC, + .pgprot = IS_ENABLED(CONFIG_STRICT_MODULE_RWX) ? + PAGE_KERNEL_READ_EXEC : + PAGE_KERNEL_EXEC, .alignment = 1, }, [EXECMEM_BPF] = { @@ -1713,7 +1626,7 @@ static void __meminit remove_pud_mapping(pud_t *pud_base, unsigned long addr, un for (; addr < end; addr = next) { next = pud_addr_end(addr, end); - pudp = pud_base + pud_index(addr); + pudp = pgtable_l4_enabled ? pud_base + pud_index(addr) : pud_base; pud = pudp_get(pudp); if (!pud_present(pud)) continue; @@ -1744,7 +1657,7 @@ static void __meminit remove_p4d_mapping(p4d_t *p4d_base, unsigned long addr, un for (; addr < end; addr = next) { next = p4d_addr_end(addr, end); - p4dp = p4d_base + p4d_index(addr); + p4dp = pgtable_l5_enabled ? p4d_base + p4d_index(addr) : p4d_base; p4d = p4dp_get(p4dp); if (!p4d_present(p4d)) continue; @@ -1826,9 +1739,10 @@ int __ref arch_add_memory(int nid, u64 start, u64 size, struct mhp_params *param return ret; } -void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap) +void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap, + struct dev_pagemap *pgmap) { - __remove_pages(start >> PAGE_SHIFT, size >> PAGE_SHIFT, altmap); + __remove_pages(start >> PAGE_SHIFT, size >> PAGE_SHIFT, altmap, pgmap); remove_linear_mapping(start, size); flush_tlb_all(); } diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c index c4a2a9e5586e..1f3aa9611187 100644 --- a/arch/riscv/mm/kasan_init.c +++ b/arch/riscv/mm/kasan_init.c @@ -512,9 +512,6 @@ void __init kasan_init(void) void *start = (void *)__va(p_start); void *end = (void *)__va(p_end); - if (start >= end) - break; - kasan_populate(kasan_mem_to_shadow(start), kasan_mem_to_shadow(end)); } diff --git a/arch/riscv/mm/pgtable.c b/arch/riscv/mm/pgtable.c index 807c0a0de182..9c4427d0b187 100644 --- a/arch/riscv/mm/pgtable.c +++ b/arch/riscv/mm/pgtable.c @@ -29,12 +29,11 @@ int ptep_set_access_flags(struct vm_area_struct *vma, return true; } -int ptep_test_and_clear_young(struct vm_area_struct *vma, - unsigned long address, - pte_t *ptep) +bool ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep) { if (!pte_young(ptep_get(ptep))) - return 0; + return false; return test_and_clear_bit(_PAGE_ACCESSED_OFFSET, &pte_val(*ptep)); } EXPORT_SYMBOL_GPL(ptep_test_and_clear_young); @@ -47,6 +46,7 @@ pud_t *pud_offset(p4d_t *p4d, unsigned long address) return (pud_t *)p4d; } +EXPORT_SYMBOL_GPL(pud_offset); p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) { @@ -55,6 +55,7 @@ p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) return (p4d_t *)pgd; } +EXPORT_SYMBOL_GPL(p4d_offset); #endif #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP @@ -163,3 +164,19 @@ pud_t pudp_invalidate(struct vm_area_struct *vma, unsigned long address, return old; } #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma) +{ + if (vma->vm_flags & VM_SHADOW_STACK) + return pte_mkwrite_shstk(pte); + + return pte_mkwrite_novma(pte); +} + +pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) +{ + if (vma->vm_flags & VM_SHADOW_STACK) + return pmd_mkwrite_shstk(pmd); + + return pmd_mkwrite_novma(pmd); +} diff --git a/arch/riscv/mm/physaddr.c b/arch/riscv/mm/physaddr.c index 559d291fac5c..b1836e9481ce 100644 --- a/arch/riscv/mm/physaddr.c +++ b/arch/riscv/mm/physaddr.c @@ -9,7 +9,7 @@ phys_addr_t __virt_to_phys(unsigned long x) { /* - * Boundary checking aginst the kernel linear mapping space. + * Boundary checking against the kernel linear mapping space. */ WARN(!is_linear_mapping(x) && !is_kernel_mapping(x), "virt_to_phys used for non-linear address: %p (%pS)\n", @@ -25,7 +25,7 @@ phys_addr_t __phys_addr_symbol(unsigned long x) unsigned long kernel_end = kernel_start + kernel_map.size; /* - * Boundary checking aginst the kernel image mapping. + * Boundary checking against the kernel image mapping. * __pa_symbol should only be used on kernel symbol addresses. */ VIRTUAL_BUG_ON(x < kernel_start || x > kernel_end); diff --git a/arch/riscv/mm/ptdump.c b/arch/riscv/mm/ptdump.c index 34299c2b231f..f4b4a9fcbbd8 100644 --- a/arch/riscv/mm/ptdump.c +++ b/arch/riscv/mm/ptdump.c @@ -7,6 +7,7 @@ #include <linux/init.h> #include <linux/debugfs.h> #include <linux/seq_file.h> +#include <linux/string.h> #include <linux/ptdump.h> #include <linux/pgtable.h> @@ -213,21 +214,21 @@ static void dump_prot(struct pg_state *st) val = st->current_prot & pte_bits[i].mask; if (val) { if (pte_bits[i].mask == _PAGE_SOFT) - sprintf(s, pte_bits[i].set, val >> 8); + snprintf(s, sizeof(s), pte_bits[i].set, val >> 8); #ifdef CONFIG_64BIT else if (pte_bits[i].mask == _PAGE_MTMASK_SVPBMT) { if (val == _PAGE_NOCACHE_SVPBMT) - sprintf(s, pte_bits[i].set, "NC"); + snprintf(s, sizeof(s), pte_bits[i].set, "NC"); else if (val == _PAGE_IO_SVPBMT) - sprintf(s, pte_bits[i].set, "IO"); + snprintf(s, sizeof(s), pte_bits[i].set, "IO"); else - sprintf(s, pte_bits[i].set, "??"); + snprintf(s, sizeof(s), pte_bits[i].set, "??"); } #endif else - sprintf(s, "%s", pte_bits[i].set); + strscpy(s, pte_bits[i].set); } else { - sprintf(s, "%s", pte_bits[i].clear); + strscpy(s, pte_bits[i].clear); } pt_dump_seq_printf(st->seq, " %s", s); diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c index 8404530ec00f..962db300a166 100644 --- a/arch/riscv/mm/tlbflush.c +++ b/arch/riscv/mm/tlbflush.c @@ -11,24 +11,6 @@ #define has_svinval() riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL) -static inline void local_sfence_inval_ir(void) -{ - asm volatile(SFENCE_INVAL_IR() ::: "memory"); -} - -static inline void local_sfence_w_inval(void) -{ - asm volatile(SFENCE_W_INVAL() ::: "memory"); -} - -static inline void local_sinval_vma(unsigned long vma, unsigned long asid) -{ - if (asid != FLUSH_TLB_NO_ASID) - asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory"); - else - asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory"); -} - /* * Flush entire TLB if number of entries to be flushed is greater * than the threshold below. @@ -110,11 +92,6 @@ static void __ipi_flush_tlb_range_asid(void *info) local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid); } -static inline unsigned long get_mm_asid(struct mm_struct *mm) -{ - return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID; -} - static void __flush_tlb_range(struct mm_struct *mm, const struct cpumask *cmask, unsigned long start, unsigned long size, diff --git a/arch/riscv/net/Makefile b/arch/riscv/net/Makefile index 9a1e5f0a94e5..6458d4d51990 100644 --- a/arch/riscv/net/Makefile +++ b/arch/riscv/net/Makefile @@ -3,7 +3,7 @@ obj-$(CONFIG_BPF_JIT) += bpf_jit_core.o ifeq ($(CONFIG_ARCH_RV64I),y) - obj-$(CONFIG_BPF_JIT) += bpf_jit_comp64.o + obj-$(CONFIG_BPF_JIT) += bpf_jit_comp64.o bpf_timed_may_goto.o else obj-$(CONFIG_BPF_JIT) += bpf_jit_comp32.o endif diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h index 632ced07bca4..419b9d795f2a 100644 --- a/arch/riscv/net/bpf_jit.h +++ b/arch/riscv/net/bpf_jit.h @@ -11,7 +11,6 @@ #include <linux/bpf.h> #include <linux/filter.h> -#include <asm/cacheflush.h> /* verify runtime detection extension status */ #define rv_ext_enabled(ext) \ @@ -82,6 +81,7 @@ struct rv_jit_context { int ex_jmp_off; unsigned long flags; int stack_size; + int tcc_offset; u64 arena_vm_start; u64 user_vm_start; }; @@ -105,11 +105,6 @@ static inline void bpf_fill_ill_insns(void *area, unsigned int size) memset(area, 0, size); } -static inline void bpf_flush_icache(void *start, void *end) -{ - flush_icache_range((unsigned long)start, (unsigned long)end); -} - /* Emit a 4-byte riscv instruction. */ static inline void emit(const u32 insn, struct rv_jit_context *ctx) { diff --git a/arch/riscv/net/bpf_jit_comp32.c b/arch/riscv/net/bpf_jit_comp32.c index 592dd86fbf81..a9e0bd5cc81d 100644 --- a/arch/riscv/net/bpf_jit_comp32.c +++ b/arch/riscv/net/bpf_jit_comp32.c @@ -509,12 +509,15 @@ static void emit_alu_r64(const s8 *dst, const s8 *src, } static void emit_alu_r32(const s8 *dst, const s8 *src, - struct rv_jit_context *ctx, const u8 op) + struct rv_jit_context *ctx, + const struct bpf_insn *insn) { const s8 *tmp1 = bpf2rv32[TMP_REG_1]; const s8 *tmp2 = bpf2rv32[TMP_REG_2]; const s8 *rd = bpf_get_reg32(dst, tmp1, ctx); const s8 *rs = bpf_get_reg32(src, tmp2, ctx); + u8 op = BPF_OP(insn->code); + bool is_signed = insn->off == 1; switch (op) { case BPF_MOV: @@ -539,10 +542,12 @@ static void emit_alu_r32(const s8 *dst, const s8 *src, emit(rv_mul(lo(rd), lo(rd), lo(rs)), ctx); break; case BPF_DIV: - emit(rv_divu(lo(rd), lo(rd), lo(rs)), ctx); + emit(is_signed ? rv_div(lo(rd), lo(rd), lo(rs)) : + rv_divu(lo(rd), lo(rd), lo(rs)), ctx); break; case BPF_MOD: - emit(rv_remu(lo(rd), lo(rd), lo(rs)), ctx); + emit(is_signed ? rv_rem(lo(rd), lo(rd), lo(rs)) : + rv_remu(lo(rd), lo(rd), lo(rs)), ctx); break; case BPF_LSH: emit(rv_sll(lo(rd), lo(rd), lo(rs)), ctx); @@ -869,14 +874,58 @@ static int emit_load_r64(const s8 *dst, const s8 *src, s16 off, return 0; } -static int emit_store_r64(const s8 *dst, const s8 *src, s16 off, - struct rv_jit_context *ctx, const u8 size, - const u8 mode) +static int emit_bpf_atomic(s8 dst, const s8 *src, const s8 *rs, + struct rv_jit_context *ctx, + const struct bpf_insn *insn) +{ + s32 imm = insn->imm; + bool is_fetch = (imm & BPF_FETCH) || (imm == BPF_XCHG); + s8 fetch_reg = is_fetch ? lo(rs) : RV_REG_ZERO; + int aq = is_fetch ? 1 : 0; + int rl = is_fetch ? 1 : 0; + + switch (imm) { + case BPF_ADD: + case BPF_ADD | BPF_FETCH: + emit(rv_amoadd_w(fetch_reg, lo(rs), dst, aq, rl), ctx); + break; + case BPF_AND: + case BPF_AND | BPF_FETCH: + emit(rv_amoand_w(fetch_reg, lo(rs), dst, aq, rl), ctx); + break; + case BPF_OR: + case BPF_OR | BPF_FETCH: + emit(rv_amoor_w(fetch_reg, lo(rs), dst, aq, rl), ctx); + break; + case BPF_XOR: + case BPF_XOR | BPF_FETCH: + emit(rv_amoxor_w(fetch_reg, lo(rs), dst, aq, rl), ctx); + break; + case BPF_XCHG: + emit(rv_amoswap_w(fetch_reg, lo(rs), dst, aq, rl), ctx); + break; + default: + return -1; + } + + if (is_fetch) { + emit(rv_addi(hi(rs), RV_REG_ZERO, 0), ctx); + bpf_put_reg64(src, rs, ctx); + } + return 0; +} + +static int emit_store_r64(const s8 *dst, const s8 *src, + struct rv_jit_context *ctx, + const struct bpf_insn *insn) { const s8 *tmp1 = bpf2rv32[TMP_REG_1]; const s8 *tmp2 = bpf2rv32[TMP_REG_2]; const s8 *rd = bpf_get_reg64(dst, tmp1, ctx); const s8 *rs = bpf_get_reg64(src, tmp2, ctx); + u8 size = BPF_SIZE(insn->code); + u8 mode = BPF_MODE(insn->code); + s16 off = insn->off; if (mode == BPF_ATOMIC && size != BPF_W) return -1; @@ -896,9 +945,9 @@ static int emit_store_r64(const s8 *dst, const s8 *src, s16 off, case BPF_MEM: emit(rv_sw(RV_REG_T0, 0, lo(rs)), ctx); break; - case BPF_ATOMIC: /* Only BPF_ADD supported */ - emit(rv_amoadd_w(RV_REG_ZERO, lo(rs), RV_REG_T0, 0, 0), - ctx); + case BPF_ATOMIC: + if (emit_bpf_atomic(RV_REG_T0, src, rs, ctx, insn)) + return -1; break; } break; @@ -967,6 +1016,24 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, switch (code) { case BPF_ALU64 | BPF_MOV | BPF_X: + if (insn->off != 0) { + const s8 *rd = bpf_get_reg64(dst, tmp1, ctx); + const s8 *rs = bpf_get_reg64(src, tmp2, ctx); + + if (insn->off == 8) { + emit(rv_slli(lo(rd), lo(rs), 24), ctx); + emit(rv_srai(lo(rd), lo(rd), 24), ctx); + } else if (insn->off == 16) { + emit(rv_slli(lo(rd), lo(rs), 16), ctx); + emit(rv_srai(lo(rd), lo(rd), 16), ctx); + } else { + emit(rv_addi(lo(rd), lo(rs), 0), ctx); + } + emit(rv_srai(hi(rd), lo(rd), 31), ctx); + bpf_put_reg64(dst, rd, ctx); + break; + } + fallthrough; case BPF_ALU64 | BPF_ADD | BPF_X: case BPF_ALU64 | BPF_ADD | BPF_K: @@ -1017,6 +1084,20 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_zext64(dst, ctx); break; } + if (insn->off != 0) { + const s8 *rd = bpf_get_reg32(dst, tmp1, ctx); + const s8 *rs = bpf_get_reg32(src, tmp2, ctx); + + if (insn->off == 8) { + emit(rv_slli(lo(rd), lo(rs), 24), ctx); + emit(rv_srai(lo(rd), lo(rd), 24), ctx); + } else if (insn->off == 16) { + emit(rv_slli(lo(rd), lo(rs), 16), ctx); + emit(rv_srai(lo(rd), lo(rd), 16), ctx); + } + bpf_put_reg32(dst, rd, ctx); + break; + } fallthrough; case BPF_ALU | BPF_ADD | BPF_X: @@ -1041,7 +1122,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_imm32(tmp2, imm, ctx); src = tmp2; } - emit_alu_r32(dst, src, ctx, BPF_OP(code)); + emit_alu_r32(dst, src, ctx, insn); break; case BPF_ALU | BPF_MOV | BPF_K: @@ -1065,7 +1146,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, * src is ignored---choose tmp2 as a dummy register since it * is not on the stack. */ - emit_alu_r32(dst, tmp2, ctx, BPF_OP(code)); + emit_alu_r32(dst, tmp2, ctx, insn); break; case BPF_ALU | BPF_END | BPF_FROM_LE: @@ -1266,21 +1347,19 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, src = tmp2; } - if (emit_store_r64(dst, src, off, ctx, BPF_SIZE(code), - BPF_MODE(code))) + if (emit_store_r64(dst, src, ctx, insn)) return -1; break; case BPF_STX | BPF_ATOMIC | BPF_W: - if (insn->imm != BPF_ADD) { + if (insn->imm == BPF_CMPXCHG) { pr_info_once( "bpf-jit: not supported: atomic operation %02x ***\n", insn->imm); return -EFAULT; } - if (emit_store_r64(dst, src, off, ctx, BPF_SIZE(code), - BPF_MODE(code))) + if (emit_store_r64(dst, src, ctx, insn)) return -1; break; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 37888abee70c..151031e97a24 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -18,12 +18,12 @@ #define RV_MAX_REG_ARGS 8 #define RV_FENTRY_NINSNS 2 #define RV_FENTRY_NBYTES (RV_FENTRY_NINSNS * 4) -#define RV_KCFI_NINSNS (IS_ENABLED(CONFIG_CFI) ? 1 : 0) /* imm that allows emit_imm to emit max count insns */ #define RV_MAX_COUNT_IMM 0x7FFF7FF7FF7FF7FF +/* fentry and TCC init insns will be skipped on tailcall */ +#define RV_TAILCALL_OFFSET ((RV_FENTRY_NINSNS + 1) * 4) #define RV_REG_TCC RV_REG_A6 -#define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */ #define RV_REG_ARENA RV_REG_S7 /* For storing arena_vm_start */ static const int regmap[] = { @@ -57,14 +57,12 @@ static const int pt_regmap[] = { }; enum { - RV_CTX_F_SEEN_TAIL_CALL = 0, RV_CTX_F_SEEN_CALL = RV_REG_RA, RV_CTX_F_SEEN_S1 = RV_REG_S1, RV_CTX_F_SEEN_S2 = RV_REG_S2, RV_CTX_F_SEEN_S3 = RV_REG_S3, RV_CTX_F_SEEN_S4 = RV_REG_S4, RV_CTX_F_SEEN_S5 = RV_REG_S5, - RV_CTX_F_SEEN_S6 = RV_REG_S6, }; static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) @@ -77,7 +75,6 @@ static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) case RV_CTX_F_SEEN_S3: case RV_CTX_F_SEEN_S4: case RV_CTX_F_SEEN_S5: - case RV_CTX_F_SEEN_S6: __set_bit(reg, &ctx->flags); } return reg; @@ -92,7 +89,6 @@ static bool seen_reg(int reg, struct rv_jit_context *ctx) case RV_CTX_F_SEEN_S3: case RV_CTX_F_SEEN_S4: case RV_CTX_F_SEEN_S5: - case RV_CTX_F_SEEN_S6: return test_bit(reg, &ctx->flags); } return false; @@ -108,32 +104,6 @@ static void mark_call(struct rv_jit_context *ctx) __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); } -static bool seen_call(struct rv_jit_context *ctx) -{ - return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); -} - -static void mark_tail_call(struct rv_jit_context *ctx) -{ - __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); -} - -static bool seen_tail_call(struct rv_jit_context *ctx) -{ - return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); -} - -static u8 rv_tail_call_reg(struct rv_jit_context *ctx) -{ - mark_tail_call(ctx); - - if (seen_call(ctx)) { - __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); - return RV_REG_S6; - } - return RV_REG_A6; -} - static bool is_32b_int(s64 val) { return -(1L << 31) <= val && val < (1L << 31); @@ -258,23 +228,20 @@ static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx); store_offset -= 8; } - if (seen_reg(RV_REG_S6, ctx)) { - emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx); - store_offset -= 8; - } if (ctx->arena_vm_start) { emit_ld(RV_REG_ARENA, store_offset, RV_REG_SP, ctx); store_offset -= 8; } + /* restore TCC from stack to RV_REG_TCC */ + emit_ld(RV_REG_TCC, ctx->tcc_offset, RV_REG_SP, ctx); + emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); /* Set return value. */ if (!is_tail_call) emit_addiw(RV_REG_A0, RV_REG_A5, 0, ctx); emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, - /* kcfi, fentry and TCC init insns will be skipped on tailcall */ - is_tail_call ? (RV_KCFI_NINSNS + RV_FENTRY_NINSNS + 1) * 4 : 0, - ctx); + is_tail_call ? RV_TAILCALL_OFFSET : 0, ctx); } static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, @@ -355,7 +322,6 @@ static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) { int tc_ninsn, off, start_insn = ctx->ninsns; - u8 tcc = rv_tail_call_reg(ctx); /* a0: &ctx * a1: &array @@ -378,7 +344,8 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) /* if (--TCC < 0) * goto out; */ - emit_addi(RV_REG_TCC, tcc, -1, ctx); + emit_ld(RV_REG_TCC, ctx->tcc_offset, RV_REG_SP, ctx); + emit_addi(RV_REG_TCC, RV_REG_TCC, -1, ctx); off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_branch(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); @@ -394,7 +361,10 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); - /* goto *(prog->bpf_func + 4); */ + /* store updated TCC back to stack */ + emit_sd(RV_REG_SP, ctx->tcc_offset, RV_REG_TCC, ctx); + + /* goto *(prog->bpf_func + RV_TAILCALL_OFFSET); */ off = offsetof(struct bpf_prog, bpf_func); if (is_12b_check(off, insn)) return -1; @@ -926,19 +896,25 @@ static void restore_stack_args(int nr_stack_args, int args_off, int stk_arg_off, } } -static int invoke_bpf_prog(struct bpf_tramp_link *l, int args_off, int retval_off, +static void emit_store_stack_imm64(u8 reg, int stack_off, u64 imm64, + struct rv_jit_context *ctx) +{ + /* Load imm64 into reg and store it at [FP + stack_off]. */ + emit_imm(reg, (s64)imm64, ctx); + emit_sd(RV_REG_FP, stack_off, reg, ctx); +} + +static int invoke_bpf_prog(struct bpf_tramp_node *node, int args_off, int retval_off, int run_ctx_off, bool save_ret, struct rv_jit_context *ctx) { int ret, branch_off; - struct bpf_prog *p = l->link.prog; + struct bpf_prog *p = node->link->prog; int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); - if (l->cookie) { - emit_imm(RV_REG_T1, l->cookie, ctx); - emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_T1, ctx); - } else { + if (node->cookie) + emit_store_stack_imm64(RV_REG_T1, -run_ctx_off + cookie_off, node->cookie, ctx); + else emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_ZERO, ctx); - } /* arg1: prog */ emit_imm(RV_REG_A0, (const s64)p, ctx); @@ -990,22 +966,48 @@ static int invoke_bpf_prog(struct bpf_tramp_link *l, int args_off, int retval_of return ret; } +static int invoke_bpf(struct bpf_tramp_nodes *tn, int args_off, int retval_off, + int run_ctx_off, int func_meta_off, bool save_ret, u64 func_meta, + int cookie_off, struct rv_jit_context *ctx) +{ + int i, cur_cookie = (cookie_off - args_off) / 8; + + for (i = 0; i < tn->nr_nodes; i++) { + int err; + + if (bpf_prog_calls_session_cookie(tn->nodes[i])) { + u64 meta = func_meta | ((u64)cur_cookie << BPF_TRAMP_COOKIE_INDEX_SHIFT); + + emit_store_stack_imm64(RV_REG_T1, -func_meta_off, meta, ctx); + cur_cookie--; + } + err = invoke_bpf_prog(tn->nodes[i], args_off, retval_off, run_ctx_off, + save_ret, ctx); + if (err) + return err; + } + return 0; +} + static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, const struct btf_func_model *m, - struct bpf_tramp_links *tlinks, + struct bpf_tramp_nodes *tnodes, void *func_addr, u32 flags, struct rv_jit_context *ctx) { int i, ret, offset; int *branches_off = NULL; int stack_size = 0, nr_arg_slots = 0; - int retval_off, args_off, nregs_off, ip_off, run_ctx_off, sreg_off, stk_arg_off; - struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; - struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; - struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; - bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT; + int retval_off, args_off, func_meta_off, ip_off; + int run_ctx_off, sreg_off, stk_arg_off, tcc_off; + int cookie_off, cookie_cnt; + struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY]; + struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT]; + struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN]; + bool is_struct_ops = is_struct_ops_tramp(fentry); void *orig_call = func_addr; bool save_ret; + u64 func_meta; u32 insn; /* Two types of generated trampoline stack layout: @@ -1036,14 +1038,20 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, * [ ... ] * FP - args_off [ arg1 ] * - * FP - nregs_off [ regs count ] + * FP - func_meta_off [ regs count, etc ] * * FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG * + * [ stack cookie N ] + * [ ... ] + * FP - cookie_off [ stack cookie 1 ] + * * FP - run_ctx_off [ bpf_tramp_run_ctx ] * * FP - sreg_off [ callee saved reg ] * + * FP - tcc_off [ tail call count ] BPF_TRAMP_F_TAIL_CALL_CTX + * * [ pads ] pads for 16 bytes alignment * * [ stack_argN ] @@ -1071,20 +1079,31 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, stack_size += nr_arg_slots * 8; args_off = stack_size; + /* function metadata, such as regs count */ stack_size += 8; - nregs_off = stack_size; + func_meta_off = stack_size; if (flags & BPF_TRAMP_F_IP_ARG) { stack_size += 8; ip_off = stack_size; } + cookie_cnt = bpf_fsession_cookie_cnt(tnodes); + /* room for session cookies */ + stack_size += cookie_cnt * 8; + cookie_off = stack_size; + stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8); run_ctx_off = stack_size; stack_size += 8; sreg_off = stack_size; + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { + stack_size += 8; + tcc_off = stack_size; + } + if ((flags & BPF_TRAMP_F_CALL_ORIG) && (nr_arg_slots - RV_MAX_REG_ARGS > 0)) stack_size += (nr_arg_slots - RV_MAX_REG_ARGS) * 8; @@ -1119,20 +1138,30 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx); } + /* store tail call count */ + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + emit_sd(RV_REG_FP, -tcc_off, RV_REG_TCC, ctx); + /* callee saved register S1 to pass start time */ emit_sd(RV_REG_FP, -sreg_off, RV_REG_S1, ctx); /* store ip address of the traced function */ - if (flags & BPF_TRAMP_F_IP_ARG) { - emit_imm(RV_REG_T1, (const s64)func_addr, ctx); - emit_sd(RV_REG_FP, -ip_off, RV_REG_T1, ctx); - } + if (flags & BPF_TRAMP_F_IP_ARG) + emit_store_stack_imm64(RV_REG_T1, -ip_off, (u64)func_addr, ctx); - emit_li(RV_REG_T1, nr_arg_slots, ctx); - emit_sd(RV_REG_FP, -nregs_off, RV_REG_T1, ctx); + func_meta = nr_arg_slots; + emit_store_stack_imm64(RV_REG_T1, -func_meta_off, func_meta, ctx); store_args(nr_arg_slots, args_off, ctx); + if (bpf_fsession_cnt(tnodes)) { + /* clear all session cookies' value */ + for (i = 0; i < cookie_cnt; i++) + emit_sd(RV_REG_FP, -cookie_off + 8 * i, RV_REG_ZERO, ctx); + /* clear return value to make sure fentry always get 0 */ + emit_sd(RV_REG_FP, -retval_off, RV_REG_ZERO, ctx); + } + if (flags & BPF_TRAMP_F_CALL_ORIG) { emit_imm(RV_REG_A0, ctx->insns ? (const s64)im : RV_MAX_COUNT_IMM, ctx); ret = emit_call((const u64)__bpf_tramp_enter, true, ctx); @@ -1140,22 +1169,22 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, return ret; } - for (i = 0; i < fentry->nr_links; i++) { - ret = invoke_bpf_prog(fentry->links[i], args_off, retval_off, run_ctx_off, - flags & BPF_TRAMP_F_RET_FENTRY_RET, ctx); + if (fentry->nr_nodes) { + ret = invoke_bpf(fentry, args_off, retval_off, run_ctx_off, func_meta_off, + flags & BPF_TRAMP_F_RET_FENTRY_RET, func_meta, cookie_off, ctx); if (ret) return ret; } - if (fmod_ret->nr_links) { - branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL); + if (fmod_ret->nr_nodes) { + branches_off = kvzalloc_objs(int, fmod_ret->nr_nodes); if (!branches_off) return -ENOMEM; /* cleanup to avoid garbage return value confusion */ emit_sd(RV_REG_FP, -retval_off, RV_REG_ZERO, ctx); - for (i = 0; i < fmod_ret->nr_links; i++) { - ret = invoke_bpf_prog(fmod_ret->links[i], args_off, retval_off, + for (i = 0; i < fmod_ret->nr_nodes; i++) { + ret = invoke_bpf_prog(fmod_ret->nodes[i], args_off, retval_off, run_ctx_off, true, ctx); if (ret) goto out; @@ -1171,9 +1200,15 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, orig_call += RV_FENTRY_NINSNS * 4; restore_args(min_t(int, nr_arg_slots, RV_MAX_REG_ARGS), args_off, ctx); restore_stack_args(nr_arg_slots - RV_MAX_REG_ARGS, args_off, stk_arg_off, ctx); + /* restore TCC to RV_REG_TCC before calling the orig bpf func */ + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + emit_ld(RV_REG_TCC, -tcc_off, RV_REG_FP, ctx); ret = emit_call((const u64)orig_call, true, ctx); if (ret) goto out; + /* store updated TCC back to stack after calling the orig bpf func */ + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + emit_sd(RV_REG_FP, -tcc_off, RV_REG_TCC, ctx); emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); im->ip_after_call = ctx->ro_insns + ctx->ninsns; @@ -1183,15 +1218,20 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, } /* update branches saved in invoke_bpf_mod_ret with bnez */ - for (i = 0; ctx->insns && i < fmod_ret->nr_links; i++) { + for (i = 0; ctx->insns && i < fmod_ret->nr_nodes; i++) { offset = ninsns_rvoff(ctx->ninsns - branches_off[i]); insn = rv_bne(RV_REG_T1, RV_REG_ZERO, offset >> 1); *(u32 *)(ctx->insns + branches_off[i]) = insn; } - for (i = 0; i < fexit->nr_links; i++) { - ret = invoke_bpf_prog(fexit->links[i], args_off, retval_off, - run_ctx_off, false, ctx); + /* set "is_return" flag for fsession */ + func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT); + if (bpf_fsession_cnt(tnodes)) + emit_store_stack_imm64(RV_REG_T1, -func_meta_off, func_meta, ctx); + + if (fexit->nr_nodes) { + ret = invoke_bpf(fexit, args_off, retval_off, run_ctx_off, func_meta_off, + false, func_meta, cookie_off, ctx); if (ret) goto out; } @@ -1221,6 +1261,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_ld(RV_REG_S1, -sreg_off, RV_REG_FP, ctx); + /* restore TCC from stack to RV_REG_TCC */ + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + emit_ld(RV_REG_TCC, -tcc_off, RV_REG_FP, ctx); + if (!is_struct_ops) { /* trampoline called from function entry */ emit_ld(RV_REG_T0, stack_size - 8, RV_REG_SP, ctx); @@ -1248,12 +1292,12 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, ret = ctx->ninsns; out: - kfree(branches_off); + kvfree(branches_off); return ret; } int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, - struct bpf_tramp_links *tlinks, void *func_addr) + struct bpf_tramp_nodes *tnodes, void *func_addr) { struct bpf_tramp_image im; struct rv_jit_context ctx; @@ -1262,14 +1306,14 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, ctx.ninsns = 0; ctx.insns = NULL; ctx.ro_insns = NULL; - ret = __arch_prepare_bpf_trampoline(&im, m, tlinks, func_addr, flags, &ctx); + ret = __arch_prepare_bpf_trampoline(&im, m, tnodes, func_addr, flags, &ctx); return ret < 0 ? ret : ninsns_rvoff(ctx.ninsns); } void *arch_alloc_bpf_trampoline(unsigned int size) { - return bpf_prog_pack_alloc(size, bpf_fill_ill_insns); + return bpf_prog_pack_alloc(size, bpf_fill_ill_insns, false); } void arch_free_bpf_trampoline(void *image, unsigned int size) @@ -1279,7 +1323,7 @@ void arch_free_bpf_trampoline(void *image, unsigned int size) int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image, void *ro_image_end, const struct btf_func_model *m, - u32 flags, struct bpf_tramp_links *tlinks, + u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr) { int ret; @@ -1294,7 +1338,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image, ctx.ninsns = 0; ctx.insns = image; ctx.ro_insns = ro_image; - ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx); + ret = __arch_prepare_bpf_trampoline(im, m, tnodes, func_addr, flags, &ctx); if (ret < 0) goto out; @@ -1756,6 +1800,13 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, break; } + /* Implement helper call to bpf_get_current_task/_btf() inline */ + if (insn->src_reg == 0 && (insn->imm == BPF_FUNC_get_current_task || + insn->imm == BPF_FUNC_get_current_task_btf)) { + emit_mv(bpf_to_rv_reg(BPF_REG_0, ctx), RV_REG_TP, ctx); + break; + } + mark_call(ctx); ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr, &fixed_addr); @@ -1772,17 +1823,31 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, for (idx = 0; idx < fm->nr_args; idx++) { u8 reg = bpf_to_rv_reg(BPF_REG_1 + idx, ctx); + bool sign = fm->arg_flags[idx] & BTF_FMODEL_SIGNED_ARG; - if (fm->arg_size[idx] == sizeof(int)) - emit_sextw(reg, reg, ctx); + if (sign_extend(reg, reg, fm->arg_size[idx], sign, ctx)) + return -EINVAL; } } + /* restore TCC to RV_REG_TCC before bpf2bpf call */ + if (aux->tail_call_reachable && insn->src_reg == BPF_PSEUDO_CALL) + emit_ld(RV_REG_TCC, ctx->tcc_offset, RV_REG_SP, ctx); + ret = emit_call(addr, fixed_addr, ctx); if (ret) return ret; - if (insn->src_reg != BPF_PSEUDO_CALL) + /* store updated TCC back to stack after bpf2bpf call */ + if (aux->tail_call_reachable && insn->src_reg == BPF_PSEUDO_CALL) + emit_sd(RV_REG_SP, ctx->tcc_offset, RV_REG_TCC, ctx); + + /* + * arch_bpf_timed_may_goto() is emitted by the verifier and + * returns its result in BPF_REG_AX instead of BPF_REG_0, so + * skip the normal "move return register into R0". + */ + if (insn->src_reg != BPF_PSEUDO_CALL && addr != (u64)arch_bpf_timed_may_goto) emit_mv(bpf_to_rv_reg(BPF_REG_0, ctx), RV_REG_A0, ctx); break; } @@ -1927,7 +1992,21 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, else ret = emit_atomic_rmw(rd, rs, insn, ctx); - ret = ret ?: add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx); + /* ret can be 1 (skip-zext); extable entry still needs to be added */ + if (ret >= 0) { + /* + * A load-acquire reads into dst_reg, and a read-modify-write + * carrying BPF_FETCH reads the old value into src_reg, or into + * r0 for a BPF_CMPXCHG. Clear that register on fault, the + * remaining atomics have no destination register. + */ + int load_reg = bpf_atomic_load_reg(insn); + + ret = add_exception_handler(insn, load_reg < 0 ? + REG_DONT_CLEAR_MARKER : regmap[load_reg], + ctx) ?: ret; + } + if (ret) return ret; break; @@ -1961,10 +2040,9 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) stack_adjust += 8; if (seen_reg(RV_REG_S5, ctx)) stack_adjust += 8; - if (seen_reg(RV_REG_S6, ctx)) - stack_adjust += 8; if (ctx->arena_vm_start) stack_adjust += 8; + stack_adjust += 8; /* RV_REG_TCC */ stack_adjust = round_up(stack_adjust, STACK_ALIGN); stack_adjust += bpf_stack_adjust; @@ -1974,15 +2052,16 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) /* emit kcfi type preamble immediately before the first insn */ emit_kcfi(is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash, ctx); + /* bpf prog starts here as kcfi skipped during prog->bpf_func setup */ + /* nops reserved for auipc+jalr pair */ for (i = 0; i < RV_FENTRY_NINSNS; i++) emit(rv_nop(), ctx); - /* First instruction is always setting the tail-call-counter - * (TCC) register. This instruction is skipped for tail calls. - * Force using a 4-byte (non-compressed) instruction. - */ - emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); + if (!is_subprog) + emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); + + /* tailcall starts here, emit insn before it must be fixed */ emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); @@ -2012,26 +2091,20 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx); store_offset -= 8; } - if (seen_reg(RV_REG_S6, ctx)) { - emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx); - store_offset -= 8; - } if (ctx->arena_vm_start) { emit_sd(RV_REG_SP, store_offset, RV_REG_ARENA, ctx); store_offset -= 8; } + /* store TCC from RV_REG_TCC to stack */ + emit_sd(RV_REG_SP, store_offset, RV_REG_TCC, ctx); + ctx->tcc_offset = store_offset; + emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx); if (bpf_stack_adjust) emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx); - /* Program contains calls and tail calls, so RV_REG_TCC need - * to be saved across calls. - */ - if (seen_tail_call(ctx) && seen_call(ctx)) - emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx); - ctx->stack_size = stack_adjust; if (ctx->arena_vm_start) @@ -2055,7 +2128,15 @@ bool bpf_jit_supports_ptr_xchg(void) bool bpf_jit_supports_arena(void) { - return true; + /* + * The arena range tree uses kmalloc_nolock(), which needs + * cmpxchg128, provided by ZACAS on riscv. + */ +#ifdef system_has_cmpxchg128 + return system_has_cmpxchg128(); +#else + return false; +#endif } bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) @@ -2086,8 +2167,25 @@ bool bpf_jit_inlines_helper_call(s32 imm) { switch (imm) { case BPF_FUNC_get_smp_processor_id: + case BPF_FUNC_get_current_task: + case BPF_FUNC_get_current_task_btf: return true; default: return false; } } + +bool bpf_jit_supports_fsession(void) +{ + return true; +} + +bool bpf_jit_supports_subprog_tailcalls(void) +{ + return true; +} + +bool bpf_jit_supports_timed_may_goto(void) +{ + return true; +} diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index f6ca5cfa6b2f..cbfcd287ea16 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -41,32 +41,22 @@ bool bpf_jit_needs_zext(void) return true; } -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) { unsigned int prog_size = 0, extable_size = 0; - bool tmp_blinded = false, extra_pass = false; - struct bpf_prog *tmp, *orig_prog = prog; + bool extra_pass = false; int pass = 0, prev_ninsns = 0, i; struct rv_jit_data *jit_data; struct rv_jit_context *ctx; if (!prog->jit_requested) - return orig_prog; - - tmp = bpf_jit_blind_constants(prog); - if (IS_ERR(tmp)) - return orig_prog; - if (tmp != prog) { - tmp_blinded = true; - prog = tmp; - } + return prog; jit_data = prog->aux->jit_data; if (!jit_data) { - jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); + jit_data = kzalloc_obj(*jit_data); if (!jit_data) { - prog = orig_prog; - goto out; + return prog; } prog->aux->jit_data = jit_data; } @@ -82,16 +72,12 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); ctx->prog = prog; - ctx->offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL); - if (!ctx->offset) { - prog = orig_prog; + ctx->offset = kvzalloc_objs(int, prog->len); + if (!ctx->offset) goto out_offset; - } - if (build_body(ctx, extra_pass, NULL)) { - prog = orig_prog; + if (build_body(ctx, extra_pass, NULL)) goto out_offset; - } for (i = 0; i < prog->len; i++) { prev_ninsns += 32; @@ -105,10 +91,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) bpf_jit_build_prologue(ctx, bpf_is_subprog(prog)); ctx->prologue_len = ctx->ninsns; - if (build_body(ctx, extra_pass, ctx->offset)) { - prog = orig_prog; + if (build_body(ctx, extra_pass, ctx->offset)) goto out_offset; - } ctx->epilogue_offset = ctx->ninsns; bpf_jit_build_epilogue(ctx); @@ -125,11 +109,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) bpf_jit_binary_pack_alloc(prog_size + extable_size, &jit_data->ro_image, sizeof(u32), &jit_data->header, &jit_data->image, - bpf_fill_ill_insns); - if (!jit_data->ro_header) { - prog = orig_prog; + bpf_fill_ill_insns, + bpf_prog_was_classic(prog)); + if (!jit_data->ro_header) goto out_offset; - } /* * Use the image(RW) for writing the JITed instructions. But also save @@ -150,7 +133,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) if (i == NR_JIT_ITERATIONS) { pr_err("bpf-jit: image did not converge in <%d passes!\n", i); - prog = orig_prog; goto out_free_hdr; } @@ -163,49 +145,44 @@ skip_init_ctx: ctx->nexentries = 0; bpf_jit_build_prologue(ctx, bpf_is_subprog(prog)); - if (build_body(ctx, extra_pass, NULL)) { - prog = orig_prog; + if (build_body(ctx, extra_pass, NULL)) goto out_free_hdr; - } bpf_jit_build_epilogue(ctx); if (bpf_jit_enable > 1) bpf_jit_dump(prog->len, prog_size, pass, ctx->insns); - prog->bpf_func = (void *)ctx->ro_insns + cfi_get_offset(); - prog->jited = 1; - prog->jited_len = prog_size - cfi_get_offset(); - if (!prog->is_func || extra_pass) { if (WARN_ON(bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header))) { /* ro_header has been freed */ jit_data->ro_header = NULL; - prog = orig_prog; - goto out_offset; + jit_data->header = NULL; + goto out_free_hdr; } - /* - * The instructions have now been copied to the ROX region from - * where they will execute. - * Write any modified data cache blocks out to memory and - * invalidate the corresponding blocks in the instruction cache. - */ - bpf_flush_icache(jit_data->ro_header, ctx->ro_insns + ctx->ninsns); + } + + prog->bpf_func = (void *)ctx->ro_insns + cfi_get_offset(); + prog->jited = 1; + prog->jited_len = prog_size - cfi_get_offset(); + + if (!prog->is_func || extra_pass) { for (i = 0; i < prog->len; i++) ctx->offset[i] = ninsns_rvoff(ctx->offset[i]); bpf_prog_fill_jited_linfo(prog, ctx->offset); out_offset: - kfree(ctx->offset); + kvfree(ctx->offset); kfree(jit_data); prog->aux->jit_data = NULL; } -out: - if (tmp_blinded) - bpf_jit_prog_release_other(prog, prog == orig_prog ? - tmp : orig_prog); return prog; out_free_hdr: + if (extra_pass) { + prog->bpf_func = NULL; + prog->jited = 0; + prog->jited_len = 0; + } if (jit_data->header) { bpf_arch_text_copy(&jit_data->ro_header->size, &jit_data->header->size, sizeof(jit_data->header->size)); @@ -257,6 +234,7 @@ void bpf_jit_free(struct bpf_prog *prog) */ if (jit_data) { bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header); + kvfree(jit_data->ctx.offset); kfree(jit_data); } hdr = bpf_jit_binary_pack_hdr(prog); diff --git a/arch/riscv/net/bpf_timed_may_goto.S b/arch/riscv/net/bpf_timed_may_goto.S new file mode 100644 index 000000000000..02c637d87420 --- /dev/null +++ b/arch/riscv/net/bpf_timed_may_goto.S @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 Feng Jiang <jiangfeng@kylinos.cn> */ + +#include <linux/linkage.h> +#include <asm/asm.h> + +/* + * Trampoline for the BPF timed may_goto loop bound. Custom calling convention: + * - input: stack offset in BPF_REG_AX (t0) + * - output: updated count in BPF_REG_AX (t0) + * + * Calls bpf_check_timed_may_goto(ptr) with the standard RISC-V ABI, where + * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (a5, a0-a4) are saved + * across the call; BPF_REG_FP (s5) is callee-saved and needs no saving. + */ + +SYM_FUNC_START(arch_bpf_timed_may_goto) + addi sp, sp, -(8*SZREG) + REG_S ra, 7*SZREG(sp) + REG_S s0, 6*SZREG(sp) + addi s0, sp, 8*SZREG + + /* Save BPF registers R0-R5 (a5, a0-a4) */ + REG_S a5, 5*SZREG(sp) + REG_S a0, 4*SZREG(sp) + REG_S a1, 3*SZREG(sp) + REG_S a2, 2*SZREG(sp) + REG_S a3, 1*SZREG(sp) + REG_S a4, 0*SZREG(sp) + + add a0, t0, s5 + call bpf_check_timed_may_goto + mv t0, a0 + + /* Restore BPF registers R0-R5 */ + REG_L a4, 0*SZREG(sp) + REG_L a3, 1*SZREG(sp) + REG_L a2, 2*SZREG(sp) + REG_L a1, 3*SZREG(sp) + REG_L a0, 4*SZREG(sp) + REG_L a5, 5*SZREG(sp) + + REG_L s0, 6*SZREG(sp) + REG_L ra, 7*SZREG(sp) + addi sp, sp, 8*SZREG + ret +SYM_FUNC_END(arch_bpf_timed_may_goto) diff --git a/arch/riscv/purgatory/Makefile b/arch/riscv/purgatory/Makefile index 530e497ca2f9..b0358a78f11a 100644 --- a/arch/riscv/purgatory/Makefile +++ b/arch/riscv/purgatory/Makefile @@ -2,7 +2,7 @@ purgatory-y := purgatory.o sha256.o entry.o string.o ctype.o memcpy.o memset.o ifeq ($(CONFIG_KASAN_GENERIC)$(CONFIG_KASAN_SW_TAGS),) -purgatory-y += strcmp.o strlen.o strncmp.o +purgatory-y += strcmp.o strlen.o strncmp.o strnlen.o strchr.o strrchr.o endif targets += $(purgatory-y) @@ -32,6 +32,15 @@ $(obj)/strncmp.o: $(srctree)/arch/riscv/lib/strncmp.S FORCE $(obj)/sha256.o: $(srctree)/lib/crypto/sha256.c FORCE $(call if_changed_rule,cc_o_c) +$(obj)/strnlen.o: $(srctree)/arch/riscv/lib/strnlen.S FORCE + $(call if_changed_rule,as_o_S) + +$(obj)/strchr.o: $(srctree)/arch/riscv/lib/strchr.S FORCE + $(call if_changed_rule,as_o_S) + +$(obj)/strrchr.o: $(srctree)/arch/riscv/lib/strrchr.S FORCE + $(call if_changed_rule,as_o_S) + CFLAGS_sha256.o := -D__DISABLE_EXPORTS -D__NO_FORTIFY CFLAGS_string.o := -D__DISABLE_EXPORTS CFLAGS_ctype.o := -D__DISABLE_EXPORTS diff --git a/arch/riscv/purgatory/kexec-purgatory.S b/arch/riscv/purgatory/kexec-purgatory.S index 0e9188815718..32c53581b8f2 100644 --- a/arch/riscv/purgatory/kexec-purgatory.S +++ b/arch/riscv/purgatory/kexec-purgatory.S @@ -6,9 +6,9 @@ kexec_purgatory: .globl kexec_purgatory .incbin "arch/riscv/purgatory/purgatory.ro" -.Lkexec_purgatroy_end: +.Lkexec_purgatory_end: .align 8 kexec_purgatory_size: .globl kexec_purgatory_size - .quad .Lkexec_purgatroy_end - kexec_purgatory + .quad .Lkexec_purgatory_end - kexec_purgatory diff --git a/arch/riscv/purgatory/purgatory.c b/arch/riscv/purgatory/purgatory.c index bbd5cfa4d741..3474e3abe5d7 100644 --- a/arch/riscv/purgatory/purgatory.c +++ b/arch/riscv/purgatory/purgatory.c @@ -8,16 +8,16 @@ * */ -#include <linux/purgatory.h> #include <linux/kernel.h> #include <linux/string.h> +#include <asm/purgatory.h> #include <asm/string.h> u8 purgatory_sha256_digest[SHA256_DIGEST_SIZE] __section(".kexec-purgatory"); struct kexec_sha_region purgatory_sha_regions[KEXEC_SEGMENT_MAX] __section(".kexec-purgatory"); -static int verify_sha256_digest(void) +static bool verify_sha256_digest(void) { struct kexec_sha_region *ptr, *end; struct sha256_ctx sctx; @@ -26,19 +26,15 @@ static int verify_sha256_digest(void) sha256_init(&sctx); end = purgatory_sha_regions + ARRAY_SIZE(purgatory_sha_regions); for (ptr = purgatory_sha_regions; ptr < end; ptr++) - sha256_update(&sctx, (uint8_t *)(ptr->start), ptr->len); + sha256_update(&sctx, (const u8 *)(ptr->start), ptr->len); sha256_final(&sctx, digest); - if (memcmp(digest, purgatory_sha256_digest, sizeof(digest)) != 0) - return 1; - return 0; -} -/* workaround for a warning with -Wmissing-prototypes */ -void purgatory(void); + return memcmp(digest, purgatory_sha256_digest, sizeof(digest)) == 0; +} void purgatory(void) { - if (verify_sha256_digest()) + if (!verify_sha256_digest()) for (;;) /* loop forever */ ; |