summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/riscv/Kconfig1
-rw-r--r--arch/riscv/include/asm/alternative.h2
-rw-r--r--arch/riscv/include/asm/cpufeature.h2
-rw-r--r--arch/riscv/include/asm/hwcap.h2
-rw-r--r--arch/riscv/include/asm/pgtable-64.h12
-rw-r--r--arch/riscv/kernel/alternative.c24
-rw-r--r--arch/riscv/kernel/cpufeature.c62
-rw-r--r--arch/riscv/mm/init.c9
8 files changed, 98 insertions, 16 deletions
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 6d4b8dae6f20..67e72bf4f91a 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -218,6 +218,7 @@ config RISCV
select PCI_ECAM if (ACPI && PCI)
select PCI_MSI if PCI
select RELOCATABLE if !MMU && !PHYS_RAM_BASE_FIXED
+ select RISCV_ALTERNATIVE_EARLY if 64BIT
select RISCV_APLIC
select RISCV_IMSIC
select RISCV_INTC
diff --git a/arch/riscv/include/asm/alternative.h b/arch/riscv/include/asm/alternative.h
index 688c7d1a9ae3..6be7b2b6ade9 100644
--- a/arch/riscv/include/asm/alternative.h
+++ b/arch/riscv/include/asm/alternative.h
@@ -33,7 +33,7 @@ void __init apply_early_boot_alternatives(void);
void apply_module_alternatives(void *start, size_t length);
void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len,
- int patch_offset);
+ int patch_offset, bool early);
struct alt_entry {
s32 old_offset; /* offset relative to original instruction or data */
diff --git a/arch/riscv/include/asm/cpufeature.h b/arch/riscv/include/asm/cpufeature.h
index 25c5da5120b5..99799cc6c198 100644
--- a/arch/riscv/include/asm/cpufeature.h
+++ b/arch/riscv/include/asm/cpufeature.h
@@ -36,6 +36,8 @@ extern const struct seq_operations cpuinfo_op;
/* Per-cpu ISA extensions. */
extern struct riscv_isainfo hart_isa[NR_CPUS];
+extern DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX);
+
extern u32 thead_vlenb_of;
void __init riscv_user_isa_enable(void);
diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h
index b103aaeafa47..b5952db9259f 100644
--- a/arch/riscv/include/asm/hwcap.h
+++ b/arch/riscv/include/asm/hwcap.h
@@ -124,6 +124,8 @@
#define RISCV_ISA_EXT_ZICCIF 114
#define RISCV_ISA_EXT_ZA64RS 115
#define RISCV_ISA_EXT_ZIC64B 116
+#define RISCV_ISA_EXT_SV48 117
+#define RISCV_ISA_EXT_SV57 118
#define RISCV_ISA_EXT_XLINUXENVCFG 127
diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h
index 72b8c63469fa..7407e36fbf7d 100644
--- a/arch/riscv/include/asm/pgtable-64.h
+++ b/arch/riscv/include/asm/pgtable-64.h
@@ -13,6 +13,7 @@
extern bool _pgtable_l4_enabled;
extern bool _pgtable_l5_enabled;
+#ifdef USE_EARLY_PGTABLE_LEVELS
static __always_inline bool pgtable_l5_enabled(void)
{
return _pgtable_l5_enabled;
@@ -22,6 +23,17 @@ static __always_inline bool pgtable_l4_enabled(void)
{
return _pgtable_l4_enabled;
}
+#else
+static __always_inline bool pgtable_l5_enabled(void)
+{
+ return riscv_has_extension_likely(RISCV_ISA_EXT_SV57);
+}
+
+static __always_inline bool pgtable_l4_enabled(void)
+{
+ return riscv_has_extension_likely(RISCV_ISA_EXT_SV48);
+}
+#endif
#define PGDIR_SHIFT_L3 30
#define PGDIR_SHIFT_L4 39
diff --git a/arch/riscv/kernel/alternative.c b/arch/riscv/kernel/alternative.c
index c0c9306022c5..bbb215349452 100644
--- a/arch/riscv/kernel/alternative.c
+++ b/arch/riscv/kernel/alternative.c
@@ -75,7 +75,8 @@ static u32 riscv_instruction_at(void *p)
}
static void riscv_alternative_fix_auipc_jalr(void *ptr, u32 auipc_insn,
- u32 jalr_insn, int patch_offset)
+ u32 jalr_insn, int patch_offset,
+ bool early)
{
u32 call[2] = { auipc_insn, jalr_insn };
s32 imm;
@@ -88,10 +89,15 @@ static void riscv_alternative_fix_auipc_jalr(void *ptr, u32 auipc_insn,
riscv_insn_insert_utype_itype_imm(&call[0], &call[1], imm);
/* patch the call place again */
- patch_text_nosync(ptr, call, sizeof(u32) * 2);
+ if (early) {
+ memcpy(ptr, call, sizeof(call));
+ } else {
+ patch_text_nosync(ptr, call, sizeof(call));
+ }
}
-static void riscv_alternative_fix_jal(void *ptr, u32 jal_insn, int patch_offset)
+static void riscv_alternative_fix_jal(void *ptr, u32 jal_insn, int patch_offset,
+ bool early)
{
s32 imm;
@@ -103,11 +109,14 @@ static void riscv_alternative_fix_jal(void *ptr, u32 jal_insn, int patch_offset)
riscv_insn_insert_jtype_imm(&jal_insn, imm);
/* patch the call place again */
- patch_text_nosync(ptr, &jal_insn, sizeof(u32));
+ if (early)
+ memcpy(ptr, &jal_insn, sizeof(u32));
+ else
+ patch_text_nosync(ptr, &jal_insn, sizeof(u32));
}
void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len,
- int patch_offset)
+ int patch_offset, bool early)
{
int num_insn = len / sizeof(u32);
int i;
@@ -131,7 +140,8 @@ void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len,
continue;
riscv_alternative_fix_auipc_jalr(alt_ptr + i * sizeof(u32),
- insn, insn2, patch_offset);
+ insn, insn2, patch_offset,
+ early);
i++;
}
@@ -144,7 +154,7 @@ void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len,
continue;
riscv_alternative_fix_jal(alt_ptr + i * sizeof(u32),
- insn, patch_offset);
+ insn, patch_offset, early);
}
}
}
diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c
index 263597b08968..72bce616491f 100644
--- a/arch/riscv/kernel/cpufeature.c
+++ b/arch/riscv/kernel/cpufeature.c
@@ -39,7 +39,7 @@ static bool any_cpu_has_zicbom;
unsigned long elf_hwcap __read_mostly;
/* Host ISA bitmap */
-static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly;
+DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly;
/* Per-cpu ISA extensions. */
struct riscv_isainfo hart_isa[NR_CPUS];
@@ -666,6 +666,8 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = {
__RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT),
__RISCV_ISA_EXT_DATA(svrsw60t59b, RISCV_ISA_EXT_SVRSW60T59B),
__RISCV_ISA_EXT_DATA(svvptc, RISCV_ISA_EXT_SVVPTC),
+ __RISCV_ISA_EXT_DATA(sv48, RISCV_ISA_EXT_SV48),
+ __RISCV_ISA_EXT_DATA(sv57, RISCV_ISA_EXT_SV57),
};
const size_t riscv_isa_ext_count = ARRAY_SIZE(riscv_isa_ext);
@@ -964,6 +966,13 @@ static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap)
set_bit(RISCV_ISA_EXT_ZIHPM, source_isa);
}
+#ifndef CONFIG_32BIT
+ if (_pgtable_l4_enabled)
+ set_bit(RISCV_ISA_EXT_SV48, source_isa);
+ if (_pgtable_l5_enabled)
+ set_bit(RISCV_ISA_EXT_SV57, source_isa);
+#endif
+
/*
* "V" in ISA strings is ambiguous in practice: it should mean
* just the standard V-1.0 but vendors aren't well behaved.
@@ -1124,6 +1133,13 @@ static int __init riscv_fill_hwcap_from_ext_list(unsigned long *isa2hwcap)
riscv_isa_set_ext(ext, source_isa);
}
+#ifndef CONFIG_32BIT
+ if (_pgtable_l4_enabled)
+ set_bit(RISCV_ISA_EXT_SV48, source_isa);
+ if (_pgtable_l5_enabled)
+ set_bit(RISCV_ISA_EXT_SV57, source_isa);
+#endif
+
riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap);
riscv_fill_cpu_vendor_ext(cpu_node, cpu);
@@ -1190,6 +1206,12 @@ void __init riscv_fill_hwcap(void)
isa2hwcap[RISCV_ISA_EXT_B] = COMPAT_HWCAP_ISA_B;
isa2hwcap[RISCV_ISA_EXT_V] = COMPAT_HWCAP_ISA_V;
+ /*
+ * riscv_isa may have been touched by "early" ISA parsing, so zero it
+ * here to get a fresh slate.
+ */
+ bitmap_zero(riscv_isa, RISCV_ISA_EXT_MAX);
+
if (!acpi_disabled) {
riscv_fill_hwcap_from_isa_string(isa2hwcap);
} else {
@@ -1305,6 +1327,17 @@ static bool riscv_cpufeature_patch_check(u16 id, u16 value)
return false;
}
+static bool __init_or_module riscv_is_isa_ext_early_id(u16 id)
+{
+ switch (id) {
+ case RISCV_ISA_EXT_SV48:
+ case RISCV_ISA_EXT_SV57:
+ return true;
+ }
+
+ return false;
+}
+
void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin,
struct alt_entry *end,
unsigned int stage)
@@ -1312,9 +1345,7 @@ void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin,
struct alt_entry *alt;
void *oldptr, *altptr;
u16 id, value, vendor;
-
- if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
- return;
+ bool early = stage == RISCV_ALTERNATIVES_EARLY_BOOT;
for (alt = begin; alt < end; alt++) {
id = PATCH_ID_CPUFEATURE_ID(alt->patch_id);
@@ -1329,6 +1360,8 @@ void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin,
* vendor extension.
*/
if (id < RISCV_ISA_EXT_MAX) {
+ if (early && !riscv_is_isa_ext_early_id(id))
+ continue;
/*
* This patch should be treated as errata so skip
* processing here.
@@ -1343,6 +1376,8 @@ void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin,
if (!riscv_cpufeature_patch_check(id, value))
continue;
} else if (id >= RISCV_VENDOR_EXT_ALTERNATIVES_BASE) {
+ if (early)
+ continue;
if (!__riscv_isa_vendor_extension_available(VENDOR_EXT_ALL_CPUS, vendor,
id - RISCV_VENDOR_EXT_ALTERNATIVES_BASE))
continue;
@@ -1354,9 +1389,20 @@ void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin,
oldptr = ALT_OLD_PTR(alt);
altptr = ALT_ALT_PTR(alt);
- mutex_lock(&text_mutex);
- patch_text_nosync(oldptr, altptr, alt->alt_len);
- riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr);
- mutex_unlock(&text_mutex);
+ if (early) {
+ /* oldptr is writable through the MMU-off kernel mapping. */
+ memcpy(oldptr, altptr, alt->alt_len);
+ riscv_alternative_fix_offsets(oldptr, alt->alt_len,
+ oldptr - altptr, true);
+ } else {
+ mutex_lock(&text_mutex);
+ patch_text_nosync(oldptr, altptr, alt->alt_len);
+ riscv_alternative_fix_offsets(oldptr, alt->alt_len,
+ oldptr - altptr, false);
+ mutex_unlock(&text_mutex);
+ }
}
+
+ if (early)
+ local_flush_icache_all();
}
diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
index 8e8352d80a90..19e8ebf95c0c 100644
--- a/arch/riscv/mm/init.c
+++ b/arch/riscv/mm/init.c
@@ -6,6 +6,11 @@
* Nick Kossifidis <mick@ics.forth.gr>
*/
+#ifdef CONFIG_64BIT
+/* riscv_has_extension_likely() cannot be used this early */
+#define USE_EARLY_PGTABLE_LEVELS
+#endif
+
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/memblock.h>
@@ -892,6 +897,10 @@ out:
memset(early_p4d, 0, PAGE_SIZE);
memset(early_pud, 0, PAGE_SIZE);
memset(early_pmd, 0, PAGE_SIZE);
+ if (pgtable_l4_enabled())
+ set_bit(RISCV_ISA_EXT_SV48, riscv_isa);
+ if (pgtable_l5_enabled())
+ set_bit(RISCV_ISA_EXT_SV57, riscv_isa);
}
#endif
generated by cgit v1.2.3 (git 2.0.4) at 2026-11-05 21:57:32 +0000