summaryrefslogtreecommitdiff
path: root/arch/x86/kernel/alternative.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/kernel/alternative.c')
-rw-r--r--arch/x86/kernel/alternative.c117
1 files changed, 78 insertions, 39 deletions
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index 91b1cdd16569..582c6d8307d1 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -6,6 +6,9 @@
#include <linux/vmalloc.h>
#include <linux/memory.h>
#include <linux/execmem.h>
+#include <linux/cleanup.h>
+#include <linux/kgdb.h>
+#include <linux/mmap_lock.h>
#include <asm/text-patching.h>
#include <asm/insn.h>
@@ -1198,6 +1201,41 @@ static bool cfi_debug __ro_after_init;
bool cfi_bhi __ro_after_init = false;
#endif
+#ifdef CONFIG_FINEIBT
+/*
+ * <fineibt_preamble_start>:
+ * 0: f3 0f 1e fa endbr64
+ * 4: 2d 78 56 34 12 sub $0x12345678, %eax
+ * 9: 2e 0f 85 03 00 00 00 jne,pn 13 <fineibt_preamble_start+0x13>
+ * 10: 0f 1f 40 d6 nopl -0x2a(%rax)
+ *
+ * Note that the JNE target is the 0xD6 byte inside the NOPL, this decodes as
+ * UDB on x86_64 and raises #UD.
+ */
+asm( ".pushsection .rodata \n"
+ "fineibt_preamble_start: \n"
+ " endbr64 \n"
+ " subl $0x12345678, %eax \n"
+ "fineibt_preamble_bhi: \n"
+ " cs jne.d32 fineibt_preamble_start+0x13 \n"
+ "#fineibt_func: \n"
+ " nopl -42(%rax) \n"
+ "fineibt_preamble_end: \n"
+ ".popsection\n"
+);
+
+extern u8 fineibt_preamble_start[];
+extern u8 fineibt_preamble_bhi[];
+extern u8 fineibt_preamble_end[];
+
+#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start)
+#define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_start)
+#define fineibt_preamble_ud 0x13
+#define fineibt_preamble_hash 5
+
+#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE)
+#endif /* CONFIG_FINEIBT */
+
#ifdef CONFIG_CFI
u32 cfi_get_func_hash(void *func)
{
@@ -1205,9 +1243,11 @@ u32 cfi_get_func_hash(void *func)
func -= cfi_get_offset();
switch (cfi_mode) {
+#ifdef CONFIG_FINEIBT
case CFI_FINEIBT:
- func += 7;
+ func += fineibt_preamble_hash;
break;
+#endif
case CFI_KCFI:
func += 1;
break;
@@ -1364,39 +1404,6 @@ early_param("cfi", cfi_parse_cmdline);
*/
/*
- * <fineibt_preamble_start>:
- * 0: f3 0f 1e fa endbr64
- * 4: 2d 78 56 34 12 sub $0x12345678, %eax
- * 9: 2e 0f 85 03 00 00 00 jne,pn 13 <fineibt_preamble_start+0x13>
- * 10: 0f 1f 40 d6 nopl -0x2a(%rax)
- *
- * Note that the JNE target is the 0xD6 byte inside the NOPL, this decodes as
- * UDB on x86_64 and raises #UD.
- */
-asm( ".pushsection .rodata \n"
- "fineibt_preamble_start: \n"
- " endbr64 \n"
- " subl $0x12345678, %eax \n"
- "fineibt_preamble_bhi: \n"
- " cs jne.d32 fineibt_preamble_start+0x13 \n"
- "#fineibt_func: \n"
- " nopl -42(%rax) \n"
- "fineibt_preamble_end: \n"
- ".popsection\n"
-);
-
-extern u8 fineibt_preamble_start[];
-extern u8 fineibt_preamble_bhi[];
-extern u8 fineibt_preamble_end[];
-
-#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start)
-#define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_start)
-#define fineibt_preamble_ud 0x13
-#define fineibt_preamble_hash 5
-
-#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE)
-
-/*
* <fineibt_caller_start>:
* 0: b8 78 56 34 12 mov $0x12345678, %eax
* 5: 4d 8d 5b f0 lea -0x10(%r11), %r11
@@ -2169,6 +2176,7 @@ int3_exception_notify(struct notifier_block *self, unsigned long val, void *data
unsigned long selftest = (unsigned long)&int3_selftest_asm;
struct die_args *args = data;
struct pt_regs *regs = args->regs;
+ unsigned long ip;
OPTIMIZER_HIDE_VAR(selftest);
@@ -2181,7 +2189,8 @@ int3_exception_notify(struct notifier_block *self, unsigned long val, void *data
if (regs->ip - INT3_INSN_SIZE != selftest)
return NOTIFY_DONE;
- int3_emulate_call(regs, (unsigned long)&int3_selftest_callee);
+ ip = regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE;
+ int3_emulate_call(regs, ip, (unsigned long)&int3_selftest_callee);
return NOTIFY_STOP;
}
@@ -2372,6 +2381,38 @@ static void text_poke_memset(void *dst, const void *src, size_t len)
typedef void text_poke_f(void *dst, const void *src, size_t len);
+static void __poke_vmalloc_pages(struct page **pages, void *addr,
+ bool cross_page_boundary)
+{
+ pages[0] = vmalloc_to_page(addr);
+ if (cross_page_boundary)
+ pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
+}
+
+static void poke_vmalloc_pages(struct page **pages, void *addr,
+ bool cross_page_boundary)
+{
+ if (in_dbg_master()) {
+ /*
+ * If called from kgdb cannot sleep, but all other CPUs stopped
+ * anyway so safe to proceed without locks
+ */
+ __poke_vmalloc_pages(pages, addr, cross_page_boundary);
+ } else {
+ /*
+ * execmem ROX ranges are shared between modules and can be
+ * collapsed to huge PMD entries, and this collapse can happen
+ * concurrently with a racing set_memory_rox().
+ *
+ * Prevent vmalloc_to_page() from racing by acquiring an
+ * init_mm read lock which pairs with the init_mm write lock in
+ * cpa_collapse_large_pages().
+ */
+ guard(mmap_read_lock)(&init_mm);
+ __poke_vmalloc_pages(pages, addr, cross_page_boundary);
+ }
+}
+
static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t len)
{
bool cross_page_boundary = offset_in_page(addr) + len > PAGE_SIZE;
@@ -2389,9 +2430,7 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l
BUG_ON(!after_bootmem);
if (!core_kernel_text((unsigned long)addr)) {
- pages[0] = vmalloc_to_page(addr);
- if (cross_page_boundary)
- pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
+ poke_vmalloc_pages(pages, addr, cross_page_boundary);
} else {
pages[0] = virt_to_page(addr);
WARN_ON(!PageReserved(pages[0]));
@@ -2721,7 +2760,7 @@ noinstr int smp_text_poke_int3_handler(struct pt_regs *regs)
break;
case CALL_INSN_OPCODE:
- int3_emulate_call(regs, (long)ip + tpl->disp);
+ int3_emulate_call(regs, (long)ip, (long)ip + tpl->disp);
break;
case JMP32_INSN_OPCODE:
generated by cgit v1.2.3 (git 2.0.4) at 2026-09-20 18:35:53 +0000