summaryrefslogtreecommitdiff
path: root/arch/x86
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86')
-rw-r--r--arch/x86/Kconfig.cpu11
-rw-r--r--arch/x86/Makefile5
-rw-r--r--arch/x86/entry/entry_fred.c11
-rw-r--r--arch/x86/include/asm/div64.h2
-rw-r--r--arch/x86/include/asm/pgtable.h2
-rw-r--r--arch/x86/include/asm/text-patching.h4
-rw-r--r--arch/x86/kernel/alternative.c117
-rw-r--r--arch/x86/kernel/amd_node.c6
-rw-r--r--arch/x86/kernel/cpu/microcode/intel.c26
-rw-r--r--arch/x86/kernel/kprobes/core.c5
-rw-r--r--arch/x86/mm/mem_encrypt.c3
-rw-r--r--arch/x86/mm/pat/set_memory.c53
12 files changed, 181 insertions, 64 deletions
diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu
index e4654388d794..6e7a366f0798 100644
--- a/arch/x86/Kconfig.cpu
+++ b/arch/x86/Kconfig.cpu
@@ -204,10 +204,21 @@ config CC_HAS_MARCH_NATIVE
# usage warnings that only appear wth '-march=native'.
depends on CC_IS_GCC || CLANG_VERSION >= 190100
+config RUSTC_HAS_APXF
+ # The kernel isn't ready for in-kernel APX instructions. Without
+ # explicit frontend gating of APX, the backend may emit those
+ # instructions in native builds.
+ #
+ # Rust 1.88 added the `apxf` feature option, but versions before 1.93
+ # emit an `apxf` target attribute that only LLVM 23+ can interpret.
+ def_bool (RUSTC_VERSION >= 108800 && RUSTC_LLVM_MAJOR_VERSION >= 23) || \
+ RUSTC_VERSION >= 109300
+
config X86_NATIVE_CPU
bool "Build and optimize for local/native CPU"
depends on X86_64
depends on CC_HAS_MARCH_NATIVE
+ depends on !RUST || RUSTC_HAS_APXF
help
Optimize for the current CPU used to compile the kernel.
Use this option if you intend to build the kernel for your
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 598f178102ee..8af6b80cffdd 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -161,6 +161,11 @@ else
ifdef CONFIG_X86_NATIVE_CPU
KBUILD_CFLAGS += -march=native
+ # Prevent the compiler from generating EGPR use. The kernel is
+ # not yet prepared for general in-kernel use.
+ KBUILD_CFLAGS += $(call cc-option,-mno-apx-features=egpr)
+
+ # generate_rust_target.rs handles Rust APX gating.
KBUILD_RUSTFLAGS += -Ctarget-cpu=native
else
KBUILD_CFLAGS += -march=x86-64 -mtune=generic
diff --git a/arch/x86/entry/entry_fred.c b/arch/x86/entry/entry_fred.c
index fb3594ddf731..854899bfec5d 100644
--- a/arch/x86/entry/entry_fred.c
+++ b/arch/x86/entry/entry_fred.c
@@ -10,6 +10,7 @@
#include <asm/desc.h>
#include <asm/fred.h>
#include <asm/idtentry.h>
+#include <asm/processor-flags.h>
#include <asm/syscall.h>
#include <asm/trapnr.h>
#include <asm/traps.h>
@@ -71,7 +72,15 @@ static noinstr void fred_intx(struct pt_regs *regs)
#endif
default:
- return exc_general_protection(regs, 0);
+ /*
+ * Reconstruct the #GP fault state that IDT delivery would produce.
+ * Clear the software event flag so ERETU with TF set does not trap
+ * before the resumed instruction. See prevent_single_step_upon_eretu().
+ */
+ regs->ip -= regs->fred_ss.insnlen;
+ regs->flags |= X86_EFLAGS_RF;
+ regs->fred_ss.swevent = 0;
+ return exc_general_protection(regs, (regs->fred_ss.vector << 3) | 2);
}
}
diff --git a/arch/x86/include/asm/div64.h b/arch/x86/include/asm/div64.h
index 30fd06ede751..8a2d343f977e 100644
--- a/arch/x86/include/asm/div64.h
+++ b/arch/x86/include/asm/div64.h
@@ -111,7 +111,7 @@ static inline u64 mul_u64_add_u64_div_u64(u64 rax, u64 mul, u64 add, u64 div)
if (!statically_true(!add))
asm ("addq %[add], %[lo]; adcq $0, %[hi]" :
- [lo] "+r" (rax), [hi] "+r" (rdx) : [add] "irm" (add));
+ [lo] "+r" (rax), [hi] "+r" (rdx) : [add] "erm" (add));
asm ("divq %[div]" : "+a" (rax), "+d" (rdx) : [div] "rm" (div));
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index d5f4917c1edc..d551120a7c88 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -806,7 +806,7 @@ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
pmdval_t val = pmd_val(pmd), oldval = val;
pmd_t pmd_result;
- val &= (_HPAGE_CHG_MASK & ~_PAGE_DIRTY);
+ val &= _HPAGE_CHG_MASK;
val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK;
val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK);
diff --git a/arch/x86/include/asm/text-patching.h b/arch/x86/include/asm/text-patching.h
index f2d142a0a862..ea09381070e8 100644
--- a/arch/x86/include/asm/text-patching.h
+++ b/arch/x86/include/asm/text-patching.h
@@ -164,9 +164,9 @@ unsigned long int3_emulate_pop(struct pt_regs *regs)
}
static __always_inline
-void int3_emulate_call(struct pt_regs *regs, unsigned long func)
+void int3_emulate_call(struct pt_regs *regs, unsigned long ip, unsigned long func)
{
- int3_emulate_push(regs, regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE);
+ int3_emulate_push(regs, ip);
int3_emulate_jmp(regs, func);
}
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index 91b1cdd16569..582c6d8307d1 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -6,6 +6,9 @@
#include <linux/vmalloc.h>
#include <linux/memory.h>
#include <linux/execmem.h>
+#include <linux/cleanup.h>
+#include <linux/kgdb.h>
+#include <linux/mmap_lock.h>
#include <asm/text-patching.h>
#include <asm/insn.h>
@@ -1198,6 +1201,41 @@ static bool cfi_debug __ro_after_init;
bool cfi_bhi __ro_after_init = false;
#endif
+#ifdef CONFIG_FINEIBT
+/*
+ * <fineibt_preamble_start>:
+ * 0: f3 0f 1e fa endbr64
+ * 4: 2d 78 56 34 12 sub $0x12345678, %eax
+ * 9: 2e 0f 85 03 00 00 00 jne,pn 13 <fineibt_preamble_start+0x13>
+ * 10: 0f 1f 40 d6 nopl -0x2a(%rax)
+ *
+ * Note that the JNE target is the 0xD6 byte inside the NOPL, this decodes as
+ * UDB on x86_64 and raises #UD.
+ */
+asm( ".pushsection .rodata \n"
+ "fineibt_preamble_start: \n"
+ " endbr64 \n"
+ " subl $0x12345678, %eax \n"
+ "fineibt_preamble_bhi: \n"
+ " cs jne.d32 fineibt_preamble_start+0x13 \n"
+ "#fineibt_func: \n"
+ " nopl -42(%rax) \n"
+ "fineibt_preamble_end: \n"
+ ".popsection\n"
+);
+
+extern u8 fineibt_preamble_start[];
+extern u8 fineibt_preamble_bhi[];
+extern u8 fineibt_preamble_end[];
+
+#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start)
+#define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_start)
+#define fineibt_preamble_ud 0x13
+#define fineibt_preamble_hash 5
+
+#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE)
+#endif /* CONFIG_FINEIBT */
+
#ifdef CONFIG_CFI
u32 cfi_get_func_hash(void *func)
{
@@ -1205,9 +1243,11 @@ u32 cfi_get_func_hash(void *func)
func -= cfi_get_offset();
switch (cfi_mode) {
+#ifdef CONFIG_FINEIBT
case CFI_FINEIBT:
- func += 7;
+ func += fineibt_preamble_hash;
break;
+#endif
case CFI_KCFI:
func += 1;
break;
@@ -1364,39 +1404,6 @@ early_param("cfi", cfi_parse_cmdline);
*/
/*
- * <fineibt_preamble_start>:
- * 0: f3 0f 1e fa endbr64
- * 4: 2d 78 56 34 12 sub $0x12345678, %eax
- * 9: 2e 0f 85 03 00 00 00 jne,pn 13 <fineibt_preamble_start+0x13>
- * 10: 0f 1f 40 d6 nopl -0x2a(%rax)
- *
- * Note that the JNE target is the 0xD6 byte inside the NOPL, this decodes as
- * UDB on x86_64 and raises #UD.
- */
-asm( ".pushsection .rodata \n"
- "fineibt_preamble_start: \n"
- " endbr64 \n"
- " subl $0x12345678, %eax \n"
- "fineibt_preamble_bhi: \n"
- " cs jne.d32 fineibt_preamble_start+0x13 \n"
- "#fineibt_func: \n"
- " nopl -42(%rax) \n"
- "fineibt_preamble_end: \n"
- ".popsection\n"
-);
-
-extern u8 fineibt_preamble_start[];
-extern u8 fineibt_preamble_bhi[];
-extern u8 fineibt_preamble_end[];
-
-#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start)
-#define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_start)
-#define fineibt_preamble_ud 0x13
-#define fineibt_preamble_hash 5
-
-#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE)
-
-/*
* <fineibt_caller_start>:
* 0: b8 78 56 34 12 mov $0x12345678, %eax
* 5: 4d 8d 5b f0 lea -0x10(%r11), %r11
@@ -2169,6 +2176,7 @@ int3_exception_notify(struct notifier_block *self, unsigned long val, void *data
unsigned long selftest = (unsigned long)&int3_selftest_asm;
struct die_args *args = data;
struct pt_regs *regs = args->regs;
+ unsigned long ip;
OPTIMIZER_HIDE_VAR(selftest);
@@ -2181,7 +2189,8 @@ int3_exception_notify(struct notifier_block *self, unsigned long val, void *data
if (regs->ip - INT3_INSN_SIZE != selftest)
return NOTIFY_DONE;
- int3_emulate_call(regs, (unsigned long)&int3_selftest_callee);
+ ip = regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE;
+ int3_emulate_call(regs, ip, (unsigned long)&int3_selftest_callee);
return NOTIFY_STOP;
}
@@ -2372,6 +2381,38 @@ static void text_poke_memset(void *dst, const void *src, size_t len)
typedef void text_poke_f(void *dst, const void *src, size_t len);
+static void __poke_vmalloc_pages(struct page **pages, void *addr,
+ bool cross_page_boundary)
+{
+ pages[0] = vmalloc_to_page(addr);
+ if (cross_page_boundary)
+ pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
+}
+
+static void poke_vmalloc_pages(struct page **pages, void *addr,
+ bool cross_page_boundary)
+{
+ if (in_dbg_master()) {
+ /*
+ * If called from kgdb cannot sleep, but all other CPUs stopped
+ * anyway so safe to proceed without locks
+ */
+ __poke_vmalloc_pages(pages, addr, cross_page_boundary);
+ } else {
+ /*
+ * execmem ROX ranges are shared between modules and can be
+ * collapsed to huge PMD entries, and this collapse can happen
+ * concurrently with a racing set_memory_rox().
+ *
+ * Prevent vmalloc_to_page() from racing by acquiring an
+ * init_mm read lock which pairs with the init_mm write lock in
+ * cpa_collapse_large_pages().
+ */
+ guard(mmap_read_lock)(&init_mm);
+ __poke_vmalloc_pages(pages, addr, cross_page_boundary);
+ }
+}
+
static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t len)
{
bool cross_page_boundary = offset_in_page(addr) + len > PAGE_SIZE;
@@ -2389,9 +2430,7 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l
BUG_ON(!after_bootmem);
if (!core_kernel_text((unsigned long)addr)) {
- pages[0] = vmalloc_to_page(addr);
- if (cross_page_boundary)
- pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
+ poke_vmalloc_pages(pages, addr, cross_page_boundary);
} else {
pages[0] = virt_to_page(addr);
WARN_ON(!PageReserved(pages[0]));
@@ -2721,7 +2760,7 @@ noinstr int smp_text_poke_int3_handler(struct pt_regs *regs)
break;
case CALL_INSN_OPCODE:
- int3_emulate_call(regs, (long)ip + tpl->disp);
+ int3_emulate_call(regs, (long)ip, (long)ip + tpl->disp);
break;
case JMP32_INSN_OPCODE:
diff --git a/arch/x86/kernel/amd_node.c b/arch/x86/kernel/amd_node.c
index 762585775b5a..b7926ba3610a 100644
--- a/arch/x86/kernel/amd_node.c
+++ b/arch/x86/kernel/amd_node.c
@@ -251,7 +251,7 @@ __setup("amd_smn_debugfs_enable", amd_smn_enable_dfs);
static int __init amd_smn_init(void)
{
u16 count, num_roots, roots_per_node, node, num_nodes;
- struct pci_dev *root;
+ struct pci_dev *root __free(pci_dev_put) = NULL;
if (!cpu_feature_enabled(X86_FEATURE_ZEN))
return 0;
@@ -262,7 +262,6 @@ static int __init amd_smn_init(void)
return 0;
num_roots = 0;
- root = NULL;
while ((root = get_next_root(root))) {
pci_dbg(root, "Reserving PCI config space\n");
@@ -299,14 +298,13 @@ static int __init amd_smn_init(void)
count = 0;
node = 0;
- root = NULL;
while (node < num_nodes && (root = get_next_root(root))) {
/* Use one root for each node and skip the rest. */
if (count++ % roots_per_node)
continue;
pci_dbg(root, "is root for AMD node %u\n", node);
- amd_roots[node++] = root;
+ amd_roots[node++] = pci_dev_get(root);
}
if (enable_dfs) {
diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c
index 1142183c950c..9f09d388ed20 100644
--- a/arch/x86/kernel/cpu/microcode/intel.c
+++ b/arch/x86/kernel/cpu/microcode/intel.c
@@ -309,6 +309,26 @@ static void save_microcode_patch(struct microcode_intel *patch)
pr_err("Unable to allocate microcode memory size: %u\n", size);
}
+static bool revision_is_safe(struct cpu_signature *sig, u32 rev)
+{
+ u32 vfm = IFM(x86_family(sig->sig), x86_model(sig->sig));
+
+ /*
+ * Erratum GNR98 can cause #MCs if "jumping over" revision 0x1000405.
+ * Avoid the jumps.
+ */
+ if (vfm == INTEL_GRANITERAPIDS_X &&
+ x86_stepping(sig->sig) == 1 &&
+ sig->pf & 0x95 &&
+ sig->rev < 0x1000405 &&
+ rev > 0x1000405) {
+ pr_err_once("Erratum GNR98: skipping revision 0x%x.\n", rev);
+ return false;
+ }
+
+ return true;
+}
+
/* Scan blob for microcode matching the boot CPUs family, model, stepping */
static __init struct microcode_intel *scan_microcode(void *data, size_t size,
struct ucode_cpu_info *uci,
@@ -330,6 +350,9 @@ static __init struct microcode_intel *scan_microcode(void *data, size_t size,
if (!intel_find_matching_signature(data, &uci->cpu_sig))
continue;
+ if (!revision_is_safe(&uci->cpu_sig, mc_header->rev))
+ continue;
+
/*
* For saving the early microcode, find the matching revision which
* was loaded on the BSP.
@@ -878,6 +901,9 @@ static enum ucode_state parse_microcode_blobs(int cpu, struct iov_iter *iter)
if (!intel_find_matching_signature(mc, &uci->cpu_sig))
continue;
+ if (!revision_is_safe(&uci->cpu_sig, mc_header.rev))
+ continue;
+
is_safe = ucode_validate_minrev(&mc_header);
if (force_minrev && !is_safe)
continue;
diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c
index 4e5f8c1736ec..133ff20caccd 100644
--- a/arch/x86/kernel/kprobes/core.c
+++ b/arch/x86/kernel/kprobes/core.c
@@ -510,10 +510,9 @@ NOKPROBE_SYMBOL(kprobe_emulate_ret);
static void kprobe_emulate_call(struct kprobe *p, struct pt_regs *regs)
{
- unsigned long func = regs->ip - INT3_INSN_SIZE + p->ainsn.size;
+ unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size;
- func += p->ainsn.rel32;
- int3_emulate_call(regs, func);
+ int3_emulate_call(regs, ip, ip + p->ainsn.rel32);
}
NOKPROBE_SYMBOL(kprobe_emulate_call);
diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c
index 95bae74fdab2..3aefdef5bcb0 100644
--- a/arch/x86/mm/mem_encrypt.c
+++ b/arch/x86/mm/mem_encrypt.c
@@ -13,6 +13,7 @@
#include <linux/cc_platform.h>
#include <linux/mem_encrypt.h>
#include <linux/virtio_anchor.h>
+#include <linux/iommu-dma.h>
#include <asm/sev.h>
@@ -30,7 +31,7 @@ bool force_dma_unencrypted(struct device *dev)
* device does not support DMA to addresses that include the
* encryption mask.
*/
- if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) {
+ if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT) && !use_dma_iommu(dev)) {
u64 dma_enc_mask = DMA_BIT_MASK(__ffs64(sme_me_mask));
u64 dma_dev_mask = min_not_zero(dev->coherent_dma_mask,
dev->bus_dma_limit);
diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c
index c38faf39ce15..4652487b5572 100644
--- a/arch/x86/mm/pat/set_memory.c
+++ b/arch/x86/mm/pat/set_memory.c
@@ -22,6 +22,7 @@
#include <linux/cc_platform.h>
#include <linux/set_memory.h>
#include <linux/memregion.h>
+#include <linux/cleanup.h>
#include <asm/e820/api.h>
#include <asm/processor.h>
@@ -49,7 +50,8 @@ struct cpa_data {
unsigned int flags;
unsigned int force_split : 1,
force_static_prot : 1,
- force_flush_all : 1;
+ force_flush_all : 1,
+ init_mm_read_locked : 1;
struct page **pages;
};
@@ -409,7 +411,7 @@ static void __cpa_flush_tlb(void *data)
static int collapse_large_pages(unsigned long addr, struct list_head *pgtables);
-static void cpa_collapse_large_pages(struct cpa_data *cpa)
+static void __cpa_collapse_large_pages(struct cpa_data *cpa)
{
unsigned long start, addr, end;
struct ptdesc *ptdesc, *tmp;
@@ -439,10 +441,30 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa)
list_for_each_entry_safe(ptdesc, tmp, &pgtables, pt_list) {
list_del(&ptdesc->pt_list);
- pagetable_free(ptdesc);
+ /*
+ * Only early alloc'd direct map should not be flagged PG_table
+ * here and those shouldn't be collapsed. However be abundantly
+ * cautious and handle the !PG_table case too.
+ */
+ if (PageTable((ptdesc_page(ptdesc))))
+ pagetable_dtor_free(ptdesc);
+ else
+ pagetable_free(ptdesc);
}
}
+static void cpa_collapse_large_pages(struct cpa_data *cpa)
+{
+ /*
+ * Take the mmap write lock on init_mm to:
+ * - Avoid a use-after-free if raced by ptdump (which takes its own
+ * write lock on init_mm).
+ * - Serialise concurrent CPA walkers.
+ */
+ scoped_guard(mmap_write_lock, &init_mm)
+ __cpa_collapse_large_pages(cpa);
+}
+
static void cpa_flush(struct cpa_data *cpa, int cache)
{
unsigned int i;
@@ -1120,11 +1142,10 @@ set:
static int
__split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,
- struct ptdesc *ptdesc)
+ pte_t *pbase)
{
unsigned long lpaddr, lpinc, ref_pfn, pfn, pfninc = 1;
- struct page *base = ptdesc_page(ptdesc);
- pte_t *pbase = (pte_t *)page_address(base);
+ struct page *base = virt_to_page(pbase);
unsigned int i, level;
pgprot_t ref_prot;
bool nx, rw;
@@ -1224,16 +1245,20 @@ __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,
static int split_large_page(struct cpa_data *cpa, pte_t *kpte,
unsigned long address)
{
- struct ptdesc *ptdesc;
+ pte_t *pte;
spin_unlock(&cpa_lock);
- ptdesc = pagetable_alloc(GFP_KERNEL, 0);
+ if (cpa->init_mm_read_locked)
+ mmap_read_unlock(&init_mm);
+ pte = pte_alloc_one_kernel(&init_mm);
+ if (cpa->init_mm_read_locked)
+ mmap_read_lock(&init_mm);
spin_lock(&cpa_lock);
- if (!ptdesc)
+ if (!pte)
return -ENOMEM;
- if (__split_large_page(cpa, kpte, address, ptdesc))
- pagetable_free(ptdesc);
+ if (__split_large_page(cpa, kpte, address, pte))
+ pte_free_kernel(&init_mm, pte);
return 0;
}
@@ -2121,7 +2146,11 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages,
cpa.curpage = 0;
cpa.force_split = force_split;
- ret = __change_page_attr_set_clr(&cpa, 1);
+ /* Avoid race with concurrent CPA collapse. */
+ cpa.init_mm_read_locked = true;
+ scoped_guard(mmap_read_lock, &init_mm)
+ ret = __change_page_attr_set_clr(&cpa, 1);
+ cpa.init_mm_read_locked = false;
/*
* Check whether we really changed something:
generated by cgit v1.2.3 (git 2.0.4) at 2026-09-24 23:09:56 +0000