diff options
Diffstat (limited to 'arch/x86/kernel')
72 files changed, 1050 insertions, 1025 deletions
diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c index be4c5e9e5ff6..bbade0da5130 100644 --- a/arch/x86/kernel/acpi/cppc.c +++ b/arch/x86/kernel/acpi/cppc.c @@ -241,7 +241,6 @@ EXPORT_SYMBOL_GPL(amd_detect_prefcore); */ int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator) { - enum x86_topology_cpu_type core_type = get_topology_cpu_type(&cpu_data(cpu)); bool prefcore; int ret; u32 tmp; @@ -273,16 +272,18 @@ int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator) /* detect if running on heterogeneous design */ if (cpu_feature_enabled(X86_FEATURE_AMD_HTR_CORES)) { - switch (core_type) { + switch (cpu_data(cpu).topo.cpu_type) { case TOPO_CPU_TYPE_UNKNOWN: + case TOPO_CPU_TYPE_ANY: pr_warn("Undefined core type found for cpu %d\n", cpu); break; case TOPO_CPU_TYPE_PERFORMANCE: /* use the max scale for performance cores */ *numerator = CPPC_HIGHEST_PERF_PERFORMANCE; return 0; + case TOPO_CPU_TYPE_LOW_POWER: case TOPO_CPU_TYPE_EFFICIENCY: - /* use the highest perf value for efficiency cores */ + /* use the highest perf value for efficiency and low-power cores */ ret = amd_get_highest_perf(cpu, &tmp); if (ret) return ret; diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c index 91fa262f0e30..8dfe98784bf9 100644 --- a/arch/x86/kernel/acpi/sleep.c +++ b/arch/x86/kernel/acpi/sleep.c @@ -61,6 +61,7 @@ int x86_acpi_suspend_lowlevel(void) { struct wakeup_header *header = (struct wakeup_header *) __va(real_mode_header->wakeup_header); + struct msr val; if (header->signature != WAKEUP_HEADER_SIGNATURE) { printk(KERN_ERR "wakeup header does not match\n"); @@ -82,13 +83,10 @@ int x86_acpi_suspend_lowlevel(void) * with 2-MB L2 Cache and Intel® Processor A100 and A110 on 90 * nm process with 512-KB L2 Cache Specification Update". */ - if (!rdmsr_safe(MSR_EFER, - &header->pmode_efer_low, - &header->pmode_efer_high) && - !wrmsr_safe(MSR_EFER, - header->pmode_efer_low, - header->pmode_efer_high)) + if (!rdmsrq_safe(MSR_EFER, &val.q) && !wrmsrq_safe(MSR_EFER, val.q)) header->pmode_behavior |= (1 << WAKEUP_BEHAVIOR_RESTORE_EFER); + header->pmode_efer_low = val.l; + header->pmode_efer_high = val.h; #endif /* !CONFIG_64BIT */ header->pmode_cr0 = read_cr0(); @@ -96,14 +94,12 @@ int x86_acpi_suspend_lowlevel(void) header->pmode_cr4 = __read_cr4(); header->pmode_behavior |= (1 << WAKEUP_BEHAVIOR_RESTORE_CR4); } - if (!rdmsr_safe(MSR_IA32_MISC_ENABLE, - &header->pmode_misc_en_low, - &header->pmode_misc_en_high) && - !wrmsr_safe(MSR_IA32_MISC_ENABLE, - header->pmode_misc_en_low, - header->pmode_misc_en_high)) + if (!rdmsrq_safe(MSR_IA32_MISC_ENABLE, &val.q) && + !wrmsrq_safe(MSR_IA32_MISC_ENABLE, val.q)) header->pmode_behavior |= (1 << WAKEUP_BEHAVIOR_RESTORE_MISC_ENABLE); + header->pmode_misc_en_low = val.l; + header->pmode_misc_en_high = val.h; header->realmode_flags = acpi_realmode_flags; header->real_magic = 0x12345678; diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 62936a3bde19..582c6d8307d1 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -6,6 +6,9 @@ #include <linux/vmalloc.h> #include <linux/memory.h> #include <linux/execmem.h> +#include <linux/cleanup.h> +#include <linux/kgdb.h> +#include <linux/mmap_lock.h> #include <asm/text-patching.h> #include <asm/insn.h> @@ -41,15 +44,6 @@ static int __init debug_alt(char *str) } __setup("debug-alternative", debug_alt); -static int noreplace_smp; - -static int __init setup_noreplace_smp(char *str) -{ - noreplace_smp = 1; - return 1; -} -__setup("noreplace-smp", setup_noreplace_smp); - #define DPRINTK(type, fmt, args...) \ do { \ if (debug_alternative & DA_##type) \ @@ -1207,6 +1201,41 @@ static bool cfi_debug __ro_after_init; bool cfi_bhi __ro_after_init = false; #endif +#ifdef CONFIG_FINEIBT +/* + * <fineibt_preamble_start>: + * 0: f3 0f 1e fa endbr64 + * 4: 2d 78 56 34 12 sub $0x12345678, %eax + * 9: 2e 0f 85 03 00 00 00 jne,pn 13 <fineibt_preamble_start+0x13> + * 10: 0f 1f 40 d6 nopl -0x2a(%rax) + * + * Note that the JNE target is the 0xD6 byte inside the NOPL, this decodes as + * UDB on x86_64 and raises #UD. + */ +asm( ".pushsection .rodata \n" + "fineibt_preamble_start: \n" + " endbr64 \n" + " subl $0x12345678, %eax \n" + "fineibt_preamble_bhi: \n" + " cs jne.d32 fineibt_preamble_start+0x13 \n" + "#fineibt_func: \n" + " nopl -42(%rax) \n" + "fineibt_preamble_end: \n" + ".popsection\n" +); + +extern u8 fineibt_preamble_start[]; +extern u8 fineibt_preamble_bhi[]; +extern u8 fineibt_preamble_end[]; + +#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start) +#define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_start) +#define fineibt_preamble_ud 0x13 +#define fineibt_preamble_hash 5 + +#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE) +#endif /* CONFIG_FINEIBT */ + #ifdef CONFIG_CFI u32 cfi_get_func_hash(void *func) { @@ -1214,9 +1243,11 @@ u32 cfi_get_func_hash(void *func) func -= cfi_get_offset(); switch (cfi_mode) { +#ifdef CONFIG_FINEIBT case CFI_FINEIBT: - func += 7; + func += fineibt_preamble_hash; break; +#endif case CFI_KCFI: func += 1; break; @@ -1356,40 +1387,21 @@ early_param("cfi", cfi_parse_cmdline); * "Make conditional jumps most often not taken: The efficiency and throughput * for not-taken branches is better than for taken branches on most * processors. Therefore, it is good to place the most frequent branch first" - */ - -/* - * <fineibt_preamble_start>: - * 0: f3 0f 1e fa endbr64 - * 4: 2d 78 56 34 12 sub $0x12345678, %eax - * 9: 2e 0f 85 03 00 00 00 jne,pn 13 <fineibt_preamble_start+0x13> - * 10: 0f 1f 40 d6 nopl -0x2a(%rax) * - * Note that the JNE target is the 0xD6 byte inside the NOPL, this decodes as - * UDB on x86_64 and raises #UD. + * NOTE: Update the kCFI caller sequence to make use of this observation: + * + * kCFI kCFI-OPT + * + * caller: caller: + * movl $(-0x12345678),%r10d // 6 movl $(-0x12345678),%r10d // 6 + * addl $-15(%r11),%r10d // 4 addl $-15(%r11),%r10d // 4 + * je 1f // 2 jne . + 3 // 2 + * ud2 // 2 test $0xd6, %al // 2 + * 1: cs call __x86_indirect_thunk_r11 // 6 1: cs call __x86_indirect_thunk_r11 // 6 + * + * This new test clobbers eflags, but those are clobbered by the hash test + * anyway. */ -asm( ".pushsection .rodata \n" - "fineibt_preamble_start: \n" - " endbr64 \n" - " subl $0x12345678, %eax \n" - "fineibt_preamble_bhi: \n" - " cs jne.d32 fineibt_preamble_start+0x13 \n" - "#fineibt_func: \n" - " nopl -42(%rax) \n" - "fineibt_preamble_end: \n" - ".popsection\n" -); - -extern u8 fineibt_preamble_start[]; -extern u8 fineibt_preamble_bhi[]; -extern u8 fineibt_preamble_end[]; - -#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start) -#define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_start) -#define fineibt_preamble_ud 0x13 -#define fineibt_preamble_hash 5 - -#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE) /* * <fineibt_caller_start>: @@ -1518,8 +1530,9 @@ static int cfi_disable_callers(s32 *start, s32 *end) static int cfi_enable_callers(s32 *start, s32 *end) { /* - * Re-enable kCFI, undo what cfi_disable_callers() did. + * Re-enable (and update) kCFI, undo what cfi_disable_callers() did. */ + const u8 udne[] = { 0x75, 0x01, 0xa8, 0xd6 }; const u8 mov[] = { 0x41, 0xba }; s32 *s; @@ -1532,6 +1545,10 @@ static int cfi_enable_callers(s32 *start, s32 *end) if (!hash) /* nocfi callers */ continue; + /* + * See the kCFI/FineIBT comment above -- update note. + */ + text_poke_early(addr + 10, udne, 4); text_poke_early(addr, mov, 2); } @@ -1775,8 +1792,8 @@ static int cfi_rewrite_callers(s32 *start, s32 *end) #define FINEIBT_WARN(_f, _v) \ WARN_ONCE((_f) != (_v), "FineIBT: " #_f " %ld != %d\n", _f, _v) -static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, - s32 *start_cfi, s32 *end_cfi, bool builtin) +static void __init_or_module __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, + s32 *start_cfi, s32 *end_cfi, bool builtin) { int ret; @@ -2088,8 +2105,8 @@ bool decode_fineibt_insn(struct pt_regs *regs, unsigned long *target, u32 *type) #else /* !CONFIG_FINEIBT: */ -static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, - s32 *start_cfi, s32 *end_cfi, bool builtin) +static void __init_or_module __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, + s32 *start_cfi, s32 *end_cfi, bool builtin) { if (IS_ENABLED(CONFIG_CFI) && builtin) pr_info("CFI: Using standard kCFI\n"); @@ -2101,167 +2118,14 @@ static void poison_cfi(void *addr) { } #endif /* !CONFIG_FINEIBT */ -void apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, - s32 *start_cfi, s32 *end_cfi) +void __init_or_module apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, + s32 *start_cfi, s32 *end_cfi) { return __apply_fineibt(start_retpoline, end_retpoline, start_cfi, end_cfi, /* .builtin = */ false); } -#ifdef CONFIG_SMP -static void alternatives_smp_lock(const s32 *start, const s32 *end, - u8 *text, u8 *text_end) -{ - const s32 *poff; - - for (poff = start; poff < end; poff++) { - u8 *ptr = (u8 *)poff + *poff; - - if (!*poff || ptr < text || ptr >= text_end) - continue; - /* turn DS segment override prefix into lock prefix */ - if (*ptr == 0x3e) - text_poke(ptr, ((unsigned char []){0xf0}), 1); - } -} - -static void alternatives_smp_unlock(const s32 *start, const s32 *end, - u8 *text, u8 *text_end) -{ - const s32 *poff; - - for (poff = start; poff < end; poff++) { - u8 *ptr = (u8 *)poff + *poff; - - if (!*poff || ptr < text || ptr >= text_end) - continue; - /* turn lock prefix into DS segment override prefix */ - if (*ptr == 0xf0) - text_poke(ptr, ((unsigned char []){0x3E}), 1); - } -} - -struct smp_alt_module { - /* what is this ??? */ - struct module *mod; - char *name; - - /* ptrs to lock prefixes */ - const s32 *locks; - const s32 *locks_end; - - /* .text segment, needed to avoid patching init code ;) */ - u8 *text; - u8 *text_end; - - struct list_head next; -}; -static LIST_HEAD(smp_alt_modules); -static bool uniproc_patched = false; /* protected by text_mutex */ - -void __init_or_module alternatives_smp_module_add(struct module *mod, - char *name, - void *locks, void *locks_end, - void *text, void *text_end) -{ - struct smp_alt_module *smp; - - mutex_lock(&text_mutex); - if (!uniproc_patched) - goto unlock; - - if (num_possible_cpus() == 1) - /* Don't bother remembering, we'll never have to undo it. */ - goto smp_unlock; - - smp = kzalloc_obj(*smp); - if (NULL == smp) - /* we'll run the (safe but slow) SMP code then ... */ - goto unlock; - - smp->mod = mod; - smp->name = name; - smp->locks = locks; - smp->locks_end = locks_end; - smp->text = text; - smp->text_end = text_end; - DPRINTK(SMP, "locks %p -> %p, text %p -> %p, name %s\n", - smp->locks, smp->locks_end, - smp->text, smp->text_end, smp->name); - - list_add_tail(&smp->next, &smp_alt_modules); -smp_unlock: - alternatives_smp_unlock(locks, locks_end, text, text_end); -unlock: - mutex_unlock(&text_mutex); -} - -void __init_or_module alternatives_smp_module_del(struct module *mod) -{ - struct smp_alt_module *item; - - mutex_lock(&text_mutex); - list_for_each_entry(item, &smp_alt_modules, next) { - if (mod != item->mod) - continue; - list_del(&item->next); - kfree(item); - break; - } - mutex_unlock(&text_mutex); -} - -void alternatives_enable_smp(void) -{ - struct smp_alt_module *mod; - - /* Why bother if there are no other CPUs? */ - BUG_ON(num_possible_cpus() == 1); - - mutex_lock(&text_mutex); - - if (uniproc_patched) { - pr_info("switching to SMP code\n"); - BUG_ON(num_online_cpus() != 1); - clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); - clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP); - list_for_each_entry(mod, &smp_alt_modules, next) - alternatives_smp_lock(mod->locks, mod->locks_end, - mod->text, mod->text_end); - uniproc_patched = false; - } - mutex_unlock(&text_mutex); -} - -/* - * Return 1 if the address range is reserved for SMP-alternatives. - * Must hold text_mutex. - */ -int alternatives_text_reserved(void *start, void *end) -{ - struct smp_alt_module *mod; - const s32 *poff; - u8 *text_start = start; - u8 *text_end = end; - - lockdep_assert_held(&text_mutex); - - list_for_each_entry(mod, &smp_alt_modules, next) { - if (mod->text > text_end || mod->text_end < text_start) - continue; - for (poff = mod->locks; poff < mod->locks_end; poff++) { - const u8 *ptr = (const u8 *)poff + *poff; - - if (text_start <= ptr && text_end > ptr) - return 1; - } - } - - return 0; -} -#endif /* CONFIG_SMP */ - /* * Self-test for the INT3 based CALL emulation code. * @@ -2312,6 +2176,7 @@ int3_exception_notify(struct notifier_block *self, unsigned long val, void *data unsigned long selftest = (unsigned long)&int3_selftest_asm; struct die_args *args = data; struct pt_regs *regs = args->regs; + unsigned long ip; OPTIMIZER_HIDE_VAR(selftest); @@ -2324,7 +2189,8 @@ int3_exception_notify(struct notifier_block *self, unsigned long val, void *data if (regs->ip - INT3_INSN_SIZE != selftest) return NOTIFY_DONE; - int3_emulate_call(regs, (unsigned long)&int3_selftest_callee); + ip = regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE; + int3_emulate_call(regs, ip, (unsigned long)&int3_selftest_callee); return NOTIFY_STOP; } @@ -2440,40 +2306,12 @@ void __init alternative_instructions(void) ibt_restore(ibt); -#ifdef CONFIG_SMP - /* Patch to UP if other cpus not imminent. */ - if (!noreplace_smp && (num_present_cpus() == 1 || setup_max_cpus <= 1)) { - uniproc_patched = true; - alternatives_smp_module_add(NULL, "core kernel", - __smp_locks, __smp_locks_end, - _text, _etext); - } -#endif - restart_nmi(); alternatives_patched = 1; alt_reloc_selftest(); } -#ifdef CONFIG_SMP -/* - * With CONFIG_DEFERRED_STRUCT_PAGE_INIT enabled we can free_init_pages() only - * after the deferred initialization of the memory map is complete. - */ -static int __init free_smp_locks(void) -{ - if (!uniproc_patched || num_possible_cpus() == 1) { - free_init_pages("SMP alternatives", - (unsigned long)__smp_locks, - (unsigned long)__smp_locks_end); - } - - return 0; -} -arch_initcall(free_smp_locks); -#endif - /** * text_poke_early - Update instructions on a live kernel at boot time * @addr: address to modify @@ -2543,6 +2381,38 @@ static void text_poke_memset(void *dst, const void *src, size_t len) typedef void text_poke_f(void *dst, const void *src, size_t len); +static void __poke_vmalloc_pages(struct page **pages, void *addr, + bool cross_page_boundary) +{ + pages[0] = vmalloc_to_page(addr); + if (cross_page_boundary) + pages[1] = vmalloc_to_page(addr + PAGE_SIZE); +} + +static void poke_vmalloc_pages(struct page **pages, void *addr, + bool cross_page_boundary) +{ + if (in_dbg_master()) { + /* + * If called from kgdb cannot sleep, but all other CPUs stopped + * anyway so safe to proceed without locks + */ + __poke_vmalloc_pages(pages, addr, cross_page_boundary); + } else { + /* + * execmem ROX ranges are shared between modules and can be + * collapsed to huge PMD entries, and this collapse can happen + * concurrently with a racing set_memory_rox(). + * + * Prevent vmalloc_to_page() from racing by acquiring an + * init_mm read lock which pairs with the init_mm write lock in + * cpa_collapse_large_pages(). + */ + guard(mmap_read_lock)(&init_mm); + __poke_vmalloc_pages(pages, addr, cross_page_boundary); + } +} + static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t len) { bool cross_page_boundary = offset_in_page(addr) + len > PAGE_SIZE; @@ -2560,9 +2430,7 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l BUG_ON(!after_bootmem); if (!core_kernel_text((unsigned long)addr)) { - pages[0] = vmalloc_to_page(addr); - if (cross_page_boundary) - pages[1] = vmalloc_to_page(addr + PAGE_SIZE); + poke_vmalloc_pages(pages, addr, cross_page_boundary); } else { pages[0] = virt_to_page(addr); WARN_ON(!PageReserved(pages[0])); @@ -2892,7 +2760,7 @@ noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) break; case CALL_INSN_OPCODE: - int3_emulate_call(regs, (long)ip + tpl->disp); + int3_emulate_call(regs, (long)ip, (long)ip + tpl->disp); break; case JMP32_INSN_OPCODE: diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c index e8000a56732e..b5f1f031d45b 100644 --- a/arch/x86/kernel/amd_gart_64.c +++ b/arch/x86/kernel/amd_gart_64.c @@ -180,22 +180,23 @@ static void iommu_full(struct device *dev, size_t size, int dir) } static inline int -need_iommu(struct device *dev, unsigned long addr, size_t size) +need_iommu(struct device *dev, unsigned long addr, size_t size, unsigned long attrs) { - return force_iommu || !dma_capable(dev, addr, size, true); + return force_iommu || !dma_capable(dev, addr, size, true, attrs); } static inline int -nonforced_iommu(struct device *dev, unsigned long addr, size_t size) +nonforced_iommu(struct device *dev, unsigned long addr, size_t size, + unsigned long attrs) { - return !dma_capable(dev, addr, size, true); + return !dma_capable(dev, addr, size, true, attrs); } /* Map a single continuous physical area into the IOMMU. * Caller needs to check if the iommu is needed and flush. */ static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem, - size_t size, int dir, unsigned long align_mask) + size_t size, int dir, unsigned long align_mask, unsigned long attrs) { unsigned long npages = iommu_num_pages(phys_mem, size, PAGE_SIZE); unsigned long iommu_page; @@ -206,7 +207,7 @@ static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem, iommu_page = alloc_iommu(dev, npages, align_mask); if (iommu_page == -1) { - if (!nonforced_iommu(dev, phys_mem, size)) + if (!nonforced_iommu(dev, phys_mem, size, attrs)) return phys_mem; if (panic_on_overflow) panic("dma_map_area overflow %lu bytes\n", size); @@ -231,10 +232,10 @@ static dma_addr_t gart_map_phys(struct device *dev, phys_addr_t paddr, if (unlikely(attrs & DMA_ATTR_MMIO)) return DMA_MAPPING_ERROR; - if (!need_iommu(dev, paddr, size)) + if (!need_iommu(dev, paddr, size, attrs)) return paddr; - bus = dma_map_area(dev, paddr, size, dir, 0); + bus = dma_map_area(dev, paddr, size, dir, 0, attrs); flush_gart(); return bus; @@ -289,7 +290,7 @@ static void gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, /* Fallback for dma_map_sg in case of overflow */ static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg, - int nents, int dir) + int nents, int dir, unsigned long attrs) { struct scatterlist *s; int i; @@ -301,8 +302,8 @@ static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg, for_each_sg(sg, s, nents, i) { unsigned long addr = sg_phys(s); - if (nonforced_iommu(dev, addr, s->length)) { - addr = dma_map_area(dev, addr, s->length, dir, 0); + if (nonforced_iommu(dev, addr, s->length, attrs)) { + addr = dma_map_area(dev, addr, s->length, dir, 0, attrs); if (addr == DMA_MAPPING_ERROR) { if (i > 0) gart_unmap_sg(dev, sg, i, dir, 0); @@ -401,7 +402,7 @@ static int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents, s->dma_address = addr; BUG_ON(s->length == 0); - nextneed = need_iommu(dev, addr, s->length); + nextneed = need_iommu(dev, addr, s->length, attrs); /* Handle the previous not yet processed entries */ if (i > start) { @@ -449,7 +450,7 @@ error: /* When it was forced or merged try again in a dumb way */ if (force_iommu || iommu_merge) { - out = dma_map_sg_nonforce(dev, sg, nents, dir); + out = dma_map_sg_nonforce(dev, sg, nents, dir, attrs); if (out > 0) return out; } @@ -473,7 +474,8 @@ gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, return vaddr; *dma_addr = dma_map_area(dev, virt_to_phys(vaddr), size, - DMA_BIDIRECTIONAL, (1UL << get_order(size)) - 1); + DMA_BIDIRECTIONAL, + (1UL << get_order(size)) - 1, attrs); flush_gart(); if (unlikely(*dma_addr == DMA_MAPPING_ERROR)) goto out_free; diff --git a/arch/x86/kernel/amd_node.c b/arch/x86/kernel/amd_node.c index 0be01725a2a4..b7926ba3610a 100644 --- a/arch/x86/kernel/amd_node.c +++ b/arch/x86/kernel/amd_node.c @@ -38,7 +38,6 @@ static struct pci_dev **amd_roots; /* Protect the PCI config register pairs used for SMN. */ static DEFINE_MUTEX(smn_mutex); -static bool smn_exclusive; #define SMN_INDEX_OFFSET 0x60 #define SMN_DATA_OFFSET 0x64 @@ -91,11 +90,16 @@ static int __amd_smn_rw(u8 i_off, u8 d_off, u16 node, u32 address, u32 *value, b if (node >= amd_num_nodes()) return err; - root = amd_roots[node]; - if (!root) + /* + * Uninitialized amd_roots indicates pci_request_config_region_exclusive() + * didn't run or failed and thus the kernel cannot rely on having + * exclusive access to SMN registers so prevent that. + */ + if (!amd_roots) return err; - if (!smn_exclusive) + root = amd_roots[node]; + if (!root) return err; guard(mutex)(&smn_mutex); @@ -247,7 +251,7 @@ __setup("amd_smn_debugfs_enable", amd_smn_enable_dfs); static int __init amd_smn_init(void) { u16 count, num_roots, roots_per_node, node, num_nodes; - struct pci_dev *root; + struct pci_dev *root __free(pci_dev_put) = NULL; if (!cpu_feature_enabled(X86_FEATURE_ZEN)) return 0; @@ -258,7 +262,6 @@ static int __init amd_smn_init(void) return 0; num_roots = 0; - root = NULL; while ((root = get_next_root(root))) { pci_dbg(root, "Reserving PCI config space\n"); @@ -287,17 +290,21 @@ static int __init amd_smn_init(void) return -ENOMEM; roots_per_node = num_roots / num_nodes; + if (!roots_per_node) { + if (!cpu_feature_enabled(X86_FEATURE_HYPERVISOR)) + pr_warn(FW_BUG "Error detecting roots per node.\n"); + roots_per_node = 1; + } count = 0; node = 0; - root = NULL; while (node < num_nodes && (root = get_next_root(root))) { /* Use one root for each node and skip the rest. */ if (count++ % roots_per_node) continue; pci_dbg(root, "is root for AMD node %u\n", node); - amd_roots[node++] = root; + amd_roots[node++] = pci_dev_get(root); } if (enable_dfs) { @@ -308,8 +315,6 @@ static int __init amd_smn_init(void) debugfs_create_file("value", 0600, debugfs_dir, NULL, &smn_value_fops); } - smn_exclusive = true; - return 0; } diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index aa1e19979aa8..90025451ace2 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -1191,11 +1191,11 @@ void disable_local_APIC(void) * restore the disabled state. */ if (enabled_via_apicbase) { - unsigned int l, h; + struct msr val; - rdmsr(MSR_IA32_APICBASE, l, h); - l &= ~MSR_IA32_APICBASE_ENABLE; - wrmsr(MSR_IA32_APICBASE, l, h); + rdmsrq(MSR_IA32_APICBASE, val.q); + val.l &= ~MSR_IA32_APICBASE_ENABLE; + wrmsrq(MSR_IA32_APICBASE, val.q); } #endif } @@ -1960,7 +1960,8 @@ static bool __init detect_init_APIC(void) static bool __init apic_verify(unsigned long addr) { - u32 features, h, l; + struct msr val; + u32 features; /* * The APIC feature bit should now be enabled @@ -1975,9 +1976,9 @@ static bool __init apic_verify(unsigned long addr) /* The BIOS may have set up the APIC at some other address */ if (boot_cpu_data.x86 >= 6) { - rdmsr(MSR_IA32_APICBASE, l, h); - if (l & MSR_IA32_APICBASE_ENABLE) - addr = l & MSR_IA32_APICBASE_BASE; + rdmsrq(MSR_IA32_APICBASE, val.q); + if (val.l & MSR_IA32_APICBASE_ENABLE) + addr = val.l & MSR_IA32_APICBASE_BASE; } register_lapic_address(addr); @@ -1987,7 +1988,7 @@ static bool __init apic_verify(unsigned long addr) bool __init apic_force_enable(unsigned long addr) { - u32 h, l; + struct msr val; if (apic_is_disabled) return false; @@ -1998,12 +1999,12 @@ bool __init apic_force_enable(unsigned long addr) * and AMD K7 (Model > 1) or later. */ if (boot_cpu_data.x86 >= 6) { - rdmsr(MSR_IA32_APICBASE, l, h); - if (!(l & MSR_IA32_APICBASE_ENABLE)) { + rdmsrq(MSR_IA32_APICBASE, val.q); + if (!(val.l & MSR_IA32_APICBASE_ENABLE)) { pr_info("Local APIC disabled by BIOS -- reenabling.\n"); - l &= ~MSR_IA32_APICBASE_BASE; - l |= MSR_IA32_APICBASE_ENABLE | addr; - wrmsr(MSR_IA32_APICBASE, l, h); + val.l &= ~MSR_IA32_APICBASE_BASE; + val.l |= MSR_IA32_APICBASE_ENABLE | addr; + wrmsrq(MSR_IA32_APICBASE, val.q); enabled_via_apicbase = 1; } } @@ -2442,7 +2443,7 @@ static int lapic_suspend(void *data) static void lapic_resume(void *data) { - unsigned int l, h; + struct msr val; unsigned long flags; int maxlvt; @@ -2475,10 +2476,10 @@ static void lapic_resume(void *data) * SMP! We'll need to do this as part of the CPU restore! */ if (boot_cpu_data.x86 >= 6) { - rdmsr(MSR_IA32_APICBASE, l, h); - l &= ~MSR_IA32_APICBASE_BASE; - l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr; - wrmsr(MSR_IA32_APICBASE, l, h); + rdmsrq(MSR_IA32_APICBASE, val.q); + val.l &= ~MSR_IA32_APICBASE_BASE; + val.l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr; + wrmsrq(MSR_IA32_APICBASE, val.q); } } diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c index 5c5be2d58242..a60c8960bbfd 100644 --- a/arch/x86/kernel/apic/apic_numachip.c +++ b/arch/x86/kernel/apic/apic_numachip.c @@ -31,7 +31,7 @@ static u32 numachip1_get_apic_id(u32 x) unsigned long value; unsigned int id = (x >> 24) & 0xff; - if (static_cpu_has(X86_FEATURE_NODEID_MSR)) { + if (cpu_feature_enabled(X86_FEATURE_NODEID_MSR)) { rdmsrq(MSR_FAM10H_NODE_ID, value); id |= (value << 2) & 0xff00; } diff --git a/arch/x86/kernel/apic/local.h b/arch/x86/kernel/apic/local.h index 998efd442063..090dd71837aa 100644 --- a/arch/x86/kernel/apic/local.h +++ b/arch/x86/kernel/apic/local.h @@ -44,9 +44,7 @@ static inline unsigned int __prepare_ICR(unsigned int shortcut, int vector, #ifdef CONFIG_X86_X2APIC static inline void __x2apic_send_IPI_dest(unsigned int apicid, int vector, unsigned int dest) { - unsigned long cfg = __prepare_ICR(0, vector, dest); - - native_x2apic_icr_write(cfg, apicid); + native_x2apic_icr_write(__prepare_ICR(0, vector, dest), apicid); } #endif diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c index 10f79026e8e3..090647cc5a78 100644 --- a/arch/x86/kernel/apic/x2apic_phys.c +++ b/arch/x86/kernel/apic/x2apic_phys.c @@ -85,11 +85,9 @@ static void static void __x2apic_send_IPI_shorthand(int vector, u32 which) { - unsigned long cfg = __prepare_ICR(which, vector, 0); - /* x2apic MSRs are special and need a special fence: */ weak_wrmsr_fence(); - native_x2apic_icr_write(cfg, 0); + native_x2apic_icr_write(__prepare_ICR(which, vector, 0), 0); } void x2apic_send_IPI_allbutself(int vector) diff --git a/arch/x86/kernel/apic/x2apic_savic.c b/arch/x86/kernel/apic/x2apic_savic.c index dbc5678bc3b6..4bc6d7e018a5 100644 --- a/arch/x86/kernel/apic/x2apic_savic.c +++ b/arch/x86/kernel/apic/x2apic_savic.c @@ -243,10 +243,7 @@ static void savic_write(u32 reg, u32 data) static void send_ipi(u32 dest, unsigned int vector, unsigned int dsh) { - unsigned int icr_low; - - icr_low = __prepare_ICR(dsh, vector, APIC_DEST_PHYSICAL); - savic_icr_write(icr_low, dest); + savic_icr_write(__prepare_ICR(dsh, vector, APIC_DEST_PHYSICAL), dest); } static void savic_send_ipi(int cpu, int vector) diff --git a/arch/x86/kernel/cfi.c b/arch/x86/kernel/cfi.c index 638eb5c933e0..1df26dfdce08 100644 --- a/arch/x86/kernel/cfi.c +++ b/arch/x86/kernel/cfi.c @@ -72,8 +72,15 @@ enum bug_trap_type handle_cfi_failure(struct pt_regs *regs) switch (cfi_mode) { case CFI_KCFI: - if (!is_cfi_trap(addr)) - return BUG_TRAP_TYPE_NONE; + if (!is_cfi_trap(addr)) { + /* + * The updated kCFI sequence has "test $0xd6, %al" instead of + * "ud2", adjust the offset. + */ + addr -= 1; + if (!is_cfi_trap(addr)) + return BUG_TRAP_TYPE_NONE; + } if (!decode_cfi_insn(regs, &target, &type)) return report_cfi_failure_noaddr(regs, addr); diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index d61df70d6875..54e14ed276b5 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -113,7 +113,7 @@ static void init_amd_k5(struct cpuinfo_x86 *c) static void init_amd_k6(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_32 - u32 l, h; + struct msr val; int mbytes = get_num_physpages() >> (20-PAGE_SHIFT); if (c->x86_model < 6) { @@ -160,13 +160,13 @@ static void init_amd_k6(struct cpuinfo_x86 *c) if (mbytes > 508) mbytes = 508; - rdmsr(MSR_K6_WHCR, l, h); - if ((l&0x0000FFFF) == 0) { + rdmsrq(MSR_K6_WHCR, val.q); + if ((val.l & 0x0000FFFF) == 0) { unsigned long flags; - l = (1<<0)|((mbytes/4)<<1); + val.l = (1 << 0) | ((mbytes / 4) << 1); local_irq_save(flags); wbinvd(); - wrmsr(MSR_K6_WHCR, l, h); + wrmsrq(MSR_K6_WHCR, val.q); local_irq_restore(flags); pr_info("Enabling old style K6 write allocation for %d Mb\n", mbytes); @@ -181,13 +181,13 @@ static void init_amd_k6(struct cpuinfo_x86 *c) if (mbytes > 4092) mbytes = 4092; - rdmsr(MSR_K6_WHCR, l, h); - if ((l&0xFFFF0000) == 0) { + rdmsrq(MSR_K6_WHCR, val.q); + if ((val.l & 0xFFFF0000) == 0) { unsigned long flags; - l = ((mbytes>>2)<<22)|(1<<16); + val.l = ((mbytes >> 2) << 22) | (1 << 16); local_irq_save(flags); wbinvd(); - wrmsr(MSR_K6_WHCR, l, h); + wrmsrq(MSR_K6_WHCR, val.q); local_irq_restore(flags); pr_info("Enabling new style K6 write allocation for %d Mb\n", mbytes); @@ -207,7 +207,7 @@ static void init_amd_k6(struct cpuinfo_x86 *c) static void init_amd_k7(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_32 - u32 l, h; + struct msr val; /* * Bit 15 of Athlon specific MSR 15, needs to be 0 @@ -228,11 +228,12 @@ static void init_amd_k7(struct cpuinfo_x86 *c) * As per AMD technical note 27212 0.2 */ if ((c->x86_model == 8 && c->x86_stepping >= 1) || (c->x86_model > 8)) { - rdmsr(MSR_K7_CLK_CTL, l, h); - if ((l & 0xfff00000) != 0x20000000) { + rdmsrq(MSR_K7_CLK_CTL, val.q); + if ((val.l & 0xfff00000) != 0x20000000) { pr_info("CPU: CLK_CTL MSR was %x. Reprogramming to %x\n", - l, ((l & 0x000fffff)|0x20000000)); - wrmsr(MSR_K7_CLK_CTL, (l & 0x000fffff)|0x20000000, h); + val.l, ((val.l & 0x000fffff) | 0x20000000)); + val.l = (val.l & 0x000fffff) | 0x20000000; + wrmsrq(MSR_K7_CLK_CTL, val.q); } } @@ -616,12 +617,13 @@ clear_sev: static void early_init_amd(struct cpuinfo_x86 *c) { - u32 dummy; + u64 val; if (c->x86 >= 0xf) set_cpu_cap(c, X86_FEATURE_K8); - rdmsr_safe(MSR_AMD64_PATCH_LEVEL, &c->microcode, &dummy); + rdmsrq_safe(MSR_AMD64_PATCH_LEVEL, &val); + c->microcode = (u32)val; /* * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 82436b3534fa..56eac5611c31 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -192,8 +192,8 @@ x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest) * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported. */ - if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) && - !static_cpu_has(X86_FEATURE_VIRT_SSBD)) + if (!cpu_feature_enabled(X86_FEATURE_LS_CFG_SSBD) && + !cpu_feature_enabled(X86_FEATURE_VIRT_SSBD)) return; /* @@ -201,7 +201,7 @@ x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest) * virtual MSR value. If its not permanently enabled, evaluate * current's TIF_SSBD thread flag. */ - if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE)) + if (cpu_feature_enabled(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE)) hostval = SPEC_CTRL_SSBD; else hostval = ssbd_tif_to_spec_ctrl(ti->flags); @@ -2499,8 +2499,8 @@ static void __init ssb_apply_mitigation(void) * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may * use a completely different MSR and bit dependent on family. */ - if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) && - !static_cpu_has(X86_FEATURE_AMD_SSBD)) { + if (!cpu_feature_enabled(X86_FEATURE_SPEC_CTRL_SSBD) && + !cpu_feature_enabled(X86_FEATURE_AMD_SSBD)) { x86_amd_ssb_disable(); } else { x86_spec_ctrl_base |= SPEC_CTRL_SSBD; diff --git a/arch/x86/kernel/cpu/bus_lock.c b/arch/x86/kernel/cpu/bus_lock.c index bba28607a59a..177f964d4d36 100644 --- a/arch/x86/kernel/cpu/bus_lock.c +++ b/arch/x86/kernel/cpu/bus_lock.c @@ -50,7 +50,7 @@ static struct ratelimit_state bld_ratelimit; static unsigned int sysctl_sld_mitigate = 1; static DEFINE_SEMAPHORE(buslock_sem, 1); -#ifdef CONFIG_PROC_SYSCTL +#ifdef CONFIG_SYSCTL static const struct ctl_table sld_sysctls[] = { { .procname = "split_lock_mitigate", diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index 51a95b07831f..13ed16527905 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -654,7 +654,7 @@ static DEFINE_RAW_SPINLOCK(cache_disable_lock); */ static void maybe_flush_caches(void) { - if (!static_cpu_has(X86_FEATURE_SELFSNOOP)) + if (!cpu_feature_enabled(X86_FEATURE_SELFSNOOP)) wbinvd(); } diff --git a/arch/x86/kernel/cpu/centaur.c b/arch/x86/kernel/cpu/centaur.c index 681d2da49341..513fa1f640f9 100644 --- a/arch/x86/kernel/cpu/centaur.c +++ b/arch/x86/kernel/cpu/centaur.c @@ -22,7 +22,7 @@ static void init_c3(struct cpuinfo_x86 *c) { - u32 lo, hi; + u64 msr; /* Test for Centaur Extended Feature Flags presence */ if (cpuid_eax(0xC0000000) >= 0xC0000001) { @@ -30,17 +30,17 @@ static void init_c3(struct cpuinfo_x86 *c) /* enable ACE unit, if present and disabled */ if ((tmp & (ACE_PRESENT | ACE_ENABLED)) == ACE_PRESENT) { - rdmsr(MSR_VIA_FCR, lo, hi); - lo |= ACE_FCR; /* enable ACE unit */ - wrmsr(MSR_VIA_FCR, lo, hi); + rdmsrq(MSR_VIA_FCR, msr); + /* enable ACE unit */ + wrmsrq(MSR_VIA_FCR, msr | ACE_FCR); pr_info("CPU: Enabled ACE h/w crypto\n"); } /* enable RNG unit, if present and disabled */ if ((tmp & (RNG_PRESENT | RNG_ENABLED)) == RNG_PRESENT) { - rdmsr(MSR_VIA_RNG, lo, hi); - lo |= RNG_ENABLE; /* enable RNG unit */ - wrmsr(MSR_VIA_RNG, lo, hi); + rdmsrq(MSR_VIA_RNG, msr); + /* enable RNG unit */ + wrmsrq(MSR_VIA_RNG, msr | RNG_ENABLE); pr_info("CPU: Enabled h/w RNG\n"); } @@ -52,9 +52,8 @@ static void init_c3(struct cpuinfo_x86 *c) #ifdef CONFIG_X86_32 /* Cyrix III family needs CX8 & PGE explicitly enabled. */ if (c->x86_model >= 6 && c->x86_model <= 13) { - rdmsr(MSR_VIA_FCR, lo, hi); - lo |= (1<<1 | 1<<7); - wrmsr(MSR_VIA_FCR, lo, hi); + rdmsrq(MSR_VIA_FCR, msr); + wrmsrq(MSR_VIA_FCR, msr | (1 << 1 | 1 << 7)); set_cpu_cap(c, X86_FEATURE_CX8); } @@ -115,8 +114,9 @@ static void init_centaur(struct cpuinfo_x86 *c) char *name; u32 fcr_set = 0; u32 fcr_clr = 0; - u32 lo, hi, newlo; + u32 newlo; u32 aa, bb, cc, dd; + struct msr val; #endif early_init_centaur(c); init_intel_cacheinfo(c); @@ -169,15 +169,16 @@ static void init_centaur(struct cpuinfo_x86 *c) name = "??"; } - rdmsr(MSR_IDT_FCR1, lo, hi); - newlo = (lo|fcr_set) & (~fcr_clr); + rdmsrq(MSR_IDT_FCR1, val.q); + newlo = (val.l | fcr_set) & (~fcr_clr); - if (newlo != lo) { + if (newlo != val.l) { pr_info("Centaur FCR was 0x%X now 0x%X\n", - lo, newlo); - wrmsr(MSR_IDT_FCR1, newlo, hi); + val.l, newlo); + val.l = newlo; + wrmsrq(MSR_IDT_FCR1, val.q); } else { - pr_info("Centaur FCR is 0x%X\n", lo); + pr_info("Centaur FCR is 0x%X\n", val.l); } /* Emulate MTRRs using Centaur's MCR. */ set_cpu_cap(c, X86_FEATURE_CENTAUR_MCR); diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index a3df21d26460..c7352827f491 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -339,16 +339,16 @@ bool cpuid_feature(void) static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c) { - unsigned long lo, hi; + struct msr val; if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr) return; /* Disable processor serial number: */ - rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi); - lo |= 0x200000; - wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi); + rdmsrq(MSR_IA32_BBL_CR_CTL, val.q); + val.l |= 0x200000; + wrmsrq(MSR_IA32_BBL_CR_CTL, val.q); pr_notice("CPU serial number disabled.\n"); clear_cpu_cap(c, X86_FEATURE_PN); @@ -942,26 +942,27 @@ void get_cpu_vendor(struct cpuinfo_x86 *c) void cpu_detect(struct cpuinfo_x86 *c) { - /* Get vendor name */ - cpuid(0x00000000, (unsigned int *)&c->cpuid_level, - (unsigned int *)&c->x86_vendor_id[0], - (unsigned int *)&c->x86_vendor_id[8], - (unsigned int *)&c->x86_vendor_id[4]); + const struct leaf_0x0_0 *l0 = cpuid_leaf(c, 0x0); + const struct leaf_0x1_0 *l1; + + c->cpuid_level = l0->max_std_leaf; + *(u32 *)&c->x86_vendor_id[0] = l0->cpu_vendorid_0; + *(u32 *)&c->x86_vendor_id[4] = l0->cpu_vendorid_1; + *(u32 *)&c->x86_vendor_id[8] = l0->cpu_vendorid_2; c->x86 = 4; - /* Intel-defined flags: level 0x00000001 */ - if (c->cpuid_level >= 0x00000001) { - u32 junk, tfms, cap0, misc; - cpuid(0x00000001, &tfms, &misc, &junk, &cap0); - c->x86 = x86_family(tfms); - c->x86_model = x86_model(tfms); - c->x86_stepping = x86_stepping(tfms); + l1 = cpuid_leaf(c, 0x1); + if (!l1) + return; - if (cap0 & (1<<19)) { - c->x86_clflush_size = ((misc >> 8) & 0xff) * 8; - c->x86_cache_alignment = c->x86_clflush_size; - } + c->x86 = cpuid_family(l1); + c->x86_model = cpuid_model(l1); + c->x86_stepping = l1->stepping; + + if (l1->clflush) { + c->x86_clflush_size = l1->clflush_size * 8; + c->x86_cache_alignment = c->x86_clflush_size; } } @@ -1251,9 +1252,6 @@ static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = { #define VULNBL_INTEL_STEPS(vfm, max_stepping, issues) \ X86_MATCH_VFM_STEPS(vfm, X86_STEP_MIN, max_stepping, issues) -#define VULNBL_INTEL_TYPE(vfm, cpu_type, issues) \ - X86_MATCH_VFM_CPU_TYPE(vfm, INTEL_CPU_TYPE_##cpu_type, issues) - #define VULNBL_AMD(family, blacklist) \ VULNBL(AMD, family, X86_MODEL_ANY, blacklist) @@ -1316,11 +1314,9 @@ static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = { VULNBL_INTEL_STEPS(INTEL_TIGERLAKE, X86_STEP_MAX, GDS | ITS | ITS_NATIVE_ONLY), VULNBL_INTEL_STEPS(INTEL_LAKEFIELD, X86_STEP_MAX, MMIO | MMIO_SBDS | RETBLEED), VULNBL_INTEL_STEPS(INTEL_ROCKETLAKE, X86_STEP_MAX, MMIO | RETBLEED | GDS | ITS | ITS_NATIVE_ONLY), - VULNBL_INTEL_TYPE(INTEL_ALDERLAKE, ATOM, RFDS | VMSCAPE), - VULNBL_INTEL_STEPS(INTEL_ALDERLAKE, X86_STEP_MAX, VMSCAPE), + VULNBL_INTEL_STEPS(INTEL_ALDERLAKE, X86_STEP_MAX, RFDS | VMSCAPE), VULNBL_INTEL_STEPS(INTEL_ALDERLAKE_L, X86_STEP_MAX, RFDS | VMSCAPE), - VULNBL_INTEL_TYPE(INTEL_RAPTORLAKE, ATOM, RFDS | VMSCAPE), - VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE, X86_STEP_MAX, VMSCAPE), + VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE, X86_STEP_MAX, RFDS | VMSCAPE), VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE_P, X86_STEP_MAX, RFDS | VMSCAPE), VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE_S, X86_STEP_MAX, RFDS | VMSCAPE), VULNBL_INTEL_STEPS(INTEL_METEORLAKE_L, X86_STEP_MAX, VMSCAPE), @@ -1388,7 +1384,21 @@ static bool __init vulnerable_to_rfds(u64 x86_arch_cap_msr) return true; /* Only consult the blacklist when there is no enumeration: */ - return cpu_matches(cpu_vuln_blacklist, RFDS); + if (!cpu_matches(cpu_vuln_blacklist, RFDS)) + return false; + + /* + * ADL and RPL are affected only if they have Atom CPUs. Hybrids have + * both Core and Atom CPUs. Mark unaffected when Atom CPUs are not + * present. + */ + if ((boot_cpu_data.x86_model == 0x97 || + boot_cpu_data.x86_model == 0xB7) && + boot_cpu_data.topo.intel_type != INTEL_CPU_TYPE_ATOM && + !boot_cpu_has(X86_FEATURE_HYBRID_CPU)) + return false; + + return true; } static bool __init vulnerable_to_its(u64 x86_arch_cap_msr) @@ -1651,7 +1661,7 @@ static inline bool parse_set_clear_cpuid(char *arg, bool set) int taint = 0; while (arg) { - bool found __maybe_unused = false; + bool found = false; unsigned int bit; opt = strsep(&arg, ","); @@ -2236,7 +2246,7 @@ DEFINE_PER_CPU_CACHE_HOT(struct task_struct *, current_task) = &init_task; EXPORT_PER_CPU_SYMBOL(current_task); EXPORT_PER_CPU_SYMBOL(const_current_task); -DEFINE_PER_CPU_CACHE_HOT(int, __preempt_count) = INIT_PREEMPT_COUNT; +DEFINE_PER_CPU_CACHE_HOT(unsigned long, __preempt_count) = INIT_PREEMPT_COUNT; EXPORT_PER_CPU_SYMBOL(__preempt_count); DEFINE_PER_CPU_CACHE_HOT(unsigned long, cpu_current_top_of_stack) = TOP_OF_INIT_STACK; @@ -2299,8 +2309,10 @@ static inline void idt_syscall_init(void) /* May not be marked __init: used by software suspend */ void syscall_init(void) { + struct msr val = { .h = (__USER32_CS << 16) | __KERNEL_CS }; + /* The default user and kernel segments */ - wrmsr(MSR_STAR, 0, (__USER32_CS << 16) | __KERNEL_CS); + wrmsrq(MSR_STAR, val.q); /* * Except the IA32_STAR MSR, there is NO need to setup SYSCALL and @@ -2664,3 +2676,13 @@ void __init arch_cpu_finalize_init(void) */ mem_encrypt_init(); } + +/* Control TLB flushing methods */ +static int __init tlbi_setup(char *str) +{ + if (!strcmp(str, "ipi")) + setup_clear_cpu_cap(X86_FEATURE_INVLPGB); + + return 1; +} +__setup("tlbi=", tlbi_setup); diff --git a/arch/x86/kernel/cpu/cpuid_parser.c b/arch/x86/kernel/cpu/cpuid_parser.c index 898b0c441431..dcf8fffbfb88 100644 --- a/arch/x86/kernel/cpu/cpuid_parser.c +++ b/arch/x86/kernel/cpu/cpuid_parser.c @@ -11,14 +11,14 @@ #include "cpuid_parser.h" /* Clear a single CPUID table entry */ -static void cpuid_clear(const struct cpuid_parse_entry *e, const struct cpuid_read_output *output) +static void cpuid_clear(const struct cpuid_parse_entry *e, const struct cpuid_output *out) { - struct cpuid_regs *regs = output->regs; + struct cpuid_regs *regs = out->regs; for (int i = 0; i < e->maxcnt; i++, regs++) memset(regs, 0, sizeof(*regs)); - memset(output->info, 0, sizeof(*output->info)); + memset(out->info, 0, sizeof(*out->info)); } /* @@ -29,12 +29,11 @@ static void cpuid_clear(const struct cpuid_parse_entry *e, const struct cpuid_re * Default CPUID read function * Satisfies the requirements stated at 'struct cpuid_parse_entry'->read(). */ -static void -cpuid_read_generic(const struct cpuid_parse_entry *e, const struct cpuid_read_output *output) +static void cpuid_read_generic(const struct cpuid_parse_entry *e, const struct cpuid_output *out) { - struct cpuid_regs *regs = output->regs; + struct cpuid_regs *regs = out->regs; - for (int i = 0; i < e->maxcnt; i++, regs++, output->info->nr_entries++) + for (int i = 0; i < e->maxcnt; i++, regs++, out->info->nr_entries++) cpuid_read_subleaf(e->leaf, e->subleaf + i, regs); } @@ -60,15 +59,14 @@ static unsigned int cpuid_range_max_leaf(const struct cpuid_table *t, unsigned i } } -static void -__cpuid_reset_table(struct cpuid_table *t, const struct cpuid_parse_entry entries[], - unsigned int nr_entries, unsigned int start, unsigned int end, bool fill) +static void __cpuid_reset_table(struct cpuid_table *t, const struct cpuid_parse_entry entries[], + unsigned int nr_entries, unsigned int start, unsigned int end, bool fill) { const struct cpuid_parse_entry *entry = entries; unsigned int range = CPUID_RANGE(start); for (unsigned int i = 0; i < nr_entries; i++, entry++) { - struct cpuid_read_output output = { + struct cpuid_output out = { .regs = cpuid_table_regs_p(t, entry->regs_offs), .info = cpuid_table_info_p(t, entry->info_offs), }; @@ -76,14 +74,14 @@ __cpuid_reset_table(struct cpuid_table *t, const struct cpuid_parse_entry entrie if (entry->leaf < start || entry->leaf > end) continue; - cpuid_clear(entry, &output); + cpuid_clear(entry, &out); /* * Read the range's anchor leaf unconditionally so that the cached * maximum valid leaf value is available for the remaining entries. */ if (fill && (entry->leaf == range || entry->leaf <= cpuid_range_max_leaf(t, range))) - entry->read(entry, &output); + entry->read(entry, &out); } } @@ -91,22 +89,20 @@ __cpuid_reset_table(struct cpuid_table *t, const struct cpuid_parse_entry entrie * Zero all cached CPUID entries within [@start-@end] range. This is needed when * certain operations like MSR writes induce changes to the CPU's CPUID layout. */ -static void -__cpuid_zero_table(struct cpuid_table *t, const struct cpuid_parse_entry entries[], - unsigned int nr_entries, unsigned int start, unsigned int end) +static void __cpuid_zero_table(struct cpuid_table *t, const struct cpuid_parse_entry entries[], + unsigned int nr_entries, unsigned int start, unsigned int end) { __cpuid_reset_table(t, entries, nr_entries, start, end, false); } -static void -__cpuid_fill_table(struct cpuid_table *t, const struct cpuid_parse_entry entries[], - unsigned int nr_entries, unsigned int start, unsigned int end) +static void __cpuid_fill_table(struct cpuid_table *t, const struct cpuid_parse_entry entries[], + unsigned int nr_entries, unsigned int start, unsigned int end) { __cpuid_reset_table(t, entries, nr_entries, start, end, true); } -static void -cpuid_fill_table(struct cpuid_table *t, const struct cpuid_parse_entry entries[], unsigned int nr_entries) +static void cpuid_fill_table(struct cpuid_table *t, const struct cpuid_parse_entry entries[], + unsigned int nr_entries) { static const struct { unsigned int start; @@ -127,8 +123,8 @@ static void __cpuid_scan_cpu_full(struct cpuinfo_x86 *c) cpuid_fill_table(table, cpuid_parse_entries, nr_entries); } -static void -__cpuid_scan_cpu_partial(struct cpuinfo_x86 *c, unsigned int start_leaf, unsigned int end_leaf) +static void __cpuid_scan_cpu_partial(struct cpuinfo_x86 *c, unsigned int start_leaf, + unsigned int end_leaf) { unsigned int nr_entries = ARRAY_SIZE(cpuid_parse_entries); struct cpuid_table *table = &c->cpuid; diff --git a/arch/x86/kernel/cpu/cpuid_parser.h b/arch/x86/kernel/cpu/cpuid_parser.h index df627306cc8c..3d8337323958 100644 --- a/arch/x86/kernel/cpu/cpuid_parser.h +++ b/arch/x86/kernel/cpu/cpuid_parser.h @@ -35,21 +35,21 @@ * Translation of compile time offsets to generic runtime pointers: */ -static inline struct cpuid_regs * -cpuid_table_regs_p(const struct cpuid_table *t, unsigned long regs_offset) +static inline struct cpuid_regs *cpuid_table_regs_p(const struct cpuid_table *t, + unsigned long regs_offset) { return (struct cpuid_regs *)((unsigned long)(&t->leaves) + regs_offset); } -static inline struct leaf_parse_info * -cpuid_table_info_p(const struct cpuid_table *t, unsigned long info_offset) +static inline struct leaf_parse_info *cpuid_table_info_p(const struct cpuid_table *t, + unsigned long info_offset) { return (struct leaf_parse_info *)((unsigned long)(&t->leaves) + info_offset); } /** - * struct cpuid_read_output - Output of a CPUID read operation - * @regs: Pointer to an array of CPUID outputs, where each array element covers the + * struct cpuid_output - Output of a CPUID operation + * @regs: Pointer to an array of CPUID results, where each array element covers the * full EAX->EDX output range. * @info: Pointer to query info; for saving the number of filled elements at @regs. * @@ -59,7 +59,7 @@ cpuid_table_info_p(const struct cpuid_table *t, unsigned long info_offset) * * See struct cpuid_parse_entry.read(). */ -struct cpuid_read_output { +struct cpuid_output { struct cpuid_regs *regs; struct leaf_parse_info *info; }; @@ -74,8 +74,8 @@ struct cpuid_read_output { * passed to cpuid_table_info_p(). * @maxcnt: Maximum number of output storage entries available for the CPUID query. * @read: Read function for this entry. It must save the parsed CPUID output to the passed - * 'struct cpuid_read_output'->regs array of size >= @maxcnt. It must set - * 'struct cpuid_read_output'->info.nr_entries to the number of CPUID output entries + * 'struct cpuid_output'->regs array of size >= @maxcnt. It must set + * 'struct cpuid_output'->info.nr_entries to the number of CPUID output entries * parsed and filled. A generic implementation is provided at cpuid_read_generic(). */ struct cpuid_parse_entry { @@ -84,7 +84,7 @@ struct cpuid_parse_entry { unsigned int regs_offs; unsigned int info_offs; unsigned int maxcnt; - void (*read)(const struct cpuid_parse_entry *e, const struct cpuid_read_output *o); + void (*read)(const struct cpuid_parse_entry *e, const struct cpuid_output *o); }; #define __CPUID_PARSE_ENTRY(_leaf, _subleaf, _suffix, _reader_fn) \ diff --git a/arch/x86/kernel/cpu/feat_ctl.c b/arch/x86/kernel/cpu/feat_ctl.c index d69757246bde..10a92927a515 100644 --- a/arch/x86/kernel/cpu/feat_ctl.c +++ b/arch/x86/kernel/cpu/feat_ctl.c @@ -25,7 +25,8 @@ enum vmx_feature_leafs { static void init_vmx_capabilities(struct cpuinfo_x86 *c) { - u32 supported, funcs, ept, vpid, ign, low, high; + struct msr val; + u32 supported, funcs, ept, vpid; BUILD_BUG_ON(NVMXINTS != NR_VMX_FEATURE_WORDS); @@ -39,25 +40,31 @@ static void init_vmx_capabilities(struct cpuinfo_x86 *c) * as they exist on any CPU that supports VMX, i.e. we want the WARN if * the RDMSR faults. */ - rdmsr(MSR_IA32_VMX_PROCBASED_CTLS, ign, supported); + rdmsrq(MSR_IA32_VMX_PROCBASED_CTLS, val.q); + supported = val.h; c->vmx_capability[PRIMARY_CTLS] = supported; - rdmsr_safe(MSR_IA32_VMX_PROCBASED_CTLS2, &ign, &supported); + rdmsrq_safe(MSR_IA32_VMX_PROCBASED_CTLS2, &val.q); + supported = val.h; c->vmx_capability[SECONDARY_CTLS] = supported; /* All 64 bits of tertiary controls MSR are allowed-1 settings. */ - rdmsr_safe(MSR_IA32_VMX_PROCBASED_CTLS3, &low, &high); - c->vmx_capability[TERTIARY_CTLS_LOW] = low; - c->vmx_capability[TERTIARY_CTLS_HIGH] = high; + rdmsrq_safe(MSR_IA32_VMX_PROCBASED_CTLS3, &val.q); + c->vmx_capability[TERTIARY_CTLS_LOW] = val.l; + c->vmx_capability[TERTIARY_CTLS_HIGH] = val.h; - rdmsr(MSR_IA32_VMX_PINBASED_CTLS, ign, supported); - rdmsr_safe(MSR_IA32_VMX_VMFUNC, &ign, &funcs); + rdmsrq(MSR_IA32_VMX_PINBASED_CTLS, val.q); + supported = val.h; + rdmsrq_safe(MSR_IA32_VMX_VMFUNC, &val.q); + funcs = val.h; /* * Except for EPT+VPID, which enumerates support for both in a single * MSR, low for EPT, high for VPID. */ - rdmsr_safe(MSR_IA32_VMX_EPT_VPID_CAP, &ept, &vpid); + rdmsrq_safe(MSR_IA32_VMX_EPT_VPID_CAP, &val.q); + ept = val.l; + vpid = val.h; /* Pin, EPT, VPID and VM-Func are merged into a single word. */ WARN_ON_ONCE(supported >> 16); diff --git a/arch/x86/kernel/cpu/hygon.c b/arch/x86/kernel/cpu/hygon.c index 3e8891a9caf2..ec51c2b9a257 100644 --- a/arch/x86/kernel/cpu/hygon.c +++ b/arch/x86/kernel/cpu/hygon.c @@ -125,11 +125,12 @@ static void bsp_init_hygon(struct cpuinfo_x86 *c) static void early_init_hygon(struct cpuinfo_x86 *c) { - u32 dummy; + u64 val; set_cpu_cap(c, X86_FEATURE_K8); - rdmsr_safe(MSR_AMD64_PATCH_LEVEL, &c->microcode, &dummy); + rdmsrq_safe(MSR_AMD64_PATCH_LEVEL, &val); + c->microcode = (u32)val; /* * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate diff --git a/arch/x86/kernel/cpu/hypervisor.c b/arch/x86/kernel/cpu/hypervisor.c index f3e9219845e8..73428afca796 100644 --- a/arch/x86/kernel/cpu/hypervisor.c +++ b/arch/x86/kernel/cpu/hypervisor.c @@ -31,7 +31,7 @@ static const __initconst struct hypervisor_x86 * const hypervisors[] = #ifdef CONFIG_XEN_PV &x86_hyper_xen_pv, #endif -#ifdef CONFIG_XEN_PVHVM +#ifdef CONFIG_XEN &x86_hyper_xen_hvm, #endif &x86_hyper_vmware, diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index abb3984336eb..4297ceb2cb24 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -199,6 +199,41 @@ void intel_unlock_cpuid_leafs(struct cpuinfo_x86 *c) c->cpuid_level = cpuid_eax(0); } +/* + * Use CPUID to generate a "vfm" value. Useful before cpuinfo_x86 + * structures are populated. + */ +static u32 intel_cpuid_vfm(void) +{ + u32 eax = cpuid_eax(1); + u32 fam = x86_family(eax); + u32 model = x86_model(eax); + + return IFM(fam, model); +} + +u32 intel_get_platform_id(void) +{ + unsigned int val[2]; + + if (x86_hypervisor_present) + return 0; + + /* + * This can be called early. Use CPUID directly instead of + * relying on cpuinfo_x86 which may not be fully initialized. + * The PII does not have MSR_IA32_PLATFORM_ID. Everything + * before _it_ has no microcode (for Linux at least). + */ + if (intel_cpuid_vfm() <= INTEL_PENTIUM_II_KLAMATH) + return 0; + + /* get processor flags from MSR 0x17 */ + native_rdmsr(MSR_IA32_PLATFORM_ID, val[0], val[1]); + + return (val[1] >> 18) & 7; +} + static void early_init_intel(struct cpuinfo_x86 *c) { u64 misc_enable; @@ -542,12 +577,12 @@ static void init_intel(struct cpuinfo_x86 *c) set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC); if (boot_cpu_has(X86_FEATURE_DS)) { - unsigned int l1, l2; + u64 l; - rdmsr(MSR_IA32_MISC_ENABLE, l1, l2); - if (!(l1 & MSR_IA32_MISC_ENABLE_BTS_UNAVAIL)) + rdmsrq(MSR_IA32_MISC_ENABLE, l); + if (!(l & MSR_IA32_MISC_ENABLE_BTS_UNAVAIL)) set_cpu_cap(c, X86_FEATURE_BTS); - if (!(l1 & MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL)) + if (!(l & MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL)) set_cpu_cap(c, X86_FEATURE_PEBS); } diff --git a/arch/x86/kernel/cpu/match.c b/arch/x86/kernel/cpu/match.c index 4604802692da..7ab077f0cc66 100644 --- a/arch/x86/kernel/cpu/match.c +++ b/arch/x86/kernel/cpu/match.c @@ -6,34 +6,6 @@ #include <linux/slab.h> /** - * x86_match_vendor_cpu_type - helper function to match the hardware defined - * cpu-type for a single entry in the x86_cpu_id - * table. Note, this function does not match the - * generic cpu-types TOPO_CPU_TYPE_EFFICIENCY and - * TOPO_CPU_TYPE_PERFORMANCE. - * @c: Pointer to the cpuinfo_x86 structure of the CPU to match. - * @m: Pointer to the x86_cpu_id entry to match against. - * - * Return: true if the cpu-type matches, false otherwise. - */ -static bool x86_match_vendor_cpu_type(struct cpuinfo_x86 *c, const struct x86_cpu_id *m) -{ - if (m->type == X86_CPU_TYPE_ANY) - return true; - - /* Hybrid CPUs are special, they are assumed to match all cpu-types */ - if (cpu_feature_enabled(X86_FEATURE_HYBRID_CPU)) - return true; - - if (c->x86_vendor == X86_VENDOR_INTEL) - return m->type == c->topo.intel_type; - if (c->x86_vendor == X86_VENDOR_AMD) - return m->type == c->topo.amd_type; - - return false; -} - -/** * x86_match_cpu - match current CPU against an array of x86_cpu_ids * @match: Pointer to array of x86_cpu_ids. Last entry terminated with * {}. @@ -81,7 +53,7 @@ const struct x86_cpu_id *x86_match_cpu(const struct x86_cpu_id *match) continue; if (m->feature != X86_FEATURE_ANY && !cpu_has(c, m->feature)) continue; - if (!x86_match_vendor_cpu_type(c, m)) + if (m->type != X86_CPU_TYPE_ANY && c->topo.cpu_type != m->type) continue; return m; } diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c index 36e0df4d1342..1cc20b855b7e 100644 --- a/arch/x86/kernel/cpu/mce/amd.c +++ b/arch/x86/kernel/cpu/mce/amd.c @@ -280,11 +280,11 @@ static void smca_configure(unsigned int bank, unsigned int cpu) u8 *bank_counts = this_cpu_ptr(smca_bank_counts); const struct smca_hwid *s_hwid; unsigned int i, hwid_mcatype; - u32 high, low; + struct msr val; u32 smca_config = MSR_AMD64_SMCA_MCx_CONFIG(bank); /* Set appropriate bits in MCA_CONFIG */ - if (!rdmsr_safe(smca_config, &low, &high)) { + if (!rdmsrq_safe(smca_config, &val.q)) { /* * OS is required to set the MCAX bit to acknowledge that it is * now using the new MSR ranges and new registers under each @@ -294,7 +294,7 @@ static void smca_configure(unsigned int bank, unsigned int cpu) * * MCA_CONFIG[MCAX] is bit 32 (0 in the high portion of the MSR.) */ - high |= BIT(0); + val.h |= BIT(0); /* * SMCA sets the Deferred Error Interrupt type per bank. @@ -307,9 +307,9 @@ static void smca_configure(unsigned int bank, unsigned int cpu) * APIC based interrupt. First, check that no interrupt has been * set. */ - if ((low & BIT(5)) && !((high >> 5) & 0x3) && data->dfr_intr_en) { + if ((val.l & BIT(5)) && !((val.h >> 5) & 0x3) && data->dfr_intr_en) { __set_bit(bank, data->dfr_intr_banks); - high |= BIT(5); + val.h |= BIT(5); } /* @@ -324,33 +324,33 @@ static void smca_configure(unsigned int bank, unsigned int cpu) * The OS should set this to inform the platform that the OS is ready * to handle the MCA Thresholding interrupt. */ - if ((low & BIT(10)) && data->thr_intr_en) { + if ((val.l & BIT(10)) && data->thr_intr_en) { __set_bit(bank, data->thr_intr_banks); - high |= BIT(8); + val.h |= BIT(8); } - this_cpu_ptr(mce_banks_array)[bank].lsb_in_status = !!(low & BIT(8)); + this_cpu_ptr(mce_banks_array)[bank].lsb_in_status = !!(val.l & BIT(8)); - if (low & MCI_CONFIG_PADDRV) + if (val.l & MCI_CONFIG_PADDRV) this_cpu_ptr(smca_banks)[bank].paddrv = 1; - wrmsr(smca_config, low, high); + wrmsrq(smca_config, val.q); } - if (rdmsr_safe(MSR_AMD64_SMCA_MCx_IPID(bank), &low, &high)) { + if (rdmsrq_safe(MSR_AMD64_SMCA_MCx_IPID(bank), &val.q)) { pr_warn("Failed to read MCA_IPID for bank %d\n", bank); return; } - hwid_mcatype = HWID_MCATYPE(high & MCI_IPID_HWID, - (high & MCI_IPID_MCATYPE) >> 16); + hwid_mcatype = HWID_MCATYPE(val.h & MCI_IPID_HWID, + (val.h & MCI_IPID_MCATYPE) >> 16); for (i = 0; i < ARRAY_SIZE(smca_hwid_mcatypes); i++) { s_hwid = &smca_hwid_mcatypes[i]; if (hwid_mcatype == s_hwid->hwid_mcatype) { this_cpu_ptr(smca_banks)[bank].hwid = s_hwid; - this_cpu_ptr(smca_banks)[bank].id = low; + this_cpu_ptr(smca_banks)[bank].id = val.l; this_cpu_ptr(smca_banks)[bank].sysfs_id = bank_counts[s_hwid->bank_type]++; break; } @@ -432,50 +432,50 @@ static bool lvt_off_valid(struct threshold_block *b, int apic, u32 lo, u32 hi) static void threshold_restart_block(void *_tr) { struct thresh_restart *tr = _tr; - u32 hi, lo; + struct msr val; /* sysfs write might race against an offline operation */ if (!this_cpu_read(threshold_banks) && !tr->set_lvt_off) return; - rdmsr(tr->b->address, lo, hi); + rdmsrq(tr->b->address, val.q); /* * Reset error count and overflow bit. * This is done during init or after handling an interrupt. */ - if (hi & MASK_OVERFLOW_HI || tr->set_lvt_off) { - hi &= ~(MASK_ERR_COUNT_HI | MASK_OVERFLOW_HI); - hi |= THRESHOLD_MAX - tr->b->threshold_limit; + if (val.h & MASK_OVERFLOW_HI || tr->set_lvt_off) { + val.h &= ~(MASK_ERR_COUNT_HI | MASK_OVERFLOW_HI); + val.h |= THRESHOLD_MAX - tr->b->threshold_limit; } else if (tr->old_limit) { /* change limit w/o reset */ - int new_count = (hi & THRESHOLD_MAX) + + int new_count = (val.h & THRESHOLD_MAX) + (tr->old_limit - tr->b->threshold_limit); - hi = (hi & ~MASK_ERR_COUNT_HI) | + val.h = (val.h & ~MASK_ERR_COUNT_HI) | (new_count & THRESHOLD_MAX); } /* clear IntType */ - hi &= ~MASK_INT_TYPE_HI; + val.h &= ~MASK_INT_TYPE_HI; if (!tr->b->interrupt_capable) goto done; if (tr->set_lvt_off) { - if (lvt_off_valid(tr->b, tr->lvt_off, lo, hi)) { + if (lvt_off_valid(tr->b, tr->lvt_off, val.l, val.h)) { /* set new lvt offset */ - hi &= ~MASK_LVTOFF_HI; - hi |= tr->lvt_off << 20; + val.h &= ~MASK_LVTOFF_HI; + val.h |= tr->lvt_off << 20; } } if (tr->b->interrupt_enable) - hi |= INT_TYPE_APIC; + val.h |= INT_TYPE_APIC; done: - hi |= MASK_COUNT_EN_HI; - wrmsr(tr->b->address, lo, hi); + val.h |= MASK_COUNT_EN_HI; + wrmsrq(tr->b->address, val.q); } static void threshold_restart_bank(unsigned int bank, bool intr_en) @@ -726,7 +726,8 @@ static void smca_enable_interrupt_vectors(void) void mce_amd_feature_init(struct cpuinfo_x86 *c) { unsigned int bank, block, cpu = smp_processor_id(); - u32 low = 0, high = 0, address = 0; + struct msr val = { .q = 0 }; + u32 address = 0; int offset = -1; amd_apply_cpu_quirks(c); @@ -746,21 +747,21 @@ void mce_amd_feature_init(struct cpuinfo_x86 *c) disable_err_thresholding(c, bank); for (block = 0; block < NR_BLOCKS; ++block) { - address = get_block_address(address, low, high, bank, block, cpu); + address = get_block_address(address, val.l, val.h, bank, block, cpu); if (!address) break; - if (rdmsr_safe(address, &low, &high)) + if (rdmsrq_safe(address, &val.q)) break; - if (!(high & MASK_VALID_HI)) + if (!(val.h & MASK_VALID_HI)) continue; - if (!(high & MASK_CNTP_HI) || - (high & MASK_LOCKED_HI)) + if (!(val.h & MASK_CNTP_HI) || + (val.h & MASK_LOCKED_HI)) continue; - offset = prepare_threshold_block(bank, block, address, offset, high); + offset = prepare_threshold_block(bank, block, address, offset, val.h); } } } @@ -864,7 +865,7 @@ static void amd_deferred_error_interrupt(void) void mce_amd_handle_storm(unsigned int bank, bool on) { - threshold_restart_bank(bank, on); + threshold_restart_bank(bank, !on); } static void amd_reset_thr_limit(unsigned int bank) @@ -1083,24 +1084,24 @@ static int allocate_threshold_blocks(unsigned int cpu, struct threshold_bank *tb u32 address) { struct threshold_block *b = NULL; - u32 low, high; + struct msr val; int err; if ((bank >= this_cpu_read(mce_num_banks)) || (block >= NR_BLOCKS)) return 0; - if (rdmsr_safe(address, &low, &high)) + if (rdmsrq_safe(address, &val.q)) return 0; - if (!(high & MASK_VALID_HI)) { + if (!(val.h & MASK_VALID_HI)) { if (block) goto recurse; else return 0; } - if (!(high & MASK_CNTP_HI) || - (high & MASK_LOCKED_HI)) + if (!(val.h & MASK_CNTP_HI) || + (val.h & MASK_LOCKED_HI)) goto recurse; b = kzalloc_obj(struct threshold_block); @@ -1112,7 +1113,7 @@ static int allocate_threshold_blocks(unsigned int cpu, struct threshold_bank *tb b->cpu = cpu; b->address = address; b->interrupt_enable = 0; - b->interrupt_capable = lvt_interrupt_supported(bank, high); + b->interrupt_capable = lvt_interrupt_supported(bank, val.h); b->threshold_limit = get_thr_limit(); if (b->interrupt_capable) { @@ -1124,13 +1125,13 @@ static int allocate_threshold_blocks(unsigned int cpu, struct threshold_bank *tb list_add(&b->miscj, &tb->miscj); - mce_threshold_block_init(b, (high & MASK_LVTOFF_HI) >> 20); + mce_threshold_block_init(b, (val.h & MASK_LVTOFF_HI) >> 20); err = kobject_init_and_add(&b->kobj, &threshold_ktype, tb->kobj, get_name(cpu, bank, b)); if (err) goto out_free; recurse: - address = get_block_address(address, low, high, bank, ++block, cpu); + address = get_block_address(address, val.l, val.h, bank, ++block, cpu); if (!address) return 0; diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 9bba1e2f03af..ab469605fc89 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -1866,7 +1866,7 @@ static void __mcheck_cpu_init_generic(void) rdmsrq(MSR_IA32_MCG_CAP, cap); if (cap & MCG_CTL_P) - wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff); + wrmsrq(MSR_IA32_MCG_CTL, ~0ULL); } static void __mcheck_cpu_init_prepare_banks(void) @@ -2266,10 +2266,10 @@ void mcheck_cpu_init(struct cpuinfo_x86 *c) mca_cfg.initialized = 1; + __mcheck_cpu_setup_timer(); __mcheck_cpu_init_generic(); __mcheck_cpu_init_vendor(c); __mcheck_cpu_init_prepare_banks(); - __mcheck_cpu_setup_timer(); cr4_set_bits(X86_CR4_MCE); } diff --git a/arch/x86/kernel/cpu/mce/p5.c b/arch/x86/kernel/cpu/mce/p5.c index 2272ad53fc33..3c2b6cc918b1 100644 --- a/arch/x86/kernel/cpu/mce/p5.c +++ b/arch/x86/kernel/cpu/mce/p5.c @@ -23,16 +23,16 @@ int mce_p5_enabled __read_mostly; /* Machine check handler for Pentium class Intel CPUs: */ noinstr void pentium_machine_check(struct pt_regs *regs) { - u32 loaddr, hi, lotype; + u64 addr, type; instrumentation_begin(); - rdmsr(MSR_IA32_P5_MC_ADDR, loaddr, hi); - rdmsr(MSR_IA32_P5_MC_TYPE, lotype, hi); + rdmsrq(MSR_IA32_P5_MC_ADDR, addr); + rdmsrq(MSR_IA32_P5_MC_TYPE, type); pr_emerg("CPU#%d: Machine Check Exception: 0x%8X (type 0x%8X).\n", - smp_processor_id(), loaddr, lotype); + smp_processor_id(), (u32)addr, (u32)type); - if (lotype & (1<<5)) { + if (type & (1<<5)) { pr_emerg("CPU#%d: Possible thermal failure (CPU on fire ?).\n", smp_processor_id()); } @@ -44,7 +44,7 @@ noinstr void pentium_machine_check(struct pt_regs *regs) /* Set up machine check reporting for processors with Intel style MCE: */ void intel_p5_mcheck_init(struct cpuinfo_x86 *c) { - u32 l, h; + u64 __maybe_unused q; /* Default P5 to off as its often misconnected: */ if (!mce_p5_enabled) @@ -55,8 +55,8 @@ void intel_p5_mcheck_init(struct cpuinfo_x86 *c) return; /* Read registers before enabling: */ - rdmsr(MSR_IA32_P5_MC_ADDR, l, h); - rdmsr(MSR_IA32_P5_MC_TYPE, l, h); + rdmsrq(MSR_IA32_P5_MC_ADDR, q); + rdmsrq(MSR_IA32_P5_MC_TYPE, q); pr_info("Intel old style machine check architecture supported.\n"); /* Enable MCE: */ diff --git a/arch/x86/kernel/cpu/mce/winchip.c b/arch/x86/kernel/cpu/mce/winchip.c index 6c99f2941909..7040243533d9 100644 --- a/arch/x86/kernel/cpu/mce/winchip.c +++ b/arch/x86/kernel/cpu/mce/winchip.c @@ -28,12 +28,12 @@ noinstr void winchip_machine_check(struct pt_regs *regs) /* Set up machine check reporting on the Winchip C6 series */ void winchip_mcheck_init(struct cpuinfo_x86 *c) { - u32 lo, hi; + struct msr val; - rdmsr(MSR_IDT_FCR1, lo, hi); - lo |= (1<<2); /* Enable EIERRINT (int 18 MCE) */ - lo &= ~(1<<4); /* Enable MCE */ - wrmsr(MSR_IDT_FCR1, lo, hi); + rdmsrq(MSR_IDT_FCR1, val.q); + val.l |= (1<<2); /* Enable EIERRINT (int 18 MCE) */ + val.l &= ~(1<<4); /* Enable MCE */ + wrmsrq(MSR_IDT_FCR1, val.q); cr4_set_bits(X86_CR4_MCE); diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c index 531dfb771c8b..6cdc410e7547 100644 --- a/arch/x86/kernel/cpu/microcode/amd.c +++ b/arch/x86/kernel/cpu/microcode/amd.c @@ -136,8 +136,7 @@ struct cont_desc { * Microcode patch container file is prepended to the initrd in cpio * format. See Documentation/arch/x86/microcode.rst */ -static const char -ucode_path[] __maybe_unused = "kernel/x86/microcode/AuthenticAMD.bin"; +static const char ucode_path[] = "kernel/x86/microcode/AuthenticAMD.bin"; /* * This is CPUID(1).EAX on the BSP. It is used in two ways: diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c index f4a444e6114d..9f09d388ed20 100644 --- a/arch/x86/kernel/cpu/microcode/intel.c +++ b/arch/x86/kernel/cpu/microcode/intel.c @@ -121,42 +121,6 @@ static inline unsigned int exttable_size(struct extended_sigtable *et) return et->count * EXT_SIGNATURE_SIZE + EXT_HEADER_SIZE; } - -/* - * Use CPUID to generate a "vfm" value. Useful before cpuinfo_x86 - * structures are populated. - */ -static u32 intel_cpuid_vfm(void) -{ - u32 eax = cpuid_eax(1); - u32 fam = x86_family(eax); - u32 model = x86_model(eax); - - return IFM(fam, model); -} - -u32 intel_get_platform_id(void) -{ - unsigned int val[2]; - - if (x86_hypervisor_present) - return 0; - - /* - * This can be called early. Use CPUID directly instead of - * relying on cpuinfo_x86 which may not be fully initialized. - * The PII does not have MSR_IA32_PLATFORM_ID. Everything - * before _it_ has no microcode (for Linux at least). - */ - if (intel_cpuid_vfm() <= INTEL_PENTIUM_II_KLAMATH) - return 0; - - /* get processor flags from MSR 0x17 */ - native_rdmsr(MSR_IA32_PLATFORM_ID, val[0], val[1]); - - return (val[1] >> 18) & 7; -} - void intel_collect_cpu_info(struct cpu_signature *sig) { sig->sig = cpuid_eax(1); @@ -345,6 +309,26 @@ static void save_microcode_patch(struct microcode_intel *patch) pr_err("Unable to allocate microcode memory size: %u\n", size); } +static bool revision_is_safe(struct cpu_signature *sig, u32 rev) +{ + u32 vfm = IFM(x86_family(sig->sig), x86_model(sig->sig)); + + /* + * Erratum GNR98 can cause #MCs if "jumping over" revision 0x1000405. + * Avoid the jumps. + */ + if (vfm == INTEL_GRANITERAPIDS_X && + x86_stepping(sig->sig) == 1 && + sig->pf & 0x95 && + sig->rev < 0x1000405 && + rev > 0x1000405) { + pr_err_once("Erratum GNR98: skipping revision 0x%x.\n", rev); + return false; + } + + return true; +} + /* Scan blob for microcode matching the boot CPUs family, model, stepping */ static __init struct microcode_intel *scan_microcode(void *data, size_t size, struct ucode_cpu_info *uci, @@ -366,6 +350,9 @@ static __init struct microcode_intel *scan_microcode(void *data, size_t size, if (!intel_find_matching_signature(data, &uci->cpu_sig)) continue; + if (!revision_is_safe(&uci->cpu_sig, mc_header->rev)) + continue; + /* * For saving the early microcode, find the matching revision which * was loaded on the BSP. @@ -914,6 +901,9 @@ static enum ucode_state parse_microcode_blobs(int cpu, struct iov_iter *iter) if (!intel_find_matching_signature(mc, &uci->cpu_sig)) continue; + if (!revision_is_safe(&uci->cpu_sig, mc_header.rev)) + continue; + is_safe = ucode_validate_minrev(&mc_header); if (force_minrev && !is_safe) continue; diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c index 185d4f677ec0..b4af7c0a70ac 100644 --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -502,17 +502,32 @@ static void hv_reserve_irq_vectors(void) #define HYPERV_DBG_ASSERT_VECTOR 0x2C #define HYPERV_DBG_SERVICE_VECTOR 0x2D + /* + * The hypervisor delivers these three to the NT HAL and refuses to + * map a device interrupt to any of them. + * + * The hypervisor will provide a hint in the future when these + * vectors become available to use. + */ + #define HAL_NT_APC_VECTOR 0x1F + #define HAL_NT_DPC_VECTOR 0x2F + #define HAL_NT_CLOCK_IPI_VECTOR 0xD2 + if (cpu_feature_enabled(X86_FEATURE_FRED)) return; if (test_and_set_bit(HYPERV_DBG_ASSERT_VECTOR, system_vectors) || test_and_set_bit(HYPERV_DBG_SERVICE_VECTOR, system_vectors) || - test_and_set_bit(HYPERV_DBG_FASTFAIL_VECTOR, system_vectors)) + test_and_set_bit(HYPERV_DBG_FASTFAIL_VECTOR, system_vectors) || + test_and_set_bit(HAL_NT_APC_VECTOR, system_vectors) || + test_and_set_bit(HAL_NT_DPC_VECTOR, system_vectors) || + test_and_set_bit(HAL_NT_CLOCK_IPI_VECTOR, system_vectors)) BUG(); - pr_info("Hyper-V: reserve vectors: 0x%x 0x%x 0x%x\n", + pr_info("Hyper-V: reserve vectors: 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n", HYPERV_DBG_ASSERT_VECTOR, HYPERV_DBG_SERVICE_VECTOR, - HYPERV_DBG_FASTFAIL_VECTOR); + HYPERV_DBG_FASTFAIL_VECTOR, HAL_NT_APC_VECTOR, + HAL_NT_DPC_VECTOR, HAL_NT_CLOCK_IPI_VECTOR); } static void __init ms_hyperv_init_platform(void) @@ -716,9 +731,7 @@ static void __init ms_hyperv_init_platform(void) } /* Install system interrupt handler for stimer0 */ - if (ms_hyperv.misc_features & HV_STIMER_DIRECT_MODE_AVAILABLE) { - sysvec_install(HYPERV_STIMER0_VECTOR, sysvec_hyperv_stimer0); - } + sysvec_install(HYPERV_STIMER0_VECTOR, sysvec_hyperv_stimer0); # ifdef CONFIG_SMP smp_ops.smp_prepare_boot_cpu = hv_smp_prepare_boot_cpu; diff --git a/arch/x86/kernel/cpu/mtrr/amd.c b/arch/x86/kernel/cpu/mtrr/amd.c index ef3e8e42b782..a73715d6f05c 100644 --- a/arch/x86/kernel/cpu/mtrr/amd.c +++ b/arch/x86/kernel/cpu/mtrr/amd.c @@ -10,20 +10,23 @@ static void amd_get_mtrr(unsigned int reg, unsigned long *base, unsigned long *size, mtrr_type *type) { - unsigned long low, high; + unsigned long val; + struct msr msr; - rdmsr(MSR_K6_UWCCR, low, high); + rdmsrq(MSR_K6_UWCCR, msr.q); /* Upper dword is region 1, lower is region 0 */ if (reg == 1) - low = high; + val = msr.h; + else + val = msr.l; /* The base masks off on the right alignment */ - *base = (low & 0xFFFE0000) >> PAGE_SHIFT; + *base = (val & 0xFFFE0000) >> PAGE_SHIFT; *type = 0; - if (low & 1) + if (val & 1) *type = MTRR_TYPE_UNCACHABLE; - if (low & 2) + if (val & 2) *type = MTRR_TYPE_WRCOMB; - if (!(low & 3)) { + if (!(val & 3)) { *size = 0; return; } @@ -42,8 +45,8 @@ amd_get_mtrr(unsigned int reg, unsigned long *base, * +1 000 0000 0000 0100 * *128K ... */ - low = (~low) & 0x1FFFC; - *size = (low + 4) << (15 - PAGE_SHIFT); + val = (~val) & 0x1FFFC; + *size = (val + 4) << (15 - PAGE_SHIFT); } /** @@ -59,12 +62,16 @@ amd_get_mtrr(unsigned int reg, unsigned long *base, static void amd_set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type type) { + struct msr msr; u32 regs[2]; /* * Low is MTRR0, High MTRR 1 */ - rdmsr(MSR_K6_UWCCR, regs[0], regs[1]); + rdmsrq(MSR_K6_UWCCR, msr.q); + regs[0] = msr.l; + regs[1] = msr.h; + /* * Blank to disable */ @@ -89,7 +96,9 @@ amd_set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type * disable local interrupts, write back the cache, set the mtrr */ wbinvd(); - wrmsr(MSR_K6_UWCCR, regs[0], regs[1]); + msr.l = regs[0]; + msr.h = regs[1]; + wrmsrq(MSR_K6_UWCCR, msr.q); } static int diff --git a/arch/x86/kernel/cpu/mtrr/centaur.c b/arch/x86/kernel/cpu/mtrr/centaur.c index 6f6c3ae92943..e32cca1caf59 100644 --- a/arch/x86/kernel/cpu/mtrr/centaur.c +++ b/arch/x86/kernel/cpu/mtrr/centaur.c @@ -65,26 +65,26 @@ static void centaur_set_mcr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type type) { - unsigned long low, high; + struct msr val; if (size == 0) { /* Disable */ - high = low = 0; + val.q = 0; } else { - high = base << PAGE_SHIFT; + val.h = base << PAGE_SHIFT; if (centaur_mcr_type == 0) { /* Only support write-combining... */ - low = -size << PAGE_SHIFT | 0x1f; + val.l = -size << PAGE_SHIFT | 0x1f; } else { if (type == MTRR_TYPE_UNCACHABLE) - low = -size << PAGE_SHIFT | 0x02; /* NC */ + val.l = -size << PAGE_SHIFT | 0x02; /* NC */ else - low = -size << PAGE_SHIFT | 0x09; /* WWO, WC */ + val.l = -size << PAGE_SHIFT | 0x09; /* WWO, WC */ } } - centaur_mcr[reg].high = high; - centaur_mcr[reg].low = low; - wrmsr(MSR_IDT_MCR0 + reg, low, high); + centaur_mcr[reg].high = val.h; + centaur_mcr[reg].low = val.l; + wrmsrq(MSR_IDT_MCR0 + reg, val.q); } static int diff --git a/arch/x86/kernel/cpu/mtrr/cleanup.c b/arch/x86/kernel/cpu/mtrr/cleanup.c index e3eee9ae4141..cd1a6dec4064 100644 --- a/arch/x86/kernel/cpu/mtrr/cleanup.c +++ b/arch/x86/kernel/cpu/mtrr/cleanup.c @@ -658,8 +658,8 @@ static int __init mtrr_search_optimal_index(void) int __init mtrr_cleanup(void) { unsigned long x_remove_base, x_remove_size; - unsigned long base, size, def, dummy; - u64 chunk_size, gran_size; + u64 def, chunk_size, gran_size; + unsigned long base, size; mtrr_type type; int index_good; int i; @@ -670,7 +670,7 @@ int __init mtrr_cleanup(void) if (!cpu_feature_enabled(X86_FEATURE_MTRR) || enable_mtrr_cleanup < 1) return 0; - rdmsr(MSR_MTRRdefType, def, dummy); + rdmsrq(MSR_MTRRdefType, def); def &= 0xff; if (def != MTRR_TYPE_UNCACHABLE) return 0; @@ -806,7 +806,7 @@ early_param("disable_mtrr_trim", disable_mtrr_trim_setup); int __init amd_special_default_mtrr(void) { - u32 l, h; + u64 q; if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) @@ -814,13 +814,13 @@ int __init amd_special_default_mtrr(void) if (boot_cpu_data.x86 < 0xf) return 0; /* In case some hypervisor doesn't pass SYSCFG through: */ - if (rdmsr_safe(MSR_AMD64_SYSCFG, &l, &h) < 0) + if (rdmsrq_safe(MSR_AMD64_SYSCFG, &q) < 0) return 0; /* * Memory between 4GB and top of mem is forced WB by this magic bit. * Reserved before K8RevF, but should be zero there. */ - if ((l & (Tom2Enabled | Tom2ForceMemTypeWB)) == + if ((q & (Tom2Enabled | Tom2ForceMemTypeWB)) == (Tom2Enabled | Tom2ForceMemTypeWB)) return 1; return 0; @@ -854,9 +854,9 @@ real_trim_memory(unsigned long start_pfn, unsigned long limit_pfn) */ int __init mtrr_trim_uncached_memory(unsigned long end_pfn) { - unsigned long i, base, size, highest_pfn = 0, def, dummy; + unsigned long i, base, size, highest_pfn = 0; mtrr_type type; - u64 total_trim_size; + u64 def, total_trim_size; /* extra one for all 0 */ int num[MTRR_NUM_TYPES + 1]; @@ -870,7 +870,7 @@ int __init mtrr_trim_uncached_memory(unsigned long end_pfn) if (!cpu_feature_enabled(X86_FEATURE_MTRR) || disable_mtrr_trim) return 0; - rdmsr(MSR_MTRRdefType, def, dummy); + rdmsrq(MSR_MTRRdefType, def); def &= MTRR_DEF_TYPE_TYPE; if (def != MTRR_TYPE_UNCACHABLE) return 0; diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c index 3a8317060732..67cf69f24b00 100644 --- a/arch/x86/kernel/cpu/mtrr/generic.c +++ b/arch/x86/kernel/cpu/mtrr/generic.c @@ -103,7 +103,7 @@ u32 phys_hi_rsvd; */ static inline void k8_check_syscfg_dram_mod_en(void) { - u32 lo, hi; + struct msr val; if (!((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) && (boot_cpu_data.x86 >= 0x0f))) @@ -112,13 +112,13 @@ static inline void k8_check_syscfg_dram_mod_en(void) if (cc_platform_has(CC_ATTR_HOST_SEV_SNP)) return; - rdmsr(MSR_AMD64_SYSCFG, lo, hi); - if (lo & K8_MTRRFIXRANGE_DRAM_MODIFY) { + rdmsrq(MSR_AMD64_SYSCFG, val.q); + if (val.l & K8_MTRRFIXRANGE_DRAM_MODIFY) { pr_err(FW_WARN "MTRR: CPU %u: SYSCFG[MtrrFixDramModEn]" " not cleared by BIOS, clearing this bit\n", smp_processor_id()); - lo &= ~K8_MTRRFIXRANGE_DRAM_MODIFY; - mtrr_wrmsr(MSR_AMD64_SYSCFG, lo, hi); + val.l &= ~K8_MTRRFIXRANGE_DRAM_MODIFY; + mtrr_wrmsr(MSR_AMD64_SYSCFG, val.l, val.h); } } @@ -557,8 +557,14 @@ u8 mtrr_type_lookup(u64 start, u64 end, u8 *uniform) static void get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr) { - rdmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi); - rdmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi); + struct msr val; + + rdmsrq(MTRRphysBase_MSR(index), val.q); + vr->base_lo = val.l; + vr->base_hi = val.h; + rdmsrq(MTRRphysMask_MSR(index), val.q); + vr->mask_lo = val.l; + vr->mask_hi = val.h; } /* Fill the MSR pair relating to a var range */ @@ -577,17 +583,17 @@ void fill_mtrr_var_range(unsigned int index, static void get_fixed_ranges(mtrr_type *frs) { - unsigned int *p = (unsigned int *)frs; + u64 *p = (u64 *)frs; int i; k8_check_syscfg_dram_mod_en(); - rdmsr(MSR_MTRRfix64K_00000, p[0], p[1]); + rdmsrq(MSR_MTRRfix64K_00000, p[0]); for (i = 0; i < 2; i++) - rdmsr(MSR_MTRRfix16K_80000 + i, p[2 + i * 2], p[3 + i * 2]); + rdmsrq(MSR_MTRRfix16K_80000 + i, p[1 + i]); for (i = 0; i < 8; i++) - rdmsr(MSR_MTRRfix4K_C0000 + i, p[6 + i * 2], p[7 + i * 2]); + rdmsrq(MSR_MTRRfix4K_C0000 + i, p[3 + i]); } void mtrr_save_fixed_ranges(void *info) @@ -689,31 +695,26 @@ static void __init print_mtrr_state(void) bool __init get_mtrr_state(void) { struct mtrr_var_range *vrs; - unsigned lo, dummy; unsigned int i; + u64 q; vrs = mtrr_state.var_ranges; - rdmsr(MSR_MTRRcap, lo, dummy); - mtrr_state.have_fixed = lo & MTRR_CAP_FIX; + rdmsrq(MSR_MTRRcap, q); + mtrr_state.have_fixed = q & MTRR_CAP_FIX; for (i = 0; i < num_var_ranges; i++) get_mtrr_var_range(i, &vrs[i]); if (mtrr_state.have_fixed) get_fixed_ranges(mtrr_state.fixed_ranges); - rdmsr(MSR_MTRRdefType, lo, dummy); - mtrr_state.def_type = lo & MTRR_DEF_TYPE_TYPE; - mtrr_state.enabled = (lo & MTRR_DEF_TYPE_ENABLE) >> MTRR_STATE_SHIFT; + rdmsrq(MSR_MTRRdefType, q); + mtrr_state.def_type = q & MTRR_DEF_TYPE_TYPE; + mtrr_state.enabled = (q & MTRR_DEF_TYPE_ENABLE) >> MTRR_STATE_SHIFT; if (amd_special_default_mtrr()) { - unsigned low, high; - /* TOP_MEM2 */ - rdmsr(MSR_K8_TOP_MEM2, low, high); - mtrr_tom2 = high; - mtrr_tom2 <<= 32; - mtrr_tom2 |= low; + rdmsrq(MSR_K8_TOP_MEM2, mtrr_tom2); mtrr_tom2 &= 0xffffff800000ULL; } @@ -750,7 +751,9 @@ void __init mtrr_state_warn(void) */ void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b) { - if (wrmsr_safe(msr, a, b) < 0) { + struct msr val = { .l = a, .h = b }; + + if (wrmsrq_safe(msr, val.q) < 0) { pr_err("MTRR: CPU %u: Writing MSR %x to %x:%x failed\n", smp_processor_id(), msr, a, b); } @@ -765,11 +768,11 @@ void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b) */ static void set_fixed_range(int msr, bool *changed, unsigned int *msrwords) { - unsigned lo, hi; + struct msr val; - rdmsr(msr, lo, hi); + rdmsrq(msr, val.q); - if (lo != msrwords[0] || hi != msrwords[1]) { + if (val.l != msrwords[0] || val.h != msrwords[1]) { mtrr_wrmsr(msr, msrwords[0], msrwords[1]); *changed = true; } @@ -806,9 +809,8 @@ generic_get_free_region(unsigned long base, unsigned long size, int replace_reg) static void generic_get_mtrr(unsigned int reg, unsigned long *base, unsigned long *size, mtrr_type *type) { - u32 mask_lo, mask_hi, base_lo, base_hi; + u64 tmp, mask, base_msr; unsigned int hi; - u64 tmp, mask; /* * get_mtrr doesn't need to update mtrr_state, also it could be called @@ -816,9 +818,9 @@ static void generic_get_mtrr(unsigned int reg, unsigned long *base, */ get_cpu(); - rdmsr(MTRRphysMask_MSR(reg), mask_lo, mask_hi); + rdmsrq(MTRRphysMask_MSR(reg), mask); - if (!(mask_lo & MTRR_PHYSMASK_V)) { + if (!(mask & MTRR_PHYSMASK_V)) { /* Invalid (i.e. free) range */ *base = 0; *size = 0; @@ -826,10 +828,10 @@ static void generic_get_mtrr(unsigned int reg, unsigned long *base, goto out_put_cpu; } - rdmsr(MTRRphysBase_MSR(reg), base_lo, base_hi); + rdmsrq(MTRRphysBase_MSR(reg), base_msr); /* Work out the shifted address mask: */ - tmp = (u64)mask_hi << 32 | (mask_lo & PAGE_MASK); + tmp = mask & PAGE_MASK; mask = (u64)phys_hi_rsvd << 32 | tmp; /* Expand tmp with high bits to all 1s: */ @@ -849,8 +851,8 @@ static void generic_get_mtrr(unsigned int reg, unsigned long *base, * contiguous range: */ *size = -mask >> PAGE_SHIFT; - *base = (u64)base_hi << (32 - PAGE_SHIFT) | base_lo >> PAGE_SHIFT; - *type = base_lo & MTRR_PHYSBASE_TYPE; + *base = base_msr >> PAGE_SHIFT; + *type = base_msr & MTRR_PHYSBASE_TYPE; out_put_cpu: put_cpu(); @@ -884,21 +886,21 @@ static int set_fixed_ranges(mtrr_type *frs) */ static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr) { - unsigned int lo, hi; bool changed = false; + struct msr val; - rdmsr(MTRRphysBase_MSR(index), lo, hi); - if ((vr->base_lo & ~MTRR_PHYSBASE_RSVD) != (lo & ~MTRR_PHYSBASE_RSVD) - || (vr->base_hi & ~phys_hi_rsvd) != (hi & ~phys_hi_rsvd)) { + rdmsrq(MTRRphysBase_MSR(index), val.q); + if ((vr->base_lo & ~MTRR_PHYSBASE_RSVD) != (val.l & ~MTRR_PHYSBASE_RSVD) + || (vr->base_hi & ~phys_hi_rsvd) != (val.h & ~phys_hi_rsvd)) { mtrr_wrmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi); changed = true; } - rdmsr(MTRRphysMask_MSR(index), lo, hi); + rdmsrq(MTRRphysMask_MSR(index), val.q); - if ((vr->mask_lo & ~MTRR_PHYSMASK_RSVD) != (lo & ~MTRR_PHYSMASK_RSVD) - || (vr->mask_hi & ~phys_hi_rsvd) != (hi & ~phys_hi_rsvd)) { + if ((vr->mask_lo & ~MTRR_PHYSMASK_RSVD) != (val.l & ~MTRR_PHYSMASK_RSVD) + || (vr->mask_hi & ~phys_hi_rsvd) != (val.h & ~phys_hi_rsvd)) { mtrr_wrmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi); changed = true; } @@ -947,8 +949,12 @@ static unsigned long set_mtrr_state(void) void mtrr_disable(void) { + struct msr val; + /* Save MTRR state */ - rdmsr(MSR_MTRRdefType, deftype_lo, deftype_hi); + rdmsrq(MSR_MTRRdefType, val.q); + deftype_lo = val.l; + deftype_hi = val.h; /* Disable MTRRs, and set the default type to uncached */ mtrr_wrmsr(MSR_MTRRdefType, deftype_lo & MTRR_DEF_TYPE_DISABLE, deftype_hi); @@ -1057,8 +1063,9 @@ int generic_validate_add_page(unsigned long base, unsigned long size, static int generic_have_wrcomb(void) { - unsigned long config, dummy; - rdmsr(MSR_MTRRcap, config, dummy); + u64 config; + + rdmsrq(MSR_MTRRcap, config); return config & MTRR_CAP_WC; } diff --git a/arch/x86/kernel/cpu/mtrr/mtrr.c b/arch/x86/kernel/cpu/mtrr/mtrr.c index 4b3d492afe17..468c53b20acf 100644 --- a/arch/x86/kernel/cpu/mtrr/mtrr.c +++ b/arch/x86/kernel/cpu/mtrr/mtrr.c @@ -547,7 +547,7 @@ void __init mtrr_bp_init(void) { bool generic_mtrrs = cpu_feature_enabled(X86_FEATURE_MTRR); const char *why = "(not available)"; - unsigned long config, dummy; + unsigned long config; phys_hi_rsvd = GENMASK(31, boot_cpu_data.x86_phys_bits - 32); @@ -571,7 +571,7 @@ void __init mtrr_bp_init(void) if (mtrr_enabled()) { /* Get the number of variable MTRR ranges. */ if (mtrr_if == &generic_mtrr_ops) - rdmsr(MSR_MTRRcap, config, dummy); + rdmsrq(MSR_MTRRcap, config); else config = mtrr_if->var_regs; num_var_ranges = config & MTRR_CAP_VCNT; diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index 9c01d2562b7a..55214d6fdc49 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -247,7 +247,11 @@ static __init bool __rdt_get_mem_config_amd(struct rdt_resource *r) cpuid_count(0x80000020, subleaf, &eax, &ebx, &ecx, &edx); hw_res->num_closid = edx + 1; - r->membw.max_bw = 1 << eax; + if (BITS_PER_TYPE(r->membw.max_bw) <= eax) { + pr_warn("Unable to support hardware's maximum bandwidth\n"); + return false; + } + r->membw.max_bw = BIT(eax); /* AMD does not use delay */ r->membw.delay_linear = false; @@ -515,14 +519,12 @@ static void domain_add_cpu_ctrl(int cpu, struct rdt_resource *r) return; } - list_add_tail_rcu(&d->hdr.list, add_pos); - err = resctrl_online_ctrl_domain(r, d); if (err) { - list_del_rcu(&d->hdr.list); - synchronize_rcu(); ctrl_domain_free(hw_dom); + return; } + list_add_tail_rcu(&d->hdr.list, add_pos); } static void l3_mon_domain_setup(int cpu, int id, struct rdt_resource *r, struct list_head *add_pos) @@ -556,14 +558,12 @@ static void l3_mon_domain_setup(int cpu, int id, struct rdt_resource *r, struct return; } - list_add_tail_rcu(&d->hdr.list, add_pos); - err = resctrl_online_mon_domain(r, &d->hdr); if (err) { - list_del_rcu(&d->hdr.list); - synchronize_rcu(); l3_mon_domain_free(hw_dom); + return; } + list_add_tail_rcu(&d->hdr.list, add_pos); } static void domain_add_cpu_mon(int cpu, struct rdt_resource *r) @@ -642,9 +642,9 @@ static void domain_remove_cpu_ctrl(int cpu, struct rdt_resource *r) d = container_of(hdr, struct rdt_ctrl_domain, hdr); hw_dom = resctrl_to_arch_ctrl_dom(d); - resctrl_offline_ctrl_domain(r, d); list_del_rcu(&hdr->list); synchronize_rcu(); + resctrl_offline_ctrl_domain(r, d); /* * rdt_ctrl_domain "d" is going to be freed below, so clear @@ -689,9 +689,9 @@ static void domain_remove_cpu_mon(int cpu, struct rdt_resource *r) d = container_of(hdr, struct rdt_l3_mon_domain, hdr); hw_dom = resctrl_to_arch_mon_dom(d); - resctrl_offline_mon_domain(r, hdr); list_del_rcu(&hdr->list); synchronize_rcu(); + resctrl_offline_mon_domain(r, hdr); l3_mon_domain_free(hw_dom); break; } @@ -702,9 +702,9 @@ static void domain_remove_cpu_mon(int cpu, struct rdt_resource *r) return; pkgd = container_of(hdr, struct rdt_perf_pkg_mon_domain, hdr); - resctrl_offline_mon_domain(r, hdr); list_del_rcu(&hdr->list); synchronize_rcu(); + resctrl_offline_mon_domain(r, hdr); kfree(pkgd); break; } @@ -725,13 +725,16 @@ static void domain_remove_cpu(int cpu, struct rdt_resource *r) static void clear_closid_rmid(int cpu) { struct resctrl_pqr_state *state = this_cpu_ptr(&pqr_state); + struct msr val = { + .l = RESCTRL_RESERVED_RMID, + .h = RESCTRL_RESERVED_CLOSID + }; state->default_closid = RESCTRL_RESERVED_CLOSID; state->default_rmid = RESCTRL_RESERVED_RMID; state->cur_closid = RESCTRL_RESERVED_CLOSID; state->cur_rmid = RESCTRL_RESERVED_RMID; - wrmsr(MSR_IA32_PQR_ASSOC, RESCTRL_RESERVED_RMID, - RESCTRL_RESERVED_CLOSID); + wrmsrq(MSR_IA32_PQR_ASSOC, val.q); } static int resctrl_arch_online_cpu(unsigned int cpu) diff --git a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c index b20e705606b8..e74f1ed54b86 100644 --- a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c +++ b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c @@ -53,7 +53,7 @@ int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid) /* Walking r->domains, ensure it can't race with cpuhp */ lockdep_assert_cpus_held(); - list_for_each_entry(d, &r->ctrl_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { hw_dom = resctrl_to_arch_ctrl_dom(d); msr_param.res = NULL; for (t = 0; t < CDP_NUM_TYPES; t++) { @@ -115,7 +115,7 @@ static void _resctrl_sdciae_enable(struct rdt_resource *r, bool enable) lockdep_assert_cpus_held(); /* Update MSR_IA32_L3_QOS_EXT_CFG MSR on all the CPUs in all domains */ - list_for_each_entry(d, &r->ctrl_domains, hdr.list) + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) on_each_cpu_mask(&d->hdr.cpu_mask, resctrl_sdciae_set_one_amd, &enable, 1); } diff --git a/arch/x86/kernel/cpu/resctrl/intel_aet.c b/arch/x86/kernel/cpu/resctrl/intel_aet.c index 89b8b619d5d5..c22c3cf5167d 100644 --- a/arch/x86/kernel/cpu/resctrl/intel_aet.c +++ b/arch/x86/kernel/cpu/resctrl/intel_aet.c @@ -398,12 +398,11 @@ void intel_aet_mon_domain_setup(int cpu, int id, struct rdt_resource *r, d->hdr.type = RESCTRL_MON_DOMAIN; d->hdr.rid = RDT_RESOURCE_PERF_PKG; cpumask_set_cpu(cpu, &d->hdr.cpu_mask); - list_add_tail_rcu(&d->hdr.list, add_pos); err = resctrl_online_mon_domain(r, &d->hdr); if (err) { - list_del_rcu(&d->hdr.list); - synchronize_rcu(); kfree(d); + return; } + list_add_tail_rcu(&d->hdr.list, add_pos); } diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index 569894d6e5c8..3838e0a13d36 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -136,7 +136,7 @@ static int logical_rmid_to_physical_rmid(int cpu, int lrmid) static int __rmid_read_phys(u32 prmid, enum resctrl_event_id eventid, u64 *val) { - u64 msr_val; + struct msr msr_val = { .l = eventid, .h = prmid }; /* * As per the SDM, when IA32_QM_EVTSEL.EvtID (bits 7:0) is configured @@ -146,15 +146,15 @@ static int __rmid_read_phys(u32 prmid, enum resctrl_event_id eventid, u64 *val) * IA32_QM_CTR.Error (bit 63) and IA32_QM_CTR.Unavailable (bit 62) * are error bits. */ - wrmsr(MSR_IA32_QM_EVTSEL, eventid, prmid); - rdmsrq(MSR_IA32_QM_CTR, msr_val); + wrmsrq(MSR_IA32_QM_EVTSEL, msr_val.q); + rdmsrq(MSR_IA32_QM_CTR, msr_val.q); - if (msr_val & RMID_VAL_ERROR) + if (msr_val.q & RMID_VAL_ERROR) return -EIO; - if (msr_val & RMID_VAL_UNAVAIL) + if (msr_val.q & RMID_VAL_UNAVAIL) return -EINVAL; - *val = msr_val; + *val = msr_val.q; return 0; } @@ -283,7 +283,10 @@ int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain_hdr *hdr, static int __cntr_id_read(u32 cntr_id, u64 *val) { - u64 msr_val; + struct msr msr_val = { + .l = ABMC_EXTENDED_EVT_ID | ABMC_EVT_ID, + .h = cntr_id + }; /* * QM_EVTSEL Register definition: @@ -306,15 +309,15 @@ static int __cntr_id_read(u32 cntr_id, u64 *val) * ID is set in the QM_EVTSEL.RMID field. The RMID_VAL_UNAVAIL bit * is set if the counter data is unavailable. */ - wrmsr(MSR_IA32_QM_EVTSEL, ABMC_EXTENDED_EVT_ID | ABMC_EVT_ID, cntr_id); - rdmsrq(MSR_IA32_QM_CTR, msr_val); + wrmsrq(MSR_IA32_QM_EVTSEL, msr_val.q); + rdmsrq(MSR_IA32_QM_CTR, msr_val.q); - if (msr_val & RMID_VAL_ERROR) + if (msr_val.q & RMID_VAL_ERROR) return -EIO; - if (msr_val & RMID_VAL_UNAVAIL) + if (msr_val.q & RMID_VAL_UNAVAIL) return -EINVAL; - *val = msr_val; + *val = msr_val.q; return 0; } @@ -510,7 +513,7 @@ static void _resctrl_abmc_enable(struct rdt_resource *r, bool enable) lockdep_assert_cpus_held(); - list_for_each_entry(d, &r->mon_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { on_each_cpu_mask(&d->hdr.cpu_mask, resctrl_abmc_set_one_amd, &enable, 1); resctrl_arch_reset_rmid_all(r, d); diff --git a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c index de580eca3363..d7caab0409b6 100644 --- a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c +++ b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c @@ -241,16 +241,16 @@ int resctrl_arch_pseudo_lock_fn(void *_plr) int resctrl_arch_measure_cycles_lat_fn(void *_plr) { struct pseudo_lock_region *plr = _plr; - u32 saved_low, saved_high; unsigned long i; u64 start, end; void *mem_r; + u64 saved; local_irq_disable(); /* * Disable hardware prefetchers. */ - rdmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high); + rdmsrq(MSR_MISC_FEATURE_CONTROL, saved); wrmsrq(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits); mem_r = READ_ONCE(plr->kmem); /* @@ -267,7 +267,7 @@ int resctrl_arch_measure_cycles_lat_fn(void *_plr) end = rdtsc_ordered(); trace_pseudo_lock_mem_latency((u32)(end - start)); } - wrmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high); + wrmsrq(MSR_MISC_FEATURE_CONTROL, saved); local_irq_enable(); plr->thread_done = 1; wake_up_interruptible(&plr->lock_thread_wq); @@ -312,11 +312,11 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr, u64 hits_before = 0, hits_after = 0, miss_before = 0, miss_after = 0; struct perf_event *miss_event, *hit_event; int hit_pmcnum, miss_pmcnum; - u32 saved_low, saved_high; unsigned int line_size; unsigned int size; unsigned long i; void *mem_r; + u64 saved; u64 tmp; miss_event = perf_event_create_kernel_counter(miss_attr, plr->cpu, @@ -346,7 +346,7 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr, /* * Disable hardware prefetchers. */ - rdmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high); + rdmsrq(MSR_MISC_FEATURE_CONTROL, saved); wrmsrq(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits); /* Initialize rest of local variables */ @@ -405,7 +405,7 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr, */ rmb(); /* Re-enable hardware prefetchers */ - wrmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high); + wrmsrq(MSR_MISC_FEATURE_CONTROL, saved); local_irq_enable(); out_hit: perf_event_release_kernel(hit_event); diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index 885026468440..5ffa39fa86fa 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -151,7 +151,7 @@ static int set_cache_qos_cfg(int level, bool enable) return -ENOMEM; r_l = &rdt_resources_all[level].r_resctrl; - list_for_each_entry(d, &r_l->ctrl_domains, hdr.list) { + list_for_each_entry_rcu(d, &r_l->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { if (r_l->cache.arch_has_per_cpu_cfg) /* Pick all the CPUs in the domain instance */ for_each_cpu(cpu, &d->hdr.cpu_mask) @@ -249,7 +249,7 @@ void resctrl_arch_reset_all_ctrls(struct rdt_resource *r) * CBMs in all ctrl_domains to the maximum mask value. Pick one CPU * from each domain to update the MSRs below. */ - list_for_each_entry(d, &r->ctrl_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { hw_dom = resctrl_to_arch_ctrl_dom(d); for (i = 0; i < hw_res->num_closid; i++) diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index 937129ce6a96..8665a6474806 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -38,9 +38,9 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_CQM_MBM_LOCAL, CPUID_EDX, 2, 0x0000000f, 1 }, { X86_FEATURE_CAT_L3, CPUID_EBX, 1, 0x00000010, 0 }, { X86_FEATURE_CAT_L2, CPUID_EBX, 2, 0x00000010, 0 }, + { X86_FEATURE_MBA, CPUID_EBX, 3, 0x00000010, 0 }, { X86_FEATURE_CDP_L3, CPUID_ECX, 2, 0x00000010, 1 }, { X86_FEATURE_CDP_L2, CPUID_ECX, 2, 0x00000010, 2 }, - { X86_FEATURE_MBA, CPUID_EBX, 3, 0x00000010, 0 }, { X86_FEATURE_PER_THREAD_MBA, CPUID_ECX, 0, 0x00000010, 3 }, { X86_FEATURE_SGX1, CPUID_EAX, 0, 0x00000012, 0 }, { X86_FEATURE_SGX2, CPUID_EAX, 1, 0x00000012, 0 }, @@ -61,9 +61,9 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_BMEC, CPUID_EBX, 3, 0x80000020, 0 }, { X86_FEATURE_ABMC, CPUID_EBX, 5, 0x80000020, 0 }, { X86_FEATURE_SDCIAE, CPUID_EBX, 6, 0x80000020, 0 }, + { X86_FEATURE_AMD_WORKLOAD_CLASS, CPUID_EAX, 22, 0x80000021, 0 }, { X86_FEATURE_TSA_SQ_NO, CPUID_ECX, 1, 0x80000021, 0 }, { X86_FEATURE_TSA_L1_NO, CPUID_ECX, 2, 0x80000021, 0 }, - { X86_FEATURE_AMD_WORKLOAD_CLASS, CPUID_EAX, 22, 0x80000021, 0 }, { X86_FEATURE_PERFMON_V2, CPUID_EAX, 0, 0x80000022, 0 }, { X86_FEATURE_AMD_LBR_V2, CPUID_EAX, 1, 0x80000022, 0 }, { X86_FEATURE_AMD_LBR_PMC_FREEZE, CPUID_EAX, 2, 0x80000022, 0 }, diff --git a/arch/x86/kernel/cpu/sgx/virt.c b/arch/x86/kernel/cpu/sgx/virt.c index db6806c40483..6a1933ddc6fc 100644 --- a/arch/x86/kernel/cpu/sgx/virt.c +++ b/arch/x86/kernel/cpu/sgx/virt.c @@ -9,6 +9,7 @@ #include <linux/miscdevice.h> #include <linux/mm.h> #include <linux/mman.h> +#include <linux/pagemap.h> #include <linux/sched/mm.h> #include <linux/sched/signal.h> #include <linux/slab.h> @@ -41,7 +42,7 @@ static int __sgx_vepc_fault(struct sgx_vepc *vepc, WARN_ON(!mutex_is_locked(&vepc->lock)); /* Calculate index of EPC page in virtual EPC's page_array */ - index = vma->vm_pgoff + PFN_DOWN(addr - vma->vm_start); + index = linear_page_index(vma, addr); epc_page = xa_load(&vepc->page_array, index); if (epc_page) diff --git a/arch/x86/kernel/cpu/topology.h b/arch/x86/kernel/cpu/topology.h index 37326297f80c..74e02bacd854 100644 --- a/arch/x86/kernel/cpu/topology.h +++ b/arch/x86/kernel/cpu/topology.h @@ -22,6 +22,7 @@ void topology_set_dom(struct topo_scan *tscan, enum x86_topology_domains dom, bool cpu_parse_topology_ext(struct topo_scan *tscan); void cpu_parse_topology_amd(struct topo_scan *tscan); void cpu_topology_fixup_amd(struct topo_scan *tscan); +enum x86_topology_cpu_type get_topology_cpu_type(struct cpuinfo_x86 *c); static inline u32 topo_shift_apicid(u32 apicid, enum x86_topology_domains dom) { diff --git a/arch/x86/kernel/cpu/topology_amd.c b/arch/x86/kernel/cpu/topology_amd.c index da080d732e10..c5a6944df86a 100644 --- a/arch/x86/kernel/cpu/topology_amd.c +++ b/arch/x86/kernel/cpu/topology_amd.c @@ -177,8 +177,10 @@ static void topoext_fixup(struct topo_scan *tscan) static void parse_topology_amd(struct topo_scan *tscan) { - if (cpu_feature_enabled(X86_FEATURE_AMD_HTR_CORES)) - tscan->c->topo.cpu_type = cpuid_ebx(0x80000026); + if (cpu_feature_enabled(X86_FEATURE_AMD_HTR_CORES)) { + tscan->c->topo.hw_cpu_type = cpuid_ebx(0x80000026); + tscan->c->topo.cpu_type = get_topology_cpu_type(tscan->c); + } /* * Try to get SMT, CORE, TILE, and DIE shifts from extended diff --git a/arch/x86/kernel/cpu/topology_common.c b/arch/x86/kernel/cpu/topology_common.c index cf7513416b70..6845e3c63fbb 100644 --- a/arch/x86/kernel/cpu/topology_common.c +++ b/arch/x86/kernel/cpu/topology_common.c @@ -42,8 +42,9 @@ enum x86_topology_cpu_type get_topology_cpu_type(struct cpuinfo_x86 *c) } if (c->x86_vendor == X86_VENDOR_AMD) { switch (c->topo.amd_type) { - case 0: return TOPO_CPU_TYPE_PERFORMANCE; - case 1: return TOPO_CPU_TYPE_EFFICIENCY; + case AMD_CPU_TYPE_PERFORMANCE: return TOPO_CPU_TYPE_PERFORMANCE; + case AMD_CPU_TYPE_EFFICIENCY: return TOPO_CPU_TYPE_EFFICIENCY; + case AMD_CPU_TYPE_LOW_POWER: return TOPO_CPU_TYPE_LOW_POWER; } } @@ -57,12 +58,14 @@ const char *get_topology_cpu_type_name(struct cpuinfo_x86 *c) return "performance"; case TOPO_CPU_TYPE_EFFICIENCY: return "efficiency"; + case TOPO_CPU_TYPE_LOW_POWER: + return "low_power"; default: return "unknown"; } } -static unsigned int __maybe_unused parse_num_cores_legacy(struct cpuinfo_x86 *c) +static unsigned int parse_num_cores_legacy(struct cpuinfo_x86 *c) { struct { u32 cache_type : 5, @@ -168,8 +171,12 @@ static void parse_topology(struct topo_scan *tscan, bool early) case X86_VENDOR_INTEL: if (!IS_ENABLED(CONFIG_CPU_SUP_INTEL) || !cpu_parse_topology_ext(tscan)) parse_legacy(tscan); - if (c->cpuid_level >= 0x1a) - c->topo.cpu_type = cpuid_eax(0x1a); + + if (c->cpuid_level >= 0x1a) { + c->topo.hw_cpu_type = cpuid_eax(0x1a); + c->topo.cpu_type = get_topology_cpu_type(c); + } + break; } } diff --git a/arch/x86/kernel/cpu/transmeta.c b/arch/x86/kernel/cpu/transmeta.c index 1fdcd69c625c..3e04b5d01bf8 100644 --- a/arch/x86/kernel/cpu/transmeta.c +++ b/arch/x86/kernel/cpu/transmeta.c @@ -24,7 +24,8 @@ static void early_init_transmeta(struct cpuinfo_x86 *c) static void init_transmeta(struct cpuinfo_x86 *c) { - unsigned int cap_mask, uk, max, dummy; + u64 msr; + unsigned int max, dummy; unsigned int cms_rev1, cms_rev2; unsigned int cpu_rev, cpu_freq = 0, cpu_flags, new_cpu_rev; char cpu_info[65]; @@ -86,10 +87,11 @@ static void init_transmeta(struct cpuinfo_x86 *c) } /* Unhide possibly hidden capability flags */ - rdmsr(0x80860004, cap_mask, uk); - wrmsr(0x80860004, ~0, uk); + rdmsrq(0x80860004, msr); + wrmsrq(0x80860004, msr | ~0U); + cpuid_refresh_leaf(c, 0x1); c->x86_capability[CPUID_1_EDX] = cpuid_edx(0x00000001); - wrmsr(0x80860004, cap_mask, uk); + wrmsrq(0x80860004, msr); /* All Transmeta CPUs have a constant TSC */ set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC); diff --git a/arch/x86/kernel/cpu/zhaoxin.c b/arch/x86/kernel/cpu/zhaoxin.c index 761aef5590ac..fe504fd43c77 100644 --- a/arch/x86/kernel/cpu/zhaoxin.c +++ b/arch/x86/kernel/cpu/zhaoxin.c @@ -21,7 +21,7 @@ static void init_zhaoxin_cap(struct cpuinfo_x86 *c) { - u32 lo, hi; + u64 msr; /* Test for Extended Feature Flags presence */ if (cpuid_eax(0xC0000000) >= 0xC0000001) { @@ -29,19 +29,17 @@ static void init_zhaoxin_cap(struct cpuinfo_x86 *c) /* Enable ACE unit, if present and disabled */ if ((tmp & (ACE_PRESENT | ACE_ENABLED)) == ACE_PRESENT) { - rdmsr(MSR_ZHAOXIN_FCR57, lo, hi); + rdmsrq(MSR_ZHAOXIN_FCR57, msr); /* Enable ACE unit */ - lo |= ACE_FCR; - wrmsr(MSR_ZHAOXIN_FCR57, lo, hi); + wrmsrq(MSR_ZHAOXIN_FCR57, msr | ACE_FCR); pr_info("CPU: Enabled ACE h/w crypto\n"); } /* Enable RNG unit, if present and disabled */ if ((tmp & (RNG_PRESENT | RNG_ENABLED)) == RNG_PRESENT) { - rdmsr(MSR_ZHAOXIN_FCR57, lo, hi); + rdmsrq(MSR_ZHAOXIN_FCR57, msr); /* Enable RNG unit */ - lo |= RNG_ENABLE; - wrmsr(MSR_ZHAOXIN_FCR57, lo, hi); + wrmsrq(MSR_ZHAOXIN_FCR57, msr | RNG_ENABLE); pr_info("CPU: Enabled h/w RNG\n"); } diff --git a/arch/x86/kernel/crash.c b/arch/x86/kernel/crash.c index 623d4474631a..e681ec9cf1dc 100644 --- a/arch/x86/kernel/crash.c +++ b/arch/x86/kernel/crash.c @@ -155,16 +155,8 @@ static int get_nr_ram_ranges_callback(struct resource *res, void *arg) return 0; } -/* Gather all the required information to prepare elf headers for ram regions */ -static struct crash_mem *fill_up_crash_elf_data(void) +unsigned int arch_get_system_nr_ranges(void) { - unsigned int nr_ranges = 0; - struct crash_mem *cmem; - - walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback); - if (!nr_ranges) - return NULL; - /* * Exclusion of crash region, crashk_low_res and/or crashk_cma_ranges * may cause range splits. So add extra slots here. @@ -179,49 +171,16 @@ static struct crash_mem *fill_up_crash_elf_data(void) * But in order to lest the low 1M could be changed in the future, * (e.g. [start, 1M]), add a extra slot. */ - nr_ranges += 3 + crashk_cma_cnt; - cmem = vzalloc(struct_size(cmem, ranges, nr_ranges)); - if (!cmem) - return NULL; - - cmem->max_nr_ranges = nr_ranges; + unsigned int nr_ranges = 3 + crashk_cma_cnt; - return cmem; + walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback); + return nr_ranges; } -/* - * Look for any unwanted ranges between mstart, mend and remove them. This - * might lead to split and split ranges are put in cmem->ranges[] array - */ -static int elf_header_exclude_ranges(struct crash_mem *cmem) +int arch_crash_exclude_ranges(struct crash_mem *cmem) { - int ret = 0; - int i; - /* Exclude the low 1M because it is always reserved */ - ret = crash_exclude_mem_range(cmem, 0, SZ_1M - 1); - if (ret) - return ret; - - /* Exclude crashkernel region */ - ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end); - if (ret) - return ret; - - if (crashk_low_res.end) - ret = crash_exclude_mem_range(cmem, crashk_low_res.start, - crashk_low_res.end); - if (ret) - return ret; - - for (i = 0; i < crashk_cma_cnt; ++i) { - ret = crash_exclude_mem_range(cmem, crashk_cma_ranges[i].start, - crashk_cma_ranges[i].end); - if (ret) - return ret; - } - - return 0; + return crash_exclude_mem_range(cmem, 0, SZ_1M - 1); } static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg) @@ -235,35 +194,9 @@ static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg) return 0; } -/* Prepare elf headers. Return addr and size */ -static int prepare_elf_headers(void **addr, unsigned long *sz, - unsigned long *nr_mem_ranges) +int arch_crash_populate_cmem(struct crash_mem *cmem) { - struct crash_mem *cmem; - int ret; - - cmem = fill_up_crash_elf_data(); - if (!cmem) - return -ENOMEM; - - ret = walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback); - if (ret) - goto out; - - /* Exclude unwanted mem ranges */ - ret = elf_header_exclude_ranges(cmem); - if (ret) - goto out; - - /* Return the computed number of memory ranges, for hotplug usage */ - *nr_mem_ranges = cmem->nr_ranges; - - /* By default prepare 64bit headers */ - ret = crash_prepare_elf64_headers(cmem, IS_ENABLED(CONFIG_X86_64), addr, sz); - -out: - vfree(cmem); - return ret; + return walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback); } #endif @@ -421,7 +354,8 @@ int crash_load_segments(struct kimage *image) .buf_max = ULONG_MAX, .top_down = false }; /* Prepare elf headers and add a segment */ - ret = prepare_elf_headers(&kbuf.buffer, &kbuf.bufsz, &pnum); + ret = crash_prepare_headers(IS_ENABLED(CONFIG_X86_64), &kbuf.buffer, + &kbuf.bufsz, &pnum); if (ret) return ret; @@ -514,7 +448,6 @@ unsigned int arch_crash_get_elfcorehdr_size(void) void arch_crash_handle_hotplug_event(struct kimage *image, void *arg) { void *elfbuf = NULL, *old_elfcorehdr; - unsigned long nr_mem_ranges; unsigned long mem, memsz; unsigned long elfsz = 0; @@ -532,7 +465,7 @@ void arch_crash_handle_hotplug_event(struct kimage *image, void *arg) * Create the new elfcorehdr reflecting the changes to CPU and/or * memory resources. */ - if (prepare_elf_headers(&elfbuf, &elfsz, &nr_mem_ranges)) { + if (crash_prepare_headers(IS_ENABLED(CONFIG_X86_64), &elfbuf, &elfsz, NULL)) { pr_err("unable to create new elfcorehdr"); goto out; } diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 584fb9913be4..d1aeecd57f5e 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -294,15 +294,15 @@ void fpu_free_guest_fpstate(struct fpu_guest *gfpu) } EXPORT_SYMBOL_FOR_KVM(fpu_free_guest_fpstate); -/* - * fpu_enable_guest_xfd_features - Check xfeatures against guest perm and enable - * @guest_fpu: Pointer to the guest FPU container - * @xfeatures: Features requested by guest CPUID - * - * Enable all dynamic xfeatures according to guest perm and requested CPUID. - * - * Return: 0 on success, error code otherwise - */ +/** + * fpu_enable_guest_xfd_features - Check xfeatures against guest perm and enable + * @guest_fpu: Pointer to the guest FPU container + * @xfeatures: Features requested by guest CPUID + * + * Enable all dynamic xfeatures according to guest perm and requested CPUID. + * + * Return: 0 on success, error code otherwise + */ int fpu_enable_guest_xfd_features(struct fpu_guest *guest_fpu, u64 xfeatures) { lockdep_assert_preemption_enabled(); @@ -874,7 +874,7 @@ void fpu_flush_thread(void) */ void switch_fpu_return(void) { - if (!static_cpu_has(X86_FEATURE_FPU)) + if (!cpu_feature_enabled(X86_FEATURE_FPU)) return; fpregs_restore_userregs(); diff --git a/arch/x86/kernel/fpu/signal.c b/arch/x86/kernel/fpu/signal.c index 20b638c507ca..33e1284bf3e4 100644 --- a/arch/x86/kernel/fpu/signal.c +++ b/arch/x86/kernel/fpu/signal.c @@ -187,7 +187,7 @@ bool copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size, u ia32_fxstate &= (IS_ENABLED(CONFIG_X86_32) || IS_ENABLED(CONFIG_IA32_EMULATION)); - if (!static_cpu_has(X86_FEATURE_FPU)) { + if (!cpu_feature_enabled(X86_FEATURE_FPU)) { struct user_i387_ia32_struct fp; fpregs_soft_get(current, NULL, (struct membuf){.p = &fp, diff --git a/arch/x86/kernel/itmt.c b/arch/x86/kernel/itmt.c index 243a769fdd97..85ebde361d6a 100644 --- a/arch/x86/kernel/itmt.c +++ b/arch/x86/kernel/itmt.c @@ -110,18 +110,14 @@ int sched_set_itmt_support(void) arch_debugfs_dir, &sysctl_sched_itmt_enabled, &dfs_sched_itmt_fops); - if (IS_ERR_OR_NULL(dfs_sched_itmt)) { + if (IS_ERR(dfs_sched_itmt)) dfs_sched_itmt = NULL; - return -ENOMEM; - } dfs_sched_core_prio = debugfs_create_file("sched_core_priority", 0644, arch_debugfs_dir, NULL, &sched_core_priority_fops); - if (IS_ERR_OR_NULL(dfs_sched_core_prio)) { + if (IS_ERR(dfs_sched_core_prio)) dfs_sched_core_prio = NULL; - return -ENOMEM; - } sched_itmt_capable = true; diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c index c1fac3a9fecc..133ff20caccd 100644 --- a/arch/x86/kernel/kprobes/core.c +++ b/arch/x86/kernel/kprobes/core.c @@ -510,10 +510,9 @@ NOKPROBE_SYMBOL(kprobe_emulate_ret); static void kprobe_emulate_call(struct kprobe *p, struct pt_regs *regs) { - unsigned long func = regs->ip - INT3_INSN_SIZE + p->ainsn.size; + unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size; - func += p->ainsn.rel32; - int3_emulate_call(regs, func); + int3_emulate_call(regs, ip, ip + p->ainsn.rel32); } NOKPROBE_SYMBOL(kprobe_emulate_call); @@ -762,9 +761,6 @@ int arch_prepare_kprobe(struct kprobe *p) { int ret; - if (alternatives_text_reserved(p->addr, p->addr)) - return -EINVAL; - if (!can_probe((unsigned long)p->addr)) return -EILSEQ; diff --git a/arch/x86/kernel/kprobes/opt.c b/arch/x86/kernel/kprobes/opt.c index 6f826a00eca2..3f8fea52619f 100644 --- a/arch/x86/kernel/kprobes/opt.c +++ b/arch/x86/kernel/kprobes/opt.c @@ -77,7 +77,7 @@ found: static void synthesize_clac(kprobe_opcode_t *addr) { /* - * Can't be static_cpu_has() due to how objtool treats this feature bit. + * Can't be cpu_feature_enabled() due to how objtool treats this feature bit. * This isn't a fast path anyway. */ if (!boot_cpu_has(X86_FEATURE_SMAP)) @@ -213,7 +213,6 @@ static int copy_optimized_instructions(u8 *dest, u8 *src, u8 *real) } /* Check whether the address range is reserved */ if (ftrace_text_reserved(src, src + len - 1) || - alternatives_text_reserved(src, src + len - 1) || jump_label_text_reserved(src, src + len - 1) || static_call_text_reserved(src, src + len - 1)) return -EBUSY; diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index dcef84da304b..6b0a5861ccb8 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -663,8 +663,10 @@ static void kvm_flush_tlb_multi(const struct cpumask *cpumask, u8 state; int cpu; struct kvm_steal_time *src; - struct cpumask *flushmask = this_cpu_cpumask_var_ptr(__pv_cpu_mask); + struct cpumask *flushmask; + guard(preempt)(); + flushmask = this_cpu_cpumask_var_ptr(__pv_cpu_mask); cpumask_copy(flushmask, cpumask); /* * We have to call flush only on online vCPUs. And @@ -1136,9 +1138,8 @@ void __init kvm_spinlock_init(void) pr_info("PV spinlocks enabled\n"); __pv_init_lock_hash(); - pv_ops_lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; - pv_ops_lock.queued_spin_unlock = - PV_CALLEE_SAVE(__pv_queued_spin_unlock); + static_call_update(queued_spin_lock_slowpath, __pv_queued_spin_lock_slowpath); + static_call_update(queued_spin_unlock, __raw_callee_save___pv_queued_spin_unlock); pv_ops_lock.wait = kvm_wait; pv_ops_lock.kick = kvm_kick_cpu; diff --git a/arch/x86/kernel/ldt.c b/arch/x86/kernel/ldt.c index 40c5bf97dd5c..4fa7a86adece 100644 --- a/arch/x86/kernel/ldt.c +++ b/arch/x86/kernel/ldt.c @@ -61,7 +61,7 @@ void load_mm_ldt(struct mm_struct *mm) */ if (unlikely(ldt)) { - if (static_cpu_has(X86_FEATURE_PTI)) { + if (cpu_feature_enabled(X86_FEATURE_PTI)) { if (WARN_ON_ONCE((unsigned long)ldt->slot > 1)) { /* * Whoops -- either the new LDT isn't mapped diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c index b5b4de4f08e6..33080ad889b7 100644 --- a/arch/x86/kernel/module.c +++ b/arch/x86/kernel/module.c @@ -227,7 +227,7 @@ int module_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *me) { - const Elf_Shdr *s, *alt = NULL, *locks = NULL, + const Elf_Shdr *s, *alt = NULL, *orc = NULL, *orc_ip = NULL, *retpolines = NULL, *returns = NULL, *ibt_endbr = NULL, *calls = NULL, *cfi = NULL; @@ -236,8 +236,6 @@ int module_finalize(const Elf_Ehdr *hdr, for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { if (!strcmp(".altinstructions", secstrings + s->sh_name)) alt = s; - if (!strcmp(".smp_locks", secstrings + s->sh_name)) - locks = s; if (!strcmp(".orc_unwind", secstrings + s->sh_name)) orc = s; if (!strcmp(".orc_unwind_ip", secstrings + s->sh_name)) @@ -300,14 +298,6 @@ int module_finalize(const Elf_Ehdr *hdr, void *iseg = (void *)ibt_endbr->sh_addr; apply_seal_endbr(iseg, iseg + ibt_endbr->sh_size); } - if (locks) { - void *lseg = (void *)locks->sh_addr; - void *text = me->mem[MOD_TEXT].base; - void *text_end = text + me->mem[MOD_TEXT].size; - alternatives_smp_module_add(me, me->name, - lseg, lseg + locks->sh_size, - text, text_end); - } if (orc && orc_ip) unwind_module_init(me, (void *)orc_ip->sh_addr, orc_ip->sh_size, @@ -318,6 +308,5 @@ int module_finalize(const Elf_Ehdr *hdr, void module_arch_cleanup(struct module *mod) { - alternatives_smp_module_del(mod); its_free_mod(mod); } diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c index 34bdb752f892..eb988df22521 100644 --- a/arch/x86/kernel/msr.c +++ b/arch/x86/kernel/msr.c @@ -13,6 +13,9 @@ * and then read/write in chunks of 8 bytes. A larger size means multiple * reads or writes of the same register. * + * Writing the same register multiple times can be useful for MSRs with + * I/O-like semantics, e.g. a virtual MSR that accepts logging information. + * * This driver uses /dev/cpu/%d/msr where %d is the minor number, and on * an SMP box will direct the access to CPU %d. */ diff --git a/arch/x86/kernel/paravirt-spinlocks.c b/arch/x86/kernel/paravirt-spinlocks.c index 95452444868f..ca12b3655307 100644 --- a/arch/x86/kernel/paravirt-spinlocks.c +++ b/arch/x86/kernel/paravirt-spinlocks.c @@ -7,6 +7,7 @@ #include <linux/spinlock.h> #include <linux/export.h> #include <linux/jump_label.h> +#include <trace/events/lock.h> DEFINE_STATIC_KEY_FALSE(virt_spin_lock_key); @@ -25,10 +26,63 @@ __visible void __native_queued_spin_unlock(struct qspinlock *lock) } PV_CALLEE_SAVE_REGS_THUNK(__native_queued_spin_unlock); +DEFINE_STATIC_CALL(queued_spin_lock_slowpath, native_queued_spin_lock_slowpath); +EXPORT_STATIC_CALL_TRAMP(queued_spin_lock_slowpath); +DEFINE_STATIC_CALL(queued_spin_unlock, __raw_callee_save___native_queued_spin_unlock); +EXPORT_STATIC_CALL_TRAMP(queued_spin_unlock); + +/* + * Traced unlock variants, swapped in via static_call while the + * contended_release tracepoint is enabled. Two of them, so each tail calls its + * own base directly. + */ +__visible void native_queued_spin_unlock_traced(struct qspinlock *lock) +{ + if (queued_spin_is_contended(lock)) + trace_call__contended_release(lock); + native_queued_spin_unlock(lock); +} +PV_CALLEE_SAVE_REGS_THUNK(native_queued_spin_unlock_traced); + +__visible void pv_queued_spin_unlock_traced(struct qspinlock *lock) +{ + if (queued_spin_is_contended(lock)) + trace_call__contended_release(lock); + __raw_callee_save___pv_queued_spin_unlock(lock); +} +PV_CALLEE_SAVE_REGS_THUNK(pv_queued_spin_unlock_traced); + bool pv_is_native_spin_unlock(void) { - return pv_ops_lock.queued_spin_unlock.func == - __raw_callee_save___native_queued_spin_unlock; + void *unlock = static_call_query(queued_spin_unlock); + + return unlock == __raw_callee_save___native_queued_spin_unlock || + unlock == __raw_callee_save_native_queued_spin_unlock_traced; +} + +int arch_contended_release_trace_reg(void) +{ + void *cur = static_call_query(queued_spin_unlock); + + if (cur == __raw_callee_save___native_queued_spin_unlock) + static_call_update(queued_spin_unlock, + __raw_callee_save_native_queued_spin_unlock_traced); + else if (cur == __raw_callee_save___pv_queued_spin_unlock) + static_call_update(queued_spin_unlock, + __raw_callee_save_pv_queued_spin_unlock_traced); + return 0; +} + +void arch_contended_release_trace_unreg(void) +{ + void *cur = static_call_query(queued_spin_unlock); + + if (cur == __raw_callee_save_native_queued_spin_unlock_traced) + static_call_update(queued_spin_unlock, + __raw_callee_save___native_queued_spin_unlock); + else if (cur == __raw_callee_save_pv_queued_spin_unlock_traced) + static_call_update(queued_spin_unlock, + __raw_callee_save___pv_queued_spin_unlock); } __visible bool __native_vcpu_is_preempted(long cpu) @@ -45,16 +99,11 @@ bool pv_is_native_vcpu_is_preempted(void) void __init paravirt_set_cap(void) { - if (!pv_is_native_spin_unlock()) - setup_force_cpu_cap(X86_FEATURE_PVUNLOCK); - if (!pv_is_native_vcpu_is_preempted()) setup_force_cpu_cap(X86_FEATURE_VCPUPREEMPT); } struct pv_lock_ops pv_ops_lock = { - .queued_spin_lock_slowpath = native_queued_spin_lock_slowpath, - .queued_spin_unlock = PV_CALLEE_SAVE(__native_queued_spin_unlock), .wait = paravirt_nop, .kick = paravirt_nop, .vcpu_is_preempted = PV_CALLEE_SAVE(__native_vcpu_is_preempted), diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c index 6267363e0189..75cf8f6ae8cd 100644 --- a/arch/x86/kernel/pci-dma.c +++ b/arch/x86/kernel/pci-dma.c @@ -59,10 +59,8 @@ static void __init pci_swiotlb_detect(void) * bounce buffers as the hypervisor can't access arbitrary VM memory * that is not explicitly shared with it. */ - if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) { + if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) x86_swiotlb_enable = true; - x86_swiotlb_flags |= SWIOTLB_FORCE; - } } #else static inline void __init pci_swiotlb_detect(void) diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 85435044e33c..346c438ac880 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -579,7 +579,7 @@ static __always_inline void amd_set_core_ssb_state(unsigned long tifn) struct ssb_state *st = this_cpu_ptr(&ssb_state); u64 msr = x86_amd_ls_cfg_base; - if (!static_cpu_has(X86_FEATURE_ZEN)) { + if (!cpu_feature_enabled(X86_FEATURE_ZEN)) { msr |= ssbd_tif_to_amd_ls_cfg(tifn); wrmsrq(MSR_AMD64_LS_CFG, msr); return; @@ -646,14 +646,14 @@ static __always_inline void __speculation_ctrl_update(unsigned long tifp, lockdep_assert_irqs_disabled(); /* Handle change of TIF_SSBD depending on the mitigation method. */ - if (static_cpu_has(X86_FEATURE_VIRT_SSBD)) { + if (cpu_feature_enabled(X86_FEATURE_VIRT_SSBD)) { if (tif_diff & _TIF_SSBD) amd_set_ssb_virt_state(tifn); - } else if (static_cpu_has(X86_FEATURE_LS_CFG_SSBD)) { + } else if (cpu_feature_enabled(X86_FEATURE_LS_CFG_SSBD)) { if (tif_diff & _TIF_SSBD) amd_set_core_ssb_state(tifn); - } else if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) || - static_cpu_has(X86_FEATURE_AMD_SSBD)) { + } else if (cpu_feature_enabled(X86_FEATURE_SPEC_CTRL_SSBD) || + cpu_feature_enabled(X86_FEATURE_AMD_SSBD)) { updmsr |= !!(tif_diff & _TIF_SSBD); msr |= ssbd_tif_to_spec_ctrl(tifn); } diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index d44afbe005bb..2bce7b3f97ed 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -277,7 +277,7 @@ static __always_inline void save_fsgs(struct task_struct *task) { savesegment(fs, task->thread.fsindex); savesegment(gs, task->thread.gsindex); - if (static_cpu_has(X86_FEATURE_FSGSBASE)) { + if (cpu_feature_enabled(X86_FEATURE_FSGSBASE)) { /* * If FSGSBASE is enabled, we can't make any useful guesses * about the base, and user code expects us to save the current @@ -391,7 +391,7 @@ static __always_inline void x86_pkru_load(struct thread_struct *prev, static __always_inline void x86_fsgsbase_load(struct thread_struct *prev, struct thread_struct *next) { - if (static_cpu_has(X86_FEATURE_FSGSBASE)) { + if (cpu_feature_enabled(X86_FEATURE_FSGSBASE)) { /* Update the FS and GS selectors if they could have changed. */ if (unlikely(prev->fsindex || next->fsindex)) loadseg(FS, next->fsindex); @@ -533,7 +533,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip, { WARN_ON_ONCE(regs != current_pt_regs()); - if (static_cpu_has(X86_BUG_NULL_SEG)) { + if (cpu_feature_enabled(X86_BUG_NULL_SEG)) { /* Loading zero below won't clear the base. */ loadsegment(fs, __USER_DS); load_gs_index(__USER_DS); diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 46882ce79c3a..cda6adb9f69c 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -6,6 +6,7 @@ * parts of early kernel initialization. */ #include <linux/acpi.h> +#include <linux/bootconfig.h> #include <linux/console.h> #include <linux/cpu.h> #include <linux/crash_dump.h> @@ -16,6 +17,7 @@ #include <linux/init_ohci1394_dma.h> #include <linux/initrd.h> #include <linux/iscsi_ibft.h> +#include <linux/kexec_handover.h> #include <linux/memblock.h> #include <linux/panic_notifier.h> #include <linux/pci.h> @@ -120,8 +122,7 @@ struct cpuinfo_x86 new_cpu_data; struct apm_info apm_info; EXPORT_SYMBOL(apm_info); -#if defined(CONFIG_X86_SPEEDSTEP_SMI) || \ - defined(CONFIG_X86_SPEEDSTEP_SMI_MODULE) +#if IS_ENABLED(CONFIG_X86_SPEEDSTEP_SMI) struct ist_info ist_info; EXPORT_SYMBOL(ist_info); #else @@ -230,7 +231,7 @@ char builtin_cmdline[COMMAND_LINE_SIZE] = CONFIG_CMDLINE; bool builtin_cmdline_added __ro_after_init; #endif -#if defined(CONFIG_EDD) || defined(CONFIG_EDD_MODULE) +#if IS_ENABLED(CONFIG_EDD) struct edd edd; #ifdef CONFIG_EDD_MODULE EXPORT_SYMBOL(edd); @@ -880,7 +881,6 @@ static void __init x86_report_nx(void) * * Note: On x86_64, fixmaps are ready for use even before this is called. */ - void __init setup_arch(char **cmdline_p) { #ifdef CONFIG_X86_32 @@ -924,6 +924,18 @@ void __init setup_arch(char **cmdline_p) builtin_cmdline_added = true; #endif +#ifdef CONFIG_CMDLINE_FROM_BOOTCONFIG + /* + * Prepend the build-time-rendered embedded "kernel" keys here so + * parse_early_param() below sees them, using the same opt-in as the + * runtime parser, plus the build-time CONFIG_BOOT_CONFIG_FORCE. + */ + if (bootconfig_cmdline_requested(boot_command_line, NULL) || + IS_ENABLED(CONFIG_BOOT_CONFIG_FORCE)) + xbc_prepend_embedded_cmdline(boot_command_line, + COMMAND_LINE_SIZE); +#endif + strscpy(command_line, boot_command_line, COMMAND_LINE_SIZE); *cmdline_p = command_line; @@ -1268,7 +1280,7 @@ void __init setup_arch(char **cmdline_p) mcheck_init(); - register_refined_jiffies(CLOCK_TICK_RATE); + register_refined_jiffies(PIT_TICK_RATE); #ifdef CONFIG_EFI if (efi_enabled(EFI_BOOT)) diff --git a/arch/x86/kernel/signal_32.c b/arch/x86/kernel/signal_32.c index e55cf19e68fe..8e87ab586437 100644 --- a/arch/x86/kernel/signal_32.c +++ b/arch/x86/kernel/signal_32.c @@ -357,7 +357,7 @@ int ia32_setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs) unsafe_put_user(ptr_to_compat(&frame->uc), &frame->puc, Efault); /* Create the ucontext. */ - if (static_cpu_has(X86_FEATURE_XSAVE)) + if (cpu_feature_enabled(X86_FEATURE_XSAVE)) unsafe_put_user(UC_FP_XSTATE, &frame->uc.uc_flags, Efault); else unsafe_put_user(0, &frame->uc.uc_flags, Efault); diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index cb999feb66b0..ba01a9e919b7 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -991,9 +991,6 @@ int common_cpu_up(unsigned int cpu, struct task_struct *idle) { int ret; - /* Just in case we booted with a single CPU. */ - alternatives_enable_smp(); - per_cpu(current_task, cpu) = idle; cpu_init_stack_canary(cpu, idle); diff --git a/arch/x86/kernel/static_call.c b/arch/x86/kernel/static_call.c index 61592e41a6b1..bab9406e6d6a 100644 --- a/arch/x86/kernel/static_call.c +++ b/arch/x86/kernel/static_call.c @@ -4,6 +4,12 @@ #include <linux/bug.h> #include <asm/text-patching.h> +/* Declared locally to avoid pulling asm/paravirt-spinlock.h header. */ +#ifdef CONFIG_PARAVIRT_SPINLOCKS +struct qspinlock; +void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock); +#endif + enum insn_type { CALL = 0, /* site call */ NOP = 1, /* site cond-call */ @@ -31,6 +37,17 @@ static const u8 retinsn[] = { RET_INSN_OPCODE, 0xcc, 0xcc, 0xcc, 0xcc }; */ static const u8 warninsn[] = { 0x67, 0x48, 0x0f, 0xb9, 0x3a }; +#ifdef CONFIG_PARAVIRT_SPINLOCKS +/* + * ds ds movb $0, (_ASM_ARG1) + */ +#ifdef CONFIG_64BIT +static const u8 unlockinsn[] = { 0x3e, 0x3e, 0xc6, 0x07, 0x00 }; +#else +static const u8 unlockinsn[] = { 0x3e, 0x3e, 0xc6, 0x00, 0x00 }; +#endif +#endif + static u8 __is_Jcc(u8 *insn) /* Jcc.d32 */ { u8 ret = 0; @@ -78,6 +95,12 @@ static void __ref __static_call_transform(void *insn, enum insn_type type, emulate = code; code = &warninsn; } +#ifdef CONFIG_PARAVIRT_SPINLOCKS + if (func == &__raw_callee_save___native_queued_spin_unlock) { + emulate = code; + code = &unlockinsn; + } +#endif break; case NOP: @@ -139,6 +162,10 @@ static void __static_call_validate(u8 *insn, bool tail, bool tramp) !memcmp(insn, xor5rax, 5) || !memcmp(insn, warninsn, 5)) return; +#ifdef CONFIG_PARAVIRT_SPINLOCKS + if (!memcmp(insn, unlockinsn, 5)) + return; +#endif } /* diff --git a/arch/x86/kernel/time.c b/arch/x86/kernel/time.c index 52e1f3f0b361..4061430ac74c 100644 --- a/arch/x86/kernel/time.c +++ b/arch/x86/kernel/time.c @@ -82,7 +82,7 @@ static __init void x86_late_time_init(void) x86_init.irqs.intr_mode_init(); tsc_init(); - if (static_cpu_has(X86_FEATURE_WAITPKG)) + if (cpu_feature_enabled(X86_FEATURE_WAITPKG)) use_tpause_delay(); } diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index ce10ae4b298b..723347e2cf7f 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -1221,11 +1221,11 @@ static void __init check_system_tsc_reliable(void) if (is_geode_lx()) { /* RTSC counts during suspend */ #define RTSC_SUSP 0x100 - unsigned long res_low, res_high; + u64 res; - rdmsr_safe(MSR_GEODE_BUSCONT_CONF0, &res_low, &res_high); + rdmsrq_safe(MSR_GEODE_BUSCONT_CONF0, &res); /* Geode_LX - the OLPC CPU has a very reliable TSC */ - if (res_low & RTSC_SUSP) + if (res & RTSC_SUSP) tsc_clocksource_reliable = 1; } #endif diff --git a/arch/x86/kernel/tsc_msr.c b/arch/x86/kernel/tsc_msr.c index 48e6cc1cb017..d74743c8d2a4 100644 --- a/arch/x86/kernel/tsc_msr.c +++ b/arch/x86/kernel/tsc_msr.c @@ -165,7 +165,8 @@ static const struct x86_cpu_id tsc_msr_cpu_ids[] = { */ unsigned long cpu_khz_from_msr(void) { - u32 lo, hi, ratio, freq, tscref; + u32 ratio, freq, tscref; + struct msr val; const struct freq_desc *freq_desc; const struct x86_cpu_id *id; const struct muldiv *md; @@ -178,16 +179,16 @@ unsigned long cpu_khz_from_msr(void) freq_desc = (struct freq_desc *)id->driver_data; if (freq_desc->use_msr_plat) { - rdmsr(MSR_PLATFORM_INFO, lo, hi); - ratio = (lo >> 8) & 0xff; + rdmsrq(MSR_PLATFORM_INFO, val.q); + ratio = (val.l >> 8) & 0xff; } else { - rdmsr(MSR_IA32_PERF_STATUS, lo, hi); - ratio = (hi >> 8) & 0x1f; + rdmsrq(MSR_IA32_PERF_STATUS, val.q); + ratio = (val.h >> 8) & 0x1f; } /* Get FSB FREQ ID */ - rdmsr(MSR_FSB_FREQ, lo, hi); - index = lo & freq_desc->mask; + rdmsrq(MSR_FSB_FREQ, val.q); + index = val.l & freq_desc->mask; md = &freq_desc->muldiv[index]; /* diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c index 3af979fb41d3..65a2de82ecd2 100644 --- a/arch/x86/kernel/uprobes.c +++ b/arch/x86/kernel/uprobes.c @@ -276,15 +276,9 @@ static bool is_prefix_bad(struct insn *insn) return false; } -static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool x86_64) +static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn) { - enum insn_mode m = x86_64 ? INSN_MODE_64 : INSN_MODE_32; u32 volatile *good_insns; - int ret; - - ret = insn_decode(insn, auprobe->insn, sizeof(auprobe->insn), m); - if (ret < 0) - return -ENOEXEC; if (is_prefix_bad(insn)) return -ENOTSUPP; @@ -293,7 +287,7 @@ static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool if (insn_masking_exception(insn)) return -ENOTSUPP; - if (x86_64) + if (insn->x86_64) good_insns = good_insns_64; else good_insns = good_insns_32; @@ -631,14 +625,29 @@ static struct vm_special_mapping tramp_mapping = { .pages = tramp_mapping_pages, }; -struct uprobe_trampoline { - struct hlist_node node; - unsigned long vaddr; -}; + +#define LEA_INSN_SIZE 5 +#define OPT_INSN_SIZE (LEA_INSN_SIZE + CALL_INSN_SIZE) +#define REDZONE_SIZE 0x80 + +static const u8 lea_rsp[] = { 0x48, 0x8d, 0x64, 0x24, 0x80 }; + +static bool is_opt_insns(const uprobe_opcode_t *insn) +{ + return !memcmp(insn, lea_rsp, LEA_INSN_SIZE) && + insn[LEA_INSN_SIZE] == CALL_INSN_OPCODE; +} + +static bool is_swbp_opt_insns(uprobe_opcode_t *insn) +{ + return is_swbp_insn(&insn[0]) && + !memcmp(&insn[1], &lea_rsp[1], LEA_INSN_SIZE - 1) && + insn[LEA_INSN_SIZE] == CALL_INSN_OPCODE; +} static bool is_reachable_by_call(unsigned long vtramp, unsigned long vaddr) { - long delta = (long)(vaddr + 5 - vtramp); + long delta = (long)(vaddr + OPT_INSN_SIZE - vtramp); return delta >= INT_MIN && delta <= INT_MAX; } @@ -651,7 +660,7 @@ static unsigned long find_nearest_trampoline(unsigned long vaddr) }; unsigned long low_limit, high_limit; unsigned long low_tramp, high_tramp; - unsigned long call_end = vaddr + 5; + unsigned long call_end = vaddr + OPT_INSN_SIZE; if (check_add_overflow(call_end, INT_MIN, &low_limit)) low_limit = PAGE_SIZE; @@ -682,83 +691,32 @@ static unsigned long find_nearest_trampoline(unsigned long vaddr) return high_tramp; } -static struct uprobe_trampoline *create_uprobe_trampoline(unsigned long vaddr) +static struct vm_area_struct *get_uprobe_trampoline(struct mm_struct *mm, unsigned long vaddr, + bool *new_mapping) { - struct pt_regs *regs = task_pt_regs(current); - struct mm_struct *mm = current->mm; - struct uprobe_trampoline *tramp; + VMA_ITERATOR(vmi, mm, 0); struct vm_area_struct *vma; - if (!user_64bit_mode(regs)) - return NULL; - - vaddr = find_nearest_trampoline(vaddr); - if (IS_ERR_VALUE(vaddr)) - return NULL; - - tramp = kzalloc_obj(*tramp); - if (unlikely(!tramp)) - return NULL; - - tramp->vaddr = vaddr; - vma = _install_special_mapping(mm, tramp->vaddr, PAGE_SIZE, - VM_READ|VM_EXEC|VM_MAYEXEC|VM_MAYREAD|VM_DONTCOPY|VM_IO, - &tramp_mapping); - if (IS_ERR(vma)) { - kfree(tramp); - return NULL; - } - return tramp; -} - -static struct uprobe_trampoline *get_uprobe_trampoline(unsigned long vaddr, bool *new) -{ - struct uprobes_state *state = ¤t->mm->uprobes_state; - struct uprobe_trampoline *tramp = NULL; + *new_mapping = false; if (vaddr > TASK_SIZE || vaddr < PAGE_SIZE) - return NULL; + return ERR_PTR(-EINVAL); - hlist_for_each_entry(tramp, &state->head_tramps, node) { - if (is_reachable_by_call(tramp->vaddr, vaddr)) { - *new = false; - return tramp; - } + for_each_vma(vmi, vma) { + if (!vma_is_special_mapping(vma, &tramp_mapping)) + continue; + if (is_reachable_by_call(vma->vm_start, vaddr)) + return vma; } - tramp = create_uprobe_trampoline(vaddr); - if (!tramp) - return NULL; - - *new = true; - hlist_add_head(&tramp->node, &state->head_tramps); - return tramp; -} - -static void destroy_uprobe_trampoline(struct uprobe_trampoline *tramp) -{ - /* - * We do not unmap and release uprobe trampoline page itself, - * because there's no easy way to make sure none of the threads - * is still inside the trampoline. - */ - hlist_del(&tramp->node); - kfree(tramp); -} - -void arch_uprobe_init_state(struct mm_struct *mm) -{ - INIT_HLIST_HEAD(&mm->uprobes_state.head_tramps); -} - -void arch_uprobe_clear_state(struct mm_struct *mm) -{ - struct uprobes_state *state = &mm->uprobes_state; - struct uprobe_trampoline *tramp; - struct hlist_node *n; + vaddr = find_nearest_trampoline(vaddr); + if (IS_ERR_VALUE(vaddr)) + return ERR_PTR(vaddr); - hlist_for_each_entry_safe(tramp, n, &state->head_tramps, node) - destroy_uprobe_trampoline(tramp); + *new_mapping = true; + return _install_special_mapping(mm, vaddr, PAGE_SIZE, + VM_READ|VM_EXEC|VM_MAYEXEC|VM_MAYREAD|VM_IO, + &tramp_mapping); } static bool __in_uprobe_trampoline(struct mm_struct *mm, unsigned long ip) @@ -810,7 +768,7 @@ SYSCALL_DEFINE0(uprobe) /* Allow execution only from uprobe trampolines. */ if (!in_uprobe_trampoline(regs->ip)) - return -ENXIO; + return -EPROTO; err = copy_from_user(&args, (void __user *)regs->sp, sizeof(args)); if (err) @@ -826,8 +784,8 @@ SYSCALL_DEFINE0(uprobe) regs->ax = args.ax; regs->r11 = args.r11; regs->cx = args.cx; - regs->ip = args.retaddr - 5; - regs->sp += sizeof(args); + regs->ip = args.retaddr - OPT_INSN_SIZE; + regs->sp += sizeof(args) + REDZONE_SIZE; regs->orig_ax = -1; sp = regs->sp; @@ -844,12 +802,12 @@ SYSCALL_DEFINE0(uprobe) */ if (regs->sp != sp) { /* skip the trampoline call */ - if (args.retaddr - 5 == regs->ip) - regs->ip += 5; + if (args.retaddr - OPT_INSN_SIZE == regs->ip) + regs->ip += OPT_INSN_SIZE; return regs->ax; } - regs->sp -= sizeof(args); + regs->sp -= sizeof(args) + REDZONE_SIZE; /* for the case uprobe_consumer has changed ax/r11/cx */ args.ax = regs->ax; @@ -857,7 +815,7 @@ SYSCALL_DEFINE0(uprobe) args.cx = regs->cx; /* keep return address unless we are instructed otherwise */ - if (args.retaddr - 5 != regs->ip) + if (args.retaddr - OPT_INSN_SIZE != regs->ip) args.retaddr = regs->ip; if (shstk_push(args.retaddr) == -EFAULT) @@ -891,7 +849,7 @@ asm ( "pop %rax\n" "pop %r11\n" "pop %rcx\n" - "ret\n" + "ret $" __stringify(REDZONE_SIZE) "\n" "int3\n" ".balign " __stringify(PAGE_SIZE) "\n" ".popsection\n" @@ -909,7 +867,8 @@ late_initcall(arch_uprobes_init); enum { EXPECT_SWBP, - EXPECT_CALL, + EXPECT_OPTIMIZED, + EXPECT_SWBP_OPTIMIZED, }; struct write_opcode_ctx { @@ -917,30 +876,29 @@ struct write_opcode_ctx { int expect; }; -static int is_call_insn(uprobe_opcode_t *insn) -{ - return *insn == CALL_INSN_OPCODE; -} - /* - * Verification callback used by int3_update uprobe_write calls to make sure - * the underlying instruction is as expected - either int3 or call. + * Verification callback used by uprobe_write calls to make sure the underlying + * instruction is in the expected stage of the INT3 update sequence. */ static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode, int nbytes, void *data) { struct write_opcode_ctx *ctx = data; - uprobe_opcode_t old_opcode[5]; + uprobe_opcode_t old_opcode[OPT_INSN_SIZE]; - uprobe_copy_from_page(page, ctx->base, (uprobe_opcode_t *) &old_opcode, 5); + uprobe_copy_from_page(page, ctx->base, old_opcode, OPT_INSN_SIZE); switch (ctx->expect) { case EXPECT_SWBP: if (is_swbp_insn(&old_opcode[0])) return 1; break; - case EXPECT_CALL: - if (is_call_insn(&old_opcode[0])) + case EXPECT_OPTIMIZED: + if (is_opt_insns(&old_opcode[0])) + return 1; + break; + case EXPECT_SWBP_OPTIMIZED: + if (is_swbp_opt_insns(&old_opcode[0])) return 1; break; } @@ -949,48 +907,137 @@ static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t * } /* - * Modify multi-byte instructions by using INT3 breakpoints on SMP. + * Modify the optimized instruction by using INT3 breakpoints on SMP. * We completely avoid using stop_machine() here, and achieve the * synchronization using INT3 breakpoints and SMP cross-calls. * (borrowed comment from smp_text_poke_batch_finish) * - * The way it is done: - * - Add an INT3 trap to the address that will be patched - * - SMP sync all CPUs - * - Update all but the first byte of the patched range - * - SMP sync all CPUs - * - Replace the first byte (INT3) by the first byte of the replacing opcode - * - SMP sync all CPUs + * For optimization (int3_update_optimize): + * 1) Start with the uprobe INT3 trap already installed + * 2) Update everything but the first byte + * 3) Replace the first INT3 by the first byte of the LEA instruction + * + * For unoptimization (int3_update_unoptimize): + * 1) Start with the optimized uprobe lea/call instructions + * 2) Add an INT3 trap to the address that will be patched + * 3) Restore the NOP bytes before the call opcode + * 4) Replace the first INT3 by the first byte of the NOP instruction + * + * Note that unoptimization deliberately keeps the call opcode and displacement + * in bytes 5..9. Those bytes become operands of the restored 10-byte NOP. + * + * Since there is only a single target uprobe-trampoline for the given nop10 + * instruction address, the CALL instruction will not be changed across + * unoptimization/optimization cycles. + * Therefore, any task that is preempted at the CALL instruction is guaranteed + * to observe that CALL and not anything else. */ -static int int3_update(struct arch_uprobe *auprobe, struct vm_area_struct *vma, - unsigned long vaddr, char *insn, bool optimize) +static int int3_update_optimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, + unsigned long vaddr, uprobe_opcode_t *insn) { - uprobe_opcode_t int3 = UPROBE_SWBP_INSN; struct write_opcode_ctx ctx = { .base = vaddr, }; int err; /* - * Write int3 trap. + * 1) Initial state after set_swbp() installed the uprobe: + * cc 2e 0f 1f 84 00 00 00 00 00 * - * The swbp_optimize path comes with breakpoint already installed, - * so we can skip this step for optimize == true. + * After a previous unoptimization bytes 5..9 may still contain the + * old call instruction, which remains valid for threads already there. */ - if (!optimize) { - ctx.expect = EXPECT_CALL; - err = uprobe_write(auprobe, vma, vaddr, &int3, 1, verify_insn, - true /* is_register */, false /* do_update_ref_ctr */, - &ctx); - if (err) - return err; - } + smp_text_poke_sync_each_cpu(); + + /* + * 2) Rewrite the LEA tail and call displacement: + * cc [8d 64 24 80 e8 d0 d1 d2 d3] + */ + ctx.expect = EXPECT_SWBP; + err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1, + OPT_INSN_SIZE - 1, verify_insn, + true /* is_register */, false /* do_update_ref_ctr */, + &ctx); + if (err) + return err; smp_text_poke_sync_each_cpu(); - /* Write all but the first byte of the patched range. */ + /* + * 3) Publish the first LEA byte: + * [48] 8d 64 24 80 e8 d0 d1 d2 d3 + * + * From offset 0 this is: + * lea -0x80(%rsp), %rsp + * call <uprobe-trampoline> + */ + ctx.expect = EXPECT_SWBP_OPTIMIZED; + err = uprobe_write(auprobe, vma, vaddr, insn, 1, verify_insn, + true /* is_register */, false /* do_update_ref_ctr */, + &ctx); + if (err) + goto error; + + smp_text_poke_sync_each_cpu(); + return 0; + +error: + /* + * In all intermediate states byte 0 is INT3, so EXPECT_SWBP covers every + * case. Restore NOP bytes 1..4, but keep the valid CALL at bytes 5..9 + * for a thread that had already executed the LEA before a previous + * unoptimization. + */ ctx.expect = EXPECT_SWBP; - err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1, 4, verify_insn, + uprobe_write(auprobe, vma, vaddr + 1, auprobe->insn + 1, + LEA_INSN_SIZE - 1, verify_insn, true, false, &ctx); + smp_text_poke_sync_each_cpu(); + return err; +} + +static int int3_update_unoptimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, + unsigned long vaddr, uprobe_opcode_t *insn) +{ + uprobe_opcode_t int3 = UPROBE_SWBP_INSN; + struct write_opcode_ctx ctx = { + .base = vaddr, + .expect = EXPECT_OPTIMIZED, + }; + int err; + + /* + * Note the first two uprobe_write calls use is_register=true, because they + * are intermediate patching states while the probe is still active, so + * we force the exclusive anonymous page for the update. + * Also we use do_update_ref_ctr=false because refctr was already updated by + * the initial int3 install. + * + * The last uprobe_write to nop10 instruction is called with is_register=false + * and do_update_ref_ctr=true to trigger the refctr update and to instruct + * uprobe_write to zap the anonymous page if it now matches the file page. + * + * 1) Initial optimized state: + * 48 8d 64 24 80 e8 d0 d1 d2 d3 + * + * 2) Trap new entries before restoring the NOP bytes: + * [cc] 8d 64 24 80 e8 d0 d1 d2 d3 + */ + err = uprobe_write(auprobe, vma, vaddr, &int3, 1, verify_insn, + true /* is_register */, false /* do_update_ref_ctr */, + &ctx); + if (err) + return err; + + smp_text_poke_sync_each_cpu(); + + /* + * 3) Restore bytes 1..4 of the original NOP while keeping byte 0 trapped + * and byte 5 as CALL: + * cc [2e 0f 1f 84] e8 d0 d1 d2 d3 + */ + ctx.expect = EXPECT_SWBP_OPTIMIZED; + err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1, + LEA_INSN_SIZE - 1, verify_insn, true /* is_register */, false /* do_update_ref_ctr */, &ctx); if (err) @@ -999,13 +1046,16 @@ static int int3_update(struct arch_uprobe *auprobe, struct vm_area_struct *vma, smp_text_poke_sync_each_cpu(); /* - * Write first byte. + * 4) Publish the first byte of the original NOP: + * [66] 2e 0f 1f 84 e8 d0 d1 d2 d3 * - * The swbp_unoptimize needs to finish uprobe removal together - * with ref_ctr update, using uprobe_write with proper flags. + * From offset 0 this is the restored 10-byte NOP; the CALL opcode and + * displacement are now only NOP operands. Offset 5 still decodes as + * CALL for a thread that was already there. */ + ctx.expect = EXPECT_SWBP; err = uprobe_write(auprobe, vma, vaddr, insn, 1, verify_insn, - optimize /* is_register */, !optimize /* do_update_ref_ctr */, + false /* is_register */, true /* do_update_ref_ctr */, &ctx); if (err) return err; @@ -1017,17 +1067,25 @@ static int int3_update(struct arch_uprobe *auprobe, struct vm_area_struct *vma, static int swbp_optimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, unsigned long vaddr, unsigned long tramp) { - u8 call[5]; + u8 insn[OPT_INSN_SIZE], *call = &insn[LEA_INSN_SIZE]; - __text_gen_insn(call, CALL_INSN_OPCODE, (const void *) vaddr, + /* + * We have nop10 instruction (with first byte overwritten to int3), + * changing it to: + * lea -0x80(%rsp), %rsp + * call tramp + */ + memcpy(insn, lea_rsp, LEA_INSN_SIZE); + __text_gen_insn(call, CALL_INSN_OPCODE, + (const void *) (vaddr + LEA_INSN_SIZE), (const void *) tramp, CALL_INSN_SIZE); - return int3_update(auprobe, vma, vaddr, call, true /* optimize */); + return int3_update_optimize(auprobe, vma, vaddr, insn); } static int swbp_unoptimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, unsigned long vaddr) { - return int3_update(auprobe, vma, vaddr, auprobe->insn, false /* optimize */); + return int3_update_unoptimize(auprobe, vma, vaddr, auprobe->insn); } static int copy_from_vaddr(struct mm_struct *mm, unsigned long vaddr, void *dst, int len) @@ -1049,19 +1107,19 @@ static bool __is_optimized(struct mm_struct *mm, uprobe_opcode_t *insn, unsigned struct __packed __arch_relative_insn { u8 op; s32 raddr; - } *call = (struct __arch_relative_insn *) insn; + } *call = (struct __arch_relative_insn *)(insn + LEA_INSN_SIZE); - if (!is_call_insn(insn)) + if (!is_opt_insns(insn)) return false; - return __in_uprobe_trampoline(mm, vaddr + 5 + call->raddr); + return __in_uprobe_trampoline(mm, vaddr + OPT_INSN_SIZE + call->raddr); } static int is_optimized(struct mm_struct *mm, unsigned long vaddr) { - uprobe_opcode_t insn[5]; + uprobe_opcode_t insn[OPT_INSN_SIZE]; int err; - err = copy_from_vaddr(mm, vaddr, &insn, 5); + err = copy_from_vaddr(mm, vaddr, &insn, OPT_INSN_SIZE); if (err) return err; return __is_optimized(mm, (uprobe_opcode_t *)&insn, vaddr); @@ -1111,27 +1169,29 @@ int set_orig_insn(struct arch_uprobe *auprobe, struct vm_area_struct *vma, static int __arch_uprobe_optimize(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr) { - struct uprobe_trampoline *tramp; - struct vm_area_struct *vma; - bool new = false; - int err = 0; + struct pt_regs *regs = task_pt_regs(current); + struct vm_area_struct *vma, *tramp; + bool new_mapping; + int ret; + if (!user_64bit_mode(regs)) + return -EINVAL; vma = find_vma(mm, vaddr); if (!vma) return -EINVAL; - tramp = get_uprobe_trampoline(vaddr, &new); - if (!tramp) - return -EINVAL; - err = swbp_optimize(auprobe, vma, vaddr, tramp->vaddr); - if (WARN_ON_ONCE(err) && new) - destroy_uprobe_trampoline(tramp); - return err; + tramp = get_uprobe_trampoline(mm, vaddr, &new_mapping); + if (IS_ERR(tramp)) + return PTR_ERR(tramp); + ret = swbp_optimize(auprobe, vma, vaddr, tramp->vm_start); + if (WARN_ON_ONCE(ret) && new_mapping) + WARN_ON_ONCE(do_munmap(mm, tramp->vm_start, PAGE_SIZE, NULL)); + return ret; } void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr) { struct mm_struct *mm = current->mm; - uprobe_opcode_t insn[5]; + uprobe_opcode_t insn[OPT_INSN_SIZE]; if (!should_optimize(auprobe)) return; @@ -1142,7 +1202,7 @@ void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr) * Check if some other thread already optimized the uprobe for us, * if it's the case just go away silently. */ - if (copy_from_vaddr(mm, vaddr, &insn, 5)) + if (copy_from_vaddr(mm, vaddr, &insn, OPT_INSN_SIZE)) goto unlock; if (!is_swbp_insn((uprobe_opcode_t*) &insn)) goto unlock; @@ -1158,16 +1218,32 @@ unlock: mmap_write_unlock(mm); } +static bool is_optimizable_nop10(struct insn *insn) +{ + static const u8 nop10_prefix[] = { + 0x66, 0x2e, 0x0f, 0x1f, 0x84 + }; + + /* + * Restrict this to the 10-byte NOP form whose last 5 bytes are + * SIB/displacement operands. Unoptimization keeps the call opcode and + * displacement in those bytes, so other NOP encodings are not safe. + */ + return insn->length == OPT_INSN_SIZE && + insn_is_nop(insn) && + !memcmp(insn->kaddr, nop10_prefix, ARRAY_SIZE(nop10_prefix)); +} + static bool can_optimize(struct insn *insn, unsigned long vaddr) { - if (!insn->x86_64 || insn->length != 5) + if (!insn->x86_64) return false; - if (!insn_is_nop(insn)) + if (!is_optimizable_nop10(insn)) return false; /* We can't do cross page atomic writes yet. */ - return PAGE_SIZE - (vaddr & ~PAGE_MASK) >= 5; + return PAGE_SIZE - (vaddr & ~PAGE_MASK) >= OPT_INSN_SIZE; } #else /* 32-bit: */ /* @@ -1549,16 +1625,26 @@ static int push_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn) */ int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long addr) { + enum insn_mode m = is_64bit_mm(mm) ? INSN_MODE_64 : INSN_MODE_32; u8 fix_ip_or_call = UPROBE_FIX_IP; struct insn insn; int ret; - ret = uprobe_init_insn(auprobe, &insn, is_64bit_mm(mm)); - if (ret) - return ret; + ret = insn_decode(&insn, auprobe->insn, sizeof(auprobe->insn), m); + if (ret < 0) + return -ENOEXEC; - if (can_optimize(&insn, addr)) + /* + * No need to check instruction in uprobe_init_insn in case we + * are on top of optimizable nop10. + */ + if (can_optimize(&insn, addr)) { set_bit(ARCH_UPROBE_FLAG_CAN_OPTIMIZE, &auprobe->flags); + } else { + ret = uprobe_init_insn(auprobe, &insn); + if (ret) + return ret; + } ret = branch_setup_xol_ops(auprobe, &insn); if (ret != -ENOSYS) diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index 74e336d7f9dd..2438b89a4620 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -65,9 +65,15 @@ const_cpu_current_top_of_stack = cpu_current_top_of_stack; __end_rodata_hpage_align = .; \ __end_rodata_aligned = .; +#if defined(CONFIG_MITIGATION_PAGE_TABLE_ISOLATION) || defined(CONFIG_MITIGATION_SRSO) #define ALIGN_ENTRY_TEXT_BEGIN . = ALIGN(PMD_SIZE); #define ALIGN_ENTRY_TEXT_END . = ALIGN(PMD_SIZE); #else +#define ALIGN_ENTRY_TEXT_BEGIN +#define ALIGN_ENTRY_TEXT_END +#endif + +#else #define X86_ALIGN_RODATA_BEGIN #define X86_ALIGN_RODATA_END \ @@ -221,7 +227,7 @@ SECTIONS * references to such code must be patched out by alternatives, normally * by using X86_FEATURE_ALWAYS CPU feature bit. * - * See static_cpu_has() for an example. + * See cpu_feature_enabled() for an example. */ .altinstr_aux : AT(ADDR(.altinstr_aux) - LOAD_OFFSET) { *(.altinstr_aux) @@ -346,18 +352,6 @@ SECTIONS __init_end = .; } - /* - * smp_locks might be freed after init - * start/end must be page aligned - */ - . = ALIGN(PAGE_SIZE); - .smp_locks : AT(ADDR(.smp_locks) - LOAD_OFFSET) { - __smp_locks = .; - *(.smp_locks) - . = ALIGN(PAGE_SIZE); - __smp_locks_end = .; - } - #ifdef CONFIG_X86_64 .data_nosave : AT(ADDR(.data_nosave) - LOAD_OFFSET) { NOSAVE_DATA |