diff options
Diffstat (limited to 'arch')
40 files changed, 346 insertions, 149 deletions
diff --git a/arch/arm64/include/asm/kvm_hypevents.h b/arch/arm64/include/asm/kvm_hypevents.h index 743c49bd878f..5f6e6789d121 100644 --- a/arch/arm64/include/asm/kvm_hypevents.h +++ b/arch/arm64/include/asm/kvm_hypevents.h @@ -12,6 +12,7 @@ enum hyp_enter_exit_reason { HYP_REASON_SMC, HYP_REASON_HVC, + HYP_REASON_SYS, HYP_REASON_PSCI, HYP_REASON_HOST_ABORT, HYP_REASON_GUEST_EXIT, diff --git a/arch/arm64/kernel/pi/relocate.c b/arch/arm64/kernel/pi/relocate.c index 2407d2696398..82592f3a5c1c 100644 --- a/arch/arm64/kernel/pi/relocate.c +++ b/arch/arm64/kernel/pi/relocate.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only // Copyright 2023 Google LLC // Authors: Ard Biesheuvel <ardb@google.com> -// Peter Collingbourne <pcc@google.com> +// Peter Collingbourne <peter@pcc.me.uk> #include <linux/elf.h> #include <linux/init.h> diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 50adfff75be8..9a6c72a18672 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -149,14 +149,27 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, set_bit(KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER, &kvm->arch.flags); break; - case KVM_CAP_ARM_MTE: - mutex_lock(&kvm->lock); - if (system_supports_mte() && !kvm->created_vcpus) { - r = 0; - set_bit(KVM_ARCH_FLAG_MTE_ENABLED, &kvm->arch.flags); + case KVM_CAP_ARM_MTE: { + struct kvm_memory_slot *memslot; + int bkt; + + guard(mutex)(&kvm->lock); + if (!system_supports_mte() || kvm->created_vcpus) + break; + + r = 0; + guard(mutex)(&kvm->slots_lock); + kvm_for_each_memslot(memslot, bkt, kvm_memslots(kvm)) { + if (kvm_slot_has_gmem(memslot)) { + r = -EINVAL; + break; + } } - mutex_unlock(&kvm->lock); + if (r == 0) + set_bit(KVM_ARCH_FLAG_MTE_ENABLED, &kvm->arch.flags); break; + + } case KVM_CAP_ARM_SYSTEM_SUSPEND: r = 0; set_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index d3c69de698f4..d3df96ed8ba4 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -932,6 +932,7 @@ void handle_trap(struct kvm_cpu_context *host_ctxt) handle_host_mem_abort(host_ctxt); break; case ESR_ELx_EC_SYS64: + trace_hyp_enter(host_ctxt, HYP_REASON_SYS); if (handle_host_mte(esr)) break; fallthrough; diff --git a/arch/arm64/kvm/hyp_trace.c b/arch/arm64/kvm/hyp_trace.c index 2411b4c32932..9644c424819b 100644 --- a/arch/arm64/kvm/hyp_trace.c +++ b/arch/arm64/kvm/hyp_trace.c @@ -37,8 +37,6 @@ static struct hyp_trace_clock { u32 shift; struct delayed_work work; struct completion ready; - struct mutex lock; - bool running; } hyp_clock; static void __hyp_clock_work(struct work_struct *work) @@ -110,12 +108,9 @@ static void hyp_trace_clock_enable(struct hyp_trace_clock *hyp_clock, bool enabl { struct system_time_snapshot snap; - if (hyp_clock->running == enable) - return; - if (!enable) { cancel_delayed_work_sync(&hyp_clock->work); - hyp_clock->running = false; + return; } ktime_get_snapshot_id(CLOCK_BOOTTIME, &snap); @@ -128,7 +123,6 @@ static void hyp_trace_clock_enable(struct hyp_trace_clock *hyp_clock, bool enabl INIT_DELAYED_WORK(&hyp_clock->work, __hyp_clock_work); schedule_delayed_work(&hyp_clock->work, msecs_to_jiffies(CLOCK_INIT_MS)); wait_for_completion(&hyp_clock->ready); - hyp_clock->running = true; } /* Access to this struct within the trace_remote_callbacks are protected by the trace_remote lock */ @@ -160,6 +154,7 @@ static int hyp_trace_buffer_alloc_bpages_backing(struct hyp_trace_buffer *trace_ int nr_bpages = (PAGE_ALIGN(size) / PAGE_SIZE) + 1; size_t backing_size; void *start; + int ret; backing_size = PAGE_ALIGN(sizeof(struct simple_buffer_page) * nr_bpages * num_possible_cpus()); @@ -168,10 +163,16 @@ static int hyp_trace_buffer_alloc_bpages_backing(struct hyp_trace_buffer *trace_ if (!start) return -ENOMEM; + ret = __map_hyp(start, backing_size); + if (ret) { + free_pages_exact(start, backing_size); + return ret; + } + trace_buffer->desc->bpages_backing_start = (unsigned long)start; trace_buffer->desc->bpages_backing_size = backing_size; - return __map_hyp(start, backing_size); + return ret; } static void hyp_trace_buffer_free_bpages_backing(struct hyp_trace_buffer *trace_buffer) @@ -228,18 +229,22 @@ static int hyp_trace_buffer_share_hyp(struct hyp_trace_buffer *trace_buffer) static struct trace_buffer_desc *hyp_trace_load(unsigned long size, void *priv) { struct hyp_trace_buffer *trace_buffer = priv; + size_t desc_size, tb_desc_size; struct hyp_trace_desc *desc; - size_t desc_size; int ret; if (WARN_ON(trace_buffer->desc)) return ERR_PTR(-EINVAL); - desc_size = trace_buffer_desc_size(size, num_possible_cpus()); + tb_desc_size = trace_buffer_desc_size(size, num_possible_cpus()); + desc_size = size_add(tb_desc_size, offsetof(struct hyp_trace_desc, trace_buffer_desc)); if (desc_size == SIZE_MAX) return ERR_PTR(-E2BIG); desc_size = PAGE_ALIGN(desc_size); + if (!desc_size) + return ERR_PTR(-E2BIG); + desc = (struct hyp_trace_desc *)alloc_pages_exact(desc_size, GFP_KERNEL); if (!desc) return ERR_PTR(-ENOMEM); @@ -255,7 +260,7 @@ static struct trace_buffer_desc *hyp_trace_load(unsigned long size, void *priv) if (ret) goto err_free_desc; - ret = trace_remote_alloc_buffer(&desc->trace_buffer_desc, desc_size, size, + ret = trace_remote_alloc_buffer(&desc->trace_buffer_desc, tb_desc_size, size, cpu_possible_mask); if (ret) goto err_free_backing; @@ -304,9 +309,15 @@ static void hyp_trace_unload(struct trace_buffer_desc *desc, void *priv) static int hyp_trace_enable_tracing(bool enable, void *priv) { + int ret; + hyp_trace_clock_enable(&hyp_clock, enable); - return kvm_call_hyp_nvhe(__tracing_enable, enable); + ret = kvm_call_hyp_nvhe(__tracing_enable, enable); + if (ret) + hyp_trace_clock_enable(&hyp_clock, !enable); + + return ret; } static int hyp_trace_swap_reader_page(unsigned int cpu, void *priv) @@ -398,6 +409,7 @@ static const char *__hyp_enter_exit_reason_str(u8 reason) static const char strs[][12] = { "smc", "hvc", + "sys", "psci", "host_abort", "guest_exit", diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 6c941aaa10c6..2d95203386ba 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -2652,6 +2652,10 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, if (kvm_slot_has_gmem(new) && !kvm_memslot_is_gmem_only(new)) return -EINVAL; + /* guest_memfd is incompatible with MTE. */ + if (kvm_slot_has_gmem(new) && kvm_has_mte(kvm)) + return -EINVAL; + hva = new->userspace_addr; reg_end = hva + (new->npages << PAGE_SHIFT); diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 740b39875728..36ab3e492915 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -116,18 +116,27 @@ static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid, kfree(irq); irq = oldirq; } else { - ret = xa_err(__xa_store(&dist->lpi_xa, intid, irq, 0)); + /* + * The entry is either empty or contains a dead LPI (refcount=0) + * from the deferred release path, pending cleanup by + * vgic_release_deleted_lpis(). Evict and free it if present. + */ + oldirq = __xa_store(&dist->lpi_xa, intid, irq, + GFP_NOWAIT | __GFP_ACCOUNT); + ret = xa_err(oldirq); + if (ret) { + xa_unlock_irqrestore(&dist->lpi_xa, flags); + kfree(irq); + + return ERR_PTR(ret); + } + + if (oldirq && !WARN_ON_ONCE(refcount_read(&oldirq->refcount))) + kfree_rcu(oldirq, rcu); } xa_unlock_irqrestore(&dist->lpi_xa, flags); - if (ret) { - xa_release(&dist->lpi_xa, intid); - kfree(irq); - - return ERR_PTR(ret); - } - /* * We "cache" the configuration table entries in our struct vgic_irq's. * However we only have those structs for mapped IRQs, so we read in diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 9e841e7afd4a..7aa417440f6a 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -275,7 +275,13 @@ void vgic_v3_deactivate(struct kvm_vcpu *vcpu, u64 val) lr = vgic_v3_compute_lr(vcpu, irq) & ~ICH_LR_ACTIVE_BIT; } - if (lr & ICH_LR_HW) + /* + * In the nested state, the irq has already been deactivated via the HW + * bit in the LR. Deactivating again would be harmless except AmpereOne + * errata AC03_CPU_57, AC04_CPU_29 could cause irq delivery to break if + * the deactivation hits the highest priority pending irq. + */ + if ((lr & ICH_LR_HW) && !vgic_state_is_nested(vcpu)) vgic_v3_deactivate_phys(FIELD_GET(ICH_LR_PHYS_ID_MASK, lr)); vgic_v3_fold_lr(vcpu, lr); diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index ccb7e3a90cd0..74bace10a22e 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -146,11 +146,7 @@ static __must_check bool __vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq) static __must_check bool vgic_put_irq_norelease(struct kvm *kvm, struct vgic_irq *irq) { - if (!__vgic_put_irq(kvm, irq)) - return false; - - irq->pending_release = true; - return true; + return __vgic_put_irq(kvm, irq); } void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq) @@ -167,12 +163,14 @@ void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq) guard(spinlock_irqsave)(&dist->lpi_xa.xa_lock); } - if (!__vgic_put_irq(kvm, irq)) + if (!irq_is_lpi(kvm, irq->intid)) return; - xa_lock_irqsave(&dist->lpi_xa, flags); - vgic_release_lpi_locked(dist, irq); - xa_unlock_irqrestore(&dist->lpi_xa, flags); + if (refcount_dec_and_lock_irqsave(&irq->refcount, + &dist->lpi_xa.xa_lock, &flags)) { + vgic_release_lpi_locked(dist, irq); + xa_unlock_irqrestore(&dist->lpi_xa, flags); + } } static void vgic_release_deleted_lpis(struct kvm *kvm) @@ -184,7 +182,7 @@ static void vgic_release_deleted_lpis(struct kvm *kvm) xa_lock_irqsave(&dist->lpi_xa, flags); xa_for_each(&dist->lpi_xa, intid, irq) { - if (irq->pending_release) + if (!refcount_read(&irq->refcount)) vgic_release_lpi_locked(dist, irq); } diff --git a/arch/powerpc/boot/simpleboot.c b/arch/powerpc/boot/simpleboot.c index c80691d83880..27591df41e9e 100644 --- a/arch/powerpc/boot/simpleboot.c +++ b/arch/powerpc/boot/simpleboot.c @@ -68,7 +68,7 @@ void platform_init(unsigned long r3, unsigned long r4, unsigned long r5, /* finally, setup the timebase */ node = fdt_node_offset_by_prop_value(_dtb_start, -1, "device_type", "cpu", sizeof("cpu")); - if (!node) + if (node < 0) fatal("Cannot find cpu node\n"); timebase = fdt_getprop(_dtb_start, node, "timebase-frequency", &size); if (timebase && (size == 4)) diff --git a/arch/powerpc/boot/treeboot-akebono.c b/arch/powerpc/boot/treeboot-akebono.c index e3cc2599869c..1b529037480f 100644 --- a/arch/powerpc/boot/treeboot-akebono.c +++ b/arch/powerpc/boot/treeboot-akebono.c @@ -146,7 +146,7 @@ void platform_init(char *userdata) node = fdt_node_offset_by_prop_value(_dtb_start, -1, "device_type", "cpu", sizeof("cpu")); - if (!node) + if (node < 0) fatal("Cannot find cpu node\n"); timebase = fdt_getprop(_dtb_start, node, "timebase-frequency", &size); if (timebase && (size == 4)) diff --git a/arch/powerpc/boot/treeboot-currituck.c b/arch/powerpc/boot/treeboot-currituck.c index d53e8a592f81..5b5363b74f9f 100644 --- a/arch/powerpc/boot/treeboot-currituck.c +++ b/arch/powerpc/boot/treeboot-currituck.c @@ -102,7 +102,7 @@ void platform_init(void) node = fdt_node_offset_by_prop_value(_dtb_start, -1, "device_type", "cpu", sizeof("cpu")); - if (!node) + if (node < 0) fatal("Cannot find cpu node\n"); timebase = fdt_getprop(_dtb_start, node, "timebase-frequency", &size); if (timebase && (size == 4)) diff --git a/arch/powerpc/include/asm/entry-common.h b/arch/powerpc/include/asm/entry-common.h index fc636c42e89a..c5adb5006361 100644 --- a/arch/powerpc/include/asm/entry-common.h +++ b/arch/powerpc/include/asm/entry-common.h @@ -66,6 +66,13 @@ static inline void srr_regs_clobbered(void) static inline void nap_adjust_return(struct pt_regs *regs) { #ifdef CONFIG_PPC_970_NAP + /* + * Adjust the nap return address before irq_exit_rcu(). irq_exit_rcu() + * may invoke softirqs with interrupts re-enabled, allowing a nested + * async interrupt to arrive. If _TLF_NAPPING is still set at that + * point, the nested interrupt would erroneously redirect its own + * return address to power4_idle_nap_return, corrupting the stack. + */ if (unlikely(test_thread_local_flags(_TLF_NAPPING))) { /* Can avoid a test-and-clear because NMIs do not call this */ clear_thread_local_flags(_TLF_NAPPING); @@ -286,14 +293,6 @@ static inline void arch_interrupt_async_enter_prepare(struct pt_regs *regs) static inline void arch_interrupt_async_exit_prepare(struct pt_regs *regs) { - /* - * Adjust at exit so the main handler sees the true NIA. This must - * come before irq_exit() because irq_exit can enable interrupts, and - * if another interrupt is taken before nap_adjust_return has run - * here, then that interrupt would return directly to idle nap return. - */ - nap_adjust_return(regs); - arch_interrupt_exit_prepare(regs); } diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h index fb42a664ae54..1b45a49e9bed 100644 --- a/arch/powerpc/include/asm/interrupt.h +++ b/arch/powerpc/include/asm/interrupt.h @@ -246,6 +246,7 @@ interrupt_handler void func(struct pt_regs *regs) \ instrumentation_begin(); \ irq_enter_rcu(); \ ____##func (regs); \ + nap_adjust_return(regs); \ irq_exit_rcu(); \ instrumentation_end(); \ arch_interrupt_async_exit_prepare(regs); \ diff --git a/arch/powerpc/include/asm/ptrace.h b/arch/powerpc/include/asm/ptrace.h index fdeb97421785..d53c4dd4d8b6 100644 --- a/arch/powerpc/include/asm/ptrace.h +++ b/arch/powerpc/include/asm/ptrace.h @@ -53,9 +53,6 @@ struct pt_regs unsigned long esr; }; unsigned long result; - unsigned long exit_flags; - /* Maintain 16 byte interrupt stack alignment */ - unsigned long __pt_regs_pad[3]; }; }; #if defined(CONFIG_PPC64) || defined(CONFIG_PPC_KUAP) diff --git a/arch/powerpc/include/asm/serial.h b/arch/powerpc/include/asm/serial.h index cd6c18d0e66e..c3eb4a64b3f5 100644 --- a/arch/powerpc/include/asm/serial.h +++ b/arch/powerpc/include/asm/serial.h @@ -1,6 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - */ #ifndef _ASM_POWERPC_SERIAL_H #define _ASM_POWERPC_SERIAL_H @@ -18,4 +16,4 @@ extern void find_legacy_serial_ports(void); #define find_legacy_serial_ports() do { } while (0) #endif -#endif /* _PPC64_SERIAL_H */ +#endif /* _ASM_POWERPC_SERIAL_H */ diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h index ee3b9adb5b67..0487e94d3416 100644 --- a/arch/powerpc/include/asm/thread_info.h +++ b/arch/powerpc/include/asm/thread_info.h @@ -57,6 +57,7 @@ struct thread_info { #ifdef CONFIG_SMP unsigned int cpu; #endif + unsigned long exit_flags; /* Exit Flags for entry/exit */ unsigned long syscall_work; /* SYSCALL_WORK_ flags */ unsigned long local_flags; /* private flags for thread */ #ifdef CONFIG_LIVEPATCH_64 diff --git a/arch/powerpc/include/uapi/asm/ptrace.h b/arch/powerpc/include/uapi/asm/ptrace.h index a393b7f2760a..01e630149d48 100644 --- a/arch/powerpc/include/uapi/asm/ptrace.h +++ b/arch/powerpc/include/uapi/asm/ptrace.h @@ -55,8 +55,6 @@ struct pt_regs unsigned long dar; /* Fault registers */ unsigned long dsisr; /* on 4xx/Book-E used for ESR */ unsigned long result; /* Result of a system call */ - unsigned long exit_flags; /* System call exit flags */ - unsigned long __pt_regs_pad[3]; /* Maintain 16 byte interrupt stack alignment */ }; #endif /* __ASSEMBLER__ */ @@ -116,12 +114,10 @@ struct pt_regs #define PT_DAR 41 #define PT_DSISR 42 #define PT_RESULT 43 -#define PT_EXIT_FLAGS 44 -#define PT_PAD 47 /* 3 times */ -#define PT_DSCR 48 -#define PT_REGS_COUNT 48 +#define PT_DSCR 44 +#define PT_REGS_COUNT 44 -#define PT_FPR0 (PT_REGS_COUNT + 4) /* each FP reg occupies 2 slots in this space */ +#define PT_FPR0 48 /* each FP reg occupies 2 slots in this space */ #ifndef __powerpc64__ @@ -133,7 +129,7 @@ struct pt_regs #define PT_FPSCR (PT_FPR0 + 32) /* each FP reg occupies 1 slot in 64-bit space */ -#define PT_VR0 (PT_FPSCR + 2) /* <82> each Vector reg occupies 2 slots in 64-bit */ +#define PT_VR0 82 /* each Vector reg occupies 2 slots in 64-bit */ #define PT_VSCR (PT_VR0 + 32*2 + 1) #define PT_VRSAVE (PT_VR0 + 33*2) @@ -141,7 +137,7 @@ struct pt_regs /* * Only store first 32 VSRs here. The second 32 VSRs in VR0-31 */ -#define PT_VSR0 (PT_VRSAVE + 2) /* each VSR reg occupies 2 slots in 64-bit */ +#define PT_VSR0 150 /* each VSR reg occupies 2 slots in 64-bit */ #define PT_VSR31 (PT_VSR0 + 2*31) #endif /* __powerpc64__ */ diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c index f04978080837..5b88bf72786c 100644 --- a/arch/powerpc/kernel/interrupt.c +++ b/arch/powerpc/kernel/interrupt.c @@ -89,15 +89,17 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, long scv) { unsigned long ti_flags; + unsigned long ret = 0; bool is_not_scv = !IS_ENABLED(CONFIG_PPC_BOOK3S_64) || !scv; kuap_assert_locked(); regs->result = r3; - regs->exit_flags = 0; - ti_flags = read_thread_flags(); + /* Clear exit_flags so only flags set during this exit are visible */ + current->thread_info.exit_flags = 0; + ti_flags = read_thread_flags(); if (unlikely(r3 >= (unsigned long)-MAX_ERRNO) && is_not_scv) { if (likely(!(ti_flags & (_TIF_NOERROR | _TIF_RESTOREALL)))) { r3 = -r3; @@ -107,7 +109,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, if (unlikely(ti_flags & _TIF_PERSYSCALL_MASK)) { if (ti_flags & _TIF_RESTOREALL) - regs->exit_flags = _TIF_RESTOREALL; + ret = _TIF_RESTOREALL; else regs->gpr[3] = r3; clear_bits(_TIF_PERSYSCALL_MASK, ¤t_thread_info()->flags); @@ -116,7 +118,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, } if (unlikely(ti_flags & _TIF_SYSCALL_DOTRACE)) { - regs->exit_flags |= _TIF_RESTOREALL; + ret |= _TIF_RESTOREALL; } syscall_exit_to_user_mode(regs); @@ -132,17 +134,19 @@ again: /* Restore user access locks last */ kuap_user_restore(regs); - + ret |= current->thread_info.exit_flags; #ifdef CONFIG_PPC64 - regs->exit_result = regs->exit_flags; + regs->exit_result = ret; #endif - return regs->exit_flags; + return ret; } #ifdef CONFIG_PPC64 notrace unsigned long syscall_exit_restart(unsigned long r3, struct pt_regs *regs) { + unsigned long ret; + /* * This is called when detecting a soft-pending interrupt as well as * an alternate-return interrupt. So we can't just have the alternate @@ -167,9 +171,11 @@ again: } kuap_user_restore(regs); - regs->exit_result |= regs->exit_flags; + ret = current_thread_info()->exit_flags & _TIF_RESTOREALL; + current_thread_info()->exit_flags &= ~_TIF_RESTOREALL; + regs->exit_result |= ret; - return regs->exit_result; + return ret; } #endif @@ -186,8 +192,10 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs) */ kuap_assert_locked(); + /* Clear exit_flags so only flags set during this exit are visible */ + current_thread_info()->exit_flags = 0; + local_irq_disable(); - regs->exit_flags = 0; again: check_return_regs_valid(regs); user_enter_irqoff(); @@ -200,9 +208,7 @@ again: /* Restore user access locks last */ kuap_user_restore(regs); - - ret = regs->exit_flags; - + ret = current_thread_info()->exit_flags & _TIF_RESTOREALL; #ifdef CONFIG_PPC64 regs->exit_result = ret; #endif diff --git a/arch/powerpc/kernel/ptrace/ptrace.c b/arch/powerpc/kernel/ptrace/ptrace.c index 316d4f5ead8e..6cd180bc36ab 100644 --- a/arch/powerpc/kernel/ptrace/ptrace.c +++ b/arch/powerpc/kernel/ptrace/ptrace.c @@ -291,7 +291,6 @@ void __init pt_regs_check(void) CHECK_REG(PT_DAR, dar); CHECK_REG(PT_DSISR, dsisr); CHECK_REG(PT_RESULT, result); - CHECK_REG(PT_EXIT_FLAGS, exit_flags); #undef CHECK_REG BUILD_BUG_ON(PT_REGS_COUNT != sizeof(struct user_pt_regs) / sizeof(unsigned long)); diff --git a/arch/powerpc/kernel/signal.c b/arch/powerpc/kernel/signal.c index bb42a8b6c642..cc6498501610 100644 --- a/arch/powerpc/kernel/signal.c +++ b/arch/powerpc/kernel/signal.c @@ -356,6 +356,6 @@ void signal_fault(struct task_struct *tsk, struct pt_regs *regs, void arch_do_signal_or_restart(struct pt_regs *regs) { BUG_ON(regs != current->thread.regs); - regs->exit_flags |= _TIF_RESTOREALL; + current_thread_info()->exit_flags |= _TIF_RESTOREALL; do_signal(current); } diff --git a/arch/powerpc/perf/imc-pmu.c b/arch/powerpc/perf/imc-pmu.c index e3822f36c419..1d2db6d3e226 100644 --- a/arch/powerpc/perf/imc-pmu.c +++ b/arch/powerpc/perf/imc-pmu.c @@ -1023,10 +1023,7 @@ static int thread_imc_event_init(struct perf_event *event) static bool is_thread_imc_pmu(struct perf_event *event) { - if (!strncmp(event->pmu->name, "thread_imc", strlen("thread_imc"))) - return true; - - return false; + return strstarts(event->pmu->name, "thread_imc"); } static __be64 *get_event_base_addr(struct perf_event *event) diff --git a/arch/powerpc/platforms/ps3/mm.c b/arch/powerpc/platforms/ps3/mm.c index 20fc5b68faee..315a32fd75b1 100644 --- a/arch/powerpc/platforms/ps3/mm.c +++ b/arch/powerpc/platforms/ps3/mm.c @@ -615,6 +615,7 @@ static int dma_ioc0_map_pages(struct ps3_dma_region *r, unsigned long phys_addr, fail_map: for (iopage--; 0 <= iopage; iopage--) { + offset = (1 << r->page_size) * iopage; lv1_put_iopte(0, c->bus_addr + offset, c->lpar_addr + offset, diff --git a/arch/powerpc/platforms/ps3/spu.c b/arch/powerpc/platforms/ps3/spu.c index e4e0b45e1b9d..a71c0b0cfb07 100644 --- a/arch/powerpc/platforms/ps3/spu.c +++ b/arch/powerpc/platforms/ps3/spu.c @@ -191,8 +191,6 @@ static void spu_unmap(struct spu *spu) static int __init setup_areas(struct spu *spu) { - struct table {char* name; unsigned long addr; unsigned long size;}; - spu_pdata(spu)->shadow = ioremap_prot(spu_pdata(spu)->shadow_addr, sizeof(struct spe_shadow), pgprot_noncached_wc(PAGE_KERNEL_RO)); diff --git a/arch/powerpc/platforms/pseries/kexec.c b/arch/powerpc/platforms/pseries/kexec.c index 431be156ca9b..29f7c97ff193 100644 --- a/arch/powerpc/platforms/pseries/kexec.c +++ b/arch/powerpc/platforms/pseries/kexec.c @@ -20,12 +20,15 @@ void pseries_kexec_cpu_down(int crash_shutdown, int secondary) { /* - * Don't risk a hypervisor call if we're crashing - * XXX: Why? The hypervisor is not crashing. It might be better - * to at least attempt unregister to avoid the hypervisor stepping - * on our memory. + * Ensure vpa/slb_shadow/dtl cleanup even while we are crashing. + * Why? The hypervisor is not crashing so at least attempt unregister to + * avoid the hypervisor stepping on our memory. If hypervisor or kexec + * kernel steps on the old memory allocated to these areas before the + * new kexec-kernel happens to allocate and register new areas, + * the hypervisor will see invalid content which may cause + * unexpected behavior. */ - if (firmware_has_feature(FW_FEATURE_SPLPAR) && !crash_shutdown) { + if (firmware_has_feature(FW_FEATURE_SPLPAR)) { int ret; int cpu = smp_processor_id(); int hwcpu = hard_smp_processor_id(); diff --git a/arch/powerpc/platforms/pseries/smp.c b/arch/powerpc/platforms/pseries/smp.c index db99725e752b..bf3d7ed3be01 100644 --- a/arch/powerpc/platforms/pseries/smp.c +++ b/arch/powerpc/platforms/pseries/smp.c @@ -128,7 +128,12 @@ static void smp_setup_cpu(int cpu) else if (cpu != boot_cpuid) xics_setup_cpu(); - if (firmware_has_feature(FW_FEATURE_SPLPAR)) + /* + * Initialize VPA on non-boot cpus since boot-cpu vpa was + * already initialized in pSeries_setup_arch() + */ + if (firmware_has_feature(FW_FEATURE_SPLPAR) && + cpu != boot_cpuid) vpa_init(cpu); cpumask_clear_cpu(cpu, of_spin_mask); diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index b6d149e989c8..01d252c741d2 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -63,7 +63,8 @@ EXPORT_SYMBOL(phys_ram_base); #ifdef CONFIG_SPARSEMEM_VMEMMAP #define VMEMMAP_ADDR_ALIGN max(1ULL << SECTION_SIZE_BITS, \ - MAX_FOLIO_VMEMMAP_ALIGN) + PFN_PHYS(MAX_FOLIO_VMEMMAP_ALIGN / \ + sizeof(struct page))) unsigned long vmemmap_start_pfn __ro_after_init; EXPORT_SYMBOL(vmemmap_start_pfn); diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index eaa34c5bd3c1..c172f9b212d1 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -476,6 +476,11 @@ struct s390_map_info { __u64 guest_addr; __u64 addr; struct page *page; + /* + * True if the page is long-term pinned. False if long-term pinning + * failed and this entry exists only to preserve MAP/UNMAP symmetry. + */ + bool pinned; }; struct s390_io_adapter { diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index 9e3e6b0d72ad..7514d9e2403c 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -2520,8 +2520,22 @@ static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr) map->addr = host_addr; map->page = pin_map_page(kvm, host_addr, FOLL_LONGTERM); if (!map->page) { - ret = -EINVAL; - goto out; + /* + * Long-term pinning may fail for memory types such as file-backed + * memory. Verify that short-term pinning succeeds so that the + * non-atomic irqfd path can handle interrupt injection. + */ + map->page = pin_map_page(kvm, host_addr, 0); + if (!map->page) { + ret = -EINVAL; + goto out; + } + unpin_user_page(map->page); + map->page = NULL; + map->pinned = false; + /* Add an entry to preserve MAP/UNMAP symmetry. */ + } else { + map->pinned = true; } spin_lock_irqsave(&adapter->maps_lock, flags); if (adapter->nr_maps < MAX_S390_ADAPTER_MAPS) { @@ -2532,7 +2546,7 @@ static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr) ret = -EINVAL; } spin_unlock_irqrestore(&adapter->maps_lock, flags); - if (ret) + if (ret && map->page) unpin_user_page(map->page); out: if (ret) @@ -2546,6 +2560,7 @@ static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr) struct s390_map_info *map, *tmp, *map_to_free; struct page *map_page_to_put = NULL; u64 map_addr_to_mark = 0; + bool map_pinned = false; unsigned long flags; int found = 0, idx; @@ -2560,6 +2575,7 @@ static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr) list_del(&map->list); map_page_to_put = map->page; map_addr_to_mark = map->guest_addr; + map_pinned = map->pinned; map_to_free = map; break; } @@ -2568,11 +2584,18 @@ static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr) if (found) { kfree(map_to_free); - idx = srcu_read_lock(&kvm->srcu); - mark_page_dirty(kvm, map_addr_to_mark >> PAGE_SHIFT); - set_page_dirty_lock(map_page_to_put); - srcu_read_unlock(&kvm->srcu, idx); - unpin_user_page(map_page_to_put); + if (map_pinned) { + /* + * Only long-term pinned pages need to be marked dirty + * and released. Fallback entries exist only for + * MAP/UNMAP symmetry. + */ + idx = srcu_read_lock(&kvm->srcu); + mark_page_dirty(kvm, map_addr_to_mark >> PAGE_SHIFT); + set_page_dirty_lock(map_page_to_put); + srcu_read_unlock(&kvm->srcu, idx); + unpin_user_page(map_page_to_put); + } } return found ? 0 : -ENOENT; @@ -2598,11 +2621,13 @@ void kvm_s390_unmap_all_adapters(struct kvm *kvm) list_for_each_entry_safe(map, tmp, &local_list, list) { list_del(&map->list); - idx = srcu_read_lock(&kvm->srcu); - mark_page_dirty(kvm, map->guest_addr >> PAGE_SHIFT); - set_page_dirty_lock(map->page); - srcu_read_unlock(&kvm->srcu, idx); - unpin_user_page(map->page); + if (map->pinned) { + idx = srcu_read_lock(&kvm->srcu); + mark_page_dirty(kvm, map->guest_addr >> PAGE_SHIFT); + set_page_dirty_lock(map->page); + srcu_read_unlock(&kvm->srcu, idx); + unpin_user_page(map->page); + } kfree(map); } } @@ -2929,8 +2954,11 @@ static struct s390_map_info *get_map_info(struct s390_io_adapter *adapter, return NULL; list_for_each_entry(map, &adapter->maps, list) { - if (map->addr == addr) + if (map->addr == addr) { + if (!map->pinned) + return NULL; return map; + } } return NULL; } diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c index 720bb58cabe2..50f5ec79600e 100644 --- a/arch/s390/kvm/pci.c +++ b/arch/s390/kvm/pci.c @@ -190,40 +190,61 @@ static int kvm_zpci_clear_airq(struct zpci_dev *zdev) return cc ? -EIO : 0; } -static inline void unaccount_mem(unsigned long nr_pages) +static inline void unaccount_mem(struct kvm_zdev *kzdev, unsigned long nr_pages) { - struct user_struct *user = get_uid(current_user()); + struct user_struct *user = kzdev->user_account; + struct mm_struct *mm_account = kzdev->mm_account; - if (user) + if (user) { atomic_long_sub(nr_pages, &user->locked_vm); - if (current->mm) - atomic64_sub(nr_pages, ¤t->mm->pinned_vm); + free_uid(user); + kzdev->user_account = NULL; + } + + if (mm_account) { + atomic64_sub(nr_pages, &mm_account->pinned_vm); + mmdrop(mm_account); + kzdev->mm_account = NULL; + } } -static inline int account_mem(unsigned long nr_pages) +static inline int account_mem(struct kvm_zdev *kzdev, unsigned long nr_pages) { struct user_struct *user = get_uid(current_user()); unsigned long page_limit, cur_pages, new_pages; + int rc = 0; page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; cur_pages = atomic_long_read(&user->locked_vm); do { new_pages = cur_pages + nr_pages; - if (new_pages > page_limit) - return -ENOMEM; + if (new_pages > page_limit) { + rc = -ENOMEM; + goto out; + } } while (!atomic_long_try_cmpxchg(&user->locked_vm, &cur_pages, new_pages)); - atomic64_add(nr_pages, ¤t->mm->pinned_vm); + if (current->mm) { + mmgrab(current->mm); + atomic64_add(nr_pages, ¤t->mm->pinned_vm); + } + + kzdev->user_account = user; + kzdev->mm_account = current->mm; return 0; + +out: + free_uid(user); + return rc; } static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, bool assist) { struct page *pages[1], *aibv_page, *aisb_page = NULL; - unsigned int msi_vecs, idx; + unsigned int msi_vecs, idx, size; struct zpci_gaite *gaite; unsigned long hva, bit; struct kvm *kvm; @@ -237,6 +258,10 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, if (zdev->gisa == 0) return -EINVAL; + /* AIF already enabled for the device */ + if (zdev->kzdev->fib.fmt0.aibv != 0) + return -EINVAL; + kvm = zdev->kzdev->kvm; msi_vecs = min_t(unsigned int, fib->fmt0.noi, zdev->max_msi); @@ -246,6 +271,14 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, return gisc; /* Replace AIBV address */ + size = BITS_TO_LONGS(msi_vecs + fib->fmt0.aibvo) * sizeof(unsigned long); + npages = DIV_ROUND_UP((fib->fmt0.aibv & ~PAGE_MASK) + size, PAGE_SIZE); + /* AIBV cannot span more than 1 page */ + if (npages > 1) { + rc = -EINVAL; + goto out; + } + idx = srcu_read_lock(&kvm->srcu); hva = gfn_to_hva(kvm, gpa_to_gfn((gpa_t)fib->fmt0.aibv)); npages = pin_user_pages_fast(hva, 1, FOLL_WRITE | FOLL_LONGTERM, pages); @@ -261,6 +294,12 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, /* Pin the guest AISB if one was specified */ if (fib->fmt0.sum == 1) { + /* AISB must be dword aligned */ + if (fib->fmt0.aisb & 0x7) { + rc = -EINVAL; + goto unpin1; + } + idx = srcu_read_lock(&kvm->srcu); hva = gfn_to_hva(kvm, gpa_to_gfn((gpa_t)fib->fmt0.aisb)); npages = pin_user_pages_fast(hva, 1, FOLL_WRITE | FOLL_LONGTERM, @@ -275,20 +314,28 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, } /* Account for pinned pages, roll back on failure */ - if (account_mem(pcount)) + rc = account_mem(zdev->kzdev, pcount); + if (rc) goto unpin2; /* AISB must be allocated before we can fill in GAITE */ mutex_lock(&aift->aift_lock); bit = airq_iv_alloc_bit(aift->sbv); - if (bit == -1UL) + if (bit == -1UL) { + rc = -ENOMEM; goto unlock; + } zdev->aisb = bit; /* store the summary bit number */ zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA | AIRQ_IV_BITLOCK | AIRQ_IV_GUESTVEC, phys_to_virt(fib->fmt0.aibv)); + if (!zdev->aibv) { + rc = -ENOMEM; + goto free_aisb; + } + spin_lock_irq(&aift->gait_lock); gaite = aift->gait + zdev->aisb; @@ -311,21 +358,39 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, aift->kzdev[zdev->aisb] = zdev->kzdev; spin_unlock_irq(&aift->gait_lock); - /* Update guest FIB for re-issue */ - fib->fmt0.aisbo = zdev->aisb & 63; - fib->fmt0.aisb = virt_to_phys(aift->sbv->vector) + (zdev->aisb / 64) * 8; - fib->fmt0.isc = gisc; - /* Save some guest fib values in the host for later use */ - zdev->kzdev->fib.fmt0.isc = fib->fmt0.isc; + zdev->kzdev->fib.fmt0.isc = gisc; zdev->kzdev->fib.fmt0.aibv = fib->fmt0.aibv; - mutex_unlock(&aift->aift_lock); /* Issue the clp to setup the irq now */ rc = kvm_zpci_set_airq(zdev); - return rc; + if (!rc) { + mutex_unlock(&aift->aift_lock); + return rc; + } + + /* Start cleanup */ + zdev->kzdev->fib.fmt0.isc = 0; + zdev->kzdev->fib.fmt0.aibv = 0; + spin_lock_irq(&aift->gait_lock); + gaite->count--; + gaite->aisb = 0; + gaite->gisc = 0; + gaite->aisbo = 0; + gaite->gisa = 0; + aift->kzdev[zdev->aisb] = NULL; + spin_unlock_irq(&aift->gait_lock); + + airq_iv_release(zdev->aibv); + zdev->aibv = NULL; + +free_aisb: + airq_iv_free_bit(aift->sbv, zdev->aisb); + zdev->aisb = 0; unlock: + if (pcount > 0) + unaccount_mem(zdev->kzdev, pcount); mutex_unlock(&aift->aift_lock); unpin2: if (fib->fmt0.sum == 1) @@ -396,7 +461,7 @@ static int kvm_s390_pci_aif_disable(struct zpci_dev *zdev, bool force) pcount++; } if (pcount > 0) - unaccount_mem(pcount); + unaccount_mem(kzdev, pcount); out: mutex_unlock(&aift->aift_lock); diff --git a/arch/s390/kvm/pci.h b/arch/s390/kvm/pci.h index ff0972dd5e71..fdf8c7bf4ed0 100644 --- a/arch/s390/kvm/pci.h +++ b/arch/s390/kvm/pci.h @@ -22,6 +22,8 @@ struct kvm_zdev { struct kvm *kvm; struct zpci_fib fib; struct list_head entry; + struct user_struct *user_account; + struct mm_struct *mm_account; }; struct zpci_gaite { diff --git a/arch/s390/pci/pci_mmio.c b/arch/s390/pci/pci_mmio.c index 51e7a28af899..f3f79ba78410 100644 --- a/arch/s390/pci/pci_mmio.c +++ b/arch/s390/pci/pci_mmio.c @@ -188,6 +188,7 @@ SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, goto out_unlock_mmap; } + ret = -EFAULT; io_addr = (void __iomem *)((args.pfn << PAGE_SHIFT) | (mmio_addr & ~PAGE_MASK)); diff --git a/arch/um/drivers/vector_kern.c b/arch/um/drivers/vector_kern.c index 2cc90055499a..8a70b3a62553 100644 --- a/arch/um/drivers/vector_kern.c +++ b/arch/um/drivers/vector_kern.c @@ -995,6 +995,9 @@ static int vector_mmsg_rx(struct vector_private *vp, int budget) */ dev_kfree_skb_irq(skb); vp->estats.rx_encaps_errors++; + (*skbuff_vector) = NULL; + mmsg_vector++; + skbuff_vector++; continue; } if (header_check > 0) { diff --git a/arch/x86/boot/string.c b/arch/x86/boot/string.c index ac0f900ebc47..1632d40e1f54 100644 --- a/arch/x86/boot/string.c +++ b/arch/x86/boot/string.c @@ -32,8 +32,15 @@ int memcmp(const void *s1, const void *s2, size_t len) { bool diff; - asm("repe cmpsb" - : "=@ccnz" (diff), "+D" (s1), "+S" (s2), "+c" (len)); + + /* + * Make sure ZF is properly set in the len==0 case because in it, + * RCX==0 and the REPE; CMPSB won't get executed. + */ + asm volatile("test %3, %3\n\t" + "repe cmpsb" + : "=@ccnz" (diff), "+D" (s1), "+S" (s2), "+c" (len) + : : "cc", "memory"); return diff; } diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 487ac147e11f..d61df70d6875 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -515,11 +515,13 @@ static void bsp_init_amd(struct cpuinfo_x86 *c) case 0x00 ... 0x2f: case 0x40 ... 0x4f: case 0x60 ... 0x7f: + case 0xd0 ... 0xd7: setup_force_cpu_cap(X86_FEATURE_ZEN5); break; case 0x50 ... 0x5f: case 0x80 ... 0xaf: - case 0xc0 ... 0xef: + case 0xc0 ... 0xcf: + case 0xd8 ... 0xef: setup_force_cpu_cap(X86_FEATURE_ZEN6); break; default: diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 22cf222d3033..66e69d2a41b3 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -722,6 +722,26 @@ static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn, return &slot->arch.lpage_info[level - 2][idx]; } +static bool kvm_gfn_is_lpage_allowed(struct kvm *kvm, + const struct kvm_memory_slot *slot, + gfn_t gfn, int level) +{ + const struct kvm_memory_slot *other_slot; + + BUILD_BUG_ON(KVM_MAX_NR_ADDRESS_SPACES > 2); + + if (lpage_info_slot(gfn, slot, level)->disallow_lpage) + return false; + + if (kvm_arch_nr_memslot_as_ids(kvm) > 1) { + other_slot = __gfn_to_memslot(__kvm_memslots(kvm, slot->as_id ^ 1), gfn); + if (other_slot && lpage_info_slot(gfn, other_slot, level)->disallow_lpage) + return false; + } + + return true; +} + /* * The most significant bit in disallow_lpage tracks whether or not memory * attributes are mixed, i.e. not identical for all gfns at the current level. @@ -2968,7 +2988,7 @@ int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot, * write-protected (see above), thus if the gfn can be mapped with a * hugepage and isn't write-tracked, it can't have a shadow page. */ - if (!lpage_info_slot(gfn, slot, PG_LEVEL_2M)->disallow_lpage) + if (kvm_gfn_is_lpage_allowed(kvm, slot, gfn, PG_LEVEL_2M)) return 0; /* diff --git a/arch/x86/kvm/mmu/page_track.c b/arch/x86/kvm/mmu/page_track.c index 1b17b12393a8..7e8195a311bb 100644 --- a/arch/x86/kvm/mmu/page_track.c +++ b/arch/x86/kvm/mmu/page_track.c @@ -130,13 +130,23 @@ void __kvm_write_track_remove_gfn(struct kvm *kvm, kvm_mmu_gfn_allow_lpage(slot, gfn); } -/* - * check if the corresponding access on the specified guest page is tracked. - */ +static bool __kvm_gfn_is_write_tracked(const struct kvm_memory_slot *slot, + gfn_t gfn) +{ + int index; + + if (!slot) + return false; + + index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K); + return !!READ_ONCE(slot->arch.gfn_write_track[index]); +} + +/* check if write access is tracked on the specified guest page. */ bool kvm_gfn_is_write_tracked(struct kvm *kvm, const struct kvm_memory_slot *slot, gfn_t gfn) { - int index; + const struct kvm_memory_slot *other_slot; if (!slot) return false; @@ -144,8 +154,18 @@ bool kvm_gfn_is_write_tracked(struct kvm *kvm, if (!kvm_page_track_write_tracking_enabled(kvm)) return false; - index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K); - return !!READ_ONCE(slot->arch.gfn_write_track[index]); + BUILD_BUG_ON(KVM_MAX_NR_ADDRESS_SPACES > 2); + + if (__kvm_gfn_is_write_tracked(slot, gfn)) + return true; + + if (kvm_arch_nr_memslot_as_ids(kvm) > 1) { + other_slot = __gfn_to_memslot(__kvm_memslots(kvm, slot->as_id ^ 1), gfn); + if (__kvm_gfn_is_write_tracked(other_slot, gfn)) + return true; + } + + return false; } #ifdef CONFIG_KVM_EXTERNAL_WRITE_TRACKING diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c index 58e493a80cb0..16e09a8fe85e 100644 --- a/arch/x86/kvm/svm/avic.c +++ b/arch/x86/kvm/svm/avic.c @@ -240,14 +240,6 @@ static void avic_deactivate_vmcb(struct vcpu_svm *svm) if (!is_sev_es_guest(&svm->vcpu)) svm_set_intercept(svm, INTERCEPT_CR8_WRITE); - /* - * If running nested and the guest uses its own MSR bitmap, there - * is no need to update L0's msr bitmap - */ - if (is_guest_mode(&svm->vcpu) && - vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT)) - return; - /* Enabling MSR intercept for x2APIC registers */ avic_set_x2apic_msr_interception(svm, true); } diff --git a/arch/x86/kvm/vmx/vmx_ops.h b/arch/x86/kvm/vmx/vmx_ops.h index 81784befaaf4..9ffe03a0c223 100644 --- a/arch/x86/kvm/vmx/vmx_ops.h +++ b/arch/x86/kvm/vmx/vmx_ops.h @@ -101,7 +101,7 @@ static __always_inline unsigned long __vmcs_readl(unsigned long field) : [output] "=r" (value) : [field] "r" (field) - : "cc" + : "cc", "memory" : do_fail, do_exception); return value; @@ -145,7 +145,7 @@ do_exception: : ASM_CALL_CONSTRAINT, [output] "=&r" (value) : [field] "r" (field) - : "cc"); + : "cc", "memory"); return value; #endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ @@ -192,7 +192,7 @@ do { \ asm goto("1: " __stringify(insn) " %0\n\t" \ "jna %l[error]\n\t" \ _ASM_EXTABLE(1b, %l[fault]) \ - : : op1 : "cc" : error, fault); \ + : : op1 : "cc", "memory" : error, fault); \ return; \ error: \ instrumentation_begin(); \ @@ -208,7 +208,7 @@ do { \ asm goto("1: " __stringify(insn) " %1, %0\n\t" \ "jna %l[error]\n\t" \ _ASM_EXTABLE(1b, %l[fault]) \ - : : op1, op2 : "cc" : error, fault); \ + : : op1, op2 : "cc", "memory" : error, fault);\ return; \ error: \ instrumentation_begin(); \ diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index afcac1042947..47cb9eba113b 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -13429,9 +13429,15 @@ void kvm_arch_pre_destroy_vm(struct kvm *kvm) * iterating over vCPUs in a different task while vCPUs are being freed * is unsafe, i.e. will lead to use-after-free. The PIT also needs to * be stopped before IRQ routing is freed. + * + * Do NOT free the in-kernel PIC or I/O APIC here (but as above, make + * sure to flush any background work), as KVM expects interrupt routing + * structures to be valid until vCPUs are destroyed. */ #ifdef CONFIG_KVM_IOAPIC kvm_free_pit(kvm); + if (kvm->arch.vioapic) + cancel_delayed_work_sync(&kvm->arch.vioapic->eoi_inject); #endif kvm_mmu_pre_destroy_vm(kvm); |