summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/s390/include/asm/kvm_host.h1
-rw-r--r--arch/s390/kvm/dat.c23
-rw-r--r--arch/s390/kvm/dat.h2
-rw-r--r--arch/s390/kvm/faultin.c6
-rw-r--r--arch/s390/kvm/interrupt.c19
-rw-r--r--arch/s390/kvm/kvm-s390.c125
-rw-r--r--arch/s390/kvm/priv.c10
-rw-r--r--arch/s390/kvm/pv.c43
-rw-r--r--arch/x86/include/asm/kvm_host.h2
-rw-r--r--arch/x86/kvm/hyperv.c7
-rw-r--r--arch/x86/kvm/mmu/mmu.c5
-rw-r--r--arch/x86/kvm/svm/sev.c34
-rw-r--r--arch/x86/kvm/svm/svm.c25
-rw-r--r--arch/x86/kvm/svm/svm.h1
-rw-r--r--arch/x86/kvm/vmx/main.c4
-rw-r--r--arch/x86/kvm/vmx/vmx.c2
-rw-r--r--arch/x86/kvm/vmx/x86_ops.h2
-rw-r--r--drivers/s390/cio/vfio_ccw_async.c16
-rw-r--r--drivers/s390/cio/vfio_ccw_chp.c31
-rw-r--r--drivers/s390/cio/vfio_ccw_cp.c74
-rw-r--r--drivers/s390/cio/vfio_ccw_cp.h10
-rw-r--r--drivers/s390/cio/vfio_ccw_drv.c21
-rw-r--r--drivers/s390/cio/vfio_ccw_fsm.c8
-rw-r--r--drivers/s390/cio/vfio_ccw_ops.c39
-rw-r--r--drivers/s390/cio/vfio_ccw_private.h10
25 files changed, 378 insertions, 142 deletions
diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
index c172f9b212d1..b4182ca4435f 100644
--- a/arch/s390/include/asm/kvm_host.h
+++ b/arch/s390/include/asm/kvm_host.h
@@ -440,6 +440,7 @@ struct kvm_vcpu_arch {
bool skey_enabled;
/* Indicator if the access registers have been loaded from guest */
bool acrs_loaded;
+ bool initialized;
struct kvm_s390_pv_vcpu pv;
union diag318_info diag318_info;
struct kvm_s390_mmu_cache *mc;
diff --git a/arch/s390/kvm/dat.c b/arch/s390/kvm/dat.c
index ed4259d17629..3f2d6e8902d7 100644
--- a/arch/s390/kvm/dat.c
+++ b/arch/s390/kvm/dat.c
@@ -755,13 +755,15 @@ int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gf
return rc;
}
-int dat_reset_reference_bit(union asce asce, gfn_t gfn)
+int dat_reset_reference_bit(union asce asce, gfn_t gfn, union skey *skey)
{
union pgste pgste, old;
union crste *crstep;
union pte *ptep;
int rc;
+ skey->skey = 0;
+
rc = dat_entry_walk(NULL, gfn, asce, DAT_WALK_ANY, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep);
if (rc)
return rc;
@@ -771,21 +773,23 @@ int dat_reset_reference_bit(union asce asce, gfn_t gfn)
if (!crste.h.fc || !crste.s.fc1.pr)
return 0;
- return page_reset_referenced(large_crste_to_phys(*crstep, gfn));
+ skey->skey = page_reset_referenced(large_crste_to_phys(*crstep, gfn)) << 1;
+ return 0;
}
old = pgste_get_lock(ptep);
pgste = old;
if (!ptep->h.i) {
- rc = page_reset_referenced(pte_origin(*ptep));
- pgste.hr = rc >> 1;
+ skey->skey = page_reset_referenced(pte_origin(*ptep)) << 1;
+ pgste.hr = skey->r;
}
- rc |= (pgste.gr << 1) | pgste.gc;
+ skey->r |= pgste.gr;
+ skey->c |= pgste.gc;
pgste.gr = 0;
dat_update_ptep_sd(old, pgste, ptep);
pgste_set_unlock(ptep, pgste);
- return rc;
+ return 0;
}
static long dat_reset_skeys_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
@@ -846,6 +850,7 @@ static long _dat_slot_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_wal
struct slot_priv *p = walk->priv;
union crste dummy = { .val = p->token };
union pte new_pte, pte = READ_ONCE(*ptep);
+ union pgste pgste;
new_pte = _PTE_TOK(dummy.tok.type, dummy.tok.par);
@@ -853,7 +858,11 @@ static long _dat_slot_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_wal
if (pte.val == new_pte.val)
return 0;
- dat_ptep_xchg(ptep, new_pte, gfn, walk->asce, false);
+ pgste = pgste_get_lock(ptep);
+ pgste = __dat_ptep_xchg(ptep, pgste, new_pte, gfn, walk->asce, false);
+ pgste.cmma_d = 0;
+ pgste_set_unlock(ptep, pgste);
+
return 0;
}
diff --git a/arch/s390/kvm/dat.h b/arch/s390/kvm/dat.h
index fad605305e05..141ee7b9f019 100644
--- a/arch/s390/kvm/dat.h
+++ b/arch/s390/kvm/dat.h
@@ -537,7 +537,7 @@ int dat_set_storage_key(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gf
union skey skey, bool nq);
int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gfn_t gfn,
union skey skey, union skey *oldkey, bool nq, bool mr, bool mc);
-int dat_reset_reference_bit(union asce asce, gfn_t gfn);
+int dat_reset_reference_bit(union asce asce, gfn_t gfn, union skey *skey);
long dat_reset_skeys(union asce asce, gfn_t start);
unsigned long dat_get_ptval(struct page_table *table, struct ptval_param param);
diff --git a/arch/s390/kvm/faultin.c b/arch/s390/kvm/faultin.c
index fee80047bd94..3cc45f7f5b2d 100644
--- a/arch/s390/kvm/faultin.c
+++ b/arch/s390/kvm/faultin.c
@@ -91,9 +91,9 @@ int kvm_s390_faultin_gfn(struct kvm_vcpu *vcpu, struct kvm *kvm, struct guest_fa
/* Access outside memory, addressing exception. */
if (is_noslot_pfn(f->pfn))
return PGM_ADDRESSING;
- /* Signal pending: try again. */
- if (f->pfn == KVM_PFN_ERR_SIGPENDING)
- return -EAGAIN;
+ /* Fatal signal pending: bail out. */
+ if (is_sigpending_pfn(f->pfn))
+ return -EINTR;
/* Check if it's read-only memory; don't try to actually handle that case. */
if (f->pfn == KVM_PFN_ERR_RO_FAULT)
return -EOPNOTSUPP;
diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c
index 7514d9e2403c..8f24bcd1a6d3 100644
--- a/arch/s390/kvm/interrupt.c
+++ b/arch/s390/kvm/interrupt.c
@@ -45,13 +45,16 @@ static struct kvm_s390_gib *gib;
static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id)
{
struct esca_block *sca = vcpu->kvm->arch.sca;
- union esca_sigp_ctrl sigp_ctrl = sca->cpu[vcpu->vcpu_id].sigp_ctrl;
+ union esca_sigp_ctrl sigp_ctrl;
if (!kvm_s390_test_cpuflags(vcpu, CPUSTAT_ECALL_PEND))
return 0;
+ if (kvm_is_ucontrol(vcpu->kvm))
+ return 0;
BUG_ON(!kvm_s390_use_sca_entries());
+ sigp_ctrl = sca->cpu[vcpu->vcpu_id].sigp_ctrl;
if (src_id)
*src_id = sigp_ctrl.scn;
@@ -60,13 +63,16 @@ static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id)
static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id)
{
- struct esca_block *sca = vcpu->kvm->arch.sca;
- union esca_sigp_ctrl *sigp_ctrl = &sca->cpu[vcpu->vcpu_id].sigp_ctrl;
union esca_sigp_ctrl old_val, new_val = {.scn = src_id, .c = 1};
+ struct esca_block *sca = vcpu->kvm->arch.sca;
+ union esca_sigp_ctrl *sigp_ctrl;
int expect, rc;
BUG_ON(!kvm_s390_use_sca_entries());
+ if (kvm_is_ucontrol(vcpu->kvm))
+ return -EINVAL;
+ sigp_ctrl = &sca->cpu[vcpu->vcpu_id].sigp_ctrl;
old_val = READ_ONCE(*sigp_ctrl);
old_val.c = 0;
@@ -84,10 +90,13 @@ static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id)
static void sca_clear_ext_call(struct kvm_vcpu *vcpu)
{
struct esca_block *sca = vcpu->kvm->arch.sca;
- union esca_sigp_ctrl *sigp_ctrl = &sca->cpu[vcpu->vcpu_id].sigp_ctrl;
+ union esca_sigp_ctrl *sigp_ctrl;
- if (!kvm_s390_use_sca_entries())
+ if (!kvm_s390_use_sca_entries() || !vcpu->arch.initialized || kvm_is_ucontrol(vcpu->kvm))
return;
+
+ /* Initialize after the above check, to prevent going out of bounds */
+ sigp_ctrl = &sca->cpu[vcpu->vcpu_id].sigp_ctrl;
kvm_s390_clear_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
WRITE_ONCE(sigp_ctrl->value, 0);
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 150b5dd2170e..518a69c55e85 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -571,7 +571,7 @@ static int kvm_s390_keyop(struct kvm_s390_mmu_cache *mc, struct kvm *kvm, int op
switch (op) {
case KVM_S390_KEYOP_SSKE:
r = dat_cond_set_storage_key(mc, asce, gfn, skey, &skey, 0, 0, 0);
- if (r >= 0)
+ if (r == 0 || r == 1)
return skey.skey;
break;
case KVM_S390_KEYOP_ISKE:
@@ -580,14 +580,14 @@ static int kvm_s390_keyop(struct kvm_s390_mmu_cache *mc, struct kvm *kvm, int op
return skey.skey;
break;
case KVM_S390_KEYOP_RRBE:
- r = dat_reset_reference_bit(asce, gfn);
- if (r > 0)
- return r << 1;
+ r = dat_reset_reference_bit(asce, gfn, &skey);
+ if (!r)
+ return skey.skey;
break;
default:
return -EINVAL;
}
- return r;
+ return r > 0 ? -EFAULT : r;
}
/* Section: device related */
@@ -1219,8 +1219,8 @@ static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
/*
* Must be called with kvm->srcu held to avoid races on memslots, and with
- * kvm->slots_lock to avoid races with ourselves, kvm_s390_vm_stop_migration(),
- * and kvm_s390_get_cmma_bits().
+ * kvm->slots_arch_lock to avoid races with ourselves,
+ * kvm_s390_vm_stop_migration(), and kvm_s390_get_cmma_bits().
*/
static int kvm_s390_vm_start_migration(struct kvm *kvm)
{
@@ -1265,7 +1265,7 @@ static int kvm_s390_vm_start_migration(struct kvm *kvm)
}
/*
- * Must be called with kvm->slots_lock to avoid races with ourselves,
+ * Must be called with kvm->slots_arch_lock to avoid races with ourselves,
* kvm_s390_vm_start_migration() and kvm_s390_get_cmma_bits().
*/
static int kvm_s390_vm_stop_migration(struct kvm *kvm)
@@ -1300,7 +1300,9 @@ static int kvm_s390_vm_set_migration(struct kvm *kvm,
{
int res = -ENXIO;
- mutex_lock(&kvm->slots_lock);
+ guard(srcu)(&kvm->srcu);
+ guard(mutex)(&kvm->slots_arch_lock);
+
switch (attr->attr) {
case KVM_S390_VM_MIGRATION_START:
res = kvm_s390_vm_start_migration(kvm);
@@ -1311,7 +1313,6 @@ static int kvm_s390_vm_set_migration(struct kvm *kvm,
default:
break;
}
- mutex_unlock(&kvm->slots_lock);
return res;
}
@@ -2214,7 +2215,7 @@ static int kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
}
kvfree(keys);
- return r;
+ return r <= 0 ? r : -EFAULT;
}
static int kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
@@ -2276,7 +2277,7 @@ static int kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
kvm_s390_free_mmu_cache(mc);
out:
kvfree(keys);
- return r;
+ return r <= 0 ? r : -EFAULT;
}
/*
@@ -2386,7 +2387,7 @@ static int kvm_s390_set_cmma_bits(struct kvm *kvm,
set_bit(GMAP_FLAG_USES_CMM, &kvm->arch.gmap->flags);
- return r;
+ return r <= 0 ? r : -EFAULT;
}
/**
@@ -2934,6 +2935,9 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
case KVM_S390_INTERRUPT: {
struct kvm_s390_interrupt s390int;
+ r = -EINVAL;
+ if (kvm_is_ucontrol(kvm))
+ break;
r = -EFAULT;
if (copy_from_user(&s390int, argp, sizeof(s390int)))
break;
@@ -2998,9 +3002,8 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
r = -EFAULT;
if (copy_from_user(&args, argp, sizeof(args)))
break;
- mutex_lock(&kvm->slots_lock);
- r = kvm_s390_get_cmma_bits(kvm, &args);
- mutex_unlock(&kvm->slots_lock);
+ scoped_guard(mutex, &kvm->slots_arch_lock)
+ r = kvm_s390_get_cmma_bits(kvm, &args);
if (!r) {
r = copy_to_user(argp, &args, sizeof(args));
if (r)
@@ -3014,9 +3017,9 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
r = -EFAULT;
if (copy_from_user(&args, argp, sizeof(args)))
break;
- mutex_lock(&kvm->slots_lock);
+ mutex_lock(&kvm->slots_arch_lock);
r = kvm_s390_set_cmma_bits(kvm, &args);
- mutex_unlock(&kvm->slots_lock);
+ mutex_unlock(&kvm->slots_arch_lock);
break;
}
case KVM_S390_PV_COMMAND: {
@@ -3247,7 +3250,8 @@ static void kvm_s390_crypto_init(struct kvm *kvm)
static void sca_dispose(struct kvm *kvm)
{
- free_pages_exact(kvm->arch.sca, sizeof(*kvm->arch.sca));
+ if (kvm->arch.sca)
+ free_pages_exact(kvm->arch.sca, sizeof(*kvm->arch.sca));
kvm->arch.sca = NULL;
}
@@ -3461,7 +3465,7 @@ static void sca_del_vcpu(struct kvm_vcpu *vcpu)
{
struct esca_block *sca = vcpu->kvm->arch.sca;
- if (!kvm_s390_use_sca_entries())
+ if (!kvm_s390_use_sca_entries() || !vcpu->arch.initialized)
return;
clear_bit_inv(vcpu->vcpu_id, (unsigned long *)sca->mcn);
@@ -3481,8 +3485,8 @@ static void sca_add_vcpu(struct kvm_vcpu *vcpu)
if (!kvm_s390_use_sca_entries())
return;
+ WRITE_ONCE(sca->cpu[vcpu->vcpu_id].sda, virt_to_phys(vcpu->arch.sie_block));
set_bit_inv(vcpu->vcpu_id, (unsigned long *)sca->mcn);
- sca->cpu[vcpu->vcpu_id].sda = virt_to_phys(vcpu->arch.sie_block);
}
static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
@@ -3613,6 +3617,9 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0 ||
vcpu->kvm->arch.user_operexec)
vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
+
+ /* Pairs with smp_load_acquire() in kvm_arch_vcpu_ioctl_run() and kvm_arch_vcpu_ioctl() */
+ smp_store_release(&vcpu->arch.initialized, true);
}
static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr)
@@ -3674,7 +3681,8 @@ static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
{
- free_page((unsigned long)phys_to_virt(vcpu->arch.sie_block->cbrlo));
+ if (vcpu->arch.sie_block->cbrlo)
+ free_page((unsigned long)phys_to_virt(vcpu->arch.sie_block->cbrlo));
vcpu->arch.sie_block->cbrlo = 0;
}
@@ -3792,21 +3800,21 @@ int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
return 0;
}
+DEFINE_FREE(sie_page, struct sie_page *, if (_T) free_page((unsigned long)(_T)))
+
int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
{
- struct sie_page *sie_page;
+ struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL;
+ struct sie_page *sie_page __free(sie_page) = NULL;
int rc;
BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
- vcpu->arch.mc = kvm_s390_new_mmu_cache();
- if (!vcpu->arch.mc)
+ mc = kvm_s390_new_mmu_cache();
+ if (!mc)
return -ENOMEM;
sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL_ACCOUNT);
- if (!sie_page) {
- kvm_s390_free_mmu_cache(vcpu->arch.mc);
- vcpu->arch.mc = NULL;
+ if (!sie_page)
return -ENOMEM;
- }
vcpu->arch.sie_block = &sie_page->sie_block;
vcpu->arch.sie_block->itdba = virt_to_phys(&sie_page->itdb);
@@ -3848,10 +3856,9 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
if (kvm_is_ucontrol(vcpu->kvm)) {
- rc = -ENOMEM;
vcpu->arch.gmap = gmap_new_child(vcpu->kvm->arch.gmap, -1UL);
if (!vcpu->arch.gmap)
- goto out_free_sie_block;
+ return -ENOMEM;
}
VM_EVENT(vcpu->kvm, 3, "create cpu %d at 0x%p, sie block at 0x%p",
@@ -3859,20 +3866,19 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
trace_kvm_s390_create_vcpu(vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
rc = kvm_s390_vcpu_setup(vcpu);
- if (rc)
- goto out_ucontrol_uninit;
+ if (rc) {
+ if (kvm_is_ucontrol(vcpu->kvm)) {
+ scoped_guard(spinlock, &vcpu->kvm->arch.gmap->children_lock)
+ gmap_remove_child(vcpu->arch.gmap);
+ vcpu->arch.gmap = gmap_put(vcpu->arch.gmap);
+ }
+ return rc;
+ }
+ vcpu->arch.mc = no_free_ptr(mc);
+ sie_page = NULL;
kvm_s390_update_topology_change_report(vcpu->kvm, 1);
return 0;
-
-out_ucontrol_uninit:
- if (kvm_is_ucontrol(vcpu->kvm)) {
- gmap_remove_child(vcpu->arch.gmap);
- vcpu->arch.gmap = gmap_put(vcpu->arch.gmap);
- }
-out_free_sie_block:
- free_page((unsigned long)(vcpu->arch.sie_block));
- return rc;
}
int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
@@ -5039,6 +5045,10 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS)
return -EINVAL;
+ /* Pairs with smp_store_release() in kvm_arch_vcpu_postcreate() */
+ if (!smp_load_acquire(&vcpu->arch.initialized))
+ return -EINVAL;
+
vcpu_load(vcpu);
if (guestdbg_exit_pending(vcpu)) {
@@ -5447,6 +5457,8 @@ long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl,
struct kvm_s390_interrupt s390int;
struct kvm_s390_irq s390irq = {};
+ if (kvm_is_ucontrol(vcpu->kvm))
+ return -EINVAL;
if (copy_from_user(&s390int, argp, sizeof(s390int)))
return -EFAULT;
if (s390int_to_s390irq(&s390int, &s390irq))
@@ -5523,6 +5535,10 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
long r;
u16 rc, rrc;
+ /* Pairs with smp_store_release() in kvm_arch_vcpu_postcreate() */
+ if (!smp_load_acquire(&vcpu->arch.initialized))
+ return -EINVAL;
+
vcpu_load(vcpu);
switch (ioctl) {
@@ -5794,14 +5810,30 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
return 0;
}
+static long cmma_d_count_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
+{
+ union pgste pgste;
+
+ pgste = pgste_get_lock(ptep);
+ if (pgste.cmma_d) {
+ pgste.cmma_d = 0;
+ atomic64_dec(walk->priv);
+ }
+ pgste_set_unlock(ptep, pgste);
+ return 0;
+}
+
void kvm_arch_commit_memory_region(struct kvm *kvm,
struct kvm_memory_slot *old,
const struct kvm_memory_slot *new,
enum kvm_mr_change change)
{
- struct kvm_s390_mmu_cache *mc = NULL;
+ const struct dat_walk_ops ops = { .pte_entry = cmma_d_count_pte, };
+ struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL;
int rc = 0;
+ guard(mutex)(&kvm->slots_arch_lock);
+
if (change == KVM_MR_FLAGS_ONLY)
return;
@@ -5812,6 +5844,12 @@ void kvm_arch_commit_memory_region(struct kvm *kvm,
}
scoped_guard(write_lock, &kvm->mmu_lock) {
+ if (kvm->arch.migration_mode && kvm->arch.use_cmma && old) {
+ _dat_walk_gfn_range(old->base_gfn, old->base_gfn + old->npages,
+ kvm->arch.gmap->asce, &ops, DAT_WALK_IGN_HOLES,
+ &kvm->arch.cmma_dirty_pages);
+ }
+
switch (change) {
case KVM_MR_DELETE:
rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages);
@@ -5833,7 +5871,6 @@ void kvm_arch_commit_memory_region(struct kvm *kvm,
out:
if (rc)
pr_warn("failed to commit memory region\n");
- kvm_s390_free_mmu_cache(mc);
return;
}
diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c
index ad0ddc433a73..b1ba24c346ef 100644
--- a/arch/s390/kvm/priv.c
+++ b/arch/s390/kvm/priv.c
@@ -289,6 +289,7 @@ static int handle_iske(struct kvm_vcpu *vcpu)
static int handle_rrbe(struct kvm_vcpu *vcpu)
{
unsigned long gaddr;
+ union skey skey;
int reg1, reg2;
int rc;
@@ -307,12 +308,12 @@ static int handle_rrbe(struct kvm_vcpu *vcpu)
gaddr = kvm_s390_logical_to_effective(vcpu, gaddr);
gaddr = kvm_s390_real_to_abs(vcpu, gaddr);
scoped_guard(read_lock, &vcpu->kvm->mmu_lock)
- rc = dat_reset_reference_bit(vcpu->arch.gmap->asce, gpa_to_gfn(gaddr));
+ rc = dat_reset_reference_bit(vcpu->arch.gmap->asce, gpa_to_gfn(gaddr), &skey);
if (rc > 0)
return kvm_s390_inject_program_int(vcpu, rc);
if (rc < 0)
return rc;
- kvm_s390_set_psw_cc(vcpu, rc);
+ kvm_s390_set_psw_cc(vcpu, (skey.skey >> 1) & 3);
return 0;
}
@@ -1260,8 +1261,9 @@ static int handle_essa(struct kvm_vcpu *vcpu)
/* Retry the ESSA instruction */
kvm_s390_retry_instr(vcpu);
} else {
- scoped_guard(read_lock, &vcpu->kvm->mmu_lock)
- i = __do_essa(vcpu, orc);
+ scoped_guard(mutex, &vcpu->kvm->slots_arch_lock)
+ scoped_guard(read_lock, &vcpu->kvm->mmu_lock)
+ i = __do_essa(vcpu, orc);
if (i < 0)
return i;
/* Account for the possible extra cbrl entry */
diff --git a/arch/s390/kvm/pv.c b/arch/s390/kvm/pv.c
index 1beacc841ca8..b02e0159d3cd 100644
--- a/arch/s390/kvm/pv.c
+++ b/arch/s390/kvm/pv.c
@@ -244,6 +244,24 @@ static void kvm_s390_clear_pv_state(struct kvm *kvm)
kvm->arch.pv.stor_var = NULL;
}
+static void kvm_s390_pv_dispose_cpu(struct kvm_vcpu *vcpu, bool free_stor_base)
+{
+ if (free_stor_base)
+ free_pages(vcpu->arch.pv.stor_base, get_order(uv_info.guest_cpu_stor_len));
+ free_page((unsigned long)sida_addr(vcpu->arch.sie_block));
+ vcpu->arch.sie_block->pv_handle_cpu = 0;
+ vcpu->arch.sie_block->pv_handle_config = 0;
+ memset(&vcpu->arch.pv, 0, sizeof(vcpu->arch.pv));
+ vcpu->arch.sie_block->sdf = 0;
+ /*
+ * The sidad field (for sdf == 2) is now the gbea field (for sdf == 0).
+ * Use the reset value of gbea to avoid leaking the kernel pointer of
+ * the just freed sida.
+ */
+ vcpu->arch.sie_block->gbea = 1;
+ kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
+}
+
int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc)
{
int cc;
@@ -258,24 +276,9 @@ int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc)
WARN_ONCE(cc, "protvirt destroy cpu failed rc %x rrc %x", *rc, *rrc);
/* Intended memory leak for something that should never happen. */
- if (!cc)
- free_pages(vcpu->arch.pv.stor_base,
- get_order(uv_info.guest_cpu_stor_len));
-
- free_page((unsigned long)sida_addr(vcpu->arch.sie_block));
- vcpu->arch.sie_block->pv_handle_cpu = 0;
- vcpu->arch.sie_block->pv_handle_config = 0;
- memset(&vcpu->arch.pv, 0, sizeof(vcpu->arch.pv));
- vcpu->arch.sie_block->sdf = 0;
- /*
- * The sidad field (for sdf == 2) is now the gbea field (for sdf == 0).
- * Use the reset value of gbea to avoid leaking the kernel pointer of
- * the just freed sida.
- */
- vcpu->arch.sie_block->gbea = 1;
- kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
+ kvm_s390_pv_dispose_cpu(vcpu, !cc);
- return cc ? EIO : 0;
+ return cc ? -EIO : 0;
}
int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc)
@@ -319,9 +322,7 @@ int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc)
uvcb.header.rrc);
if (cc) {
- u16 dummy;
-
- kvm_s390_pv_destroy_cpu(vcpu, &dummy, &dummy);
+ kvm_s390_pv_dispose_cpu(vcpu, true);
return -EIO;
}
@@ -809,7 +810,7 @@ static int unpack_one(struct kvm *kvm, unsigned long addr, u64 tweak,
return -EAGAIN;
}
- if (ret && ret != -EAGAIN)
+ if (ret && ret != -EAGAIN && ret != -EINTR)
KVM_UV_EVENT(kvm, 3, "PROTVIRT VM UNPACK: failed addr %llx with rc %x rrc %x",
uvcb.gaddr, *rc, *rrc);
return ret;
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 5f6c1ce9673b..6db5b5f79df9 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1858,7 +1858,7 @@ struct kvm_x86_ops {
* Can potentially get non-canonical addresses through INVLPGs, which
* the implementation may choose to ignore if appropriate.
*/
- void (*flush_tlb_gva)(struct kvm_vcpu *vcpu, gva_t addr);
+ void (*flush_tlb_gva)(struct kvm_vcpu *vcpu, gva_t addr, bool *full);
/*
* Flush any TLB entries created by the guest. Like tlb_flush_gva(),
diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c
index fd4eb1e561f7..d8736b5dfcd3 100644
--- a/arch/x86/kvm/hyperv.c
+++ b/arch/x86/kvm/hyperv.c
@@ -1974,6 +1974,7 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu)
u64 entries[KVM_HV_TLB_FLUSH_FIFO_SIZE];
int i, j, count;
gva_t gva;
+ bool full = false;
if (!tdp_enabled || !hv_vcpu)
return -EINVAL;
@@ -1982,7 +1983,7 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu)
count = kfifo_out(&tlb_flush_fifo->entries, entries, KVM_HV_TLB_FLUSH_FIFO_SIZE);
- for (i = 0; i < count; i++) {
+ for (i = 0; i < count && !full; i++) {
if (entries[i] == KVM_HV_TLB_FLUSHALL_ENTRY)
goto out_flush_all;
@@ -1991,11 +1992,11 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu)
* pages to flush.
*/
gva = entries[i] & PAGE_MASK;
- for (j = 0; j < (entries[i] & ~PAGE_MASK) + 1; j++) {
+ for (j = 0; j < (entries[i] & ~PAGE_MASK) + 1 && !full; j++) {
if (is_noncanonical_invlpg_address(gva + j * PAGE_SIZE, vcpu))
continue;
- kvm_x86_call(flush_tlb_gva)(vcpu, gva + j * PAGE_SIZE);
+ kvm_x86_call(flush_tlb_gva)(vcpu, gva + j * PAGE_SIZE, &full);
}
++vcpu->stat.tlb_flush;
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 66e69d2a41b3..a61750f8e1e3 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -2442,6 +2442,9 @@ static union kvm_mmu_page_role kvm_mmu_child_role(u64 *sptep, bool direct,
role.direct = direct;
role.passthrough = 0;
+ WARN_ON_ONCE(role.invalid);
+ role.invalid = 0;
+
/*
* If the guest has 4-byte PTEs then that means it's using 32-bit,
* 2-level, non-PAE paging. KVM shadows such guests with PAE paging
@@ -6652,7 +6655,7 @@ void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
if (is_noncanonical_invlpg_address(addr, vcpu))
return;
- kvm_x86_call(flush_tlb_gva)(vcpu, addr);
+ kvm_x86_call(flush_tlb_gva)(vcpu, addr, NULL);
}
if (!mmu->sync_spte)
diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c
index 944aaea6501f..0f0ea7896af5 100644
--- a/arch/x86/kvm/svm/sev.c
+++ b/arch/x86/kvm/svm/sev.c
@@ -97,6 +97,8 @@ static u64 sev_supported_vmsa_features __ro_after_init;
static u8 sev_enc_bit;
static DECLARE_RWSEM(sev_deactivate_lock);
static DEFINE_MUTEX(sev_bitmap_lock);
+/* Protects kvm_sev_info's enc_context_owner, mirror_vms and mirror_entry. */
+static DEFINE_MUTEX(sev_mirror_lock);
unsigned int max_sev_asid;
static unsigned int min_sev_asid;
static unsigned int max_sev_es_asid;
@@ -2018,7 +2020,6 @@ static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm)
dst->asid = src->asid;
dst->handle = src->handle;
dst->pages_locked = src->pages_locked;
- dst->enc_context_owner = src->enc_context_owner;
dst->es_active = src->es_active;
dst->vmsa_features = src->vmsa_features;
@@ -2026,11 +2027,12 @@ static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm)
src->active = false;
src->handle = 0;
src->pages_locked = 0;
- src->enc_context_owner = NULL;
src->es_active = false;
list_cut_before(&dst->regions_list, &src->regions_list, &src->regions_list);
+ mutex_lock(&sev_mirror_lock);
+
/*
* If this VM has mirrors, "transfer" each mirror's refcount of the
* source to the destination (this KVM). The caller holds a reference
@@ -2047,12 +2049,15 @@ static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm)
* If this VM is a mirror, remove the old mirror from the owners list
* and add the new mirror to the list.
*/
- if (is_mirroring_enc_context(dst_kvm)) {
- struct kvm_sev_info *owner_sev_info = to_kvm_sev_info(dst->enc_context_owner);
+ if (is_mirroring_enc_context(src_kvm)) {
+ struct kvm_sev_info *owner_sev_info = to_kvm_sev_info(src->enc_context_owner);
+ dst->enc_context_owner = src->enc_context_owner;
+ src->enc_context_owner = NULL;
list_del(&src->mirror_entry);
list_add_tail(&dst->mirror_entry, &owner_sev_info->mirror_vms);
}
+ mutex_unlock(&sev_mirror_lock);
kvm_for_each_vcpu(i, dst_vcpu, dst_kvm) {
dst_svm = to_svm(dst_vcpu);
@@ -2871,11 +2876,14 @@ int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd)
* disappear until we're done with it
*/
source_sev = to_kvm_sev_info(source_kvm);
- kvm_get_kvm(source_kvm);
- list_add_tail(&mirror_sev->mirror_entry, &source_sev->mirror_vms);
/* Set enc_context_owner and copy its encryption context over */
+ mutex_lock(&sev_mirror_lock);
+ kvm_get_kvm(source_kvm);
+ list_add_tail(&mirror_sev->mirror_entry, &source_sev->mirror_vms);
mirror_sev->enc_context_owner = source_kvm;
+ mutex_unlock(&sev_mirror_lock);
+
mirror_sev->active = true;
mirror_sev->asid = source_sev->asid;
mirror_sev->fd = source_sev->fd;
@@ -2963,11 +2971,19 @@ void sev_vm_destroy(struct kvm *kvm)
* Note, mirror VMs don't support registering encrypted regions.
*/
if (is_mirroring_enc_context(kvm)) {
- struct kvm *owner_kvm = sev->enc_context_owner;
+ struct kvm *owner_kvm;
- mutex_lock(&owner_kvm->lock);
+ mutex_lock(&sev_mirror_lock);
+ owner_kvm = sev->enc_context_owner;
list_del(&sev->mirror_entry);
- mutex_unlock(&owner_kvm->lock);
+ sev->enc_context_owner = NULL;
+
+ /*
+ * The reference to owner_kvm cannot move after sev_mirror_lock is
+ * released. Release it before kvm_put_kvm() so that owner_kvm is
+ * never destroyed inside sev_mirror_lock.
+ */
+ mutex_unlock(&sev_mirror_lock);
kvm_put_kvm(owner_kvm);
return;
}
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index d0971685034b..60495447f0ca 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -4227,18 +4227,31 @@ static void svm_flush_tlb_all(struct kvm_vcpu *vcpu)
svm_flush_tlb_asid(vcpu);
}
-static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
+static void svm_flush_tlb_guest(struct kvm_vcpu *vcpu)
{
- struct vcpu_svm *svm = to_svm(vcpu);
+ kvm_register_mark_dirty(vcpu, VCPU_REG_ERAPS);
- invlpga(gva, svm->vmcb->control.asid);
+ svm_flush_tlb_asid(vcpu);
}
-static void svm_flush_tlb_guest(struct kvm_vcpu *vcpu)
+static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva, bool *full)
{
- kvm_register_mark_dirty(vcpu, VCPU_REG_ERAPS);
+ struct vcpu_svm *svm = to_svm(vcpu);
- svm_flush_tlb_asid(vcpu);
+ /*
+ * INVLPGA has had errata on Genoa and Turin, and even on older
+ * generations there were reports of Windows BSODs if INVLPGA
+ * was used for Hyper-V tlbflush. Use it only for shadow paging
+ * where it seems to be okay.
+ */
+ if (!npt_enabled) {
+ invlpga(gva, svm->vmcb->control.asid);
+ return;
+ }
+
+ svm_flush_tlb_guest(vcpu);
+ if (full)
+ *full = true;
}
static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h
index 716be21fba33..d63e5878988a 100644
--- a/arch/x86/kvm/svm/svm.h
+++ b/arch/x86/kvm/svm/svm.h
@@ -109,6 +109,7 @@ struct kvm_sev_info {
u64 ap_jump_table; /* SEV-ES AP Jump Table address */
u64 vmsa_features;
u16 ghcb_version; /* Highest guest GHCB protocol version allowed */
+ /* The three fields below are protected by sev_mirror_lock */
struct kvm *enc_context_owner; /* Owner of copied encryption context */
struct list_head mirror_vms; /* List of VMs mirroring */
struct list_head mirror_entry; /* Use as a list entry of mirrors */
diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c
index 83d9921277ea..f204a0fc0a57 100644
--- a/arch/x86/kvm/vmx/main.c
+++ b/arch/x86/kvm/vmx/main.c
@@ -535,12 +535,12 @@ static void vt_flush_tlb_current(struct kvm_vcpu *vcpu)
vmx_flush_tlb_current(vcpu);
}
-static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr)
+static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full)
{
if (is_td_vcpu(vcpu))
return;
- vmx_flush_tlb_gva(vcpu, addr);
+ vmx_flush_tlb_gva(vcpu, addr, full);
}
static void vt_flush_tlb_guest(struct kvm_vcpu *vcpu)
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index cc75feec05da..b8d745f6fd22 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -3361,7 +3361,7 @@ void vmx_flush_tlb_current(struct kvm_vcpu *vcpu)
vpid_sync_context(vmx_get_current_vpid(vcpu));
}
-void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr)
+void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full)
{
/*
* vpid_sync_vcpu_addr() is a nop if vpid==0, see the comment in
diff --git a/arch/x86/kvm/vmx/x86_ops.h b/arch/x86/kvm/vmx/x86_ops.h
index 409858074246..17595d52985c 100644
--- a/arch/x86/kvm/vmx/x86_ops.h
+++ b/arch/x86/kvm/vmx/x86_ops.h
@@ -82,7 +82,7 @@ void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
bool vmx_get_if_flag(struct kvm_vcpu *vcpu);
void vmx_flush_tlb_all(struct kvm_vcpu *vcpu);
void vmx_flush_tlb_current(struct kvm_vcpu *vcpu);
-void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr);
+void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full);
void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu);
void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask);
u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu);
diff --git a/drivers/s390/cio/vfio_ccw_async.c b/drivers/s390/cio/vfio_ccw_async.c
index 420d89ba7f83..4aff0b58fa5d 100644
--- a/drivers/s390/cio/vfio_ccw_async.c
+++ b/drivers/s390/cio/vfio_ccw_async.c
@@ -8,6 +8,7 @@
*/
#include <linux/vfio.h>
+#include <linux/nospec.h>
#include "vfio_ccw_private.h"
@@ -24,11 +25,20 @@ static ssize_t vfio_ccw_async_region_read(struct vfio_ccw_private *private,
return -EINVAL;
mutex_lock(&private->io_mutex);
+
+ if (i >= private->num_regions) {
+ ret = -EINVAL;
+ goto out_unlock;
+ }
+
+ i = array_index_nospec(i, private->num_regions);
region = private->region[i].data;
if (copy_to_user(buf, (void *)region + pos, count))
ret = -EFAULT;
else
ret = count;
+
+out_unlock:
mutex_unlock(&private->io_mutex);
return ret;
}
@@ -48,6 +58,12 @@ static ssize_t vfio_ccw_async_region_write(struct vfio_ccw_private *private,
if (!mutex_trylock(&private->io_mutex))
return -EAGAIN;
+ if (i >= private->num_regions) {
+ ret = -EINVAL;
+ goto out_unlock;
+ }
+
+ i = array_index_nospec(i, private->num_regions);
region = private->region[i].data;
if (copy_from_user((void *)region + pos, buf, count)) {
ret = -EFAULT;
diff --git a/drivers/s390/cio/vfio_ccw_chp.c b/drivers/s390/cio/vfio_ccw_chp.c
index 38c176cf6295..7708eb4d6de0 100644
--- a/drivers/s390/cio/vfio_ccw_chp.c
+++ b/drivers/s390/cio/vfio_ccw_chp.c
@@ -9,6 +9,7 @@
*/
#include <linux/slab.h>
+#include <linux/nospec.h>
#include <linux/vfio.h>
#include "vfio_ccw_private.h"
@@ -26,6 +27,13 @@ static ssize_t vfio_ccw_schib_region_read(struct vfio_ccw_private *private,
return -EINVAL;
mutex_lock(&private->io_mutex);
+
+ if (i >= private->num_regions) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ i = array_index_nospec(i, private->num_regions);
region = private->region[i].data;
if (cio_update_schib(sch)) {
@@ -85,19 +93,30 @@ static ssize_t vfio_ccw_crw_region_read(struct vfio_ccw_private *private,
loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK;
struct ccw_crw_region *region;
struct vfio_ccw_crw *crw;
+ unsigned long flags;
int ret;
if (pos + count > sizeof(*region))
return -EINVAL;
+ mutex_lock(&private->io_mutex);
+ if (i >= private->num_regions) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ i = array_index_nospec(i, private->num_regions);
+ region = private->region[i].data;
+
+ spin_lock_irqsave(&private->crw_lock, flags);
crw = list_first_entry_or_null(&private->crw,
struct vfio_ccw_crw, next);
if (crw)
list_del(&crw->next);
- mutex_lock(&private->io_mutex);
- region = private->region[i].data;
+ /* Drop CRW lock while copying to userspace */
+ spin_unlock_irqrestore(&private->crw_lock, flags);
if (crw)
memcpy(&region->crw, &crw->crw, sizeof(region->crw));
@@ -108,14 +127,16 @@ static ssize_t vfio_ccw_crw_region_read(struct vfio_ccw_private *private,
ret = count;
region->crw = 0;
-
- mutex_unlock(&private->io_mutex);
-
kfree(crw);
/* Notify the guest if more CRWs are on our queue */
+ spin_lock_irqsave(&private->crw_lock, flags);
if (!list_empty(&private->crw) && private->crw_trigger)
eventfd_signal(private->crw_trigger);
+ spin_unlock_irqrestore(&private->crw_lock, flags);
+
+out:
+ mutex_unlock(&private->io_mutex);
return ret;
}
diff --git a/drivers/s390/cio/vfio_ccw_cp.c b/drivers/s390/cio/vfio_ccw_cp.c
index 7561aa7d3e01..58722c4baa25 100644
--- a/drivers/s390/cio/vfio_ccw_cp.c
+++ b/drivers/s390/cio/vfio_ccw_cp.c
@@ -233,6 +233,7 @@ static void convert_ccw0_to_ccw1(struct ccw1 *source, unsigned long len)
}
#define idal_is_2k(_cp) (!(_cp)->orb.cmd.c64 || (_cp)->orb.cmd.i2k)
+#define get_idaw_size(_cp) ((_cp)->orb.cmd.c64 ? sizeof(u64) : sizeof(u32))
/*
* Helpers to operate ccwchain.
@@ -332,6 +333,7 @@ static struct ccwchain *ccwchain_alloc(struct channel_program *cp, int len)
goto out_err;
list_add_tail(&chain->next, &cp->ccwchain_list);
+ cp->ccwchain_count++;
return chain;
@@ -376,11 +378,9 @@ static void ccwchain_cda_free(struct ccwchain *chain, int idx)
static int ccwchain_calc_length(u64 iova, struct channel_program *cp)
{
struct ccw1 *ccw = cp->guest_cp;
- int cnt = 0;
-
- do {
- cnt++;
+ int cnt;
+ for (cnt = 1; cnt <= CCWCHAIN_LEN_MAX; cnt++, ccw++) {
/*
* We want to keep counting if the current CCW has the
* command-chaining flag enabled, or if it is a TIC CCW
@@ -390,15 +390,10 @@ static int ccwchain_calc_length(u64 iova, struct channel_program *cp)
* after the TIC, depending on the results of its operation.
*/
if (!ccw_is_chain(ccw) && !is_tic_within_range(ccw, iova, cnt))
- break;
-
- ccw++;
- } while (cnt < CCWCHAIN_LEN_MAX + 1);
-
- if (cnt == CCWCHAIN_LEN_MAX + 1)
- cnt = -EINVAL;
+ return cnt;
+ }
- return cnt;
+ return -EINVAL;
}
static int tic_target_chain_exists(struct ccw1 *tic, struct channel_program *cp)
@@ -441,6 +436,10 @@ static int ccwchain_handle_ccw(dma32_t cda, struct channel_program *cp)
if (len < 0)
return len;
+ /* Limit number of chains in a single channel program */
+ if (cp->ccwchain_count >= CCWCHAIN_COUNT_MAX)
+ return -EINVAL;
+
/* Need alloc a new chain for this one. */
chain = ccwchain_alloc(cp, len);
if (!chain)
@@ -455,9 +454,6 @@ static int ccwchain_handle_ccw(dma32_t cda, struct channel_program *cp)
/* Loop for tics on this new chain. */
ret = ccwchain_loop_tic(chain, cp);
- if (ret)
- ccwchain_free(chain);
-
return ret;
}
@@ -486,6 +482,23 @@ static int ccwchain_loop_tic(struct ccwchain *chain, struct channel_program *cp)
return 0;
}
+static int ccwchain_build_ccws(dma32_t cda, struct channel_program *cp)
+{
+ struct ccwchain *chain, *temp;
+ int ret;
+
+ ret = ccwchain_handle_ccw(cda, cp);
+
+ if (ret) {
+ /* Cleanup if an error occurred */
+ list_for_each_entry_safe(chain, temp, &cp->ccwchain_list, next) {
+ ccwchain_free(chain);
+ }
+ }
+
+ return ret;
+}
+
static int ccwchain_fetch_tic(struct ccw1 *ccw,
struct channel_program *cp)
{
@@ -511,7 +524,8 @@ static dma64_t *get_guest_idal(struct ccw1 *ccw, struct channel_program *cp, int
&container_of(cp, struct vfio_ccw_private, cp)->vdev;
dma64_t *idaws;
dma32_t *idaws_f1;
- int idal_len = idaw_nr * sizeof(*idaws);
+ u64 first_idaw;
+ int idal_len = idaw_nr * get_idaw_size(cp);
int idaw_size = idal_is_2k(cp) ? PAGE_SIZE / 2 : PAGE_SIZE;
int idaw_mask = ~(idaw_size - 1);
int i, ret;
@@ -527,6 +541,18 @@ static dma64_t *get_guest_idal(struct ccw1 *ccw, struct channel_program *cp, int
kfree(idaws);
return ERR_PTR(ret);
}
+
+ idaws_f1 = (dma32_t *)idaws;
+ if (cp->orb.cmd.c64)
+ first_idaw = dma64_to_u64(idaws[0]);
+ else
+ first_idaw = dma32_to_u32(idaws_f1[0]);
+
+ /* Unexpected mismatch from earlier read */
+ if (first_idaw != cp->guest_iova) {
+ kfree(idaws);
+ return ERR_PTR(-EINVAL);
+ }
} else {
/* Fabricate an IDAL based off CCW data address */
if (cp->orb.cmd.c64) {
@@ -568,7 +594,7 @@ static int ccw_count_idaws(struct ccw1 *ccw,
struct vfio_device *vdev =
&container_of(cp, struct vfio_ccw_private, cp)->vdev;
u64 iova;
- int size = cp->orb.cmd.c64 ? sizeof(u64) : sizeof(u32);
+ int size = get_idaw_size(cp);
int ret;
int bytes = 1;
@@ -592,6 +618,9 @@ static int ccw_count_idaws(struct ccw1 *ccw,
iova = dma32_to_u32(ccw->cda);
}
+ /* Save the read address for later */
+ cp->guest_iova = iova;
+
/* Format-1 IDAWs operate on 2K each */
if (!cp->orb.cmd.c64)
return idal_2k_nr_words((void *)iova, bytes);
@@ -731,11 +760,12 @@ int cp_init(struct channel_program *cp, union orb *orb)
vdev->dev,
"Prefetching channel program even though prefetch not specified in ORB");
+ cp->ccwchain_count = 0;
INIT_LIST_HEAD(&cp->ccwchain_list);
memcpy(&cp->orb, orb, sizeof(*orb));
/* Build a ccwchain for the first CCW segment */
- ret = ccwchain_handle_ccw(orb->cmd.cpa, cp);
+ ret = ccwchain_build_ccws(orb->cmd.cpa, cp);
if (!ret)
cp->initialized = true;
@@ -947,17 +977,23 @@ void cp_update_scsw(struct channel_program *cp, union scsw *scsw)
*/
bool cp_iova_pinned(struct channel_program *cp, u64 iova, u64 length)
{
+ struct vfio_ccw_private *private =
+ container_of(cp, struct vfio_ccw_private, cp);
struct ccwchain *chain;
int i;
if (!cp->initialized)
return false;
+ mutex_lock(&private->io_mutex);
list_for_each_entry(chain, &cp->ccwchain_list, next) {
for (i = 0; i < chain->ch_len; i++)
- if (page_array_iova_pinned(&chain->ch_pa[i], iova, length))
+ if (page_array_iova_pinned(&chain->ch_pa[i], iova, length)) {
+ mutex_unlock(&private->io_mutex);
return true;
+ }
}
+ mutex_unlock(&private->io_mutex);
return false;
}
diff --git a/drivers/s390/cio/vfio_ccw_cp.h b/drivers/s390/cio/vfio_ccw_cp.h
index fc31eb699807..9af98ff12d67 100644
--- a/drivers/s390/cio/vfio_ccw_cp.h
+++ b/drivers/s390/cio/vfio_ccw_cp.h
@@ -23,11 +23,19 @@
*/
#define CCWCHAIN_LEN_MAX 256
+/*
+ * Maximum number of chains
+ */
+#define CCWCHAIN_COUNT_MAX 16
+
/**
* struct channel_program - manage information for channel program
* @ccwchain_list: list head of ccwchains
* @orb: orb for the currently processed ssch request
* @initialized: whether this instance is actually initialized
+ * @guest_cp: copy of guest channel program
+ * @ccwchain_count: number of channel program segments (linked by TIC)
+ * @guest_iova: first data address of a guest channel program
*
* @ccwchain_list is the head of a ccwchain list, that contents the
* translated result of the guest channel program that pointed out by
@@ -38,6 +46,8 @@ struct channel_program {
union orb orb;
bool initialized;
struct ccw1 *guest_cp;
+ unsigned int ccwchain_count;
+ u64 guest_iova;
};
int cp_init(struct channel_program *cp, union orb *orb);
diff --git a/drivers/s390/cio/vfio_ccw_drv.c b/drivers/s390/cio/vfio_ccw_drv.c
index 1a095085bc72..ab6b518cc353 100644
--- a/drivers/s390/cio/vfio_ccw_drv.c
+++ b/drivers/s390/cio/vfio_ccw_drv.c
@@ -91,6 +91,7 @@ void vfio_ccw_sch_io_todo(struct work_struct *work)
is_final = !(scsw_actl(&irb->scsw) &
(SCSW_ACTL_DEVACT | SCSW_ACTL_SCHACT));
+ mutex_lock(&private->io_mutex);
if (scsw_is_solicited(&irb->scsw)) {
cp_update_scsw(&private->cp, &irb->scsw);
if (is_final && private->state == VFIO_CCW_STATE_CP_PENDING) {
@@ -98,9 +99,7 @@ void vfio_ccw_sch_io_todo(struct work_struct *work)
cp_is_finished = true;
}
}
- mutex_lock(&private->io_mutex);
memcpy(private->io_region->irb_area, irb, sizeof(*irb));
- mutex_unlock(&private->io_mutex);
/*
* Reset to IDLE only if processing of a channel program
@@ -110,6 +109,7 @@ void vfio_ccw_sch_io_todo(struct work_struct *work)
*/
if (cp_is_finished)
private->state = VFIO_CCW_STATE_IDLE;
+ mutex_unlock(&private->io_mutex);
if (private->io_trigger)
eventfd_signal(private->io_trigger);
@@ -118,11 +118,25 @@ void vfio_ccw_sch_io_todo(struct work_struct *work)
void vfio_ccw_crw_todo(struct work_struct *work)
{
struct vfio_ccw_private *private;
+ unsigned long flags;
private = container_of(work, struct vfio_ccw_private, crw_work);
+ spin_lock_irqsave(&private->crw_lock, flags);
if (!list_empty(&private->crw) && private->crw_trigger)
eventfd_signal(private->crw_trigger);
+ spin_unlock_irqrestore(&private->crw_lock, flags);
+}
+
+void vfio_ccw_notoper_todo(struct work_struct *work)
+{
+ struct vfio_ccw_private *private;
+
+ private = container_of(work, struct vfio_ccw_private, notoper_work);
+
+ mutex_lock(&private->io_mutex);
+ cp_free(&private->cp);
+ mutex_unlock(&private->io_mutex);
}
/*
@@ -275,6 +289,7 @@ static void vfio_ccw_queue_crw(struct vfio_ccw_private *private,
unsigned int rsid)
{
struct vfio_ccw_crw *crw;
+ unsigned long flags;
/*
* If unable to allocate a CRW, just drop the event and
@@ -292,7 +307,9 @@ static void vfio_ccw_queue_crw(struct vfio_ccw_private *private,
crw->crw.erc = erc;
crw->crw.rsid = rsid;
+ spin_lock_irqsave(&private->crw_lock, flags);
list_add_tail(&crw->next, &private->crw);
+ spin_unlock_irqrestore(&private->crw_lock, flags);
queue_work(vfio_ccw_work_q, &private->crw_work);
}
diff --git a/drivers/s390/cio/vfio_ccw_fsm.c b/drivers/s390/cio/vfio_ccw_fsm.c
index 4d7988ea47ef..5fd94e9d5c61 100644
--- a/drivers/s390/cio/vfio_ccw_fsm.c
+++ b/drivers/s390/cio/vfio_ccw_fsm.c
@@ -170,8 +170,8 @@ static void fsm_notoper(struct vfio_ccw_private *private,
css_sched_sch_todo(sch, SCH_TODO_UNREG);
private->state = VFIO_CCW_STATE_NOT_OPER;
- /* This is usually handled during CLOSE event */
- cp_free(&private->cp);
+ /* This routine could be called from IRQ context, so defer */
+ queue_work(vfio_ccw_work_q, &private->notoper_work);
}
/*
@@ -410,7 +410,11 @@ static void fsm_close(struct vfio_ccw_private *private,
private->state = VFIO_CCW_STATE_STANDBY;
spin_unlock_irq(&sch->lock);
+
+ mutex_lock(&private->io_mutex);
cp_free(&private->cp);
+ mutex_unlock(&private->io_mutex);
+
return;
err_unlock:
diff --git a/drivers/s390/cio/vfio_ccw_ops.c b/drivers/s390/cio/vfio_ccw_ops.c
index 45ec722d25ea..5ce91285c7d5 100644
--- a/drivers/s390/cio/vfio_ccw_ops.c
+++ b/drivers/s390/cio/vfio_ccw_ops.c
@@ -54,6 +54,8 @@ static int vfio_ccw_mdev_init_dev(struct vfio_device *vdev)
INIT_LIST_HEAD(&private->crw);
INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo);
INIT_WORK(&private->crw_work, vfio_ccw_crw_todo);
+ INIT_WORK(&private->notoper_work, vfio_ccw_notoper_todo);
+ spin_lock_init(&private->crw_lock);
private->cp.guest_cp = kzalloc_objs(struct ccw1, CCWCHAIN_LEN_MAX);
if (!private->cp.guest_cp)
@@ -130,11 +132,28 @@ static void vfio_ccw_mdev_release_dev(struct vfio_device *vdev)
struct vfio_ccw_private *private =
container_of(vdev, struct vfio_ccw_private, vdev);
struct vfio_ccw_crw *crw, *temp;
+ unsigned long flags;
+ /*
+ * Ensure these work items are fully drained, so none can
+ * fire after being released.
+ *
+ * notoper_work should have nothing to do here, because only
+ * open devices could have channel_program resources in use
+ * and those would be released during close. Nevertheless,
+ * call flush here as well to be certain anything that was
+ * allocated is freed.
+ */
+ cancel_work_sync(&private->io_work);
+ cancel_work_sync(&private->crw_work);
+ flush_work(&private->notoper_work);
+
+ spin_lock_irqsave(&private->crw_lock, flags);
list_for_each_entry_safe(crw, temp, &private->crw, next) {
list_del(&crw->next);
kfree(crw);
}
+ spin_unlock_irqrestore(&private->crw_lock, flags);
kmem_cache_free(vfio_ccw_crw_region, private->crw_region);
kmem_cache_free(vfio_ccw_schib_region, private->schib_region);
@@ -202,6 +221,19 @@ static void vfio_ccw_mdev_close_device(struct vfio_device *vdev)
container_of(vdev, struct vfio_ccw_private, vdev);
vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE);
+
+ /*
+ * Ensure these work items are drained, in the event the
+ * device is re-opened instead of released.
+ *
+ * notoper_work needs to be given a chance to run if it
+ * is queued, so any memory associated with the channel
+ * program can be returned.
+ */
+ cancel_work_sync(&private->io_work);
+ cancel_work_sync(&private->crw_work);
+ flush_work(&private->notoper_work);
+
vfio_ccw_unregister_dev_regions(private);
}
@@ -243,6 +275,7 @@ static ssize_t vfio_ccw_mdev_read(struct vfio_device *vdev,
return vfio_ccw_mdev_read_io_region(private, buf, count, ppos);
default:
index -= VFIO_CCW_NUM_REGIONS;
+ index = array_index_nospec(index, private->num_regions);
return private->region[index].ops->read(private, buf, count,
ppos);
}
@@ -295,6 +328,7 @@ static ssize_t vfio_ccw_mdev_write(struct vfio_device *vdev,
return vfio_ccw_mdev_write_io_region(private, buf, count, ppos);
default:
index -= VFIO_CCW_NUM_REGIONS;
+ index = array_index_nospec(index, private->num_regions);
return private->region[index].ops->write(private, buf, count,
ppos);
}
@@ -338,11 +372,8 @@ static int vfio_ccw_mdev_ioctl_get_region_info(struct vfio_device *vdev,
VFIO_CCW_NUM_REGIONS + private->num_regions)
return -EINVAL;
- info->index = array_index_nospec(info->index,
- VFIO_CCW_NUM_REGIONS +
- private->num_regions);
-
i = info->index - VFIO_CCW_NUM_REGIONS;
+ i = array_index_nospec(i, private->num_regions);
info->offset = VFIO_CCW_INDEX_TO_OFFSET(info->index);
info->size = private->region[i].size;
diff --git a/drivers/s390/cio/vfio_ccw_private.h b/drivers/s390/cio/vfio_ccw_private.h
index 0501d4bbcdbd..3bd0171d38d0 100644
--- a/drivers/s390/cio/vfio_ccw_private.h
+++ b/drivers/s390/cio/vfio_ccw_private.h
@@ -88,7 +88,8 @@ struct vfio_ccw_parent {
* @state: internal state of the device
* @completion: synchronization helper of the I/O completion
* @io_region: MMIO region to input/output I/O arguments/results
- * @io_mutex: protect against concurrent update of I/O regions
+ * @io_mutex: protect against concurrent update of I/O resources
+ * and @cp lifecycle
* @region: additional regions for other subchannel operations
* @cmd_region: MMIO region for asynchronous I/O commands other than START
* @schib_region: MMIO region for SCHIB information
@@ -97,11 +98,14 @@ struct vfio_ccw_parent {
* @cp: channel program for the current I/O operation
* @irb: irb info received from interrupt
* @scsw: scsw info
+ * @crw_lock: serialization of CRW list information
+ * @crw: list of Channel Report Word elements
* @io_trigger: eventfd ctx for signaling userspace I/O results
* @crw_trigger: eventfd ctx for signaling userspace CRW information
* @req_trigger: eventfd ctx for signaling userspace to return device
* @io_work: work for deferral process of I/O handling
* @crw_work: work for deferral process of CRW handling
+ * @notoper_work: work for deferred processing in not-operational state
*/
struct vfio_ccw_private {
struct vfio_device vdev;
@@ -118,6 +122,8 @@ struct vfio_ccw_private {
struct channel_program cp;
struct irb irb;
union scsw scsw;
+
+ spinlock_t crw_lock;
struct list_head crw;
struct eventfd_ctx *io_trigger;
@@ -125,11 +131,13 @@ struct vfio_ccw_private {
struct eventfd_ctx *req_trigger;
struct work_struct io_work;
struct work_struct crw_work;
+ struct work_struct notoper_work;
} __aligned(8);
int vfio_ccw_sch_quiesce(struct subchannel *sch);
void vfio_ccw_sch_io_todo(struct work_struct *work);
void vfio_ccw_crw_todo(struct work_struct *work);
+void vfio_ccw_notoper_todo(struct work_struct *work);
extern struct mdev_driver vfio_ccw_mdev_driver;
generated by cgit v1.2.3 (git 2.0.4) at 2026-08-07 22:57:02 +0000