diff options
Diffstat (limited to 'arch/s390')
116 files changed, 3079 insertions, 2110 deletions
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 84404e6778d5..4b51bc6e8948 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -20,6 +20,9 @@ config ARCH_HAS_ILOG2_U64 config ARCH_PROC_KCORE_TEXT def_bool y +config ARCH_HAS_CPU_RELAX + def_bool y + config GENERIC_HWEIGHT def_bool !HAVE_MARCH_Z196_FEATURES @@ -82,8 +85,9 @@ config S390 select ARCH_CORRECT_STACKTRACE_ON_KRETPROBE select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE select ARCH_HAS_CC_CAN_LINK + select ARCH_HAS_CC_PLATFORM select ARCH_HAS_CPU_FINALIZE_INIT select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DEBUG_VIRTUAL @@ -144,13 +148,16 @@ config S390 select ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE select ARCH_STACKWALK select ARCH_SUPPORTS_ATOMIC_RMW + select ARCH_SUPPORTS_CFI select ARCH_SUPPORTS_DEBUG_PAGEALLOC select ARCH_SUPPORTS_HUGETLBFS select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && CC_IS_CLANG select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS + select ARCH_HAS_PTE_PROTNONE select ARCH_SUPPORTS_NUMA_BALANCING select ARCH_SUPPORTS_PAGE_TABLE_CHECK select ARCH_SUPPORTS_PER_VMA_LOCK + select ARCH_USES_CFI_GENERIC_LLVM_PASS if CC_IS_CLANG select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF select ARCH_USE_SYM_ANNOTATIONS @@ -163,6 +170,7 @@ config S390 select ARCH_WANTS_THP_SWAP select BUILDTIME_TABLE_SORT select CLONE_BACKWARDS2 + select DCACHE_WORD_ACCESS if !KMSAN select DYNAMIC_FTRACE if FUNCTION_TRACER select FUNCTION_ALIGNMENT_8B if CC_IS_GCC select FUNCTION_ALIGNMENT_16B if !CC_IS_GCC @@ -215,7 +223,6 @@ config S390 select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_TRACER select HAVE_GCC_PLUGINS - select HAVE_GENERIC_VDSO select HAVE_IOREMAP_PROT if PCI select HAVE_KERNEL_BZIP2 select HAVE_KERNEL_GZIP @@ -273,8 +280,11 @@ config S390 select PCI_MSI if PCI select PCI_MSI_ARCH_FALLBACKS if PCI_MSI select PCI_QUIRKS if PCI + select HAS_SEPARATE_PREEMPT_RESCHED_BITS select SPARSE_IRQ select SWIOTLB + select SWIOTLB_DYNAMIC + select DMA_COHERENT_POOL select SYSCTL_EXCEPTION_TRACE select SYSTEM_DATA_VERIFICATION if KEXEC_SIG select THREAD_INFO_IN_TASK @@ -575,7 +585,7 @@ config HIPERDISPATCH_ON def_bool y bool "Use hiperdispatch on vertical polarization by default" depends on SCHED_TOPOLOGY - depends on PROC_SYSCTL + depends on SYSCTL help Hiperdispatch aims to improve the CPU scheduler's decision making when using vertical polarization by adjusting CPU @@ -705,6 +715,8 @@ config KERNEL_IMAGE_BASE endmenu +source "drivers/cpuidle/Kconfig" + menu "Memory setup" config ARCH_SPARSEMEM_ENABLE @@ -893,7 +905,7 @@ config CMM_IUCV config APPLDATA_BASE def_bool n prompt "Linux - VM Monitor Stream, base infrastructure" - depends on PROC_SYSCTL + depends on SYSCTL help This provides a kernel interface for creating and updating z/VM APPLDATA monitor records. The monitor records are updated at certain time diff --git a/arch/s390/appldata/Makefile b/arch/s390/appldata/Makefile index b06def4a4f2f..132381f16fb1 100644 --- a/arch/s390/appldata/Makefile +++ b/arch/s390/appldata/Makefile @@ -3,6 +3,8 @@ # Makefile for the Linux - z/VM Monitor Stream. # +CONTEXT_ANALYSIS := y + obj-$(CONFIG_APPLDATA_BASE) += appldata_base.o obj-$(CONFIG_APPLDATA_MEM) += appldata_mem.o obj-$(CONFIG_APPLDATA_OS) += appldata_os.o diff --git a/arch/s390/boot/alternative.c b/arch/s390/boot/alternative.c index 19ea7934b918..77e8bad560c5 100644 --- a/arch/s390/boot/alternative.c +++ b/arch/s390/boot/alternative.c @@ -45,11 +45,12 @@ static void alt_debug_modify(int type, unsigned int nr, bool clear) static char *alt_debug_parse(int type, char *str) { - unsigned long val, endval; + unsigned long val, endval, limit; char *endp; bool clear; int i; + limit = type == ALT_TYPE_FACILITY ? MAX_FACILITY_BIT : MAX_MFEATURE_BIT; if (*str == ':') { str++; } else { @@ -73,7 +74,7 @@ static char *alt_debug_parse(int type, char *str) if (str == endp) break; str = endp; - while (val <= endval) { + while (val <= endval && val < limit) { alt_debug_modify(type, val, clear); val++; } diff --git a/arch/s390/boot/ipl_parm.c b/arch/s390/boot/ipl_parm.c index 6bc950b92be7..c1b43e5e688a 100644 --- a/arch/s390/boot/ipl_parm.c +++ b/arch/s390/boot/ipl_parm.c @@ -23,6 +23,7 @@ struct parmarea parmarea __section(".parmarea") = { }; char __bootdata(early_command_line)[COMMAND_LINE_SIZE]; +static char command_line_buf[COMMAND_LINE_SIZE]; unsigned int __bootdata_preserved(zlib_dfltcc_support) = ZLIB_DFLTCC_FULL; struct ipl_parameter_block __bootdata_preserved(ipl_block); @@ -135,31 +136,29 @@ out: static void append_ipl_block_parm(void) { - char *parm, *delim; - size_t len, rc = 0; + size_t len, extra = 0; + char *delim; len = strlen(early_command_line); - - delim = early_command_line + len; /* '\0' character position */ - parm = early_command_line + len + 1; /* append right after '\0' */ + delim = early_command_line + len; /* '\0' character position */ switch (ipl_block.pb0_hdr.pbt) { case IPL_PBT_CCW: - rc = ipl_block_get_ascii_vmparm( - parm, COMMAND_LINE_SIZE - len - 1, &ipl_block); + extra = ipl_block_get_ascii_vmparm(command_line_buf, sizeof(command_line_buf), &ipl_block); break; case IPL_PBT_FCP: case IPL_PBT_NVME: case IPL_PBT_ECKD: - rc = ipl_block_get_ascii_scpdata( - parm, COMMAND_LINE_SIZE - len - 1, &ipl_block); + extra = ipl_block_get_ascii_scpdata(command_line_buf, sizeof(command_line_buf), &ipl_block); break; } - if (rc) { - if (*parm == '=') - memmove(early_command_line, parm + 1, rc); - else + if (extra) { + if (command_line_buf[0] == '=') { + memmove(early_command_line, command_line_buf + 1, extra); + } else if (len < COMMAND_LINE_SIZE - 2) { *delim = ' '; /* replace '\0' with space */ + sized_strscpy(delim + 1, command_line_buf, COMMAND_LINE_SIZE - len - 1); + } } } @@ -231,7 +230,7 @@ static void modify_fac_list(char *str) if (str == endp) break; str = endp; - while (val <= endval) { + while (val <= endval && val < MAX_FACILITY_BIT) { modify_facility(val, clear); val++; } @@ -245,7 +244,6 @@ static void modify_fac_list(char *str) check_cleared_facilities(); } -static char command_line_buf[COMMAND_LINE_SIZE]; void parse_boot_command_line(void) { char *param, *val; diff --git a/arch/s390/boot/physmem_info.c b/arch/s390/boot/physmem_info.c index 1f2ca5435838..0ebb2174713f 100644 --- a/arch/s390/boot/physmem_info.c +++ b/arch/s390/boot/physmem_info.c @@ -141,7 +141,7 @@ static int tprot(unsigned long addr) static unsigned long search_mem_end(void) { - unsigned long range = 1 << (MAX_PHYSMEM_BITS - 20); /* in 1MB blocks */ + unsigned long range = 1UL << (MAX_PHYSMEM_BITS - 20); /* in 1MB blocks */ unsigned long offset = 0; unsigned long pivot; diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index 54637be87fb7..3dae71474333 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -29,6 +29,8 @@ CONFIG_CGROUP_PIDS=y CONFIG_CGROUP_RDMA=y CONFIG_CGROUP_FREEZER=y CONFIG_CGROUP_HUGETLB=y +CONFIG_CPU_IDLE=y +CONFIG_CPU_IDLE_GOV_TEO=y CONFIG_CPUSETS=y CONFIG_CGROUP_DEVICE=y CONFIG_CGROUP_CPUACCT=y @@ -867,7 +869,7 @@ CONFIG_DEBUG_PREEMPT=y CONFIG_PROVE_LOCKING=y CONFIG_LOCK_STAT=y CONFIG_LOCKDEP_BITS=16 -CONFIG_LOCKDEP_CHAINS_BITS=17 +CONFIG_LOCKDEP_CHAINS_BITS=18 CONFIG_DEBUG_ATOMIC_SLEEP=y CONFIG_DEBUG_LOCKING_API_SELFTESTS=y CONFIG_DEBUG_IRQFLAGS=y diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index 5f5114a253cf..6f5722634b4d 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -27,6 +27,8 @@ CONFIG_CGROUP_PIDS=y CONFIG_CGROUP_RDMA=y CONFIG_CGROUP_FREEZER=y CONFIG_CGROUP_HUGETLB=y +CONFIG_CPU_IDLE=y +CONFIG_CPU_IDLE_GOV_TEO=y CONFIG_CPUSETS=y CONFIG_CGROUP_DEVICE=y CONFIG_CGROUP_CPUACCT=y diff --git a/arch/s390/crypto/Makefile b/arch/s390/crypto/Makefile index 48aeb0c0ffbd..1d6420813935 100644 --- a/arch/s390/crypto/Makefile +++ b/arch/s390/crypto/Makefile @@ -3,6 +3,8 @@ # Cryptographic API # +CONTEXT_ANALYSIS := y + obj-$(CONFIG_CRYPTO_AES_S390) += aes_s390.o obj-$(CONFIG_CRYPTO_PAES_S390) += paes_s390.o obj-$(CONFIG_S390_PRNG) += prng.o diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index 62edc66d5478..0be6fa779d2c 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -26,14 +26,14 @@ #include <linux/module.h> #include <linux/cpufeature.h> #include <linux/init.h> -#include <linux/mutex.h> #include <linux/fips.h> +#include <linux/semaphore.h> #include <linux/string.h> #include <crypto/xts.h> #include <asm/cpacf.h> static u8 *ctrblk; -static DEFINE_MUTEX(ctrblk_lock); +static DEFINE_SEMAPHORE(ctrblk_sem, 1); static cpacf_mask_t km_functions, kmc_functions, kmctr_functions, kma_functions; @@ -129,7 +129,7 @@ static int ecb_aes_crypt(struct skcipher_request *req, unsigned long modifier) return fallback_skcipher_crypt(sctx, req, modifier); ret = skcipher_walk_virt(&walk, req, false); - while ((nbytes = walk.nbytes) != 0) { + while (!ret && ((nbytes = walk.nbytes) != 0)) { /* only use complete blocks */ n = nbytes & ~(AES_BLOCK_SIZE - 1); cpacf_km(sctx->fc | modifier, sctx->key, @@ -233,7 +233,7 @@ static int cbc_aes_crypt(struct skcipher_request *req, unsigned long modifier) return ret; memcpy(param.iv, walk.iv, AES_BLOCK_SIZE); memcpy(param.key, sctx->key, sctx->key_len); - while ((nbytes = walk.nbytes) != 0) { + while (!ret && ((nbytes = walk.nbytes) != 0)) { /* only use complete blocks */ n = nbytes & ~(AES_BLOCK_SIZE - 1); cpacf_kmc(sctx->fc | modifier, ¶m, @@ -359,7 +359,7 @@ static int xts_aes_crypt(struct skcipher_request *req, unsigned long modifier) memcpy(xts_param.key + offset, xts_ctx->key, xts_ctx->key_len); memcpy(xts_param.init, pcc_param.xts, 16); - while ((nbytes = walk.nbytes) != 0) { + while (!ret && ((nbytes = walk.nbytes) != 0)) { /* only use complete blocks */ n = nbytes & ~(AES_BLOCK_SIZE - 1); cpacf_km(xts_ctx->fc | modifier, xts_param.key + offset, @@ -487,7 +487,7 @@ static int fullxts_aes_crypt(struct skcipher_request *req, unsigned long modifi memcpy(fxts_param.tweak, req->iv, AES_BLOCK_SIZE); fxts_param.nap[0] = 0x01; /* initial alpha power (1, little-endian) */ - while ((nbytes = walk.nbytes) != 0) { + while (!ret && ((nbytes = walk.nbytes) != 0)) { /* only use complete blocks */ n = nbytes & ~(AES_BLOCK_SIZE - 1); cpacf_km(xts_ctx->fc | modifier, fxts_param.key + offset, @@ -562,48 +562,64 @@ static unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes) return n; } +static int __ctr_aes_crypt(struct s390_aes_ctx *sctx, + struct skcipher_walk *walk, bool locked) +{ + unsigned int n, nbytes; + int ret = 0; + u8 *ctrptr; + + while (!ret && ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE)) { + n = AES_BLOCK_SIZE; + if (nbytes >= 2 * AES_BLOCK_SIZE && locked) + n = __ctrblk_init(ctrblk, walk->iv, nbytes); + ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk->iv; + cpacf_kmctr(sctx->fc, sctx->key, walk->dst.virt.addr, + walk->src.virt.addr, n, ctrptr); + if (ctrptr == ctrblk) + memcpy(walk->iv, ctrptr + n - AES_BLOCK_SIZE, + AES_BLOCK_SIZE); + crypto_inc(walk->iv, AES_BLOCK_SIZE); + ret = skcipher_walk_done(walk, nbytes - n); + } + + return ret; +} + static int ctr_aes_crypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct s390_aes_ctx *sctx = crypto_skcipher_ctx(tfm); - u8 buf[AES_BLOCK_SIZE], *ctrptr; struct skcipher_walk walk; - unsigned int n, nbytes; - int ret, locked; + u8 buf[AES_BLOCK_SIZE]; + int ret; if (unlikely(!sctx->fc)) return fallback_skcipher_crypt(sctx, req, 0); - locked = mutex_trylock(&ctrblk_lock); - ret = skcipher_walk_virt(&walk, req, false); - while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { - n = AES_BLOCK_SIZE; + if (ret) + return ret; - if (nbytes >= 2*AES_BLOCK_SIZE && locked) - n = __ctrblk_init(ctrblk, walk.iv, nbytes); - ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk.iv; - cpacf_kmctr(sctx->fc, sctx->key, walk.dst.virt.addr, - walk.src.virt.addr, n, ctrptr); - if (ctrptr == ctrblk) - memcpy(walk.iv, ctrptr + n - AES_BLOCK_SIZE, - AES_BLOCK_SIZE); - crypto_inc(walk.iv, AES_BLOCK_SIZE); - ret = skcipher_walk_done(&walk, nbytes - n); + if (down_trylock(&ctrblk_sem) == 0) { + ret = __ctr_aes_crypt(sctx, &walk, true); + up(&ctrblk_sem); + } else { + ret = __ctr_aes_crypt(sctx, &walk, false); } - if (locked) - mutex_unlock(&ctrblk_lock); + /* * final block may be < AES_BLOCK_SIZE, copy only nbytes */ - if (nbytes) { + if (!ret && walk.nbytes > 0) { memset(buf, 0, AES_BLOCK_SIZE); - memcpy(buf, walk.src.virt.addr, nbytes); + memcpy(buf, walk.src.virt.addr, walk.nbytes); cpacf_kmctr(sctx->fc, sctx->key, buf, buf, AES_BLOCK_SIZE, walk.iv); - memcpy(walk.dst.virt.addr, buf, nbytes); + memcpy(walk.dst.virt.addr, buf, walk.nbytes); crypto_inc(walk.iv, AES_BLOCK_SIZE); ret = skcipher_walk_done(&walk, 0); + memzero_explicit(buf, sizeof(buf)); } return ret; @@ -895,10 +911,14 @@ static int gcm_aes_crypt(struct aead_request *req, unsigned int flags) gw_in.ptr, aad_bytes); n = aad_bytes + pc_bytes; - if (gcm_in_walk_done(&gw_in, n) != n) - return -ENOMEM; - if (gcm_out_walk_done(&gw_out, n) != n) - return -ENOMEM; + if (gcm_in_walk_done(&gw_in, n) != n) { + ret = -ENOMEM; + goto out; + } + if (gcm_out_walk_done(&gw_out, n) != n) { + ret = -ENOMEM; + goto out; + } aadlen -= aad_bytes; pclen -= pc_bytes; } while (aadlen + pclen > 0); @@ -910,7 +930,10 @@ static int gcm_aes_crypt(struct aead_request *req, unsigned int flags) } else scatterwalk_map_and_copy(param.t, req->dst, len, taglen, 1); +out: memzero_explicit(¶m, sizeof(param)); + memzero_explicit(gw_in.buf, sizeof(gw_in.buf)); + memzero_explicit(gw_out.buf, sizeof(gw_out.buf)); return ret; } diff --git a/arch/s390/crypto/hmac_s390.c b/arch/s390/crypto/hmac_s390.c index f8cd09f341d4..445fa7bbd958 100644 --- a/arch/s390/crypto/hmac_s390.c +++ b/arch/s390/crypto/hmac_s390.c @@ -150,7 +150,10 @@ static int hash_data(const u8 *in, unsigned int inlen, #undef PARAM_INIT - cpacf_klmd(func, ¶m, in, inlen); + if (final) + cpacf_klmd(func, ¶m, in, inlen); + else + cpacf_kimd(func, ¶m, in, inlen); memcpy(digest, ¶m, digestsize); diff --git a/arch/s390/crypto/paes_s390.c b/arch/s390/crypto/paes_s390.c index 8cfe6166c193..f987bcbe8f35 100644 --- a/arch/s390/crypto/paes_s390.c +++ b/arch/s390/crypto/paes_s390.c @@ -19,7 +19,7 @@ #include <linux/init.h> #include <linux/miscdevice.h> #include <linux/module.h> -#include <linux/mutex.h> +#include <linux/semaphore.h> #include <linux/spinlock.h> #include <crypto/aes.h> #include <crypto/algapi.h> @@ -45,7 +45,7 @@ module_param_named(clrkey, pkey_clrkey_allowed, bool, 0444); MODULE_PARM_DESC(clrkey, "Allow clear key material (default N)"); static u8 *ctrblk; -static DEFINE_MUTEX(ctrblk_lock); +static DEFINE_SEMAPHORE(ctrblk_sem, 1); static cpacf_mask_t km_functions, kmc_functions, kmctr_functions; @@ -220,6 +220,10 @@ static inline int convert_key(const u8 *key, unsigned int keylen, xflags); } + /* But finally map -EBUSY to -EIO to indicate an IO failure */ + if (rc == -EBUSY) + rc = -EIO; + out: pr_debug("rc=%d\n", rc); return rc; @@ -432,8 +436,11 @@ static int ecb_paes_do_crypt(struct s390_paes_ctx *ctx, n = nbytes & ~(AES_BLOCK_SIZE - 1); k = cpacf_km(ctx->fc | req_ctx->modifier, param, walk->dst.virt.addr, walk->src.virt.addr, n); - if (k) + if (k) { rc = skcipher_walk_done(walk, nbytes - k); + if (rc) + goto out; + } if (k < n) { if (!maysleep) { rc = -EKEYEXPIRED; @@ -460,6 +467,7 @@ static int ecb_paes_crypt(struct skcipher_request *req, unsigned long modifier) struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); struct skcipher_walk *walk = &req_ctx->walk; bool tested = crypto_skcipher_tested(tfm); + bool cleanup = true; int rc; /* @@ -491,15 +499,17 @@ static int ecb_paes_crypt(struct skcipher_request *req, unsigned long modifier) if (rc == 0 || rc == -EKEYEXPIRED) { atomic_inc(&ctx->via_engine_ctr); rc = crypto_transfer_skcipher_request_to_engine(paes_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) + if (cleanup && walk->nbytes) skcipher_walk_done(walk, rc); out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(&req_ctx->param, sizeof(req_ctx->param)); pr_debug("rc=%d\n", rc); return rc; @@ -548,17 +558,8 @@ static int ecb_paes_do_one_request(struct crypto_engine *engine, void *areq) rc = ecb_paes_do_crypt(ctx, req_ctx, tested, true); if (rc == -EKEYEXPIRED) { - /* - * Protected key expired, conversion is in process. - * Trigger a re-schedule of this request by returning - * -ENOSPC ("hardware queue is full") to the crypto engine. - * To avoid immediately re-invocation of this callback, - * tell the scheduler to voluntarily give up the CPU here. - */ - cond_resched(); - pr_debug("rescheduling request\n"); - return -ENOSPC; - } else if (rc) { + return pkey_handle_expired(); + } else if (rc && walk->nbytes) { skcipher_walk_done(walk, rc); } @@ -568,7 +569,7 @@ static int ecb_paes_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&ctx->via_engine_ctr); crypto_finalize_skcipher_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } static struct skcipher_engine_alg ecb_paes_alg = { @@ -576,6 +577,7 @@ static struct skcipher_engine_alg ecb_paes_alg = { .base.cra_name = "ecb(paes)", .base.cra_driver_name = "ecb-paes-s390", .base.cra_priority = 401, /* combo: aes + ecb + 1 */ + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct s390_paes_ctx), .base.cra_module = THIS_MODULE, @@ -699,6 +701,8 @@ static int cbc_paes_do_crypt(struct s390_paes_ctx *ctx, if (k) { memcpy(walk->iv, param->iv, AES_BLOCK_SIZE); rc = skcipher_walk_done(walk, nbytes - k); + if (rc) + goto out; } if (k < n) { if (!maysleep) { @@ -726,6 +730,7 @@ static int cbc_paes_crypt(struct skcipher_request *req, unsigned long modifier) struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); struct skcipher_walk *walk = &req_ctx->walk; bool tested = crypto_skcipher_tested(tfm); + bool cleanup = true; int rc; /* @@ -757,15 +762,17 @@ static int cbc_paes_crypt(struct skcipher_request *req, unsigned long modifier) if (rc == 0 || rc == -EKEYEXPIRED) { atomic_inc(&ctx->via_engine_ctr); rc = crypto_transfer_skcipher_request_to_engine(paes_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) + if (cleanup && walk->nbytes) skcipher_walk_done(walk, rc); out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(&req_ctx->param, sizeof(req_ctx->param)); pr_debug("rc=%d\n", rc); return rc; @@ -814,17 +821,8 @@ static int cbc_paes_do_one_request(struct crypto_engine *engine, void *areq) rc = cbc_paes_do_crypt(ctx, req_ctx, tested, true); if (rc == -EKEYEXPIRED) { - /* - * Protected key expired, conversion is in process. - * Trigger a re-schedule of this request by returning - * -ENOSPC ("hardware queue is full") to the crypto engine. - * To avoid immediately re-invocation of this callback, - * tell the scheduler to voluntarily give up the CPU here. - */ - cond_resched(); - pr_debug("rescheduling request\n"); - return -ENOSPC; - } else if (rc) { + return pkey_handle_expired(); + } else if (rc && walk->nbytes) { skcipher_walk_done(walk, rc); } @@ -834,7 +832,7 @@ static int cbc_paes_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&ctx->via_engine_ctr); crypto_finalize_skcipher_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } static struct skcipher_engine_alg cbc_paes_alg = { @@ -842,6 +840,7 @@ static struct skcipher_engine_alg cbc_paes_alg = { .base.cra_name = "cbc(paes)", .base.cra_driver_name = "cbc-paes-s390", .base.cra_priority = 402, /* cbc-paes-s390 + 1 */ + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct s390_paes_ctx), .base.cra_module = THIS_MODULE, @@ -932,41 +931,14 @@ static inline unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes return n; } -static int ctr_paes_do_crypt(struct s390_paes_ctx *ctx, - struct s390_pctr_req_ctx *req_ctx, - bool tested, bool maysleep) +static int __ctr_paes_do_crypt(struct s390_paes_ctx *ctx, + struct ctr_param *param, + struct skcipher_walk *walk, + bool tested, bool maysleep, bool locked) { - struct ctr_param *param = &req_ctx->param; - struct skcipher_walk *walk = &req_ctx->walk; - u8 buf[AES_BLOCK_SIZE], *ctrptr; unsigned int nbytes, n, k; - int pk_state, locked, rc = 0; - - if (!req_ctx->param_init_done) { - /* fetch and check protected key state */ - spin_lock_bh(&ctx->pk_lock); - pk_state = ctx->pk_state; - switch (pk_state) { - case PK_STATE_NO_KEY: - rc = -ENOKEY; - break; - case PK_STATE_CONVERT_IN_PROGRESS: - rc = -EKEYEXPIRED; - break; - case PK_STATE_VALID: - memcpy(param->key, ctx->pk.protkey, sizeof(param->key)); - req_ctx->param_init_done = true; - break; - default: - rc = pk_state < 0 ? pk_state : -EIO; - break; - } - spin_unlock_bh(&ctx->pk_lock); - } - if (rc) - goto out; - - locked = mutex_trylock(&ctrblk_lock); + u8 *ctrptr; + int rc = 0; /* * Note that in case of partial processing or failure the walk @@ -986,32 +958,71 @@ static int ctr_paes_do_crypt(struct s390_paes_ctx *ctx, AES_BLOCK_SIZE); crypto_inc(walk->iv, AES_BLOCK_SIZE); rc = skcipher_walk_done(walk, nbytes - k); + if (rc) + goto out; } if (k < n) { if (!maysleep) { - if (locked) - mutex_unlock(&ctrblk_lock); rc = -EKEYEXPIRED; goto out; } rc = paes_convert_key(ctx, tested); - if (rc) { - if (locked) - mutex_unlock(&ctrblk_lock); + if (rc) goto out; - } spin_lock_bh(&ctx->pk_lock); memcpy(param->key, ctx->pk.protkey, sizeof(param->key)); spin_unlock_bh(&ctx->pk_lock); } } - if (locked) - mutex_unlock(&ctrblk_lock); + +out: + return rc; +} + +static int ctr_paes_do_crypt(struct s390_paes_ctx *ctx, + struct s390_pctr_req_ctx *req_ctx, + bool tested, bool maysleep) +{ + struct ctr_param *param = &req_ctx->param; + struct skcipher_walk *walk = &req_ctx->walk; + u8 buf[AES_BLOCK_SIZE]; + int pk_state, rc = 0; + + if (!req_ctx->param_init_done) { + /* fetch and check protected key state */ + spin_lock_bh(&ctx->pk_lock); + pk_state = ctx->pk_state; + switch (pk_state) { + case PK_STATE_NO_KEY: + rc = -ENOKEY; + break; + case PK_STATE_CONVERT_IN_PROGRESS: + rc = -EKEYEXPIRED; + break; + case PK_STATE_VALID: + memcpy(param->key, ctx->pk.protkey, sizeof(param->key)); + req_ctx->param_init_done = true; + break; + default: + rc = pk_state < 0 ? pk_state : -EIO; + break; + } + spin_unlock_bh(&ctx->pk_lock); + } + if (rc) + goto out; + + if (down_trylock(&ctrblk_sem) == 0) { + rc = __ctr_paes_do_crypt(ctx, param, walk, tested, maysleep, true); + up(&ctrblk_sem); + } else { + rc = __ctr_paes_do_crypt(ctx, param, walk, tested, maysleep, false); + } /* final block may be < AES_BLOCK_SIZE, copy only nbytes */ - if (nbytes) { + if (!rc && walk->nbytes > 0) { memset(buf, 0, AES_BLOCK_SIZE); - memcpy(buf, walk->src.virt.addr, nbytes); + memcpy(buf, walk->src.virt.addr, walk->nbytes); while (1) { if (cpacf_kmctr(ctx->fc, param, buf, buf, AES_BLOCK_SIZE, @@ -1028,12 +1039,13 @@ static int ctr_paes_do_crypt(struct s390_paes_ctx *ctx, memcpy(param->key, ctx->pk.protkey, sizeof(param->key)); spin_unlock_bh(&ctx->pk_lock); } - memcpy(walk->dst.virt.addr, buf, nbytes); + memcpy(walk->dst.virt.addr, buf, walk->nbytes); crypto_inc(walk->iv, AES_BLOCK_SIZE); rc = skcipher_walk_done(walk, 0); } out: + memzero_explicit(buf, sizeof(buf)); pr_debug("rc=%d\n", rc); return rc; } @@ -1045,6 +1057,7 @@ static int ctr_paes_crypt(struct skcipher_request *req) struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); struct skcipher_walk *walk = &req_ctx->walk; bool tested = crypto_skcipher_tested(tfm); + bool cleanup = true; int rc; /* @@ -1075,15 +1088,17 @@ static int ctr_paes_crypt(struct skcipher_request *req) if (rc == 0 || rc == -EKEYEXPIRED) { atomic_inc(&ctx->via_engine_ctr); rc = crypto_transfer_skcipher_request_to_engine(paes_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) + if (cleanup && walk->nbytes) skcipher_walk_done(walk, rc); out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(&req_ctx->param, sizeof(req_ctx->param)); pr_debug("rc=%d\n", rc); return rc; @@ -1122,17 +1137,8 @@ static int ctr_paes_do_one_request(struct crypto_engine *engine, void *areq) rc = ctr_paes_do_crypt(ctx, req_ctx, tested, true); if (rc == -EKEYEXPIRED) { - /* - * Protected key expired, conversion is in process. - * Trigger a re-schedule of this request by returning - * -ENOSPC ("hardware queue is full") to the crypto engine. - * To avoid immediately re-invocation of this callback, - * tell the scheduler to voluntarily give up the CPU here. - */ - cond_resched(); - pr_debug("rescheduling request\n"); - return -ENOSPC; - } else if (rc) { + return pkey_handle_expired(); + } else if (rc && walk->nbytes) { skcipher_walk_done(walk, rc); } @@ -1142,7 +1148,7 @@ static int ctr_paes_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&ctx->via_engine_ctr); crypto_finalize_skcipher_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } static struct skcipher_engine_alg ctr_paes_alg = { @@ -1150,6 +1156,7 @@ static struct skcipher_engine_alg ctr_paes_alg = { .base.cra_name = "ctr(paes)", .base.cra_driver_name = "ctr-paes-s390", .base.cra_priority = 402, /* ecb-paes-s390 + 1 */ + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct s390_paes_ctx), .base.cra_module = THIS_MODULE, @@ -1310,8 +1317,11 @@ static int xts_paes_do_crypt_fullkey(struct s390_pxts_ctx *ctx, n = nbytes & ~(AES_BLOCK_SIZE - 1); k = cpacf_km(ctx->fc | req_ctx->modifier, param->key + offset, walk->dst.virt.addr, walk->src.virt.addr, n); - if (k) + if (k) { rc = skcipher_walk_done(walk, nbytes - k); + if (rc) + goto out; + } if (k < n) { if (!maysleep) { rc = -EKEYEXPIRED; @@ -1364,7 +1374,7 @@ static inline int __xts_2keys_prep_param(struct s390_pxts_ctx *ctx, memcpy(param->init, pcc_param.xts, 16); } - memzero_explicit(pcc_param.key, sizeof(pcc_param.key)); + memzero_explicit(&pcc_param, sizeof(pcc_param)); return rc; } @@ -1404,8 +1414,11 @@ static int xts_paes_do_crypt_2keys(struct s390_pxts_ctx *ctx, n = nbytes & ~(AES_BLOCK_SIZE - 1); k = cpacf_km(ctx->fc | req_ctx->modifier, param->key + offset, walk->dst.virt.addr, walk->src.virt.addr, n); - if (k) + if (k) { rc = skcipher_walk_done(walk, nbytes - k); + if (rc) + goto out; + } if (k < n) { if (!maysleep) { rc = -EKEYEXPIRED; @@ -1477,6 +1490,7 @@ static inline int xts_paes_crypt(struct skcipher_request *req, unsigned long mod struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm); struct skcipher_walk *walk = &req_ctx->walk; bool tested = crypto_skcipher_tested(tfm); + bool cleanup = true; int rc; /* @@ -1508,15 +1522,17 @@ static inline int xts_paes_crypt(struct skcipher_request *req, unsigned long mod if (rc == 0 || rc == -EKEYEXPIRED) { atomic_inc(&ctx->via_engine_ctr); rc = crypto_transfer_skcipher_request_to_engine(paes_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) + if (cleanup && walk->nbytes) skcipher_walk_done(walk, rc); out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(&req_ctx->param, sizeof(req_ctx->param)); pr_debug("rc=%d\n", rc); return rc; @@ -1565,17 +1581,8 @@ static int xts_paes_do_one_request(struct crypto_engine *engine, void *areq) rc = xts_paes_do_crypt(ctx, req_ctx, tested, true); if (rc == -EKEYEXPIRED) { - /* - * Protected key expired, conversion is in process. - * Trigger a re-schedule of this request by returning - * -ENOSPC ("hardware queue is full") to the crypto engine. - * To avoid immediately re-invocation of this callback, - * tell the scheduler to voluntarily give up the CPU here. - */ - cond_resched(); - pr_debug("rescheduling request\n"); - return -ENOSPC; - } else if (rc) { + return pkey_handle_expired(); + } else if (rc && walk->nbytes) { skcipher_walk_done(walk, rc); } @@ -1585,7 +1592,7 @@ static int xts_paes_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&ctx->via_engine_ctr); crypto_finalize_skcipher_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } static struct skcipher_engine_alg xts_paes_alg = { @@ -1593,6 +1600,7 @@ static struct skcipher_engine_alg xts_paes_alg = { .base.cra_name = "xts(paes)", .base.cra_driver_name = "xts-paes-s390", .base.cra_priority = 402, /* ecb-paes-s390 + 1 */ + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct s390_pxts_ctx), .base.cra_module = THIS_MODULE, diff --git a/arch/s390/crypto/phmac_s390.c b/arch/s390/crypto/phmac_s390.c index 03ca33ffe6cc..bbf8a6809ecb 100644 --- a/arch/s390/crypto/phmac_s390.c +++ b/arch/s390/crypto/phmac_s390.c @@ -62,8 +62,10 @@ static inline int hwh_prepare(struct ahash_request *req, */ static inline int hwh_advance(struct hash_walk_helper *hwh, int n) { - if (n < 0) + if (n < 0) { + hwh->walkbytes = n; return crypto_hash_walk_done(&hwh->walk, n); + } hwh->walkbytes -= n; hwh->walkaddr += n; @@ -339,6 +341,10 @@ static inline int convert_key(const u8 *key, unsigned int keylen, xflags); } + /* But finally map -EBUSY to -EIO to indicate an IO failure */ + if (rc == -EBUSY) + rc = -EIO; + out: pr_debug("rc=%d\n", rc); return rc; @@ -606,6 +612,7 @@ static int phmac_update(struct ahash_request *req) struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm); struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx; struct hash_walk_helper *hwh = &req_ctx->hwh; + bool cleanup = true; int rc; /* prep the walk in the request context */ @@ -629,12 +636,15 @@ static int phmac_update(struct ahash_request *req) req_ctx->async_op = OP_UPDATE; atomic_inc(&tfm_ctx->via_engine_ctr); rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&tfm_ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) { - hwh_advance(hwh, rc); + if (cleanup) { + if (hwh->walkbytes > 0) + hwh_advance(hwh, rc); memzero_explicit(kmac_ctx, sizeof(*kmac_ctx)); } @@ -649,6 +659,7 @@ static int phmac_final(struct ahash_request *req) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm); struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx; + bool cleanup = true; int rc = 0; /* Try synchronous operation if no active engine usage */ @@ -667,12 +678,14 @@ static int phmac_final(struct ahash_request *req) req_ctx->async_op = OP_FINAL; atomic_inc(&tfm_ctx->via_engine_ctr); rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&tfm_ctx->via_engine_ctr); } out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(kmac_ctx, sizeof(*kmac_ctx)); pr_debug("rc=%d\n", rc); return rc; @@ -685,6 +698,7 @@ static int phmac_finup(struct ahash_request *req) struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm); struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx; struct hash_walk_helper *hwh = &req_ctx->hwh; + bool cleanup = true; int rc; /* prep the walk in the request context */ @@ -716,15 +730,17 @@ static int phmac_finup(struct ahash_request *req) /* req->async_op has been set to either OP_FINUP or OP_FINAL */ atomic_inc(&tfm_ctx->via_engine_ctr); rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&tfm_ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) + if (cleanup && hwh->walkbytes > 0) hwh_advance(hwh, rc); out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(kmac_ctx, sizeof(*kmac_ctx)); pr_debug("rc=%d\n", rc); return rc; @@ -887,16 +903,7 @@ static int phmac_do_one_request(struct crypto_engine *engine, void *areq) case OP_FINUP: rc = phmac_kmac_update(req, true); if (rc == -EKEYEXPIRED) { - /* - * Protected key expired, conversion is in process. - * Trigger a re-schedule of this request by returning - * -ENOSPC ("hardware queue full") to the crypto engine. - * To avoid immediately re-invocation of this callback, - * tell scheduler to voluntarily give up the CPU here. - */ - pr_debug("rescheduling request\n"); - cond_resched(); - return -ENOSPC; + return pkey_handle_expired(); } else if (rc) { hwh_advance(hwh, rc); goto out; @@ -907,18 +914,8 @@ static int phmac_do_one_request(struct crypto_engine *engine, void *areq) fallthrough; case OP_FINAL: rc = phmac_kmac_final(req, true); - if (rc == -EKEYEXPIRED) { - /* - * Protected key expired, conversion is in process. - * Trigger a re-schedule of this request by returning - * -ENOSPC ("hardware queue full") to the crypto engine. - * To avoid immediately re-invocation of this callback, - * tell scheduler to voluntarily give up the CPU here. - */ - pr_debug("rescheduling request\n"); - cond_resched(); - return -ENOSPC; - } + if (rc == -EKEYEXPIRED) + return pkey_handle_expired(); break; default: /* unknown/unsupported/unimplemented asynch op */ @@ -933,7 +930,7 @@ out: atomic_dec(&tfm_ctx->via_engine_ctr); crypto_finalize_hash_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } #define S390_ASYNC_PHMAC_ALG(x) \ diff --git a/arch/s390/hypfs/Makefile b/arch/s390/hypfs/Makefile index c34854d298f8..5a3daeafe926 100644 --- a/arch/s390/hypfs/Makefile +++ b/arch/s390/hypfs/Makefile @@ -3,6 +3,8 @@ # Makefile for the linux hypfs filesystem routines. # +CONTEXT_ANALYSIS := y + obj-$(CONFIG_S390_HYPFS) += hypfs_dbfs.o obj-$(CONFIG_S390_HYPFS) += hypfs_diag.o obj-$(CONFIG_S390_HYPFS) += hypfs_diag0c.o diff --git a/arch/s390/include/asm/ap.h b/arch/s390/include/asm/ap.h index 3b95c6531a67..c91b6ace199d 100644 --- a/arch/s390/include/asm/ap.h +++ b/arch/s390/include/asm/ap.h @@ -125,8 +125,8 @@ struct ap_tapq_hwinfo { unsigned int nd : 8; /* nr of domains */ unsigned int : 4; unsigned int ml : 4; /* apxl ml */ - unsigned int : 4; - unsigned int qd : 4; /* queue depth */ + unsigned int : 3; + unsigned int qd : 5; /* queue depth */ }; }; }; diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h index 99748c20e767..d23ea0c94e4e 100644 --- a/arch/s390/include/asm/asm-extable.h +++ b/arch/s390/include/asm/asm-extable.h @@ -12,6 +12,7 @@ #define EX_TYPE_UA_FAULT 3 #define EX_TYPE_UA_LOAD_REG 5 #define EX_TYPE_UA_LOAD_REGPAIR 6 +#define EX_TYPE_ZEROPAD 7 #define EX_TYPE_FPC 8 #define EX_TYPE_UA_MVCOS_TO 9 #define EX_TYPE_UA_MVCOS_FROM 10 @@ -79,6 +80,9 @@ #define EX_TABLE_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0) +#define EX_TABLE_ZEROPAD(_fault, _target, _regdata, _regaddr) \ + __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_ZEROPAD, _regdata, _regaddr, 0) + #define EX_TABLE_FPC(_fault, _target) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FPC, __stringify(%%r0), __stringify(%%r0), 0) diff --git a/arch/s390/include/asm/cfi.h b/arch/s390/include/asm/cfi.h new file mode 100644 index 000000000000..9af2c7cb70ca --- /dev/null +++ b/arch/s390/include/asm/cfi.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_S390_CFI_H +#define _ASM_S390_CFI_H + +#define __bpfcall + +#endif /* _ASM_S390_CFI_H */ diff --git a/arch/s390/include/asm/cpacf.h b/arch/s390/include/asm/cpacf.h index a83683169d98..6174552d856d 100644 --- a/arch/s390/include/asm/cpacf.h +++ b/arch/s390/include/asm/cpacf.h @@ -301,6 +301,7 @@ static __always_inline void __cpacf_query(unsigned int opcode, cpacf_mask_t *mask) { __cpacf_query_insn(opcode, mask, CPACF_FC_QUERY); + kmsan_unpoison_memory(mask, sizeof(*mask)); } static __always_inline int __cpacf_check_opcode(unsigned int opcode) @@ -370,6 +371,7 @@ static __always_inline int cpacf_query_func(unsigned int opcode, static __always_inline void __cpacf_qai(unsigned int opcode, cpacf_qai_t *qai) { __cpacf_query_insn(opcode, qai, CPACF_FC_QUERY_AUTH_INFO); + kmsan_unpoison_memory(qai, sizeof(*qai)); } /** @@ -422,6 +424,7 @@ static inline int cpacf_km(unsigned long func, void *param, [opc] "i" (CPACF_KM) : "cc", "memory", "0", "1"); + kmsan_unpoison_memory(dest, src_len - s.odd); return src_len - s.odd; } @@ -454,6 +457,7 @@ static inline int cpacf_kmc(unsigned long func, void *param, [opc] "i" (CPACF_KMC) : "cc", "memory", "0", "1"); + kmsan_unpoison_memory(dest, src_len - s.odd); return src_len - s.odd; } @@ -587,6 +591,7 @@ static inline int cpacf_kmctr(unsigned long func, void *param, u8 *dest, [opc] "i" (CPACF_KMCTR) : "cc", "memory", "0", "1"); + kmsan_unpoison_memory(dest, src_len - s.odd); return src_len - s.odd; } @@ -619,6 +624,7 @@ static inline void cpacf_prno(unsigned long func, void *param, : [fc] "d" (func), [pba] "d" ((unsigned long)param), [seed] "d" (s.pair), [opc] "i" (CPACF_PRNO) : "cc", "memory", "0", "1"); + kmsan_unpoison_memory(dest, dest_len); } /** diff --git a/arch/s390/include/asm/hugetlb.h b/arch/s390/include/asm/hugetlb.h index e33a5b587ee4..aea754b67c89 100644 --- a/arch/s390/include/asm/hugetlb.h +++ b/arch/s390/include/asm/hugetlb.h @@ -42,9 +42,9 @@ static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned long sz) { if ((pte_val(ptep_get(ptep)) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) - set_pte(ptep, __pte(_REGION3_ENTRY_EMPTY)); + set_pud((pud_t *)ptep, __pud(_REGION3_ENTRY_EMPTY)); else - set_pte(ptep, __pte(_SEGMENT_ENTRY_EMPTY)); + set_pmd((pmd_t *)ptep, __pmd(_SEGMENT_ENTRY_EMPTY)); } #define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH @@ -77,20 +77,20 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, __set_huge_pte_at(mm, addr, ptep, pte_wrprotect(pte)); } -#define __HAVE_ARCH_HUGE_PTE_MKUFFD_WP -static inline pte_t huge_pte_mkuffd_wp(pte_t pte) +#define __HAVE_ARCH_HUGE_PTE_MKUFFD +static inline pte_t huge_pte_mkuffd(pte_t pte) { return pte; } -#define __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD_WP -static inline pte_t huge_pte_clear_uffd_wp(pte_t pte) +#define __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD +static inline pte_t huge_pte_clear_uffd(pte_t pte) { return pte; } -#define __HAVE_ARCH_HUGE_PTE_UFFD_WP -static inline int huge_pte_uffd_wp(pte_t pte) +#define __HAVE_ARCH_HUGE_PTE_UFFD +static inline int huge_pte_uffd(pte_t pte) { return 0; } diff --git a/arch/s390/include/asm/irqflags.h b/arch/s390/include/asm/irqflags.h index bcab456dfb80..6eb6de6a99f2 100644 --- a/arch/s390/include/asm/irqflags.h +++ b/arch/s390/include/asm/irqflags.h @@ -37,18 +37,24 @@ static __always_inline void __arch_local_irq_ssm(unsigned long flags) asm volatile("ssm %0" : : "Q" (flags) : "memory"); } -#ifdef CONFIG_KMSAN -#define arch_local_irq_attributes noinline notrace __no_sanitize_memory __maybe_unused +#if defined(CONFIG_KMSAN) && !defined(__DECOMPRESSOR) +unsigned long arch_local_save_flags(void); +unsigned long arch_local_irq_save(void); +void arch_local_irq_enable_external(void); +void arch_local_irq_enable(void); #else -#define arch_local_irq_attributes __always_inline +#define arch_local_save_flags __arch_local_save_flags +#define arch_local_irq_save __arch_local_irq_save +#define arch_local_irq_enable_external __arch_local_irq_enable_external +#define arch_local_irq_enable __arch_local_irq_enable #endif -static arch_local_irq_attributes unsigned long arch_local_save_flags(void) +static __always_inline unsigned long __arch_local_save_flags(void) { return __arch_local_irq_stnsm(0xff); } -static arch_local_irq_attributes unsigned long arch_local_irq_save(void) +static __always_inline unsigned long __arch_local_irq_save(void) { return __arch_local_irq_stnsm(0xfc); } @@ -58,12 +64,12 @@ static __always_inline void arch_local_irq_disable(void) arch_local_irq_save(); } -static arch_local_irq_attributes void arch_local_irq_enable_external(void) +static __always_inline void __arch_local_irq_enable_external(void) { __arch_local_irq_stosm(0x01); } -static arch_local_irq_attributes void arch_local_irq_enable(void) +static __always_inline void __arch_local_irq_enable(void) { __arch_local_irq_stosm(0x03); } diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index b4182ca4435f..147325978225 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -1,154 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* - * definition for kernel virtual machines on s390 - * - * Copyright IBM Corp. 2008, 2018 - * - * Author(s): Carsten Otte <cotte@de.ibm.com> - */ - #ifndef ASM_KVM_HOST_H #define ASM_KVM_HOST_H -#include <linux/types.h> -#include <linux/hrtimer.h> -#include <linux/interrupt.h> -#include <linux/kvm_types.h> -#include <linux/kvm.h> -#include <linux/seqlock.h> -#include <linux/module.h> -#include <linux/pci.h> -#include <linux/mmu_notifier.h> -#include <asm/kvm_host_types.h> -#include <asm/debug.h> -#include <asm/cpu.h> -#include <asm/fpu.h> -#include <asm/isc.h> -#include <asm/guarded_storage.h> - -#define KVM_HAVE_MMU_RWLOCK -#define KVM_MAX_VCPUS 255 - -#define KVM_INTERNAL_MEM_SLOTS 1 - -/* - * These seem to be used for allocating ->chip in the routing table, which we - * don't use. 1 is as small as we can get to reduce the needed memory. If we - * need to look at ->chip later on, we'll need to revisit this. - */ -#define KVM_NR_IRQCHIPS 1 -#define KVM_IRQCHIP_NUM_PINS 1 -#define KVM_HALT_POLL_NS_DEFAULT 50000 - -/* s390-specific vcpu->requests bit members */ -#define KVM_REQ_ENABLE_IBS KVM_ARCH_REQ(0) -#define KVM_REQ_DISABLE_IBS KVM_ARCH_REQ(1) -#define KVM_REQ_ICPT_OPEREXC KVM_ARCH_REQ(2) -#define KVM_REQ_START_MIGRATION KVM_ARCH_REQ(3) -#define KVM_REQ_STOP_MIGRATION KVM_ARCH_REQ(4) -#define KVM_REQ_VSIE_RESTART KVM_ARCH_REQ(5) -#define KVM_REQ_REFRESH_GUEST_PREFIX \ - KVM_ARCH_REQ_FLAGS(6, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) - -struct kvm_vcpu_stat { - struct kvm_vcpu_stat_generic generic; - u64 exit_userspace; - u64 exit_null; - u64 exit_external_request; - u64 exit_io_request; - u64 exit_external_interrupt; - u64 exit_stop_request; - u64 exit_validity; - u64 exit_instruction; - u64 exit_pei; - u64 halt_no_poll_steal; - u64 instruction_lctl; - u64 instruction_lctlg; - u64 instruction_stctl; - u64 instruction_stctg; - u64 exit_program_interruption; - u64 exit_instr_and_program; - u64 exit_operation_exception; - u64 deliver_ckc; - u64 deliver_cputm; - u64 deliver_external_call; - u64 deliver_emergency_signal; - u64 deliver_service_signal; - u64 deliver_virtio; - u64 deliver_stop_signal; - u64 deliver_prefix_signal; - u64 deliver_restart_signal; - u64 deliver_program; - u64 deliver_io; - u64 deliver_machine_check; - u64 exit_wait_state; - u64 inject_ckc; - u64 inject_cputm; - u64 inject_external_call; - u64 inject_emergency_signal; - u64 inject_mchk; - u64 inject_pfault_init; - u64 inject_program; - u64 inject_restart; - u64 inject_set_prefix; - u64 inject_stop_signal; - u64 instruction_epsw; - u64 instruction_gs; - u64 instruction_io_other; - u64 instruction_lpsw; - u64 instruction_lpswe; - u64 instruction_lpswey; - u64 instruction_pfmf; - u64 instruction_ptff; - u64 instruction_sck; - u64 instruction_sckpf; - u64 instruction_stidp; - u64 instruction_spx; - u64 instruction_stpx; - u64 instruction_stap; - u64 instruction_iske; - u64 instruction_ri; - u64 instruction_rrbe; - u64 instruction_sske; - u64 instruction_ipte_interlock; - u64 instruction_stsi; - u64 instruction_stfl; - u64 instruction_tb; - u64 instruction_tpi; - u64 instruction_tprot; - u64 instruction_tsch; - u64 instruction_sie; - u64 instruction_essa; - u64 instruction_sthyi; - u64 instruction_sigp_sense; - u64 instruction_sigp_sense_running; - u64 instruction_sigp_external_call; - u64 instruction_sigp_emergency; - u64 instruction_sigp_cond_emergency; - u64 instruction_sigp_start; - u64 instruction_sigp_stop; - u64 instruction_sigp_stop_store_status; - u64 instruction_sigp_store_status; - u64 instruction_sigp_store_adtl_status; - u64 instruction_sigp_arch; - u64 instruction_sigp_prefix; - u64 instruction_sigp_restart; - u64 instruction_sigp_init_cpu_reset; - u64 instruction_sigp_cpu_reset; - u64 instruction_sigp_unknown; - u64 instruction_diagnose_10; - u64 instruction_diagnose_44; - u64 instruction_diagnose_9c; - u64 diag_9c_ignored; - u64 diag_9c_forward; - u64 instruction_diagnose_258; - u64 instruction_diagnose_308; - u64 instruction_diagnose_500; - u64 instruction_diagnose_other; - u64 pfault_sync; - u64 signal_exits; -}; +#include <asm/kvm_host_s390.h> #define PGM_OPERATION 0x01 #define PGM_PRIVILEGED_OP 0x02 @@ -207,578 +62,4 @@ struct kvm_vcpu_stat { #define PGM_PER 0x80 #define PGM_CRYPTO_OPERATION 0x119 -/* irq types in ascend order of priorities */ -enum irq_types { - IRQ_PEND_SET_PREFIX = 0, - IRQ_PEND_RESTART, - IRQ_PEND_SIGP_STOP, - IRQ_PEND_IO_ISC_7, - IRQ_PEND_IO_ISC_6, - IRQ_PEND_IO_ISC_5, - IRQ_PEND_IO_ISC_4, - IRQ_PEND_IO_ISC_3, - IRQ_PEND_IO_ISC_2, - IRQ_PEND_IO_ISC_1, - IRQ_PEND_IO_ISC_0, - IRQ_PEND_VIRTIO, - IRQ_PEND_PFAULT_DONE, - IRQ_PEND_PFAULT_INIT, - IRQ_PEND_EXT_HOST, - IRQ_PEND_EXT_SERVICE, - IRQ_PEND_EXT_SERVICE_EV, - IRQ_PEND_EXT_TIMING, - IRQ_PEND_EXT_CPU_TIMER, - IRQ_PEND_EXT_CLOCK_COMP, - IRQ_PEND_EXT_EXTERNAL, - IRQ_PEND_EXT_EMERGENCY, - IRQ_PEND_EXT_MALFUNC, - IRQ_PEND_EXT_IRQ_KEY, - IRQ_PEND_MCHK_REP, - IRQ_PEND_PROG, - IRQ_PEND_SVC, - IRQ_PEND_MCHK_EX, - IRQ_PEND_COUNT -}; - -/* We have 2M for virtio device descriptor pages. Smallest amount of - * memory per page is 24 bytes (1 queue), so (2048*1024) / 24 = 87381 - */ -#define KVM_S390_MAX_VIRTIO_IRQS 87381 - -/* - * Repressible (non-floating) machine check interrupts - * subclass bits in MCIC - */ -#define MCHK_EXTD_BIT 58 -#define MCHK_DEGR_BIT 56 -#define MCHK_WARN_BIT 55 -#define MCHK_REP_MASK ((1UL << MCHK_DEGR_BIT) | \ - (1UL << MCHK_EXTD_BIT) | \ - (1UL << MCHK_WARN_BIT)) - -/* Exigent machine check interrupts subclass bits in MCIC */ -#define MCHK_SD_BIT 63 -#define MCHK_PD_BIT 62 -#define MCHK_EX_MASK ((1UL << MCHK_SD_BIT) | (1UL << MCHK_PD_BIT)) - -#define IRQ_PEND_EXT_MASK ((1UL << IRQ_PEND_EXT_IRQ_KEY) | \ - (1UL << IRQ_PEND_EXT_CLOCK_COMP) | \ - (1UL << IRQ_PEND_EXT_CPU_TIMER) | \ - (1UL << IRQ_PEND_EXT_MALFUNC) | \ - (1UL << IRQ_PEND_EXT_EMERGENCY) | \ - (1UL << IRQ_PEND_EXT_EXTERNAL) | \ - (1UL << IRQ_PEND_EXT_TIMING) | \ - (1UL << IRQ_PEND_EXT_HOST) | \ - (1UL << IRQ_PEND_EXT_SERVICE) | \ - (1UL << IRQ_PEND_EXT_SERVICE_EV) | \ - (1UL << IRQ_PEND_VIRTIO) | \ - (1UL << IRQ_PEND_PFAULT_INIT) | \ - (1UL << IRQ_PEND_PFAULT_DONE)) - -#define IRQ_PEND_IO_MASK ((1UL << IRQ_PEND_IO_ISC_0) | \ - (1UL << IRQ_PEND_IO_ISC_1) | \ - (1UL << IRQ_PEND_IO_ISC_2) | \ - (1UL << IRQ_PEND_IO_ISC_3) | \ - (1UL << IRQ_PEND_IO_ISC_4) | \ - (1UL << IRQ_PEND_IO_ISC_5) | \ - (1UL << IRQ_PEND_IO_ISC_6) | \ - (1UL << IRQ_PEND_IO_ISC_7)) - -#define IRQ_PEND_MCHK_MASK ((1UL << IRQ_PEND_MCHK_REP) | \ - (1UL << IRQ_PEND_MCHK_EX)) - -#define IRQ_PEND_EXT_II_MASK ((1UL << IRQ_PEND_EXT_CPU_TIMER) | \ - (1UL << IRQ_PEND_EXT_CLOCK_COMP) | \ - (1UL << IRQ_PEND_EXT_EMERGENCY) | \ - (1UL << IRQ_PEND_EXT_EXTERNAL) | \ - (1UL << IRQ_PEND_EXT_SERVICE) | \ - (1UL << IRQ_PEND_EXT_SERVICE_EV)) - -struct kvm_s390_interrupt_info { - struct list_head list; - u64 type; - union { - struct kvm_s390_io_info io; - struct kvm_s390_ext_info ext; - struct kvm_s390_pgm_info pgm; - struct kvm_s390_emerg_info emerg; - struct kvm_s390_extcall_info extcall; - struct kvm_s390_prefix_info prefix; - struct kvm_s390_stop_info stop; - struct kvm_s390_mchk_info mchk; - }; -}; - -struct kvm_s390_irq_payload { - struct kvm_s390_io_info io; - struct kvm_s390_ext_info ext; - struct kvm_s390_pgm_info pgm; - struct kvm_s390_emerg_info emerg; - struct kvm_s390_extcall_info extcall; - struct kvm_s390_prefix_info prefix; - struct kvm_s390_stop_info stop; - struct kvm_s390_mchk_info mchk; -}; - -struct kvm_s390_local_interrupt { - spinlock_t lock; - DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS); - struct kvm_s390_irq_payload irq; - unsigned long pending_irqs; -}; - -#define FIRQ_LIST_IO_ISC_0 0 -#define FIRQ_LIST_IO_ISC_1 1 -#define FIRQ_LIST_IO_ISC_2 2 -#define FIRQ_LIST_IO_ISC_3 3 -#define FIRQ_LIST_IO_ISC_4 4 -#define FIRQ_LIST_IO_ISC_5 5 -#define FIRQ_LIST_IO_ISC_6 6 -#define FIRQ_LIST_IO_ISC_7 7 -#define FIRQ_LIST_PFAULT 8 -#define FIRQ_LIST_VIRTIO 9 -#define FIRQ_LIST_COUNT 10 -#define FIRQ_CNTR_IO 0 -#define FIRQ_CNTR_SERVICE 1 -#define FIRQ_CNTR_VIRTIO 2 -#define FIRQ_CNTR_PFAULT 3 -#define FIRQ_MAX_COUNT 4 - -/* mask the AIS mode for a given ISC */ -#define AIS_MODE_MASK(isc) (0x80 >> isc) - -#define KVM_S390_AIS_MODE_ALL 0 -#define KVM_S390_AIS_MODE_SINGLE 1 - -struct kvm_s390_float_interrupt { - unsigned long pending_irqs; - unsigned long masked_irqs; - spinlock_t lock; - struct list_head lists[FIRQ_LIST_COUNT]; - int counters[FIRQ_MAX_COUNT]; - struct kvm_s390_mchk_info mchk; - struct kvm_s390_ext_info srv_signal; - int last_sleep_cpu; - spinlock_t ais_lock; - u8 simm; - u8 nimm; -}; - -struct kvm_hw_wp_info_arch { - unsigned long addr; - unsigned long phys_addr; - int len; - char *old_data; -}; - -struct kvm_hw_bp_info_arch { - unsigned long addr; - int len; -}; - -/* - * Only the upper 16 bits of kvm_guest_debug->control are arch specific. - * Further KVM_GUESTDBG flags which an be used from userspace can be found in - * arch/s390/include/uapi/asm/kvm.h - */ -#define KVM_GUESTDBG_EXIT_PENDING 0x10000000 - -#define guestdbg_enabled(vcpu) \ - (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) -#define guestdbg_sstep_enabled(vcpu) \ - (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) -#define guestdbg_hw_bp_enabled(vcpu) \ - (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) -#define guestdbg_exit_pending(vcpu) (guestdbg_enabled(vcpu) && \ - (vcpu->guest_debug & KVM_GUESTDBG_EXIT_PENDING)) - -#define KVM_GUESTDBG_VALID_MASK \ - (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP |\ - KVM_GUESTDBG_USE_HW_BP | KVM_GUESTDBG_EXIT_PENDING) - -struct kvm_guestdbg_info_arch { - unsigned long cr0; - unsigned long cr9; - unsigned long cr10; - unsigned long cr11; - struct kvm_hw_bp_info_arch *hw_bp_info; - struct kvm_hw_wp_info_arch *hw_wp_info; - int nr_hw_bp; - int nr_hw_wp; - unsigned long last_bp; -}; - -struct kvm_s390_pv_vcpu { - u64 handle; - unsigned long stor_base; -}; - -struct kvm_vcpu_arch { - struct kvm_s390_sie_block *sie_block; - /* if vsie is active, currently executed shadow sie control block */ - struct kvm_s390_sie_block *vsie_block; - unsigned int host_acrs[NUM_ACRS]; - struct gs_cb *host_gscb; - struct kvm_s390_local_interrupt local_int; - struct hrtimer ckc_timer; - struct kvm_s390_pgm_info pgm; - struct gmap *gmap; - struct kvm_guestdbg_info_arch guestdbg; - unsigned long pfault_token; - unsigned long pfault_select; - unsigned long pfault_compare; - bool cputm_enabled; - /* - * The seqcount protects updates to cputm_start and sie_block.cputm, - * this way we can have non-blocking reads with consistent values. - * Only the owning VCPU thread (vcpu->cpu) is allowed to change these - * values and to start/stop/enable/disable cpu timer accounting. - */ - seqcount_t cputm_seqcount; - __u64 cputm_start; - bool gs_enabled; - bool skey_enabled; - /* Indicator if the access registers have been loaded from guest */ - bool acrs_loaded; - bool initialized; - struct kvm_s390_pv_vcpu pv; - union diag318_info diag318_info; - struct kvm_s390_mmu_cache *mc; -}; - -struct kvm_vm_stat { - struct kvm_vm_stat_generic generic; - u64 inject_io; - u64 io_390_adapter_map; - u64 io_390_adapter_unmap; - u64 io_390_inatomic; - u64 io_flic_inject_airq; - u64 io_set_adapter_int; - u64 io_390_inatomic_no_inject; - u64 inject_float_mchk; - u64 inject_pfault_done; - u64 inject_service_signal; - u64 inject_virtio; - u64 aen_forward; - u64 gmap_shadow_create; - u64 gmap_shadow_reuse; - u64 gmap_shadow_r1_entry; - u64 gmap_shadow_r2_entry; - u64 gmap_shadow_r3_entry; - u64 gmap_shadow_sg_entry; - u64 gmap_shadow_pg_entry; -}; - -struct kvm_arch_memory_slot { -}; - -struct s390_map_info { - struct list_head list; - __u64 guest_addr; - __u64 addr; - struct page *page; - /* - * True if the page is long-term pinned. False if long-term pinning - * failed and this entry exists only to preserve MAP/UNMAP symmetry. - */ - bool pinned; -}; - -struct s390_io_adapter { - unsigned int id; - int isc; - bool maskable; - bool masked; - bool swap; - bool suppressible; - spinlock_t maps_lock; - struct list_head maps; - unsigned int nr_maps; -}; - -#define MAX_S390_IO_ADAPTERS ((MAX_ISC + 1) * 8) -#define MAX_S390_ADAPTER_MAPS 256 - -/* maximum size of facilities and facility mask is 2k bytes */ -#define S390_ARCH_FAC_LIST_SIZE_BYTE (1<<11) -#define S390_ARCH_FAC_LIST_SIZE_U64 \ - (S390_ARCH_FAC_LIST_SIZE_BYTE / sizeof(u64)) -#define S390_ARCH_FAC_MASK_SIZE_BYTE S390_ARCH_FAC_LIST_SIZE_BYTE -#define S390_ARCH_FAC_MASK_SIZE_U64 \ - (S390_ARCH_FAC_MASK_SIZE_BYTE / sizeof(u64)) - -struct kvm_s390_cpu_model { - /* facility mask supported by kvm & hosting machine */ - __u64 fac_mask[S390_ARCH_FAC_MASK_SIZE_U64]; - struct kvm_s390_vm_cpu_subfunc subfuncs; - /* facility list requested by guest (in dma page) */ - __u64 *fac_list; - u64 cpuid; - unsigned short ibc; - /* subset of available UV-features for pv-guests enabled by user space */ - struct kvm_s390_vm_cpu_uv_feat uv_feat_guest; -}; - -#define S390_ARCH_FAC_FORMAT_2 2 -struct kvm_s390_flcb2 { - union { - struct { - u8 reserved0[7]; - u8 length; - }; - u64 header_val; - }; - u64 facilities[S390_ARCH_FAC_LIST_SIZE_U64]; -}; - -typedef int (*crypto_hook)(struct kvm_vcpu *vcpu); - -struct kvm_s390_crypto { - struct kvm_s390_crypto_cb *crycb; - struct rw_semaphore pqap_hook_rwsem; - crypto_hook *pqap_hook; - __u32 crycbd; - __u8 aes_kw; - __u8 dea_kw; - __u8 apie; -}; - -#define APCB0_MASK_SIZE 1 -struct kvm_s390_apcb0 { - __u64 apm[APCB0_MASK_SIZE]; /* 0x0000 */ - __u64 aqm[APCB0_MASK_SIZE]; /* 0x0008 */ - __u64 adm[APCB0_MASK_SIZE]; /* 0x0010 */ - __u64 reserved18; /* 0x0018 */ -}; - -#define APCB1_MASK_SIZE 4 -struct kvm_s390_apcb1 { - __u64 apm[APCB1_MASK_SIZE]; /* 0x0000 */ - __u64 aqm[APCB1_MASK_SIZE]; /* 0x0020 */ - __u64 adm[APCB1_MASK_SIZE]; /* 0x0040 */ - __u64 reserved60[4]; /* 0x0060 */ -}; - -struct kvm_s390_crypto_cb { - struct kvm_s390_apcb0 apcb0; /* 0x0000 */ - __u8 reserved20[0x0048 - 0x0020]; /* 0x0020 */ - __u8 dea_wrapping_key_mask[24]; /* 0x0048 */ - __u8 aes_wrapping_key_mask[32]; /* 0x0060 */ - struct kvm_s390_apcb1 apcb1; /* 0x0080 */ -}; - -struct kvm_s390_gisa { - union { - struct { /* common to all formats */ - u32 next_alert; - u8 ipm; - u8 reserved01[2]; - u8 iam; - }; - struct { /* format 0 */ - u32 next_alert; - u8 ipm; - u8 reserved01; - u8 : 6; - u8 g : 1; - u8 c : 1; - u8 iam; - u8 reserved02[4]; - u32 airq_count; - } g0; - struct { /* format 1 */ - u32 next_alert; - u8 ipm; - u8 simm; - u8 nimm; - u8 iam; - u8 aism[8]; - u8 : 6; - u8 g : 1; - u8 c : 1; - u8 reserved03[11]; - u32 airq_count; - } g1; - struct { - u64 word[4]; - } u64; - }; -}; - -struct kvm_s390_gib { - u32 alert_list_origin; - u32 reserved01; - u8:5; - u8 nisc:3; - u8 reserved03[3]; - u32 reserved04[5]; -}; - -/* - * sie_page2 has to be allocated as DMA because fac_list, crycb and - * gisa need 31bit addresses in the sie control block. - */ -struct sie_page2 { - __u64 fac_list[S390_ARCH_FAC_LIST_SIZE_U64]; /* 0x0000 */ - struct kvm_s390_crypto_cb crycb; /* 0x0800 */ - struct kvm_s390_gisa gisa; /* 0x0900 */ - struct kvm *kvm; /* 0x0920 */ - u8 reserved928[0x1000 - 0x928]; /* 0x0928 */ -}; - -struct vsie_page; - -struct kvm_s390_vsie { - struct mutex mutex; - struct radix_tree_root addr_to_page; - int page_count; - int next; - struct vsie_page *pages[KVM_MAX_VCPUS]; -}; - -struct kvm_s390_gisa_iam { - u8 mask; - spinlock_t ref_lock; - u32 ref_count[MAX_ISC + 1]; -}; - -struct kvm_s390_gisa_interrupt { - struct kvm_s390_gisa *origin; - struct kvm_s390_gisa_iam alert; - struct hrtimer timer; - u64 expires; - DECLARE_BITMAP(kicked_mask, KVM_MAX_VCPUS); -}; - -struct kvm_s390_pv { - u64 handle; - u64 guest_len; - unsigned long stor_base; - void *stor_var; - bool dumping; - void *set_aside; - struct list_head need_cleanup; - struct mmu_notifier mmu_notifier; - /* Protects against concurrent import-like operations */ - struct mutex import_lock; -}; - -struct kvm_s390_mmu_cache; - -struct kvm_arch { - struct esca_block *sca; - debug_info_t *dbf; - struct kvm_s390_float_interrupt float_int; - struct kvm_device *flic; - struct gmap *gmap; - unsigned long mem_limit; - int css_support; - int use_irqchip; - int use_cmma; - int use_pfmfi; - int use_skf; - int use_zpci_interp; - int user_cpu_state_ctrl; - int user_sigp; - int user_stsi; - int user_instr0; - int user_operexec; - int allow_vsie_esamode; - struct s390_io_adapter *adapters[MAX_S390_IO_ADAPTERS]; - wait_queue_head_t ipte_wq; - int ipte_lock_count; - struct mutex ipte_mutex; - spinlock_t start_stop_lock; - struct sie_page2 *sie_page2; - struct kvm_s390_cpu_model model; - struct kvm_s390_crypto crypto; - struct kvm_s390_vsie vsie; - u8 epdx; - u64 epoch; - int migration_mode; - atomic64_t cmma_dirty_pages; - /* subset of available cpu features enabled by user space */ - DECLARE_BITMAP(cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS); - /* indexed by vcpu_idx */ - DECLARE_BITMAP(idle_mask, KVM_MAX_VCPUS); - struct kvm_s390_gisa_interrupt gisa_int; - struct kvm_s390_pv pv; - struct list_head kzdev_list; - spinlock_t kzdev_list_lock; - struct kvm_s390_mmu_cache *mc; -}; - -#define KVM_HVA_ERR_BAD (-1UL) -#define KVM_HVA_ERR_RO_BAD (-2UL) - -static inline bool kvm_is_error_hva(unsigned long addr) -{ - return IS_ERR_VALUE(addr); -} - -#define ASYNC_PF_PER_VCPU 64 -struct kvm_arch_async_pf { - unsigned long pfault_token; -}; - -bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu); - -void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, - struct kvm_async_pf *work); - -bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, - struct kvm_async_pf *work); - -void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, - struct kvm_async_pf *work); - -static inline void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu) {} - -void kvm_arch_crypto_clear_masks(struct kvm *kvm); -void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm, - unsigned long *aqm, unsigned long *adm); - -#define SIE64_RETURN_NORMAL 0 -#define SIE64_RETURN_MCCK 1 - -int __sie64a(phys_addr_t sie_block_phys, struct kvm_s390_sie_block *sie_block, u64 *rsa, - unsigned long gasce); - -static inline int sie64a(struct kvm_s390_sie_block *sie_block, u64 *rsa, unsigned long gasce) -{ - return __sie64a(virt_to_phys(sie_block), sie_block, rsa, gasce); -} - -extern char sie_exit; - -bool kvm_s390_pv_is_protected(struct kvm *kvm); -bool kvm_s390_pv_cpu_is_protected(struct kvm_vcpu *vcpu); - -extern int kvm_s390_enter_exit_sie(struct kvm_s390_sie_block *scb, - u64 *gprs, unsigned long gasce); - -extern int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc); -extern int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc); - -bool kvm_s390_is_gpa_in_memslot(struct kvm *kvm, gpa_t gpa); - -static inline void kvm_arch_free_memslot(struct kvm *kvm, - struct kvm_memory_slot *slot) {} -static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} -static inline void kvm_arch_flush_shadow_all(struct kvm *kvm) {} -static inline void kvm_arch_flush_shadow_memslot(struct kvm *kvm, - struct kvm_memory_slot *slot) {} -static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} -static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} - -#define __KVM_HAVE_ARCH_VM_FREE -void kvm_arch_free_vm(struct kvm *kvm); - -struct zpci_kvm_hook { - int (*kvm_register)(void *opaque, struct kvm *kvm); - void (*kvm_unregister)(void *opaque); -}; - -extern struct zpci_kvm_hook zpci_kvm_hook; - -#endif +#endif /* ASM_KVM_HOST_H */ diff --git a/arch/s390/include/asm/kvm_host_s390.h b/arch/s390/include/asm/kvm_host_s390.h new file mode 100644 index 000000000000..cd692f8fb764 --- /dev/null +++ b/arch/s390/include/asm/kvm_host_s390.h @@ -0,0 +1,729 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * definition for kernel virtual machines on s390 + * + * Copyright IBM Corp. 2008, 2018 + * + * Author(s): Carsten Otte <cotte@de.ibm.com> + */ + + +#ifndef ASM_KVM_HOST_S390_H +#define ASM_KVM_HOST_S390_H + +#include <linux/types.h> +#include <linux/hrtimer.h> +#include <linux/interrupt.h> +#include <linux/kvm_types.h> +#include <linux/kvm.h> +#include <linux/seqlock.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/mmu_notifier.h> +#include <asm/kvm_host_types.h> +#include <asm/debug.h> +#include <asm/cpu.h> +#include <asm/fpu.h> +#include <asm/isc.h> +#include <asm/guarded_storage.h> + +#define KVM_HAVE_MMU_RWLOCK +#define KVM_MAX_VCPUS 255 + +#define KVM_INTERNAL_MEM_SLOTS 1 + +#define KVM_S390_MANAGES_S390_GUEST 1 + +/* + * These seem to be used for allocating ->chip in the routing table, which we + * don't use. 1 is as small as we can get to reduce the needed memory. If we + * need to look at ->chip later on, we'll need to revisit this. + */ +#define KVM_NR_IRQCHIPS 1 +#define KVM_IRQCHIP_NUM_PINS 1 +#define KVM_HALT_POLL_NS_DEFAULT 50000 + +/* s390-specific vcpu->requests bit members */ +#define KVM_REQ_ENABLE_IBS KVM_ARCH_REQ(0) +#define KVM_REQ_DISABLE_IBS KVM_ARCH_REQ(1) +#define KVM_REQ_ICPT_OPEREXC KVM_ARCH_REQ(2) +#define KVM_REQ_START_MIGRATION KVM_ARCH_REQ(3) +#define KVM_REQ_STOP_MIGRATION KVM_ARCH_REQ(4) +#define KVM_REQ_VSIE_RESTART KVM_ARCH_REQ(5) +#define KVM_REQ_REFRESH_GUEST_PREFIX \ + KVM_ARCH_REQ_FLAGS(6, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) + +struct kvm_vcpu_stat { + struct kvm_vcpu_stat_generic generic; + u64 exit_userspace; + u64 exit_null; + u64 exit_external_request; + u64 exit_io_request; + u64 exit_external_interrupt; + u64 exit_stop_request; + u64 exit_validity; + u64 exit_instruction; + u64 exit_pei; + u64 halt_no_poll_steal; + u64 instruction_lctl; + u64 instruction_lctlg; + u64 instruction_stctl; + u64 instruction_stctg; + u64 exit_program_interruption; + u64 exit_instr_and_program; + u64 exit_operation_exception; + u64 deliver_ckc; + u64 deliver_cputm; + u64 deliver_external_call; + u64 deliver_emergency_signal; + u64 deliver_service_signal; + u64 deliver_virtio; + u64 deliver_stop_signal; + u64 deliver_prefix_signal; + u64 deliver_restart_signal; + u64 deliver_program; + u64 deliver_io; + u64 deliver_machine_check; + u64 exit_wait_state; + u64 inject_ckc; + u64 inject_cputm; + u64 inject_external_call; + u64 inject_emergency_signal; + u64 inject_mchk; + u64 inject_pfault_init; + u64 inject_program; + u64 inject_restart; + u64 inject_set_prefix; + u64 inject_stop_signal; + u64 instruction_epsw; + u64 instruction_gs; + u64 instruction_io_other; + u64 instruction_lpsw; + u64 instruction_lpswe; + u64 instruction_lpswey; + u64 instruction_pfmf; + u64 instruction_ptff; + u64 instruction_sck; + u64 instruction_sckpf; + u64 instruction_stidp; + u64 instruction_spx; + u64 instruction_stpx; + u64 instruction_stap; + u64 instruction_iske; + u64 instruction_ri; + u64 instruction_rrbe; + u64 instruction_sske; + u64 instruction_ipte_interlock; + u64 instruction_stsi; + u64 instruction_stfl; + u64 instruction_tb; + u64 instruction_tpi; + u64 instruction_tprot; + u64 instruction_tsch; + u64 instruction_sie; + u64 instruction_essa; + u64 instruction_sthyi; + u64 instruction_sigp_sense; + u64 instruction_sigp_sense_running; + u64 instruction_sigp_external_call; + u64 instruction_sigp_emergency; + u64 instruction_sigp_cond_emergency; + u64 instruction_sigp_start; + u64 instruction_sigp_stop; + u64 instruction_sigp_stop_store_status; + u64 instruction_sigp_store_status; + u64 instruction_sigp_store_adtl_status; + u64 instruction_sigp_arch; + u64 instruction_sigp_prefix; + u64 instruction_sigp_restart; + u64 instruction_sigp_init_cpu_reset; + u64 instruction_sigp_cpu_reset; + u64 instruction_sigp_unknown; + u64 instruction_diagnose_10; + u64 instruction_diagnose_44; + u64 instruction_diagnose_9c; + u64 diag_9c_ignored; + u64 diag_9c_forward; + u64 instruction_diagnose_258; + u64 instruction_diagnose_308; + u64 instruction_diagnose_500; + u64 instruction_diagnose_other; + u64 pfault_sync; + u64 signal_exits; +}; + +/* irq types in ascend order of priorities */ +enum irq_types { + IRQ_PEND_SET_PREFIX = 0, + IRQ_PEND_RESTART, + IRQ_PEND_SIGP_STOP, + IRQ_PEND_IO_ISC_7, + IRQ_PEND_IO_ISC_6, + IRQ_PEND_IO_ISC_5, + IRQ_PEND_IO_ISC_4, + IRQ_PEND_IO_ISC_3, + IRQ_PEND_IO_ISC_2, + IRQ_PEND_IO_ISC_1, + IRQ_PEND_IO_ISC_0, + IRQ_PEND_VIRTIO, + IRQ_PEND_PFAULT_DONE, + IRQ_PEND_PFAULT_INIT, + IRQ_PEND_EXT_HOST, + IRQ_PEND_EXT_SERVICE, + IRQ_PEND_EXT_SERVICE_EV, + IRQ_PEND_EXT_TIMING, + IRQ_PEND_EXT_CPU_TIMER, + IRQ_PEND_EXT_CLOCK_COMP, + IRQ_PEND_EXT_EXTERNAL, + IRQ_PEND_EXT_EMERGENCY, + IRQ_PEND_EXT_MALFUNC, + IRQ_PEND_EXT_IRQ_KEY, + IRQ_PEND_MCHK_REP, + IRQ_PEND_PROG, + IRQ_PEND_SVC, + IRQ_PEND_MCHK_EX, + IRQ_PEND_COUNT +}; + +/* We have 2M for virtio device descriptor pages. Smallest amount of + * memory per page is 24 bytes (1 queue), so (2048*1024) / 24 = 87381 + */ +#define KVM_S390_MAX_VIRTIO_IRQS 87381 + +/* + * Repressible (non-floating) machine check interrupts + * subclass bits in MCIC + */ +#define MCHK_EXTD_BIT 58 +#define MCHK_DEGR_BIT 56 +#define MCHK_WARN_BIT 55 +#define MCHK_REP_MASK ((1UL << MCHK_DEGR_BIT) | \ + (1UL << MCHK_EXTD_BIT) | \ + (1UL << MCHK_WARN_BIT)) + +/* Exigent machine check interrupts subclass bits in MCIC */ +#define MCHK_SD_BIT 63 +#define MCHK_PD_BIT 62 +#define MCHK_EX_MASK ((1UL << MCHK_SD_BIT) | (1UL << MCHK_PD_BIT)) + +#define IRQ_PEND_EXT_MASK ((1UL << IRQ_PEND_EXT_IRQ_KEY) | \ + (1UL << IRQ_PEND_EXT_CLOCK_COMP) | \ + (1UL << IRQ_PEND_EXT_CPU_TIMER) | \ + (1UL << IRQ_PEND_EXT_MALFUNC) | \ + (1UL << IRQ_PEND_EXT_EMERGENCY) | \ + (1UL << IRQ_PEND_EXT_EXTERNAL) | \ + (1UL << IRQ_PEND_EXT_TIMING) | \ + (1UL << IRQ_PEND_EXT_HOST) | \ + (1UL << IRQ_PEND_EXT_SERVICE) | \ + (1UL << IRQ_PEND_EXT_SERVICE_EV) | \ + (1UL << IRQ_PEND_VIRTIO) | \ + (1UL << IRQ_PEND_PFAULT_INIT) | \ + (1UL << IRQ_PEND_PFAULT_DONE)) + +#define IRQ_PEND_IO_MASK ((1UL << IRQ_PEND_IO_ISC_0) | \ + (1UL << IRQ_PEND_IO_ISC_1) | \ + (1UL << IRQ_PEND_IO_ISC_2) | \ + (1UL << IRQ_PEND_IO_ISC_3) | \ + (1UL << IRQ_PEND_IO_ISC_4) | \ + (1UL << IRQ_PEND_IO_ISC_5) | \ + (1UL << IRQ_PEND_IO_ISC_6) | \ + (1UL << IRQ_PEND_IO_ISC_7)) + +#define IRQ_PEND_MCHK_MASK ((1UL << IRQ_PEND_MCHK_REP) | \ + (1UL << IRQ_PEND_MCHK_EX)) + +#define IRQ_PEND_EXT_II_MASK ((1UL << IRQ_PEND_EXT_CPU_TIMER) | \ + (1UL << IRQ_PEND_EXT_CLOCK_COMP) | \ + (1UL << IRQ_PEND_EXT_EMERGENCY) | \ + (1UL << IRQ_PEND_EXT_EXTERNAL) | \ + (1UL << IRQ_PEND_EXT_SERVICE) | \ + (1UL << IRQ_PEND_EXT_SERVICE_EV)) + +struct kvm_s390_interrupt_info { + struct list_head list; + u64 type; + union { + struct kvm_s390_io_info io; + struct kvm_s390_ext_info ext; + struct kvm_s390_pgm_info pgm; + struct kvm_s390_emerg_info emerg; + struct kvm_s390_extcall_info extcall; + struct kvm_s390_prefix_info prefix; + struct kvm_s390_stop_info stop; + struct kvm_s390_mchk_info mchk; + }; +}; + +struct kvm_s390_irq_payload { + struct kvm_s390_io_info io; + struct kvm_s390_ext_info ext; + struct kvm_s390_pgm_info pgm; + struct kvm_s390_emerg_info emerg; + struct kvm_s390_extcall_info extcall; + struct kvm_s390_prefix_info prefix; + struct kvm_s390_stop_info stop; + struct kvm_s390_mchk_info mchk; +}; + +struct kvm_s390_local_interrupt { + spinlock_t lock; + DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS); + struct kvm_s390_irq_payload irq; + unsigned long pending_irqs; +}; + +#define FIRQ_LIST_IO_ISC_0 0 +#define FIRQ_LIST_IO_ISC_1 1 +#define FIRQ_LIST_IO_ISC_2 2 +#define FIRQ_LIST_IO_ISC_3 3 +#define FIRQ_LIST_IO_ISC_4 4 +#define FIRQ_LIST_IO_ISC_5 5 +#define FIRQ_LIST_IO_ISC_6 6 +#define FIRQ_LIST_IO_ISC_7 7 +#define FIRQ_LIST_PFAULT 8 +#define FIRQ_LIST_VIRTIO 9 +#define FIRQ_LIST_COUNT 10 +#define FIRQ_CNTR_IO 0 +#define FIRQ_CNTR_SERVICE 1 +#define FIRQ_CNTR_VIRTIO 2 +#define FIRQ_CNTR_PFAULT 3 +#define FIRQ_MAX_COUNT 4 + +/* mask the AIS mode for a given ISC */ +#define AIS_MODE_MASK(isc) (0x80 >> isc) + +#define KVM_S390_AIS_MODE_ALL 0 +#define KVM_S390_AIS_MODE_SINGLE 1 + +struct kvm_s390_float_interrupt { + unsigned long pending_irqs; + unsigned long masked_irqs; + spinlock_t lock; + struct list_head lists[FIRQ_LIST_COUNT]; + int counters[FIRQ_MAX_COUNT]; + struct kvm_s390_mchk_info mchk; + struct kvm_s390_ext_info srv_signal; + int last_sleep_cpu; + spinlock_t ais_lock; + u8 simm; + u8 nimm; +}; + +struct kvm_hw_wp_info_arch { + unsigned long addr; + unsigned long phys_addr; + int len; + char *old_data; +}; + +struct kvm_hw_bp_info_arch { + unsigned long addr; + int len; +}; + +/* + * Only the upper 16 bits of kvm_guest_debug->control are arch specific. + * Further KVM_GUESTDBG flags which an be used from userspace can be found in + * arch/s390/include/uapi/asm/kvm.h + */ +#define KVM_GUESTDBG_EXIT_PENDING 0x10000000 + +#define guestdbg_enabled(vcpu) \ + (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) +#define guestdbg_sstep_enabled(vcpu) \ + (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) +#define guestdbg_hw_bp_enabled(vcpu) \ + (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) +#define guestdbg_exit_pending(vcpu) (guestdbg_enabled(vcpu) && \ + (vcpu->guest_debug & KVM_GUESTDBG_EXIT_PENDING)) + +#define KVM_GUESTDBG_VALID_MASK \ + (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP |\ + KVM_GUESTDBG_USE_HW_BP | KVM_GUESTDBG_EXIT_PENDING) + +struct kvm_guestdbg_info_arch { + unsigned long cr0; + unsigned long cr9; + unsigned long cr10; + unsigned long cr11; + struct kvm_hw_bp_info_arch *hw_bp_info; + struct kvm_hw_wp_info_arch *hw_wp_info; + int nr_hw_bp; + int nr_hw_wp; + unsigned long last_bp; +}; + +struct kvm_s390_pv_vcpu { + u64 handle; + unsigned long stor_base; +}; + +struct kvm_vcpu_arch { + struct kvm_s390_sie_block *sie_block; + /* if vsie is active, currently executed shadow sie control block */ + struct kvm_s390_sie_block *vsie_block; + unsigned int host_acrs[NUM_ACRS]; + struct gs_cb *host_gscb; + struct kvm_s390_local_interrupt local_int; + struct hrtimer ckc_timer; + struct kvm_s390_pgm_info pgm; + struct gmap *gmap; + struct kvm_guestdbg_info_arch guestdbg; + unsigned long pfault_token; + unsigned long pfault_select; + unsigned long pfault_compare; + bool cputm_enabled; + /* + * The seqcount protects updates to cputm_start and sie_block.cputm, + * this way we can have non-blocking reads with consistent values. + * Only the owning VCPU thread (vcpu->cpu) is allowed to change these + * values and to start/stop/enable/disable cpu timer accounting. + */ + seqcount_t cputm_seqcount; + __u64 cputm_start; + bool gs_enabled; + bool skey_enabled; + /* Indicator if the access registers have been loaded from guest */ + bool acrs_loaded; + bool initialized; + struct kvm_s390_pv_vcpu pv; + union diag318_info diag318_info; + struct kvm_s390_mmu_cache *mc; +}; + +struct kvm_vm_stat { + struct kvm_vm_stat_generic generic; + u64 inject_io; + u64 io_390_adapter_map; + u64 io_390_adapter_unmap; + u64 io_390_inatomic; + u64 io_flic_inject_airq; + u64 io_set_adapter_int; + u64 io_390_inatomic_no_inject; + u64 inject_float_mchk; + u64 inject_pfault_done; + u64 inject_service_signal; + u64 inject_virtio; + u64 aen_forward; + u64 gmap_shadow_create; + u64 gmap_shadow_reuse; + u64 gmap_shadow_r1_entry; + u64 gmap_shadow_r2_entry; + u64 gmap_shadow_r3_entry; + u64 gmap_shadow_sg_entry; + u64 gmap_shadow_pg_entry; +}; + +struct kvm_arch_memory_slot { +}; + +struct s390_map_info { + struct list_head list; + __u64 guest_addr; + __u64 addr; + struct page *page; + /* + * True if the page is long-term pinned. False if long-term pinning + * failed and this entry exists only to preserve MAP/UNMAP symmetry. + */ + bool pinned; +}; + +struct s390_io_adapter { + unsigned int id; + int isc; + bool maskable; + bool masked; + bool swap; + bool suppressible; + spinlock_t maps_lock; + struct list_head maps; + unsigned int nr_maps; +}; + +#define MAX_S390_IO_ADAPTERS ((MAX_ISC + 1) * 8) +#define MAX_S390_ADAPTER_MAPS 256 + +/* maximum size of facilities and facility mask is 2k bytes */ +#define S390_ARCH_FAC_LIST_SIZE_BYTE (1<<11) +#define S390_ARCH_FAC_LIST_SIZE_U64 \ + (S390_ARCH_FAC_LIST_SIZE_BYTE / sizeof(u64)) +#define S390_ARCH_FAC_MASK_SIZE_BYTE S390_ARCH_FAC_LIST_SIZE_BYTE +#define S390_ARCH_FAC_MASK_SIZE_U64 \ + (S390_ARCH_FAC_MASK_SIZE_BYTE / sizeof(u64)) + +struct kvm_s390_cpu_model { + /* facility mask supported by kvm & hosting machine */ + __u64 fac_mask[S390_ARCH_FAC_MASK_SIZE_U64]; + struct kvm_s390_vm_cpu_subfunc subfuncs; + /* facility list requested by guest (in dma page) */ + __u64 *fac_list; + u64 cpuid; + unsigned short ibc; + /* subset of available UV-features for pv-guests enabled by user space */ + struct kvm_s390_vm_cpu_uv_feat uv_feat_guest; +}; + +#define S390_ARCH_FAC_FORMAT_2 2 +struct kvm_s390_flcb2 { + union { + struct { + u8 reserved0[7]; + u8 length; + }; + u64 header_val; + }; + u64 facilities[S390_ARCH_FAC_LIST_SIZE_U64]; +}; + +typedef int (*crypto_hook)(struct kvm_vcpu *vcpu); + +struct kvm_s390_crypto { + struct kvm_s390_crypto_cb *crycb; + struct rw_semaphore pqap_hook_rwsem; + crypto_hook *pqap_hook; + __u32 crycbd; + __u8 aes_kw; + __u8 dea_kw; + __u8 apie; +}; + +#define APCB0_MASK_SIZE 1 +struct kvm_s390_apcb0 { + __u64 apm[APCB0_MASK_SIZE]; /* 0x0000 */ + __u64 aqm[APCB0_MASK_SIZE]; /* 0x0008 */ + __u64 adm[APCB0_MASK_SIZE]; /* 0x0010 */ + __u64 reserved18; /* 0x0018 */ +}; + +#define APCB1_MASK_SIZE 4 +struct kvm_s390_apcb1 { + __u64 apm[APCB1_MASK_SIZE]; /* 0x0000 */ + __u64 aqm[APCB1_MASK_SIZE]; /* 0x0020 */ + __u64 adm[APCB1_MASK_SIZE]; /* 0x0040 */ + __u64 reserved60[4]; /* 0x0060 */ +}; + +struct kvm_s390_crypto_cb { + struct kvm_s390_apcb0 apcb0; /* 0x0000 */ + __u8 reserved20[0x0048 - 0x0020]; /* 0x0020 */ + __u8 dea_wrapping_key_mask[24]; /* 0x0048 */ + __u8 aes_wrapping_key_mask[32]; /* 0x0060 */ + struct kvm_s390_apcb1 apcb1; /* 0x0080 */ +}; + +struct kvm_s390_gisa { + union { + struct { /* common to all formats */ + u32 next_alert; + u8 ipm; + u8 reserved01[2]; + u8 iam; + }; + struct { /* format 0 */ + u32 next_alert; + u8 ipm; + u8 reserved01; + u8 : 6; + u8 g : 1; + u8 c : 1; + u8 iam; + u8 reserved02[4]; + u32 airq_count; + } g0; + struct { /* format 1 */ + u32 next_alert; + u8 ipm; + u8 simm; + u8 nimm; + u8 iam; + u8 aism[8]; + u8 : 6; + u8 g : 1; + u8 c : 1; + u8 reserved03[11]; + u32 airq_count; + } g1; + struct { + u64 word[4]; + } u64; + }; +}; + +struct kvm_s390_gib { + u32 alert_list_origin; + u32 reserved01; + u8:5; + u8 nisc:3; + u8 reserved03[3]; + u32 reserved04[5]; +}; + +/* + * sie_page2 has to be allocated as DMA because fac_list, crycb and + * gisa need 31bit addresses in the sie control block. + */ +struct sie_page2 { + __u64 fac_list[S390_ARCH_FAC_LIST_SIZE_U64]; /* 0x0000 */ + struct kvm_s390_crypto_cb crycb; /* 0x0800 */ + struct kvm_s390_gisa gisa; /* 0x0900 */ + struct kvm *kvm; /* 0x0920 */ + u8 reserved928[0x1000 - 0x928]; /* 0x0928 */ +}; + +struct vsie_page; + +struct kvm_s390_vsie { + struct mutex mutex; + struct radix_tree_root addr_to_page; + int page_count; + int next; + struct vsie_page *pages[KVM_MAX_VCPUS]; +}; + +struct kvm_s390_gisa_iam { + u8 mask; + spinlock_t ref_lock; + u32 ref_count[MAX_ISC + 1]; +}; + +struct kvm_s390_gisa_interrupt { + struct kvm_s390_gisa *origin; + struct kvm_s390_gisa_iam alert; + struct hrtimer timer; + u64 expires; + DECLARE_BITMAP(kicked_mask, KVM_MAX_VCPUS); +}; + +struct kvm_s390_pv { + u64 handle; + u64 guest_len; + unsigned long stor_base; + void *stor_var; + bool dumping; + void *set_aside; + struct list_head need_cleanup; + struct mmu_notifier mmu_notifier; + /* Protects against concurrent import-like operations */ + struct mutex import_lock; +}; + +struct kvm_s390_mmu_cache; + +struct kvm_arch { + struct esca_block *sca; + debug_info_t *dbf; + struct kvm_s390_float_interrupt float_int; + struct kvm_device *flic; + struct gmap *gmap; + unsigned long mem_limit; + int css_support; + int use_irqchip; + int use_cmma; + int use_pfmfi; + int use_skf; + int use_zpci_interp; + int user_cpu_state_ctrl; + int user_sigp; + int user_stsi; + int user_instr0; + int user_operexec; + int allow_vsie_esamode; + struct s390_io_adapter *adapters[MAX_S390_IO_ADAPTERS]; + wait_queue_head_t ipte_wq; + int ipte_lock_count; + struct mutex ipte_mutex; + spinlock_t start_stop_lock; + struct sie_page2 *sie_page2; + struct kvm_s390_cpu_model model; + struct kvm_s390_crypto crypto; + struct kvm_s390_vsie vsie; + u8 epdx; + u64 epoch; + int migration_mode; + atomic64_t cmma_dirty_pages; + /* subset of available cpu features enabled by user space */ + DECLARE_BITMAP(cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS); + /* indexed by vcpu_idx */ + DECLARE_BITMAP(idle_mask, KVM_MAX_VCPUS); + struct kvm_s390_gisa_interrupt gisa_int; + struct kvm_s390_pv pv; + struct list_head kzdev_list; + spinlock_t kzdev_list_lock; + struct kvm_s390_mmu_cache *mc; +}; + +#define KVM_HVA_ERR_BAD (-1UL) +#define KVM_HVA_ERR_RO_BAD (-2UL) + +static inline bool kvm_is_error_hva(unsigned long addr) +{ + return IS_ERR_VALUE(addr); +} + +#define ASYNC_PF_PER_VCPU 64 +struct kvm_arch_async_pf { + unsigned long pfault_token; +}; + +bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu); + +void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work); + +bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work); + +void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work); + +static inline void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu) {} + +void kvm_arch_crypto_clear_masks(struct kvm *kvm); +void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm, + unsigned long *aqm, unsigned long *adm); + +#define SIE64_RETURN_NORMAL 0 +#define SIE64_RETURN_MCCK 1 + +int __sie64a(phys_addr_t sie_block_phys, struct kvm_s390_sie_block *sie_block, u64 *rsa, + unsigned long gasce); + +static inline int sie64a(struct kvm_s390_sie_block *sie_block, u64 *rsa, unsigned long gasce) +{ + return __sie64a(virt_to_phys(sie_block), sie_block, rsa, gasce); +} + +extern char sie_exit; + +bool kvm_s390_pv_is_protected(struct kvm *kvm); +bool kvm_s390_pv_cpu_is_protected(struct kvm_vcpu *vcpu); + +extern int kvm_s390_enter_exit_sie(struct kvm_s390_sie_block *scb, + u64 *gprs, unsigned long gasce); + +extern int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc); +extern int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc); + +bool kvm_s390_is_gpa_in_memslot(struct kvm *kvm, gpa_t gpa); + +static inline void kvm_arch_free_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot) {} +static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} +static inline void kvm_arch_flush_shadow_all(struct kvm *kvm) {} +static inline void kvm_arch_flush_shadow_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot) {} +static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} +static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} + +#define __KVM_HAVE_ARCH_VM_FREE +void kvm_arch_free_vm(struct kvm *kvm); + +struct zpci_kvm_hook { + int (*kvm_register)(void *opaque, struct kvm *kvm); + void (*kvm_unregister)(void *opaque); +}; + +extern struct zpci_kvm_hook zpci_kvm_hook; + +#endif /* ASM_KVM_HOST_S390_H */ diff --git a/arch/s390/include/asm/kvm_host_s390_types.h b/arch/s390/include/asm/kvm_host_s390_types.h new file mode 100644 index 000000000000..9e348a530421 --- /dev/null +++ b/arch/s390/include/asm/kvm_host_s390_types.h @@ -0,0 +1,349 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_KVM_HOST_S390_TYPES_H +#define _ASM_KVM_HOST_S390_TYPES_H + +#include <linux/atomic.h> +#include <linux/types.h> + +#define KVM_S390_BSCA_CPU_SLOTS 64 +#define KVM_S390_ESCA_CPU_SLOTS 248 + +#define SCB_ALIGNMENT_SHIFT 9 + +#define SIGP_CTRL_C 0x80 +#define SIGP_CTRL_SCN_MASK 0x3f + +union bsca_sigp_ctrl { + __u8 value; + struct { + __u8 c : 1; + __u8 r : 1; + __u8 scn : 6; + }; +}; + +union esca_sigp_ctrl { + __u16 value; + struct { + __u8 c : 1; + __u8 reserved: 7; + __u8 scn; + }; +}; + +struct esca_entry { + union esca_sigp_ctrl sigp_ctrl; + __u16 reserved1[3]; + __u64 sda; + __u64 reserved2[6]; +}; + +struct bsca_entry { + __u8 reserved0; + union bsca_sigp_ctrl sigp_ctrl; + __u16 reserved[3]; + __u64 sda; + __u64 reserved2[2]; +}; + +union ipte_control { + unsigned long val; + struct { + unsigned long k : 1; + unsigned long kh : 31; + unsigned long kg : 32; + }; +}; + +/* + * Utility is defined as two bytes but having it four bytes wide + * generates more efficient code. Since the following bytes are + * reserved this makes no functional difference. + */ +union sca_utility { + __u32 val; + struct { + __u32 mtcr : 1; + __u32 : 31; + }; +}; + +struct bsca_block { + union ipte_control ipte_control; + __u64 reserved[5]; + __u64 mcn; + union sca_utility utility; + __u8 reserved2[4]; + struct bsca_entry cpu[KVM_S390_BSCA_CPU_SLOTS]; +}; + +struct esca_block { + union ipte_control ipte_control; + __u64 reserved1[6]; + union sca_utility utility; + __u8 reserved2[4]; + __u64 mcn[4]; + __u64 reserved3[20]; + struct esca_entry cpu[KVM_S390_ESCA_CPU_SLOTS]; +}; + +/* + * This struct is used to store some machine check info from lowcore + * for machine checks that happen while the guest is running. + * This info in host's lowcore might be overwritten by a second machine + * check from host when host is in the machine check's high-level handling. + * The size is 24 bytes. + */ +struct mcck_volatile_info { + __u64 mcic; + __u64 failing_storage_address; + __u32 ext_damage_code; + __u32 reserved; +}; + +#define CR0_INITIAL_MASK (CR0_UNUSED_56 | CR0_INTERRUPT_KEY_SUBMASK | \ + CR0_MEASUREMENT_ALERT_SUBMASK) +#define CR14_INITIAL_MASK (CR14_UNUSED_32 | CR14_UNUSED_33 | \ + CR14_EXTERNAL_DAMAGE_SUBMASK) + +#define SIDAD_SIZE_MASK 0xff +#define sida_addr(sie_block) phys_to_virt((sie_block)->sidad & PAGE_MASK) +#define sida_size(sie_block) \ + ((((sie_block)->sidad & SIDAD_SIZE_MASK) + 1) * PAGE_SIZE) + +#define CPUSTAT_STOPPED 0x80000000 +#define CPUSTAT_WAIT 0x10000000 +#define CPUSTAT_ECALL_PEND 0x08000000 +#define CPUSTAT_STOP_INT 0x04000000 +#define CPUSTAT_IO_INT 0x02000000 +#define CPUSTAT_EXT_INT 0x01000000 +#define CPUSTAT_RUNNING 0x00800000 +#define CPUSTAT_RETAINED 0x00400000 +#define CPUSTAT_TIMING_SUB 0x00020000 +#define CPUSTAT_SIE_SUB 0x00010000 +#define CPUSTAT_RRF 0x00008000 +#define CPUSTAT_SLSV 0x00004000 +#define CPUSTAT_SLSR 0x00002000 +#define CPUSTAT_ZARCH 0x00000800 +#define CPUSTAT_MCDS 0x00000100 +#define CPUSTAT_KSS 0x00000200 +#define CPUSTAT_SM 0x00000080 +#define CPUSTAT_IBS 0x00000040 +#define CPUSTAT_GED2 0x00000010 +#define CPUSTAT_G 0x00000008 +#define CPUSTAT_GED 0x00000004 +#define CPUSTAT_J 0x00000002 +#define CPUSTAT_P 0x00000001 + +struct kvm_s390_sie_block { + atomic_t cpuflags; /* 0x0000 */ + __u32 : 1; /* 0x0004 */ + __u32 prefix : 19; + __u32 ibc : 12; + __u8 reserved08[4]; /* 0x0008 */ +#define PROG_IN_SIE (1<<0) + __u32 prog0c; /* 0x000c */ + union { + __u8 reserved10[16]; /* 0x0010 */ + struct { + __u64 pv_handle_cpu; + __u64 pv_handle_config; + }; + }; +#define PROG_BLOCK_SIE (1<<0) +#define PROG_REQUEST (1<<1) + atomic_t prog20; /* 0x0020 */ + __u8 reserved24[4]; /* 0x0024 */ + __u64 cputm; /* 0x0028 */ + __u64 ckc; /* 0x0030 */ + __u64 epoch; /* 0x0038 */ + __u32 svcc; /* 0x0040 */ +#define LCTL_CR0 0x8000 +#define LCTL_CR6 0x0200 +#define LCTL_CR9 0x0040 +#define LCTL_CR10 0x0020 +#define LCTL_CR11 0x0010 +#define LCTL_CR14 0x0002 + __u16 lctl; /* 0x0044 */ + __s16 icpua; /* 0x0046 */ +#define ICTL_OPEREXC 0x80000000 +#define ICTL_PINT 0x20000000 +#define ICTL_LPSW 0x00400000 +#define ICTL_STCTL 0x00040000 +#define ICTL_ISKE 0x00004000 +#define ICTL_SSKE 0x00002000 +#define ICTL_RRBE 0x00001000 +#define ICTL_TPROT 0x00000200 + __u32 ictl; /* 0x0048 */ +#define ECA_CEI 0x80000000 +#define ECA_IB 0x40000000 +#define ECA_SIGPI 0x10000000 +#define ECA_MVPGI 0x01000000 +#define ECA_AIV 0x00200000 +#define ECA_VX 0x00020000 +#define ECA_PROTEXCI 0x00002000 +#define ECA_APIE 0x00000008 +#define ECA_SII 0x00000001 + __u32 eca; /* 0x004c */ +#define ICPT_INST 0x04 +#define ICPT_PROGI 0x08 +#define ICPT_INSTPROGI 0x0C +#define ICPT_EXTREQ 0x10 +#define ICPT_EXTINT 0x14 +#define ICPT_IOREQ 0x18 +#define ICPT_WAIT 0x1c +#define ICPT_VALIDITY 0x20 +#define ICPT_STOP 0x28 +#define ICPT_OPEREXC 0x2C +#define ICPT_PARTEXEC 0x38 +#define ICPT_IOINST 0x40 +#define ICPT_KSS 0x5c +#define ICPT_MCHKREQ 0x60 +#define ICPT_INT_ENABLE 0x64 +#define ICPT_PV_INSTR 0x68 +#define ICPT_PV_NOTIFY 0x6c +#define ICPT_PV_PREF 0x70 + __u8 icptcode; /* 0x0050 */ + __u8 icptstatus; /* 0x0051 */ + __u16 ihcpu; /* 0x0052 */ + __u8 reserved54; /* 0x0054 */ +#define IICTL_CODE_NONE 0x00 +#define IICTL_CODE_MCHK 0x01 +#define IICTL_CODE_EXT 0x02 +#define IICTL_CODE_IO 0x03 +#define IICTL_CODE_RESTART 0x04 +#define IICTL_CODE_SPECIFICATION 0x10 +#define IICTL_CODE_OPERAND 0x11 + __u8 iictl; /* 0x0055 */ + __u16 ipa; /* 0x0056 */ + __u32 ipb; /* 0x0058 */ + __u32 scaoh; /* 0x005c */ +#define FPF_BPBC 0x20 + __u8 fpf; /* 0x0060 */ +#define ECB_GS 0x40 +#define ECB_TE 0x10 +#define ECB_SPECI 0x08 +#define ECB_SRSI 0x04 +#define ECB_HOSTPROTINT 0x02 +#define ECB_PTF 0x01 + __u8 ecb; /* 0x0061 */ +#define ECB2_CMMA 0x80 +#define ECB2_IEP 0x20 +#define ECB2_PFMFI 0x08 +#define ECB2_ESCA 0x04 +#define ECB2_ZPCI_LSI 0x02 + __u8 ecb2; /* 0x0062 */ +#define ECB3_AISI 0x20 +#define ECB3_AISII 0x10 +#define ECB3_DEA 0x08 +#define ECB3_AES 0x04 +#define ECB3_RI 0x01 + __u8 ecb3; /* 0x0063 */ +#define ESCA_SCAOL_MASK ~0x3fU + __u32 scaol; /* 0x0064 */ + __u8 sdf; /* 0x0068 */ + __u8 epdx; /* 0x0069 */ + __u8 cpnc; /* 0x006a */ + __u8 reserved6b; /* 0x006b */ + __u32 todpr; /* 0x006c */ +#define GISA_FORMAT1 0x00000001 + __u32 gd; /* 0x0070 */ + __u8 reserved74[12]; /* 0x0074 */ + __u64 mso; /* 0x0080 */ + __u64 msl; /* 0x0088 */ + psw_t gpsw; /* 0x0090 */ + __u64 gg14; /* 0x00a0 */ + __u64 gg15; /* 0x00a8 */ + __u8 reservedb0[8]; /* 0x00b0 */ +#define HPID_KVM 0x4 +#define HPID_VSIE 0x5 + __u8 hpid; /* 0x00b8 */ + __u8 reservedb9[7]; /* 0x00b9 */ + union { + struct { + __u32 eiparams; /* 0x00c0 */ + __u16 extcpuaddr; /* 0x00c4 */ + __u16 eic; /* 0x00c6 */ + }; + __u64 mcic; /* 0x00c0 */ + } __packed; + __u32 reservedc8; /* 0x00c8 */ + union { + struct { + __u16 pgmilc; /* 0x00cc */ + __u16 iprcc; /* 0x00ce */ + }; + __u32 edc; /* 0x00cc */ + } __packed; + union { + struct { + __u32 dxc; /* 0x00d0 */ + __u16 mcn; /* 0x00d4 */ + __u8 perc; /* 0x00d6 */ + __u8 peratmid; /* 0x00d7 */ + }; + __u64 faddr; /* 0x00d0 */ + } __packed; + __u64 peraddr; /* 0x00d8 */ + __u8 eai; /* 0x00e0 */ + __u8 peraid; /* 0x00e1 */ + __u8 oai; /* 0x00e2 */ + __u8 armid; /* 0x00e3 */ + __u8 reservede4[4]; /* 0x00e4 */ + union { + __u64 tecmc; /* 0x00e8 */ + struct { + __u16 subchannel_id; /* 0x00e8 */ + __u16 subchannel_nr; /* 0x00ea */ + __u32 io_int_parm; /* 0x00ec */ + __u32 io_int_word; /* 0x00f0 */ + }; + } __packed; + __u8 reservedf4[8]; /* 0x00f4 */ +#define CRYCB_FORMAT_MASK 0x00000003 +#define CRYCB_FORMAT0 0x00000000 +#define CRYCB_FORMAT1 0x00000001 +#define CRYCB_FORMAT2 0x00000003 + __u32 crycbd; /* 0x00fc */ + __u64 gcr[16]; /* 0x0100 */ + union { + __u64 gbea; /* 0x0180 */ + __u64 sidad; + }; + __u8 reserved188[8]; /* 0x0188 */ + __u64 sdnxo; /* 0x0190 */ + __u8 reserved198[8]; /* 0x0198 */ + __u32 fac; /* 0x01a0 */ + __u8 reserved1a4[20]; /* 0x01a4 */ + __u64 cbrlo; /* 0x01b8 */ + __u8 reserved1c0[8]; /* 0x01c0 */ +#define ECD_HOSTREGMGMT 0x20000000 +#define ECD_MEF 0x08000000 +#define ECD_ETOKENF 0x02000000 +#define ECD_ECC 0x00200000 +#define ECD_HMAC 0x00004000 + __u32 ecd; /* 0x01c8 */ + __u8 reserved1cc[18]; /* 0x01cc */ + __u64 pp; /* 0x01de */ + __u8 reserved1e6[2]; /* 0x01e6 */ + __u64 itdba; /* 0x01e8 */ + __u64 riccbd; /* 0x01f0 */ + __u64 gvrd; /* 0x01f8 */ +} __packed __aligned(512); + +struct kvm_s390_itdb { + __u8 data[256]; +}; + +struct sie_page { + struct kvm_s390_sie_block sie_block; + struct mcck_volatile_info mcck_info; /* 0x0200 */ + __u8 reserved218[360]; /* 0x0218 */ + __u64 pv_grregs[16]; /* 0x0380 */ + __u8 reserved400[512]; /* 0x0400 */ + struct kvm_s390_itdb itdb; /* 0x0600 */ + __u8 reserved700[2304]; /* 0x0700 */ +}; + +#endif /* _ASM_KVM_HOST_S390_TYPES_H */ diff --git a/arch/s390/include/asm/kvm_host_types.h b/arch/s390/include/asm/kvm_host_types.h index 3f50942bdfe6..e5bdba07cab0 100644 --- a/arch/s390/include/asm/kvm_host_types.h +++ b/arch/s390/include/asm/kvm_host_types.h @@ -1,347 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_KVM_HOST_TYPES_H -#define _ASM_KVM_HOST_TYPES_H +#ifndef ASM_KVM_HOST_TYPES_H +#define ASM_KVM_HOST_TYPES_H -#include <linux/atomic.h> -#include <linux/types.h> +#include <asm/kvm_host_s390_types.h> -#define KVM_S390_BSCA_CPU_SLOTS 64 -#define KVM_S390_ESCA_CPU_SLOTS 248 - -#define SIGP_CTRL_C 0x80 -#define SIGP_CTRL_SCN_MASK 0x3f - -union bsca_sigp_ctrl { - __u8 value; - struct { - __u8 c : 1; - __u8 r : 1; - __u8 scn : 6; - }; -}; - -union esca_sigp_ctrl { - __u16 value; - struct { - __u8 c : 1; - __u8 reserved: 7; - __u8 scn; - }; -}; - -struct esca_entry { - union esca_sigp_ctrl sigp_ctrl; - __u16 reserved1[3]; - __u64 sda; - __u64 reserved2[6]; -}; - -struct bsca_entry { - __u8 reserved0; - union bsca_sigp_ctrl sigp_ctrl; - __u16 reserved[3]; - __u64 sda; - __u64 reserved2[2]; -}; - -union ipte_control { - unsigned long val; - struct { - unsigned long k : 1; - unsigned long kh : 31; - unsigned long kg : 32; - }; -}; - -/* - * Utility is defined as two bytes but having it four bytes wide - * generates more efficient code. Since the following bytes are - * reserved this makes no functional difference. - */ -union sca_utility { - __u32 val; - struct { - __u32 mtcr : 1; - __u32 : 31; - }; -}; - -struct bsca_block { - union ipte_control ipte_control; - __u64 reserved[5]; - __u64 mcn; - union sca_utility utility; - __u8 reserved2[4]; - struct bsca_entry cpu[KVM_S390_BSCA_CPU_SLOTS]; -}; - -struct esca_block { - union ipte_control ipte_control; - __u64 reserved1[6]; - union sca_utility utility; - __u8 reserved2[4]; - __u64 mcn[4]; - __u64 reserved3[20]; - struct esca_entry cpu[KVM_S390_ESCA_CPU_SLOTS]; -}; - -/* - * This struct is used to store some machine check info from lowcore - * for machine checks that happen while the guest is running. - * This info in host's lowcore might be overwritten by a second machine - * check from host when host is in the machine check's high-level handling. - * The size is 24 bytes. - */ -struct mcck_volatile_info { - __u64 mcic; - __u64 failing_storage_address; - __u32 ext_damage_code; - __u32 reserved; -}; - -#define CR0_INITIAL_MASK (CR0_UNUSED_56 | CR0_INTERRUPT_KEY_SUBMASK | \ - CR0_MEASUREMENT_ALERT_SUBMASK) -#define CR14_INITIAL_MASK (CR14_UNUSED_32 | CR14_UNUSED_33 | \ - CR14_EXTERNAL_DAMAGE_SUBMASK) - -#define SIDAD_SIZE_MASK 0xff -#define sida_addr(sie_block) phys_to_virt((sie_block)->sidad & PAGE_MASK) -#define sida_size(sie_block) \ - ((((sie_block)->sidad & SIDAD_SIZE_MASK) + 1) * PAGE_SIZE) - -#define CPUSTAT_STOPPED 0x80000000 -#define CPUSTAT_WAIT 0x10000000 -#define CPUSTAT_ECALL_PEND 0x08000000 -#define CPUSTAT_STOP_INT 0x04000000 -#define CPUSTAT_IO_INT 0x02000000 -#define CPUSTAT_EXT_INT 0x01000000 -#define CPUSTAT_RUNNING 0x00800000 -#define CPUSTAT_RETAINED 0x00400000 -#define CPUSTAT_TIMING_SUB 0x00020000 -#define CPUSTAT_SIE_SUB 0x00010000 -#define CPUSTAT_RRF 0x00008000 -#define CPUSTAT_SLSV 0x00004000 -#define CPUSTAT_SLSR 0x00002000 -#define CPUSTAT_ZARCH 0x00000800 -#define CPUSTAT_MCDS 0x00000100 -#define CPUSTAT_KSS 0x00000200 -#define CPUSTAT_SM 0x00000080 -#define CPUSTAT_IBS 0x00000040 -#define CPUSTAT_GED2 0x00000010 -#define CPUSTAT_G 0x00000008 -#define CPUSTAT_GED 0x00000004 -#define CPUSTAT_J 0x00000002 -#define CPUSTAT_P 0x00000001 - -struct kvm_s390_sie_block { - atomic_t cpuflags; /* 0x0000 */ - __u32 : 1; /* 0x0004 */ - __u32 prefix : 19; - __u32 ibc : 12; - __u8 reserved08[4]; /* 0x0008 */ -#define PROG_IN_SIE (1<<0) - __u32 prog0c; /* 0x000c */ - union { - __u8 reserved10[16]; /* 0x0010 */ - struct { - __u64 pv_handle_cpu; - __u64 pv_handle_config; - }; - }; -#define PROG_BLOCK_SIE (1<<0) -#define PROG_REQUEST (1<<1) - atomic_t prog20; /* 0x0020 */ - __u8 reserved24[4]; /* 0x0024 */ - __u64 cputm; /* 0x0028 */ - __u64 ckc; /* 0x0030 */ - __u64 epoch; /* 0x0038 */ - __u32 svcc; /* 0x0040 */ -#define LCTL_CR0 0x8000 -#define LCTL_CR6 0x0200 -#define LCTL_CR9 0x0040 -#define LCTL_CR10 0x0020 -#define LCTL_CR11 0x0010 -#define LCTL_CR14 0x0002 - __u16 lctl; /* 0x0044 */ - __s16 icpua; /* 0x0046 */ -#define ICTL_OPEREXC 0x80000000 -#define ICTL_PINT 0x20000000 -#define ICTL_LPSW 0x00400000 -#define ICTL_STCTL 0x00040000 -#define ICTL_ISKE 0x00004000 -#define ICTL_SSKE 0x00002000 -#define ICTL_RRBE 0x00001000 -#define ICTL_TPROT 0x00000200 - __u32 ictl; /* 0x0048 */ -#define ECA_CEI 0x80000000 -#define ECA_IB 0x40000000 -#define ECA_SIGPI 0x10000000 -#define ECA_MVPGI 0x01000000 -#define ECA_AIV 0x00200000 -#define ECA_VX 0x00020000 -#define ECA_PROTEXCI 0x00002000 -#define ECA_APIE 0x00000008 -#define ECA_SII 0x00000001 - __u32 eca; /* 0x004c */ -#define ICPT_INST 0x04 -#define ICPT_PROGI 0x08 -#define ICPT_INSTPROGI 0x0C -#define ICPT_EXTREQ 0x10 -#define ICPT_EXTINT 0x14 -#define ICPT_IOREQ 0x18 -#define ICPT_WAIT 0x1c -#define ICPT_VALIDITY 0x20 -#define ICPT_STOP 0x28 -#define ICPT_OPEREXC 0x2C -#define ICPT_PARTEXEC 0x38 -#define ICPT_IOINST 0x40 -#define ICPT_KSS 0x5c -#define ICPT_MCHKREQ 0x60 -#define ICPT_INT_ENABLE 0x64 -#define ICPT_PV_INSTR 0x68 -#define ICPT_PV_NOTIFY 0x6c -#define ICPT_PV_PREF 0x70 - __u8 icptcode; /* 0x0050 */ - __u8 icptstatus; /* 0x0051 */ - __u16 ihcpu; /* 0x0052 */ - __u8 reserved54; /* 0x0054 */ -#define IICTL_CODE_NONE 0x00 -#define IICTL_CODE_MCHK 0x01 -#define IICTL_CODE_EXT 0x02 -#define IICTL_CODE_IO 0x03 -#define IICTL_CODE_RESTART 0x04 -#define IICTL_CODE_SPECIFICATION 0x10 -#define IICTL_CODE_OPERAND 0x11 - __u8 iictl; /* 0x0055 */ - __u16 ipa; /* 0x0056 */ - __u32 ipb; /* 0x0058 */ - __u32 scaoh; /* 0x005c */ -#define FPF_BPBC 0x20 - __u8 fpf; /* 0x0060 */ -#define ECB_GS 0x40 -#define ECB_TE 0x10 -#define ECB_SPECI 0x08 -#define ECB_SRSI 0x04 -#define ECB_HOSTPROTINT 0x02 -#define ECB_PTF 0x01 - __u8 ecb; /* 0x0061 */ -#define ECB2_CMMA 0x80 -#define ECB2_IEP 0x20 -#define ECB2_PFMFI 0x08 -#define ECB2_ESCA 0x04 -#define ECB2_ZPCI_LSI 0x02 - __u8 ecb2; /* 0x0062 */ -#define ECB3_AISI 0x20 -#define ECB3_AISII 0x10 -#define ECB3_DEA 0x08 -#define ECB3_AES 0x04 -#define ECB3_RI 0x01 - __u8 ecb3; /* 0x0063 */ -#define ESCA_SCAOL_MASK ~0x3fU - __u32 scaol; /* 0x0064 */ - __u8 sdf; /* 0x0068 */ - __u8 epdx; /* 0x0069 */ - __u8 cpnc; /* 0x006a */ - __u8 reserved6b; /* 0x006b */ - __u32 todpr; /* 0x006c */ -#define GISA_FORMAT1 0x00000001 - __u32 gd; /* 0x0070 */ - __u8 reserved74[12]; /* 0x0074 */ - __u64 mso; /* 0x0080 */ - __u64 msl; /* 0x0088 */ - psw_t gpsw; /* 0x0090 */ - __u64 gg14; /* 0x00a0 */ - __u64 gg15; /* 0x00a8 */ - __u8 reservedb0[8]; /* 0x00b0 */ -#define HPID_KVM 0x4 -#define HPID_VSIE 0x5 - __u8 hpid; /* 0x00b8 */ - __u8 reservedb9[7]; /* 0x00b9 */ - union { - struct { - __u32 eiparams; /* 0x00c0 */ - __u16 extcpuaddr; /* 0x00c4 */ - __u16 eic; /* 0x00c6 */ - }; - __u64 mcic; /* 0x00c0 */ - } __packed; - __u32 reservedc8; /* 0x00c8 */ - union { - struct { - __u16 pgmilc; /* 0x00cc */ - __u16 iprcc; /* 0x00ce */ - }; - __u32 edc; /* 0x00cc */ - } __packed; - union { - struct { - __u32 dxc; /* 0x00d0 */ - __u16 mcn; /* 0x00d4 */ - __u8 perc; /* 0x00d6 */ - __u8 peratmid; /* 0x00d7 */ - }; - __u64 faddr; /* 0x00d0 */ - } __packed; - __u64 peraddr; /* 0x00d8 */ - __u8 eai; /* 0x00e0 */ - __u8 peraid; /* 0x00e1 */ - __u8 oai; /* 0x00e2 */ - __u8 armid; /* 0x00e3 */ - __u8 reservede4[4]; /* 0x00e4 */ - union { - __u64 tecmc; /* 0x00e8 */ - struct { - __u16 subchannel_id; /* 0x00e8 */ - __u16 subchannel_nr; /* 0x00ea */ - __u32 io_int_parm; /* 0x00ec */ - __u32 io_int_word; /* 0x00f0 */ - }; - } __packed; - __u8 reservedf4[8]; /* 0x00f4 */ -#define CRYCB_FORMAT_MASK 0x00000003 -#define CRYCB_FORMAT0 0x00000000 -#define CRYCB_FORMAT1 0x00000001 -#define CRYCB_FORMAT2 0x00000003 - __u32 crycbd; /* 0x00fc */ - __u64 gcr[16]; /* 0x0100 */ - union { - __u64 gbea; /* 0x0180 */ - __u64 sidad; - }; - __u8 reserved188[8]; /* 0x0188 */ - __u64 sdnxo; /* 0x0190 */ - __u8 reserved198[8]; /* 0x0198 */ - __u32 fac; /* 0x01a0 */ - __u8 reserved1a4[20]; /* 0x01a4 */ - __u64 cbrlo; /* 0x01b8 */ - __u8 reserved1c0[8]; /* 0x01c0 */ -#define ECD_HOSTREGMGMT 0x20000000 -#define ECD_MEF 0x08000000 -#define ECD_ETOKENF 0x02000000 -#define ECD_ECC 0x00200000 -#define ECD_HMAC 0x00004000 - __u32 ecd; /* 0x01c8 */ - __u8 reserved1cc[18]; /* 0x01cc */ - __u64 pp; /* 0x01de */ - __u8 reserved1e6[2]; /* 0x01e6 */ - __u64 itdba; /* 0x01e8 */ - __u64 riccbd; /* 0x01f0 */ - __u64 gvrd; /* 0x01f8 */ -} __packed __aligned(512); - -struct kvm_s390_itdb { - __u8 data[256]; -}; - -struct sie_page { - struct kvm_s390_sie_block sie_block; - struct mcck_volatile_info mcck_info; /* 0x0200 */ - __u8 reserved218[360]; /* 0x0218 */ - __u64 pv_grregs[16]; /* 0x0380 */ - __u8 reserved400[512]; /* 0x0400 */ - struct kvm_s390_itdb itdb; /* 0x0600 */ - __u8 reserved700[2304]; /* 0x0700 */ -}; - -#endif /* _ASM_KVM_HOST_TYPES_H */ +#endif /* ASM_KVM_HOST_TYPES_H */ diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 3b3ecc647993..5cef215d30e7 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -160,10 +160,15 @@ struct lowcore { /* SMP info area */ __u32 cpu_nr; /* 0x03a0 */ __u32 softirq_pending; /* 0x03a4 */ - __s32 preempt_count; /* 0x03a8 */ - __u32 spinlock_lockval; /* 0x03ac */ - __u32 spinlock_index; /* 0x03b0 */ - __u8 pad_0x03b4[0x03b8-0x03b4]; /* 0x03b4 */ + union { + struct { + __u32 need_resched; /* 0x03a8 */ + __u32 count; /* 0x03ac */ + } preempt; + __u64 preempt_count; /* 0x03a8 */ + }; + __u32 spinlock_lockval; /* 0x03b0 */ + __u32 spinlock_index; /* 0x03b4 */ __u64 percpu_offset; /* 0x03b8 */ __u8 percpu_register; /* 0x03c0 */ __u8 pad_0x03c1[0x0400-0x03c1]; /* 0x03c1 */ diff --git a/arch/s390/include/asm/nmi.h b/arch/s390/include/asm/nmi.h index 6454c1531854..7919b2b9ac9c 100644 --- a/arch/s390/include/asm/nmi.h +++ b/arch/s390/include/asm/nmi.h @@ -22,6 +22,7 @@ #define MCCK_CODE_SYSTEM_DAMAGE BIT(63) #define MCCK_CODE_EXT_DAMAGE BIT(63 - 5) #define MCCK_CODE_CP BIT(63 - 9) +#define MCCK_CODE_CK BIT(63 - 11) #define MCCK_CODE_STG_ERROR BIT(63 - 16) #define MCCK_CODE_STG_KEY_ERROR BIT(63 - 18) #define MCCK_CODE_STG_DEGRAD BIT(63 - 19) @@ -33,6 +34,8 @@ #define MCCK_CODE_FC_VALID BIT(63 - 43) #define MCCK_CODE_CPU_TIMER_VALID BIT(63 - 46) +#define MCCK_CODE_NO_GUEST (MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE | MCCK_CODE_CK) + #ifndef __ASSEMBLER__ union mci { diff --git a/arch/s390/include/asm/pai.h b/arch/s390/include/asm/pai.h index 534d0320e2aa..a3456a36aaa7 100644 --- a/arch/s390/include/asm/pai.h +++ b/arch/s390/include/asm/pai.h @@ -76,7 +76,6 @@ static __always_inline void pai_kernel_exit(struct pt_regs *regs) } #define PAI_SAVE_AREA(x) ((x)->hw.event_base) -#define PAI_CPU_MASK(x) ((x)->hw.addr_filters) #define PAI_PMU_IDX(x) ((x)->hw.last_tag) #define PAI_SWLIST(x) (&(x)->hw.tp_list) diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h index 5dcf35f0f325..88a125b92bdd 100644 --- a/arch/s390/include/asm/pci.h +++ b/arch/s390/include/asm/pci.h @@ -118,6 +118,32 @@ struct zpci_bus { enum pci_bus_speed max_bus_speed; }; +/* Content Code Description for PCI Function Error */ +struct zpci_ccdf_err { + u32 reserved1; + u32 fh; /* function handle */ + u32 fid; /* function id */ + u32 ett : 4; /* expected table type */ + u32 mvn : 12; /* MSI vector number */ + u32 dmaas : 8; /* DMA address space */ + u32 reserved2 : 6; + u32 q : 1; /* event qualifier */ + u32 rw : 1; /* read/write */ + u64 faddr; /* failing address */ + u32 reserved3; + u16 reserved4; + u16 pec; /* PCI event code */ +} __packed; + +#define ZPCI_ERR_PENDING_MAX 4 +struct zpci_ccdf_pending { + bool mediated_recovery; + u8 count; + u8 head; + u8 tail; + struct zpci_ccdf_err err[ZPCI_ERR_PENDING_MAX]; +}; + /* Private data per function */ struct zpci_dev { struct zpci_bus *zbus; @@ -192,6 +218,8 @@ struct zpci_dev { struct iommu_domain *s390_domain; /* attached IOMMU domain */ struct kvm_zdev *kzdev; struct mutex kzdev_lock; + struct zpci_ccdf_pending pending_errs; + struct mutex pending_errs_lock; spinlock_t dom_lock; /* protect s390_domain change */ }; @@ -334,6 +362,10 @@ void zpci_debug_exit_device(struct zpci_dev *); int zpci_report_error(struct pci_dev *, struct zpci_report_error_header *); int zpci_clear_error_state(struct zpci_dev *zdev); int zpci_reset_load_store_blocked(struct zpci_dev *zdev); +void zpci_start_mediated_recovery(struct zpci_dev *zdev); +void zpci_stop_mediated_recovery(struct zpci_dev *zdev); +int zpci_get_pending_error(struct zpci_dev *zdev, + struct zpci_ccdf_err *ccdf); #ifdef CONFIG_NUMA diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index 1d955dd0defa..feb58acf87d6 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -107,8 +107,8 @@ " .endif\n" \ ".endr\n" \ ".endm\n" \ - ALTERNATIVE("GEN_MVIY " __stringify(disp) " " __stringify(reg) "\n", \ - "GEN_MVIY " __stringify(dispalt) " " __stringify(reg) "\n", \ + ALTERNATIVE("GEN_MVIY " disp ", " reg "\n", \ + "GEN_MVIY " dispalt ", " reg "\n", \ ALT_FEATURE(MFEATURE_LOWCORE)) \ ".purgem GEN_MVIY\n" diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 859ce7c7d454..e882663a58e7 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -842,7 +842,7 @@ static inline int pte_same(pte_t a, pte_t b) return pte_val(a) == pte_val(b); } -#ifdef CONFIG_NUMA_BALANCING +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pte_protnone(pte_t pte) { return pte_present(pte) && !(pte_val(pte) & _PAGE_READ); @@ -853,7 +853,7 @@ static inline int pmd_protnone(pmd_t pmd) /* pmd_leaf(pmd) implies pmd_present(pmd) */ return pmd_leaf(pmd) && !(pmd_val(pmd) & _SEGMENT_ENTRY_READ); } -#endif +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ static inline bool pte_swp_exclusive(pte_t pte) { @@ -903,7 +903,7 @@ static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) return clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_SOFT_DIRTY)); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES #define pmd_swp_soft_dirty(pmd) pmd_soft_dirty(pmd) #define pmd_swp_mksoft_dirty(pmd) pmd_mksoft_dirty(pmd) #define pmd_swp_clear_soft_dirty(pmd) pmd_clear_soft_dirty(pmd) diff --git a/arch/s390/include/asm/pkey.h b/arch/s390/include/asm/pkey.h index 0af5ac4f646b..82e403322e89 100644 --- a/arch/s390/include/asm/pkey.h +++ b/arch/s390/include/asm/pkey.h @@ -13,6 +13,7 @@ #include <linux/ioctl.h> #include <linux/types.h> +#include <linux/delay.h> #include <uapi/asm/pkey.h> /* @@ -44,4 +45,19 @@ int pkey_key2protkey(const u8 *key, u32 keylen, */ #define PKEY_XFLAG_NOCLEARKEY 0x0002 +static inline int pkey_handle_expired(void) +{ + /* + * Protected key expired due to relocation to another host. The long + * running re-wrap has no asynchronous completion notification, so + * polling is required. Trigger a re-schedule of this request by + * returning -ENOSPC ("hardware queue full") to the crypto engine. + * To avoid immediately re-invocation of this callback, + * tell the scheduler to voluntarily give up the CPU here. + */ + msleep(1); + pr_debug("rescheduling request\n"); + return -ENOSPC; +} + #endif /* _KAPI_PKEY_H */ diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preempt.h index 6e5821bb047e..5560d5fca2a3 100644 --- a/arch/s390/include/asm/preempt.h +++ b/arch/s390/include/asm/preempt.h @@ -8,11 +8,8 @@ #include <asm/cmpxchg.h> #include <asm/march.h> -/* - * Use MSB so it is possible to read preempt_count with LLGT which - * reads the least significant 31 bits with a single instruction. - */ -#define PREEMPT_NEED_RESCHED 0x80000000 +/* Use MSB for PREEMPT_NEED_RESCHED mostly because it is available. */ +#define PREEMPT_NEED_RESCHED 0x8000000000000000UL /* * We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such @@ -26,25 +23,25 @@ */ static __always_inline int preempt_count(void) { - unsigned long lc_preempt, count; + unsigned long lc_preempt; + int count; - BUILD_BUG_ON(sizeof_field(struct lowcore, preempt_count) != sizeof(int)); - lc_preempt = offsetof(struct lowcore, preempt_count); - /* READ_ONCE(get_lowcore()->preempt_count) & ~PREEMPT_NEED_RESCHED */ + lc_preempt = offsetof(struct lowcore, preempt.count); + /* READ_ONCE(get_lowcore()->preempt.count) (without PREEMPT_NEED_RESCHED) */ asm_inline( - ALTERNATIVE("llgt %[count],%[offzero](%%r0)\n", - "llgt %[count],%[offalt](%%r0)\n", + ALTERNATIVE("ly %[count],%[offzero](%%r0)\n", + "ly %[count],%[offalt](%%r0)\n", ALT_FEATURE(MFEATURE_LOWCORE)) : [count] "=d" (count) : [offzero] "i" (lc_preempt), [offalt] "i" (lc_preempt + LOWCORE_ALT_ADDRESS), - "m" (((struct lowcore *)0)->preempt_count)); + "m" (((struct lowcore *)0)->preempt.count)); return count; } -static __always_inline void preempt_count_set(int pc) +static __always_inline void preempt_count_set(unsigned long pc) { - int old, new; + unsigned long old, new; old = READ_ONCE(get_lowcore()->preempt_count); do { @@ -63,12 +60,12 @@ static __always_inline void preempt_count_set(int pc) static __always_inline void set_preempt_need_resched(void) { - __atomic_and(~PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count); + __atomic64_and(~PREEMPT_NEED_RESCHED, (long *)&get_lowcore()->preempt_count); } static __always_inline void clear_preempt_need_resched(void) { - __atomic_or(PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count); + __atomic64_or(PREEMPT_NEED_RESCHED, (long *)&get_lowcore()->preempt_count); } static __always_inline bool test_preempt_need_resched(void) @@ -88,8 +85,8 @@ static __always_inline void __preempt_count_add(int val) lc_preempt = offsetof(struct lowcore, preempt_count); asm_inline( - ALTERNATIVE("asi %[offzero](%%r0),%[val]\n", - "asi %[offalt](%%r0),%[val]\n", + ALTERNATIVE("agsi %[offzero](%%r0),%[val]\n", + "agsi %[offalt](%%r0),%[val]\n", ALT_FEATURE(MFEATURE_LOWCORE)) : "+m" (((struct lowcore *)0)->preempt_count) : [offzero] "i" (lc_preempt), [val] "i" (val), @@ -98,7 +95,7 @@ static __always_inline void __preempt_count_add(int val) return; } } - __atomic_add(val, &get_lowcore()->preempt_count); + __atomic64_add(val, (long *)&get_lowcore()->preempt_count); } static __always_inline void __preempt_count_sub(int val) @@ -119,15 +116,15 @@ static __always_inline bool __preempt_count_dec_and_test(void) lc_preempt = offsetof(struct lowcore, preempt_count); asm_inline( - ALTERNATIVE("alsi %[offzero](%%r0),%[val]\n", - "alsi %[offalt](%%r0),%[val]\n", + ALTERNATIVE("algsi %[offzero](%%r0),%[val]\n", + "algsi %[offalt](%%r0),%[val]\n", ALT_FEATURE(MFEATURE_LOWCORE)) : "=@cc" (cc), "+m" (((struct lowcore *)0)->preempt_count) : [offzero] "i" (lc_preempt), [val] "i" (-1), [offalt] "i" (lc_preempt + LOWCORE_ALT_ADDRESS)); return (cc == 0) || (cc == 2); #else - return __atomic_add_const_and_test(-1, &get_lowcore()->preempt_count); + return __atomic64_add_const_and_test(-1, (long *)&get_lowcore()->preempt_count); #endif } @@ -139,6 +136,16 @@ static __always_inline bool should_resched(int preempt_offset) return unlikely(READ_ONCE(get_lowcore()->preempt_count) == preempt_offset); } +static __always_inline int __preempt_count_add_return(int val) +{ + return val + __atomic64_add(val, (long *)&get_lowcore()->preempt_count); +} + +static __always_inline int __preempt_count_sub_return(int val) +{ + return __preempt_count_add_return(-val); +} + #define init_task_preempt_count(p) do { } while (0) /* Deferred to CPU bringup time */ #define init_idle_preempt_count(p, cpu) do { } while (0) diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index ecd3341686eb..9434c76c25b8 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -14,12 +14,10 @@ #include <linux/bits.h> -#define CIF_NOHZ_DELAY 2 /* delay HZ disable for a tick */ #define CIF_ENABLED_WAIT 5 /* in enabled wait state */ #define CIF_MCCK_GUEST 6 /* machine check happening in guest */ #define CIF_DEDICATED_CPU 7 /* this CPU is dedicated */ -#define _CIF_NOHZ_DELAY BIT(CIF_NOHZ_DELAY) #define _CIF_ENABLED_WAIT BIT(CIF_ENABLED_WAIT) #define _CIF_MCCK_GUEST BIT(CIF_MCCK_GUEST) #define _CIF_DEDICATED_CPU BIT(CIF_DEDICATED_CPU) @@ -48,7 +46,6 @@ struct pcpu { unsigned long ec_mask; /* bit mask for ec_xxx functions */ unsigned long ec_clk; /* sigp timestamp for ec_xxx */ unsigned long flags; /* per CPU flags */ - unsigned long capacity; /* cpu capacity for scheduler */ signed char state; /* physical cpu state */ signed char polarization; /* physical polarization */ u16 address; /* physical cpu address */ @@ -97,8 +94,6 @@ static __always_inline bool test_cpu_flag_of(int flag, int cpu) return test_bit(flag, &per_cpu(pcpu_devices, cpu).flags); } -#define arch_needs_cpu() test_cpu_flag(CIF_NOHZ_DELAY) - static inline void get_cpu_id(struct cpuid *ptr) { asm volatile("stidp %0" : "=Q" (*ptr)); diff --git a/arch/s390/include/asm/ptrace.h b/arch/s390/include/asm/ptrace.h index 495e310c3d6d..6411e3584283 100644 --- a/arch/s390/include/asm/ptrace.h +++ b/arch/s390/include/asm/ptrace.h @@ -16,13 +16,11 @@ #define PIF_SYSCALL 0 /* inside a system call */ #define PIF_PSW_ADDR_ADJUSTED 1 /* psw address has been adjusted */ #define PIF_SYSCALL_RET_SET 2 /* return value was set via ptrace */ -#define PIF_GUEST_FAULT 3 /* indicates program check in sie64a */ -#define PIF_FTRACE_FULL_REGS 4 /* all register contents valid (ftrace) */ +#define PIF_FTRACE_FULL_REGS 3 /* all register contents valid (ftrace) */ #define _PIF_SYSCALL BIT(PIF_SYSCALL) #define _PIF_ADDR_PSW_ADJUSTED BIT(PIF_PSW_ADDR_ADJUSTED) #define _PIF_SYSCALL_RET_SET BIT(PIF_SYSCALL_RET_SET) -#define _PIF_GUEST_FAULT BIT(PIF_GUEST_FAULT) #define _PIF_FTRACE_FULL_REGS BIT(PIF_FTRACE_FULL_REGS) #define PSW32_MASK_PER _AC(0x40000000, UL) diff --git a/arch/s390/include/asm/runtime-const.h b/arch/s390/include/asm/runtime-const.h index 17878b1d048c..7b71156031ec 100644 --- a/arch/s390/include/asm/runtime-const.h +++ b/arch/s390/include/asm/runtime-const.h @@ -33,6 +33,20 @@ __ret; \ }) +#define runtime_const_mask_32(val, sym) \ +({ \ + unsigned int __ret = (val); \ + \ + asm_inline( \ + "0: nilf %[__ret],12\n" \ + ".pushsection runtime_mask_" #sym ",\"a\"\n" \ + ".long 0b - .\n" \ + ".popsection" \ + : [__ret] "+d" (__ret) \ + : : "cc"); \ + __ret; \ +}) + #define runtime_const_init(type, sym) do { \ extern s32 __start_runtime_##type##_##sym[]; \ extern s32 __stop_runtime_##type##_##sym[]; \ @@ -43,12 +57,12 @@ __stop_runtime_##type##_##sym); \ } while (0) -/* 32-bit immediate for iihf and iilf in bits in I2 field */ static inline void __runtime_fixup_32(u32 *p, unsigned int val) { s390_kernel_write(p, &val, sizeof(val)); } +/* 32-bit immediate for iihf and iilf in bits in I2 field */ static inline void __runtime_fixup_ptr(void *where, unsigned long val) { __runtime_fixup_32(where + 2, val >> 32); @@ -65,6 +79,12 @@ static inline void __runtime_fixup_shift(void *where, unsigned long val) s390_kernel_write(where, &insn, sizeof(insn)); } +/* 32-bit immediate for nilf in bits in I2 field */ +static inline void __runtime_fixup_mask(void *where, unsigned long val) +{ + __runtime_fixup_32(where + 2, val); +} + static inline void runtime_const_fixup(void (*fn)(void *, unsigned long), unsigned long val, s32 *start, s32 *end) { diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h index f2d490558054..44066550b9b1 100644 --- a/arch/s390/include/asm/sclp.h +++ b/arch/s390/include/asm/sclp.h @@ -21,6 +21,9 @@ #define SCLP_ERRNOTIFY_AQ_INFO_LOG 2 #define SCLP_ERRNOTIFY_AQ_OPTICS_DATA 3 #define SCLP_ERRNOTIFY_AQ_NVME_SMART_LOG 4 +#define SCLP_ERRNOTIFY_AQ_ADAPTER_INITIALIZED 5 +#define SCLP_ERRNOTIFY_AQ_RECOVERABLE_ERROR 6 +#define SCLP_ERRNOTIFY_AQ_TELEMETRY_DATA 7 #ifndef __ASSEMBLER__ #include <linux/uio.h> diff --git a/arch/s390/include/asm/smp.h b/arch/s390/include/asm/smp.h index fb2bdbf35da5..a6c621e0491c 100644 --- a/arch/s390/include/asm/smp.h +++ b/arch/s390/include/asm/smp.h @@ -30,7 +30,7 @@ static __always_inline unsigned int raw_smp_processor_id(void) return cpu; } -#define arch_scale_cpu_capacity smp_cpu_get_capacity +#define arch_scale_cpu_capacity topology_get_cpu_scale extern struct mutex smp_cpu_state_mutex; extern unsigned int smp_cpu_mt_shift; @@ -53,9 +53,7 @@ extern void smp_save_dump_secondary_cpus(void); extern void smp_yield_cpu(int cpu); extern void smp_cpu_set_polarization(int cpu, int val); extern int smp_cpu_get_polarization(int cpu); -extern void smp_cpu_set_capacity(int cpu, unsigned long val); extern void smp_set_core_capacity(int cpu, unsigned long val); -extern unsigned long smp_cpu_get_capacity(int cpu); extern int smp_cpu_get_cpu_address(int cpu); extern void smp_fill_possible_mask(void); extern void smp_detect_cpus(void); diff --git a/arch/s390/include/asm/syscall.h b/arch/s390/include/asm/syscall.h index 4271e4169f45..5f310caad1fc 100644 --- a/arch/s390/include/asm/syscall.h +++ b/arch/s390/include/asm/syscall.h @@ -89,11 +89,6 @@ static inline int syscall_get_arch(struct task_struct *task) return AUDIT_ARCH_S390X; } -static inline bool arch_syscall_is_vdso_sigreturn(struct pt_regs *regs) -{ - return false; -} - #define SYSCALL_FMT_0 #define SYSCALL_FMT_1 , "0" (r2) #define SYSCALL_FMT_2 , "d" (r3) SYSCALL_FMT_1 diff --git a/arch/s390/include/asm/thread_info.h b/arch/s390/include/asm/thread_info.h index 6a548a819400..2554cef3a6a5 100644 --- a/arch/s390/include/asm/thread_info.h +++ b/arch/s390/include/asm/thread_info.h @@ -61,6 +61,7 @@ void arch_setup_new_exec(void); */ #define HAVE_TIF_NEED_RESCHED_LAZY #define HAVE_TIF_RESTORE_SIGMASK +#define HAVE_TIF_POLLING_NRFLAG #include <asm-generic/thread_info_tif.h> diff --git a/arch/s390/include/asm/timex.h b/arch/s390/include/asm/timex.h index ac3ab6c29912..23421b6eb225 100644 --- a/arch/s390/include/asm/timex.h +++ b/arch/s390/include/asm/timex.h @@ -159,8 +159,6 @@ static inline void local_tick_enable(unsigned long comp) set_clock_comparator(get_lowcore()->clock_comparator); } -#define CLOCK_TICK_RATE 1193180 /* Underlying HZ */ - typedef unsigned long cycles_t; static __always_inline unsigned long get_tod_clock(void) diff --git a/arch/s390/include/asm/uv.h b/arch/s390/include/asm/uv.h index d919e69662f5..508075e9e8a9 100644 --- a/arch/s390/include/asm/uv.h +++ b/arch/s390/include/asm/uv.h @@ -454,18 +454,6 @@ static inline int uv_call(unsigned long r1, unsigned long r2) return cc; } -/* Low level uv_call that avoids stalls for long running busy conditions */ -static inline int uv_call_sched(unsigned long r1, unsigned long r2) -{ - int cc; - - do { - cc = __uv_call(r1, r2); - cond_resched(); - } while (cc > 1); - return cc; -} - /* * special variant of uv_call that only transports the cpu or guest * handle and the command, like destroy or verify. @@ -480,7 +468,7 @@ static inline int uv_cmd_nodata(u64 handle, u16 cmd, u16 *rc, u16 *rrc) int cc; WARN(!handle, "No handle provided to Ultravisor call cmd %x\n", cmd); - cc = uv_call_sched(0, (u64)&uvcb); + cc = uv_call(0, (u64)&uvcb); *rc = uvcb.header.rc; *rrc = uvcb.header.rrc; return cc ? -EINVAL : 0; @@ -518,7 +506,7 @@ static inline int uv_list_secrets(struct uv_secret_list *buf, u16 start_idx, .start_idx = start_idx, .list_addr = (u64)buf, }; - int cc = uv_call_sched(0, (u64)&uvcb); + int cc = uv_call(0, (u64)&uvcb); if (rc) *rc = uvcb.header.rc; @@ -635,6 +623,8 @@ int s390_wiggle_split_folio(struct mm_struct *mm, struct folio *folio); int __make_folio_secure(struct folio *folio, struct uv_cb_header *uvcb); int uv_convert_from_secure(unsigned long paddr); int uv_convert_from_secure_folio(struct folio *folio); +void *uv_alloc_stor_var(unsigned long size); +void uv_free_stor_var(void *stor_var); void setup_uv(void); diff --git a/arch/s390/include/asm/word-at-a-time.h b/arch/s390/include/asm/word-at-a-time.h index e9287036392d..eaa19dee7699 100644 --- a/arch/s390/include/asm/word-at-a-time.h +++ b/arch/s390/include/asm/word-at-a-time.h @@ -4,6 +4,7 @@ #include <linux/bitops.h> #include <linux/wordpart.h> +#include <asm/asm-extable.h> #include <asm/bitsperlong.h> struct word_at_a_time { @@ -40,4 +41,25 @@ static inline unsigned long zero_bytemask(unsigned long data) return ~1UL << data; } +/* + * Load an unaligned word from kernel space. + * + * In the (very unlikely) case of the word being a page-crosser + * and the next page not being mapped, take the exception and + * return zeroes in the non-existing part. + */ +static inline unsigned long load_unaligned_zeropad(const void *addr) +{ + unsigned long data; + + asm_inline volatile( + "0: lg %[data],0(%[addr])\n" + "1: nopr %%r7\n" + EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr]) + EX_TABLE_ZEROPAD(1b, 1b, %[data], %[addr]) + : [data] "=d" (data) + : [addr] "a" (addr), "m" (*(unsigned long *)addr)); + return data; +} + #endif /* _ASM_WORD_AT_A_TIME_H */ diff --git a/arch/s390/include/uapi/asm/Kbuild b/arch/s390/include/uapi/asm/Kbuild index 46c1ff0b842a..378151cef3cc 100644 --- a/arch/s390/include/uapi/asm/Kbuild +++ b/arch/s390/include/uapi/asm/Kbuild @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 -generated-y += unistd_32.h generated-y += unistd_64.h diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 6c88476d79a3..f84b8fb41331 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -3,6 +3,8 @@ # Makefile for the linux kernel. # +CONTEXT_ANALYSIS := y + ifdef CONFIG_FUNCTION_TRACER # Do not trace tracer code @@ -72,6 +74,7 @@ obj-$(CONFIG_STACKPROTECTOR) += stackprotector.o obj-$(CONFIG_KEXEC_FILE) += machine_kexec_file.o kexec_image.o obj-$(CONFIG_KEXEC_FILE) += kexec_elf.o obj-$(CONFIG_CERT_STORE) += cert_store.o +obj-$(CONFIG_KMSAN) += irqflags.o obj-$(CONFIG_PERF_EVENTS) += perf_event.o obj-$(CONFIG_PERF_EVENTS) += perf_cpum_cf.o perf_cpum_sf.o diff --git a/arch/s390/kernel/debug.c b/arch/s390/kernel/debug.c index dbf430f479bd..b5bf8284dbfc 100644 --- a/arch/s390/kernel/debug.c +++ b/arch/s390/kernel/debug.c @@ -182,7 +182,7 @@ static struct debug_param_t { static int debug_param_num; /* functions */ -static void debug_get_param(const char *name, int *level, int *pages) +static void debug_get_param(const char *name, int *level, int *pages, bool quiet) { struct debug_param_t *p; int i; @@ -192,11 +192,13 @@ static void debug_get_param(const char *name, int *level, int *pages) if (!glob_match(p->name, name)) continue; if (level && p->level != PARAM_UNSET) { - pr_info("%s: override level to %d\n", name, p->level); + if (!quiet) + pr_info("%s: override level to %d\n", name, p->level); *level = p->level; } if (pages && p->pages != PARAM_UNSET) { - pr_info("%s: override pages to %d\n", name, p->pages); + if (!quiet) + pr_info("%s: override pages to %d\n", name, p->pages); *pages = p->pages; } } @@ -251,7 +253,7 @@ static int __init s390dbf_parse(char *arg) * regular memory allocations are possible. */ for (i = 0, id = __s390dbf_info; &id[i] < __s390dbf_info_end; i++) - debug_get_param(id[i]->name, &id[i]->level, NULL); + debug_get_param(id[i]->name, &id[i]->level, NULL, false); return rc; } @@ -395,7 +397,7 @@ static debug_info_t *debug_info_create(const char *name, int pages_per_area, int level = DEBUG_DEFAULT_LEVEL; debug_info_t *rc; - debug_get_param(name, &level, &pages_per_area); + debug_get_param(name, &level, &pages_per_area, false); rc = debug_info_alloc(name, pages_per_area, nr_areas, buf_size, level, ALL_AREAS); if (!rc) goto out; @@ -960,7 +962,7 @@ void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas) return; } - debug_get_param(id->name, &id->level, &pages_per_area); + debug_get_param(id->name, &id->level, &pages_per_area, false); copy = debug_info_alloc("", pages_per_area, nr_areas, id->buf_size, id->level, ALL_AREAS); if (!copy) { @@ -993,8 +995,8 @@ void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas) mutex_unlock(&debug_mutex); } -/* Remove debugfs entries and remove from internal list. */ -static void _debug_unregister(debug_info_t *id) +/* Remove debugfs entries. */ +static void _debug_unregister_debugfs(debug_info_t *id) { int i; @@ -1004,6 +1006,11 @@ static void _debug_unregister(debug_info_t *id) debugfs_remove(id->debugfs_entries[i]); } debugfs_remove(id->debugfs_root_entry); +} + +/* Remove from internal list. */ +static void _debug_unregister(debug_info_t *id) +{ if (id == debug_area_first) debug_area_first = id->next; if (id == debug_area_last) @@ -1029,6 +1036,7 @@ void debug_unregister(debug_info_t *id) mutex_lock(&debug_mutex); _debug_unregister(id); mutex_unlock(&debug_mutex); + _debug_unregister_debugfs(id); debug_info_put(id); } @@ -1068,9 +1076,6 @@ static void _debug_set_level(debug_info_t *id, int new_level) { unsigned long flags; - if (!id) - return; - if (new_level == DEBUG_OFF_LEVEL) { pr_info("%s: switched off\n", id->name); } else if ((new_level > DEBUG_MAX_LEVEL) || (new_level < 0)) { @@ -1095,8 +1100,14 @@ static void _debug_set_level(debug_info_t *id, int new_level) */ void debug_set_level(debug_info_t *id, int new_level) { - /* Level specified via kernel parameter takes precedence */ - debug_get_param(id->name, &new_level, NULL); + if (!id) + return; + + /* + * Level specified via kernel parameter takes precedence. The override + * was already announced during registration, so stay quiet here. + */ + debug_get_param(id->name, &new_level, NULL, true); _debug_set_level(id, new_level); } @@ -1272,7 +1283,7 @@ void debug_set_critical(void) debug_entry_t *debug_event_common(debug_info_t *id, int level, const void *buf, int len) { - debug_entry_t *active; + debug_entry_t *active = NULL; unsigned long flags; if (!debug_active || !id->areas) @@ -1283,6 +1294,8 @@ debug_entry_t *debug_event_common(debug_info_t *id, int level, const void *buf, } else { raw_spin_lock_irqsave(&id->lock, flags); } + if (!id->areas) + goto out; do { active = get_active_entry(id); memcpy(DEBUG_DATA(active), buf, min(len, id->buf_size)); @@ -1292,7 +1305,7 @@ debug_entry_t *debug_event_common(debug_info_t *id, int level, const void *buf, len -= id->buf_size; buf += id->buf_size; } while (len > 0); - +out: raw_spin_unlock_irqrestore(&id->lock, flags); return active; } @@ -1305,7 +1318,7 @@ EXPORT_SYMBOL(debug_event_common); debug_entry_t *debug_exception_common(debug_info_t *id, int level, const void *buf, int len) { - debug_entry_t *active; + debug_entry_t *active = NULL; unsigned long flags; if (!debug_active || !id->areas) @@ -1316,6 +1329,8 @@ debug_entry_t *debug_exception_common(debug_info_t *id, int level, } else { raw_spin_lock_irqsave(&id->lock, flags); } + if (!id->areas) + goto out; do { active = get_active_entry(id); memcpy(DEBUG_DATA(active), buf, min(len, id->buf_size)); @@ -1325,7 +1340,7 @@ debug_entry_t *debug_exception_common(debug_info_t *id, int level, len -= id->buf_size; buf += id->buf_size; } while (len > 0); - +out: raw_spin_unlock_irqrestore(&id->lock, flags); return active; } @@ -1351,7 +1366,7 @@ static inline int debug_count_numargs(char *string) debug_entry_t *__debug_sprintf_event(debug_info_t *id, int level, char *string, ...) { debug_sprintf_entry_t *curr_event; - debug_entry_t *active; + debug_entry_t *active = NULL; unsigned long flags; int numargs, idx; va_list ap; @@ -1366,6 +1381,8 @@ debug_entry_t *__debug_sprintf_event(debug_info_t *id, int level, char *string, } else { raw_spin_lock_irqsave(&id->lock, flags); } + if (!id->areas) + goto out; active = get_active_entry(id); curr_event = (debug_sprintf_entry_t *) DEBUG_DATA(active); va_start(ap, string); @@ -1374,6 +1391,7 @@ debug_entry_t *__debug_sprintf_event(debug_info_t *id, int level, char *string, curr_event->args[idx] = va_arg(ap, long); va_end(ap); debug_finish_entry(id, active, level, 0); +out: raw_spin_unlock_irqrestore(&id->lock, flags); return active; @@ -1386,7 +1404,7 @@ EXPORT_SYMBOL(__debug_sprintf_event); debug_entry_t *__debug_sprintf_exception(debug_info_t *id, int level, char *string, ...) { debug_sprintf_entry_t *curr_event; - debug_entry_t *active; + debug_entry_t *active = NULL; unsigned long flags; int numargs, idx; va_list ap; @@ -1402,6 +1420,8 @@ debug_entry_t *__debug_sprintf_exception(debug_info_t *id, int level, char *stri } else { raw_spin_lock_irqsave(&id->lock, flags); } + if (!id->areas) + goto out; active = get_active_entry(id); curr_event = (debug_sprintf_entry_t *)DEBUG_DATA(active); va_start(ap, string); @@ -1410,6 +1430,7 @@ debug_entry_t *__debug_sprintf_exception(debug_info_t *id, int level, char *stri curr_event->args[idx] = va_arg(ap, long); va_end(ap); debug_finish_entry(id, active, level, 1); +out: raw_spin_unlock_irqrestore(&id->lock, flags); return active; @@ -1652,9 +1673,11 @@ static void debug_flush(debug_info_t *id, int area) unsigned long flags; int i, j; - if (!id || !id->areas) + if (!id) return; raw_spin_lock_irqsave(&id->lock, flags); + if (!id->areas) + goto out; if (area == DEBUG_FLUSH_ALL) { id->active_area = 0; memset(id->active_entries, 0, id->nr_areas * sizeof(int)); @@ -1669,6 +1692,7 @@ static void debug_flush(debug_info_t *id, int area) for (i = 0; i < id->pages_per_area; i++) memset(id->areas[area][i], 0, PAGE_SIZE); } +out: raw_spin_unlock_irqrestore(&id->lock, flags); } diff --git a/arch/s390/kernel/diag/diag324.c b/arch/s390/kernel/diag/diag324.c index fe325c2a2d0d..3eec0cc8fb9e 100644 --- a/arch/s390/kernel/diag/diag324.c +++ b/arch/s390/kernel/diag/diag324.c @@ -182,8 +182,7 @@ long diag324_pibbuf(unsigned long arg) goto out; rc = copy_to_user((void __user *)address, data->pib, data->pib->len); rc |= put_user(data->sequence, &udata->sequence); - if (rc) - rc = -EFAULT; + rc = rc ? -EFAULT : data->rc; out: mutex_unlock(&pibmutex); return rc; diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 79a45efae23d..10dd9bbdf985 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -31,6 +31,7 @@ #include <asm/nospec-insn.h> #include <asm/lowcore.h> #include <asm/machine.h> +#include "entry.h" _LPP_OFFSET = __LC_LPP @@ -321,7 +322,7 @@ SYM_CODE_START(pgm_check_handler) jz 1f BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST SIEEXIT __SF_SIE_CONTROL(%r15),%r13 - lghi %r10,_PIF_GUEST_FAULT + lghi %r10,PGM_FLAG_GUEST_FAULT #endif 1: tmhh %r8,0x4000 # PER bit set in old PSW ? jnz 2f # -> enabled, can't be a double fault @@ -332,7 +333,7 @@ SYM_CODE_START(pgm_check_handler) CHECK_VMAP_STACK __LC_SAVE_AREA,%r13,4f 3: lg %r15,__LC_KERNEL_STACK(%r13) 4: la %r11,STACK_FRAME_OVERHEAD(%r15) - stg %r10,__PT_FLAGS(%r11) + xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) stmg %r0,%r7,__PT_R0(%r11) mvc __PT_R8(64,%r11),__LC_SAVE_AREA(%r13) @@ -341,13 +342,13 @@ SYM_CODE_START(pgm_check_handler) # clear user controlled registers to prevent speculative use xgr %r0,%r0 xgr %r1,%r1 - xgr %r3,%r3 xgr %r4,%r4 xgr %r5,%r5 xgr %r6,%r6 xgr %r7,%r7 xgr %r12,%r12 lgr %r2,%r11 + lgr %r3,%r10 brasl %r14,__do_pgm_check tmhh %r8,0x0001 # returning to user space? jno .Lpgm_exit_kernel @@ -367,7 +368,7 @@ SYM_CODE_START(pgm_check_handler) mvc __LC_RETURN_PSW(8,%r13),__LC_SVC_NEW_PSW(%r13) larl %r14,.Lsysc_per stg %r14,__LC_RETURN_PSW+8(%r13) - lghi %r14,1 + lghi %r14,SYSCALL_FLAG_PER_TRAP LBEAR __LC_PGM_LAST_BREAK(%r13) LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE # branch to .Lsysc_per SYM_CODE_END(pgm_check_handler) diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h index fb67b4abe68c..dc84ab497251 100644 --- a/arch/s390/kernel/entry.h +++ b/arch/s390/kernel/entry.h @@ -2,6 +2,11 @@ #ifndef _ENTRY_H #define _ENTRY_H +#define PGM_FLAG_GUEST_FAULT 1 +#define SYSCALL_FLAG_PER_TRAP 1 + +#ifndef __ASSEMBLER__ + #include <linux/percpu.h> #include <linux/types.h> #include <linux/signal.h> @@ -21,8 +26,8 @@ void early_pgm_check_handler(void); struct task_struct *__switch_to_asm(struct task_struct *prev, struct task_struct *next); void __ret_from_fork(struct task_struct *prev, struct pt_regs *regs); -void __do_pgm_check(struct pt_regs *regs); -void __do_syscall(struct pt_regs *regs, int per_trap); +void __do_pgm_check(struct pt_regs *regs, unsigned long flags); +void __do_syscall(struct pt_regs *regs, unsigned long flags); void __do_early_pgm_check(struct pt_regs *regs); void do_protection_exception(struct pt_regs *regs); @@ -70,4 +75,5 @@ extern struct exception_table_entry _stop_amode31_ex_table[]; #define __amode31_ref __section(".amode31.refs") extern long _start_amode31_refs[], _end_amode31_refs[]; +#endif /* __ASSEMBLER__ */ #endif /* _ENTRY_H */ diff --git a/arch/s390/kernel/idle.c b/arch/s390/kernel/idle.c index 08f3520c6785..dd12efb58201 100644 --- a/arch/s390/kernel/idle.c +++ b/arch/s390/kernel/idle.c @@ -136,7 +136,6 @@ void noinstr arch_cpu_idle(void) /* Wait for external, I/O or machine check interrupt. */ psw_mask = PSW_KERNEL_BITS | PSW_MASK_WAIT | PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK; - clear_cpu_flag(CIF_NOHZ_DELAY); set_cpu_flag(CIF_ENABLED_WAIT); if (smp_cpu_mtid) stcctm(MT_DIAG, smp_cpu_mtid, (u64 *)&idle->mt_cycles_enter); diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index 3c346b02ceb9..b1e798f8e1dd 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -1157,6 +1157,8 @@ static struct attribute_group reipl_nss_attr_group = { void set_os_info_reipl_block(void) { + if (!reipl_block_actual) + return; os_info_entry_add_data(OS_INFO_REIPL_BLOCK, reipl_block_actual, reipl_block_actual->hdr.len); } @@ -1927,7 +1929,8 @@ static struct shutdown_action __refdata dump_action = { static void dump_reipl_run(struct shutdown_trigger *trigger) { struct lowcore *abs_lc; - unsigned int csum; + unsigned long ipib = 0; + unsigned int csum = 0; /* * Set REIPL_CLEAR flag in os_info flags entry indicating @@ -1943,9 +1946,12 @@ static void dump_reipl_run(struct shutdown_trigger *trigger) reipl_type == IPL_TYPE_UNKNOWN) os_info_flags |= OS_INFO_FLAG_REIPL_CLEAR; os_info_entry_add_data(OS_INFO_FLAGS_ENTRY, &os_info_flags, sizeof(os_info_flags)); - csum = (__force unsigned int)cksm(reipl_block_actual, reipl_block_actual->hdr.len, 0); + if (reipl_block_actual) { + ipib = __pa(reipl_block_actual); + csum = (__force unsigned int)cksm(reipl_block_actual, reipl_block_actual->hdr.len, 0); + } abs_lc = get_abs_lowcore(); - abs_lc->ipib = __pa(reipl_block_actual); + abs_lc->ipib = ipib; abs_lc->ipib_checksum = csum; put_abs_lowcore(abs_lc); dump_run(trigger); @@ -2021,8 +2027,11 @@ static int vmcmd_init(void) return sysfs_create_group(&vmcmd_kset->kobj, &vmcmd_attr_group); } -static struct shutdown_action vmcmd_action = {SHUTDOWN_ACTION_VMCMD_STR, - vmcmd_run, vmcmd_init}; +static struct shutdown_action vmcmd_action = { + .name = SHUTDOWN_ACTION_VMCMD_STR, + .fn = vmcmd_run, + .init = vmcmd_init +}; /* * stop shutdown action: Stop Linux on shutdown. @@ -2036,15 +2045,21 @@ static void stop_run(struct shutdown_trigger *trigger) smp_stop_cpu(); } -static struct shutdown_action stop_action = {SHUTDOWN_ACTION_STOP_STR, - stop_run, NULL}; +static struct shutdown_action stop_action = { + .name = SHUTDOWN_ACTION_STOP_STR, + .fn = stop_run +}; /* action list */ static struct shutdown_action *shutdown_actions_list[] = { - &ipl_action, &reipl_action, &dump_reipl_action, &dump_action, - &vmcmd_action, &stop_action}; -#define SHUTDOWN_ACTIONS_COUNT (sizeof(shutdown_actions_list) / sizeof(void *)) + &ipl_action, + &reipl_action, + &dump_reipl_action, + &dump_action, + &vmcmd_action, + &stop_action +}; /* * Trigger section @@ -2057,7 +2072,7 @@ static int set_trigger(const char *buf, struct shutdown_trigger *trigger, { int i; - for (i = 0; i < SHUTDOWN_ACTIONS_COUNT; i++) { + for (i = 0; i < ARRAY_SIZE(shutdown_actions_list); i++) { if (sysfs_streq(buf, shutdown_actions_list[i]->name)) { if (shutdown_actions_list[i]->init_rc) { return shutdown_actions_list[i]->init_rc; @@ -2072,8 +2087,10 @@ static int set_trigger(const char *buf, struct shutdown_trigger *trigger, /* on reipl */ -static struct shutdown_trigger on_reboot_trigger = {ON_REIPL_STR, - &reipl_action}; +static struct shutdown_trigger on_reboot_trigger = { + .name = ON_REIPL_STR, + .action = &reipl_action +}; static ssize_t on_reboot_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) @@ -2098,8 +2115,10 @@ static void do_machine_restart(char *__unused) void (*_machine_restart)(char *command) = do_machine_restart; /* on panic */ - -static struct shutdown_trigger on_panic_trigger = {ON_PANIC_STR, &stop_action}; +static struct shutdown_trigger on_panic_trigger = { + .name = ON_PANIC_STR, + .action = &stop_action +}; static ssize_t on_panic_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) @@ -2123,9 +2142,10 @@ static void do_panic(void) } /* on restart */ - -static struct shutdown_trigger on_restart_trigger = {ON_RESTART_STR, - &stop_action}; +static struct shutdown_trigger on_restart_trigger = { + .name = ON_RESTART_STR, + .action = &stop_action +}; static ssize_t on_restart_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) @@ -2160,8 +2180,10 @@ void do_restart(void *arg) } /* on halt */ - -static struct shutdown_trigger on_halt_trigger = {ON_HALT_STR, &stop_action}; +static struct shutdown_trigger on_halt_trigger = { + .name = ON_HALT_STR, + .action = &stop_action +}; static ssize_t on_halt_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) @@ -2186,8 +2208,10 @@ static void do_machine_halt(void) void (*_machine_halt)(void) = do_machine_halt; /* on power off */ - -static struct shutdown_trigger on_poff_trigger = {ON_POFF_STR, &stop_action}; +static struct shutdown_trigger on_poff_trigger = { + .name = ON_POFF_STR, + .action = &stop_action +}; static ssize_t on_poff_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) @@ -2242,7 +2266,7 @@ static void __init shutdown_actions_init(void) { int i; - for (i = 0; i < SHUTDOWN_ACTIONS_COUNT; i++) { + for (i = 0; i < ARRAY_SIZE(shutdown_actions_list); i++) { if (!shutdown_actions_list[i]->init) continue; shutdown_actions_list[i]->init_rc = diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index 04d528639b94..c923496aa7b4 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -166,7 +166,6 @@ void noinstr do_io_irq(struct pt_regs *regs) if (from_idle) account_idle_time_irq(); - set_cpu_flag(CIF_NOHZ_DELAY); do { regs->tpi_info = get_lowcore()->tpi_info; if (get_lowcore()->tpi_info.adapter_IO) @@ -369,9 +368,6 @@ static irqreturn_t do_ext_interrupt(int irq, void *dummy) int index; ext_code.int_code = regs->int_code; - if (ext_code.code != EXT_IRQ_CLK_COMP) - set_cpu_flag(CIF_NOHZ_DELAY); - index = ext_hash(ext_code.code); rcu_read_lock(); hlist_for_each_entry_rcu(p, &ext_int_hash[index], entry) { diff --git a/arch/s390/kernel/irqflags.c b/arch/s390/kernel/irqflags.c new file mode 100644 index 000000000000..e192f59f8918 --- /dev/null +++ b/arch/s390/kernel/irqflags.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/export.h> +#include <asm/irqflags.h> + +noinstr unsigned long arch_local_save_flags(void) +{ + return __arch_local_save_flags(); +} +EXPORT_SYMBOL(arch_local_save_flags); + +noinstr unsigned long arch_local_irq_save(void) +{ + return __arch_local_irq_save(); +} +EXPORT_SYMBOL(arch_local_irq_save); + +noinstr void arch_local_irq_enable_external(void) +{ + __arch_local_irq_enable_external(); +} +EXPORT_SYMBOL(arch_local_irq_enable_external); + +noinstr void arch_local_irq_enable(void) +{ + __arch_local_irq_enable(); +} +EXPORT_SYMBOL(arch_local_irq_enable); diff --git a/arch/s390/kernel/mcount.S b/arch/s390/kernel/mcount.S index 1fec370fecf4..6bc44c767642 100644 --- a/arch/s390/kernel/mcount.S +++ b/arch/s390/kernel/mcount.S @@ -4,6 +4,7 @@ * */ +#include <linux/cfi_types.h> #include <linux/linkage.h> #include <asm/asm-offsets.h> #include <asm/ftrace.h> @@ -34,10 +35,16 @@ .section .kprobes.text, "ax" -SYM_FUNC_START(ftrace_stub) +SYM_TYPED_FUNC_START(ftrace_stub) BR_EX %r14 SYM_FUNC_END(ftrace_stub) +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +SYM_TYPED_FUNC_START(ftrace_stub_graph) + BR_EX %r14 +SYM_FUNC_END(ftrace_stub_graph) +#endif + SYM_CODE_START(ftrace_stub_direct_tramp) lgr %r1, %r0 BR_EX %r1 diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index e17a59d4d5a4..17297a8b63d9 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -344,8 +344,7 @@ static void notrace s390_backup_mcck_info(struct pt_regs *regs) sie_page = container_of(sie_block, struct sie_page, sie_block); mcck_backup = &sie_page->mcck_info; - mcck_backup->mcic = get_lowcore()->mcck_interruption_code & - ~(MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE); + mcck_backup->mcic = get_lowcore()->mcck_interruption_code & ~MCCK_CODE_NO_GUEST; mcck_backup->ext_damage_code = get_lowcore()->external_damage_code; mcck_backup->failing_storage_address = get_lowcore()->failing_storage_address; } @@ -357,8 +356,6 @@ NOKPROBE_SYMBOL(s390_backup_mcck_info); #define ED_STP_ISLAND 6 /* External damage STP island check */ #define ED_STP_SYNC 7 /* External damage STP sync check */ -#define MCCK_CODE_NO_GUEST (MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE) - /* * machine check handler. */ diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index 2076ac22e2c4..69fa303cd479 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -110,6 +110,7 @@ struct cpu_cf_ptr { static struct cpu_cf_root { /* Anchor to per CPU data */ refcount_t refcnt; /* Overall active events */ + unsigned int tskctx; /* Users tracking all CPUs (cpu == -1) */ struct cpu_cf_ptr __percpu *cfptr; } cpu_cf_root; @@ -118,13 +119,15 @@ static struct cpu_cf_root { /* Anchor to per CPU data */ * user space in task context with perf_event_open() and close() * system calls. * - * This mutex serializes functions cpum_cf_alloc_cpu() called at event - * initialization via cpumf_pmu_event_init() and function cpum_cf_free_cpu() - * called at event removal via call back function hw_perf_event_destroy() - * when the event is deleted. They are serialized to enforce correct - * bookkeeping of pointer and reference counts anchored by - * struct cpu_cf_root and the access to cpu_cf_root::refcnt and the - * per CPU pointers stored in cpu_cf_root::cfptr. + * This mutex serializes the allocation and removal of the per CPU counter + * data via cpum_cf_alloc_cpu() and cpum_cf_free_cpu(). They are called with + * this mutex held at event initialization via cpumf_pmu_event_init(), at + * event removal via call back function hw_perf_event_destroy() when the + * event is deleted, and from the CPU hotplug prepare/dead callbacks. The + * mutex enforces correct bookkeeping of pointer and reference counts + * anchored by struct cpu_cf_root and protects the access to + * cpu_cf_root::refcnt, cpu_cf_root::tskctx and the per CPU pointers + * stored in cpu_cf_root::cfptr. */ static DEFINE_MUTEX(pmc_reserve_mutex); @@ -167,12 +170,14 @@ static void cpum_cf_reset_cpu(void *flags) } /* Free per CPU data when the last event is removed. */ -static void cpum_cf_free_root(void) +static void cpum_cf_free_root(unsigned int num) { - if (!refcount_dec_and_test(&cpu_cf_root.refcnt)) + struct cpu_cf_ptr __percpu *p = cpu_cf_root.cfptr; + + if (!refcount_sub_and_test(num, &cpu_cf_root.refcnt)) return; - free_percpu(cpu_cf_root.cfptr); cpu_cf_root.cfptr = NULL; + free_percpu(p); irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT); on_each_cpu(cpum_cf_reset_cpu, NULL, 1); debug_sprintf_event(cf_dbg, 4, "%s root.refcnt %u cfptr %d\n", @@ -186,17 +191,17 @@ static void cpum_cf_free_root(void) * CPUs possible, which might be larger than the number of CPUs currently * online. */ -static int cpum_cf_alloc_root(void) +static int cpum_cf_alloc_root(unsigned int num) { int rc = 0; - if (refcount_inc_not_zero(&cpu_cf_root.refcnt)) + if (refcount_add_not_zero(num, &cpu_cf_root.refcnt)) return rc; /* The memory is already zeroed. */ cpu_cf_root.cfptr = alloc_percpu(struct cpu_cf_ptr); if (cpu_cf_root.cfptr) { - refcount_set(&cpu_cf_root.refcnt, 1); + refcount_set(&cpu_cf_root.refcnt, num); on_each_cpu(cpum_cf_reset_cpu, NULL, 1); irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT); } else { @@ -206,20 +211,23 @@ static int cpum_cf_alloc_root(void) return rc; } -/* Free CPU counter data structure for a PMU */ -static void cpum_cf_free_cpu(int cpu) +/* + * Remove num references to the CPU counter data structure of a PMU. + * Called with pmc_reserve_mutex held. + */ +static void cpum_cf_free_cpu(int cpu, unsigned int num) { struct cpu_cf_events *cpuhw; struct cpu_cf_ptr *p; - mutex_lock(&pmc_reserve_mutex); + lockdep_assert_held(&pmc_reserve_mutex); /* * When invoked via CPU hotplug handler, there might be no events * installed or that particular CPU might not have an * event installed. This anchor pointer can be NULL! */ if (!cpu_cf_root.cfptr) - goto out; + return; p = per_cpu_ptr(cpu_cf_root.cfptr, cpu); cpuhw = p->cpucf; /* @@ -227,28 +235,29 @@ static void cpum_cf_free_cpu(int cpu) * installed on that CPU, but on different CPUs. */ if (!cpuhw) - goto out; + return; - if (refcount_dec_and_test(&cpuhw->refcnt)) { - kfree(cpuhw); + if (refcount_sub_and_test(num, &cpuhw->refcnt)) { p->cpucf = NULL; + kfree(cpuhw); } - cpum_cf_free_root(); -out: - mutex_unlock(&pmc_reserve_mutex); + cpum_cf_free_root(num); } -/* Allocate CPU counter data structure for a PMU. Called under mutex lock. */ -static int cpum_cf_alloc_cpu(int cpu) +/* + * Add num references to the CPU counter data structure of a PMU and + * allocate it when necessary. Called with pmc_reserve_mutex held. + */ +static int cpum_cf_alloc_cpu(int cpu, unsigned int num) { struct cpu_cf_events *cpuhw; struct cpu_cf_ptr *p; int rc; - mutex_lock(&pmc_reserve_mutex); - rc = cpum_cf_alloc_root(); + lockdep_assert_held(&pmc_reserve_mutex); + rc = cpum_cf_alloc_root(num); if (rc) - goto unlock; + return rc; p = per_cpu_ptr(cpu_cf_root.cfptr, cpu); cpuhw = p->cpucf; @@ -256,12 +265,12 @@ static int cpum_cf_alloc_cpu(int cpu) cpuhw = kzalloc_obj(*cpuhw); if (cpuhw) { p->cpucf = cpuhw; - refcount_set(&cpuhw->refcnt, 1); + refcount_set(&cpuhw->refcnt, num); } else { rc = -ENOMEM; } } else { - refcount_inc(&cpuhw->refcnt); + refcount_add(num, &cpuhw->refcnt); } if (rc) { /* @@ -269,10 +278,8 @@ static int cpum_cf_alloc_cpu(int cpu) * cpu_cf_event in not created, its destroy() function is not * invoked. Adjust the reference counter for the anchor. */ - cpum_cf_free_root(); + cpum_cf_free_root(num); } -unlock: - mutex_unlock(&pmc_reserve_mutex); return rc; } @@ -284,39 +291,70 @@ unlock: * perf_event_open() with task context and /dev/hwctr interface. * If cpu is non-zero install event on this CPU only. This setup handles * perf_event_open() with CPU context. + * Users with cpu == -1 are counted in cpu_cf_root::tskctx. The CPU hotplug + * prepare and dead callbacks use this count to install and remove the per + * CPU counter data on a new or dying CPU. */ -static int cpum_cf_alloc(int cpu) +static int cpum_cf_alloc_cpuslocked(int cpu) { cpumask_var_t mask; int rc; + lockdep_assert_cpus_held(); if (cpu == -1) { if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) return -ENOMEM; + mutex_lock(&pmc_reserve_mutex); for_each_online_cpu(cpu) { - rc = cpum_cf_alloc_cpu(cpu); + rc = cpum_cf_alloc_cpu(cpu, 1); if (rc) { for_each_cpu(cpu, mask) - cpum_cf_free_cpu(cpu); + cpum_cf_free_cpu(cpu, 1); break; } cpumask_set_cpu(cpu, mask); } + if (!rc) + cpu_cf_root.tskctx++; + mutex_unlock(&pmc_reserve_mutex); free_cpumask_var(mask); } else { - rc = cpum_cf_alloc_cpu(cpu); + mutex_lock(&pmc_reserve_mutex); + rc = cpum_cf_alloc_cpu(cpu, 1); + mutex_unlock(&pmc_reserve_mutex); } return rc; } -static void cpum_cf_free(int cpu) +static int cpum_cf_alloc(int cpu) +{ + int rc; + + cpus_read_lock(); + rc = cpum_cf_alloc_cpuslocked(cpu); + cpus_read_unlock(); + return rc; +} + +static void cpum_cf_free_cpuslocked(int cpu) { + lockdep_assert_cpus_held(); + mutex_lock(&pmc_reserve_mutex); if (cpu == -1) { + cpu_cf_root.tskctx--; for_each_online_cpu(cpu) - cpum_cf_free_cpu(cpu); + cpum_cf_free_cpu(cpu, 1); } else { - cpum_cf_free_cpu(cpu); + cpum_cf_free_cpu(cpu, 1); } + mutex_unlock(&pmc_reserve_mutex); +} + +static void cpum_cf_free(int cpu) +{ + cpus_read_lock(); + cpum_cf_free_cpuslocked(cpu); + cpus_read_unlock(); } #define CF_DIAG_CTRSET_DEF 0xfeef /* Counter set header mark */ @@ -1090,53 +1128,67 @@ static refcount_t cfset_opencnt = REFCOUNT_INIT(0); /* Access count */ static DEFINE_MUTEX(cfset_ctrset_mutex); /* - * CPU hotplug handles only /dev/hwctr device. - * For perf_event_open() the CPU hotplug handling is done on kernel common - * code: + * CPU hotplug handling: + * + * cpum_cf_prepare_cpu() and cpum_cf_dead_cpu() run while the new or dying + * CPU is offline. They create and remove the per CPU counter data for all + * users tracking every CPU (cpu == -1), that is perf_event_open() events + * with task context and /dev/hwctr device sessions. Each such user holds + * one reference to the per CPU counter data of each CPU. Therefore install + * and remove one reference per user, tracked in cpu_cf_root::tskctx. This + * guarantees the per CPU counter data exists before the new CPU executes + * its first task and is removed only after the dying CPU is gone. + * + * cpum_cf_online_cpu() and cpum_cf_offline_cpu() run while the new or + * dying CPU is online. They handle only the counter set state of open + * /dev/hwctr device sessions on that CPU. For perf_event_open() events + * nothing is done: * - CPU add: Nothing is done since a file descriptor can not be created * and returned to the user. * - CPU delete: Handled by common code via pmu_disable(), pmu_stop() and - * pmu_delete(). The event itself is removed when the file descriptor is - * closed. + * pmu_delete(). During task exit processing of grouped perf events + * triggered by CPU hotplug processing, pmu_disable() is called as part + * of perf context removal process. The event itself is removed when the + * event file descriptor is closed. */ +static int cpum_cf_prepare_cpu(unsigned int cpu) +{ + int rc = 0; + + mutex_lock(&pmc_reserve_mutex); + if (cpu_cf_root.tskctx) + rc = cpum_cf_alloc_cpu(cpu, cpu_cf_root.tskctx); + mutex_unlock(&pmc_reserve_mutex); + return rc; +} + +static int cpum_cf_dead_cpu(unsigned int cpu) +{ + mutex_lock(&pmc_reserve_mutex); + if (cpu_cf_root.tskctx) + cpum_cf_free_cpu(cpu, cpu_cf_root.tskctx); + mutex_unlock(&pmc_reserve_mutex); + return 0; +} + static int cfset_online_cpu(unsigned int cpu); static int cpum_cf_online_cpu(unsigned int cpu) { - int rc = 0; - - /* - * Ignore notification for perf_event_open(). - * Handle only /dev/hwctr device sessions. - */ mutex_lock(&cfset_ctrset_mutex); - if (refcount_read(&cfset_opencnt)) { - rc = cpum_cf_alloc_cpu(cpu); - if (!rc) - cfset_online_cpu(cpu); - } + if (refcount_read(&cfset_opencnt)) + cfset_online_cpu(cpu); mutex_unlock(&cfset_ctrset_mutex); - return rc; + return 0; } static int cfset_offline_cpu(unsigned int cpu); static int cpum_cf_offline_cpu(unsigned int cpu) { - /* - * During task exit processing of grouped perf events triggered by CPU - * hotplug processing, pmu_disable() is called as part of perf context - * removal process. Therefore do not trigger event removal now for - * perf_event_open() created events. Perf common code triggers event - * destruction when the event file descriptor is closed. - * - * Handle only /dev/hwctr device sessions. - */ mutex_lock(&cfset_ctrset_mutex); - if (refcount_read(&cfset_opencnt)) { + if (refcount_read(&cfset_opencnt)) cfset_offline_cpu(cpu); - cpum_cf_free_cpu(cpu); - } mutex_unlock(&cfset_ctrset_mutex); return 0; } @@ -1183,7 +1235,7 @@ static void cpumf_measurement_alert(struct ext_code ext_code, static int cfset_init(void); static int __init cpumf_pmu_init(void) { - int rc; + int state, rc; /* Extract counter measurement facility information */ if (!cpum_cf_avail() || qctri(&cpumf_ctr_info)) @@ -1225,11 +1277,24 @@ static int __init cpumf_pmu_init(void) cfset_init(); } + rc = cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, + "perf/s390/cf:prepare", + cpum_cf_prepare_cpu, cpum_cf_dead_cpu); + if (rc < 0) + goto out3; + state = rc; + rc = cpuhp_setup_state(CPUHP_AP_PERF_S390_CF_ONLINE, "perf/s390/cf:online", cpum_cf_online_cpu, cpum_cf_offline_cpu); - return rc; + if (rc < 0) + goto out4; + return 0; +out4: + cpuhp_remove_state(state); +out3: + perf_pmu_unregister(&cpumf_pmu); out2: debug_unregister_view(cf_dbg, &debug_sprintf_view); debug_unregister(cf_dbg); @@ -1385,6 +1450,7 @@ static void cfset_all_stop(struct cfset_request *req) */ static int cfset_release(struct inode *inode, struct file *file) { + cpus_read_lock(); mutex_lock(&cfset_ctrset_mutex); /* Open followed by close/exit has no private_data */ if (file->private_data) { @@ -1395,9 +1461,10 @@ static int cfset_release(struct inode *inode, struct file *file) } if (refcount_dec_and_test(&cfset_opencnt)) { /* Last close */ on_each_cpu(cfset_release_cpu, NULL, 1); - cpum_cf_free(-1); + cpum_cf_free_cpuslocked(-1); } mutex_unlock(&cfset_ctrset_mutex); + cpus_read_unlock(); return 0; } @@ -1416,15 +1483,17 @@ static int cfset_open(struct inode *inode, struct file *file) return -EPERM; file->private_data = NULL; + cpus_read_lock(); mutex_lock(&cfset_ctrset_mutex); if (!refcount_inc_not_zero(&cfset_opencnt)) { /* First open */ - rc = cpum_cf_alloc(-1); + rc = cpum_cf_alloc_cpuslocked(-1); if (!rc) { cfset_session_init(); refcount_set(&cfset_opencnt, 1); } } mutex_unlock(&cfset_ctrset_mutex); + cpus_read_unlock(); /* nonseekable_open() never fails */ return rc ?: nonseekable_open(inode, file); @@ -1496,7 +1565,6 @@ static int cfset_all_copy(unsigned long arg, cpumask_t *mask) goto out; } uptr += sizeof(struct s390_ctrset_cpudata) + cpuhw->used; - cond_resched(); } cpus = cpumask_weight(mask); if (put_user(cpus, &ctrset_read->no_cpus)) diff --git a/arch/s390/kernel/perf_pai.c b/arch/s390/kernel/perf_pai.c index cdb8006220ca..013c3dae21ec 100644 --- a/arch/s390/kernel/perf_pai.c +++ b/arch/s390/kernel/perf_pai.c @@ -67,6 +67,7 @@ struct pai_mapptr { static struct pai_root { /* Anchor to per CPU data */ refcount_t refcnt; /* Overall active events */ + atomic_t tskctx; /* Overall per-task events */ struct pai_mapptr __percpu *mapptr; } pai_root[PAI_PMU_MAX]; @@ -93,14 +94,15 @@ struct pai_pmu { /* Define PAI PMU characteristics */ static struct pai_pmu pai_pmu[]; /* Forward declaration */ /* Free per CPU data when the last event is removed. */ -static void pai_root_free(int idx) +static void pai_root_free(int idx, int tasks) { - if (refcount_dec_and_test(&pai_root[idx].refcnt)) { + if (refcount_sub_and_test(tasks, &pai_root[idx].refcnt)) { free_percpu(pai_root[idx].mapptr); pai_root[idx].mapptr = NULL; } - debug_sprintf_event(paidbg, 5, "%s root[%d].refcount %d\n", __func__, - idx, refcount_read(&pai_root[idx].refcnt)); + debug_sprintf_event(paidbg, 5, "%s root[%d].refcount %d tskctx %d\n", + __func__, idx, refcount_read(&pai_root[idx].refcnt), + atomic_read(&pai_root[idx].tskctx)); } /* @@ -137,40 +139,54 @@ static void pai_free(struct pai_mapptr *mp) mp->mapptr = NULL; } -/* Adjust usage counters and remove allocated memory when all users are - * gone. - */ -static void pai_event_destroy_cpu(struct perf_event *event, int cpu) +/* Called under mutex_lock */ +static void pai_event_destroy_cpu(int idx, int cpu, bool hotplug) { - int idx = PAI_PMU_IDX(event); - struct pai_mapptr *mp = per_cpu_ptr(pai_root[idx].mapptr, cpu); - struct pai_map *cpump = mp->mapptr; + struct pai_mapptr *mp; + struct pai_map *cpump; + int tasks = 1; - mutex_lock(&pai_reserve_mutex); - debug_sprintf_event(paidbg, 5, "%s event %#llx idx %d cpu %d users %d " - "refcnt %u\n", __func__, event->attr.config, idx, - event->cpu, cpump->active_events, - refcount_read(&cpump->refcnt)); - if (refcount_dec_and_test(&cpump->refcnt)) + /* Check reference count and return when all gone. + * 1. An event is installed on online CPU X. + * 2. CPU x is offlined and the per-CPU data is removed. + * 3. Event is destroyed via close system call. + */ + if (!refcount_read(&pai_root[idx].refcnt)) + return; /* No events at all */ + mp = per_cpu_ptr(pai_root[idx].mapptr, cpu); + if (!mp || !mp->mapptr) /* No events on that CPU */ + return; + + /* When hotplug is true, invocation is from CPU hotplug callback. + * Delete per-CPU resource and adjust refcnt when per-task events + * are currently active. This can be more than one. + * In this case adjust counters. + */ + if (hotplug) + tasks = atomic_read(&pai_root[idx].tskctx); + + cpump = mp->mapptr; + if (refcount_sub_and_test(tasks, &cpump->refcnt)) pai_free(mp); - pai_root_free(idx); - mutex_unlock(&pai_reserve_mutex); + pai_root_free(idx, tasks); } static void pai_event_destroy(struct perf_event *event) { - int cpu; + int cpu = 0, idx = PAI_PMU_IDX(event); free_page(PAI_SAVE_AREA(event)); + cpus_read_lock(); + mutex_lock(&pai_reserve_mutex); if (event->cpu == -1) { - struct cpumask *mask = PAI_CPU_MASK(event); - - for_each_cpu(cpu, mask) - pai_event_destroy_cpu(event, cpu); - kfree(mask); + atomic_dec(&pai_root[idx].tskctx); + for_each_online_cpu(cpu) + pai_event_destroy_cpu(idx, cpu, false); } else { - pai_event_destroy_cpu(event, event->cpu); + pai_event_destroy_cpu(idx, event->cpu, false); } + mutex_unlock(&pai_reserve_mutex); + cpus_read_unlock(); } static void paicrypt_event_destroy(struct perf_event *event) @@ -234,25 +250,30 @@ static u64 paicrypt_getall(struct perf_event *event) return sum; } -/* Check concurrent access of counting and sampling for crypto events. - * This function is called in process context and it is save to block. - * When the event initialization functions fails, no other call back will - * be invoked. - * - * Allocate the memory for the event. - */ -static int pai_alloc_cpu(struct perf_event *event, int cpu) +/* Called under mutex_lock */ +static int pai_alloc_cpu(int idx, int cpu, bool hotplug) { - int rc, idx = PAI_PMU_IDX(event); struct pai_map *cpump = NULL; bool need_paiext_cb = false; struct pai_mapptr *mp; + int tasks = 1, rc = 0; + + /* When hotplug is true, invocation is from CPU hotplug callback. + * Allocate per-CPU resource when per-task events are currently active. + * This can be more than one. In this case adjust all reference + * counters. Otherwise return, this ensures memory is only allocated + * when needed. + */ + if (hotplug) { + tasks = atomic_read(&pai_root[idx].tskctx); + if (!tasks) + goto out; + } - mutex_lock(&pai_reserve_mutex); /* Allocate root node */ rc = pai_root_alloc(idx); if (rc) - goto unlock; + goto out; /* Allocate node for this event */ mp = per_cpu_ptr(pai_root[idx].mapptr, cpu); @@ -296,28 +317,45 @@ static int pai_alloc_cpu(struct perf_event *event, int cpu) goto undo; } INIT_LIST_HEAD(&cpump->syswide_list); - refcount_set(&cpump->refcnt, 1); + refcount_set(&cpump->refcnt, tasks); rc = 0; } else { - refcount_inc(&cpump->refcnt); + refcount_add(tasks, &cpump->refcnt); } + /* If tasks is greater than 1, we are called from CPU hotplug path + * and need to adjust the pai_root[idx].refcnt by the number of + * per-process events. Function pai_root_alloc(idx) already + * incremented by one. Adjust for the rest. + */ + if (tasks > 1) + refcount_add(tasks - 1, &pai_root[idx].refcnt); undo: if (rc) { /* Error in allocation of event, decrement anchor. Since * the event in not created, its destroy() function is never * invoked. Adjust the reference counter for the anchor. + * The failure happened in the case of variable + * cpump == NULL branch above. The pai_root[XXX].refcnt has + * been incremented by one. Then the per-CPU allocation + * failed, so decrement it by one, regardless of tasks. */ - pai_root_free(idx); + pai_root_free(idx, 1); } -unlock: - mutex_unlock(&pai_reserve_mutex); +out: /* If rc is non-zero, no increment of counter/sampler was done. */ return rc; } +/* Check concurrent access of counting and sampling for PAI events. + * This function is called in process context and it is safe to block. + * When the event initialization functions fails, no other call back will + * be invoked. + * Called under mutex_lock. + */ static int pai_alloc(struct perf_event *event) { + int idx = PAI_PMU_IDX(event); struct cpumask *maskptr; int cpu, rc = -ENOMEM; @@ -326,24 +364,20 @@ static int pai_alloc(struct perf_event *event) goto out; for_each_online_cpu(cpu) { - rc = pai_alloc_cpu(event, cpu); + rc = pai_alloc_cpu(idx, cpu, false); if (rc) { for_each_cpu(cpu, maskptr) - pai_event_destroy_cpu(event, cpu); - kfree(maskptr); - goto out; + pai_event_destroy_cpu(idx, cpu, false); + goto undo; } cpumask_set_cpu(cpu, maskptr); } - /* - * On error all cpumask are freed and all events have been destroyed. - * Save of which CPUs data structures have been allocated for. - * Release them in pai_event_destroy call back function - * for this event. - */ - PAI_CPU_MASK(event) = maskptr; rc = 0; + /* Trace per-task events for CPU hotplug. */ + atomic_inc(&pai_root[idx].tskctx); +undo: + kfree(maskptr); out: return rc; } @@ -391,10 +425,14 @@ static int pai_event_init(struct perf_event *event, int idx) } } + cpus_read_lock(); + mutex_lock(&pai_reserve_mutex); if (event->cpu >= 0) - rc = pai_alloc_cpu(event, event->cpu); + rc = pai_alloc_cpu(idx, event->cpu, false); else rc = pai_alloc(event); + mutex_unlock(&pai_reserve_mutex); + cpus_read_unlock(); if (rc) { free_page(PAI_SAVE_AREA(event)); goto out; @@ -464,6 +502,7 @@ static void pai_start(struct perf_event *event, int flags, cpump->event = event; } } + event->hw.state &= ~PERF_HES_STOPPED; } static void paicrypt_start(struct perf_event *event, int flags) @@ -510,6 +549,13 @@ static void pai_stop(struct perf_event *event, int flags) struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); struct pai_map *cpump = mp->mapptr; + /* Cope with multiple invocations: + * 1. perf_event_throttle() --> PMU->stop() + * 2. task schedules out --> PMU->stop() + * Check for event already stopped. + */ + if (event->hw.state & PERF_HES_STOPPED) + return; if (!event->attr.sample_period) { /* Counting */ pai_pmu[idx].pmu->read(event); } else { /* Sampling */ @@ -672,9 +718,9 @@ static void pai_have_samples(int idx) { struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr); struct pai_map *cpump = mp->mapptr; - struct perf_event *event; + struct perf_event *event, *e2; - list_for_each_entry(event, &cpump->syswide_list, hw.tp_list) + list_for_each_entry_safe(event, e2, &cpump->syswide_list, hw.tp_list) pai_have_sample(event, cpump); } @@ -691,6 +737,17 @@ static void paicrypt_sched_task(struct perf_event_pmu_context *pmu_ctx, pai_have_samples(PAI_PMU_CRYPTO); } +/* Prevent ioctl(fd, PERF_EVENT_IOC_PERIOD, ...) call. + * It sets perf_event::event_limit to a positive value and causes + * perf_event_overflow() to invoke pai_stop() call back function when + * perf_event::event_limit hits zero. This is not supported because the + * sample events CRYPTO_ALL and NNPA_ALL are always taken at schedule out + * of a task. + */ +static int pai_check_period(struct perf_event *event, u64 value) +{ + return -EINVAL; +} /* ============================= paiext ====================================*/ static void paiext_event_destroy(struct perf_event *event) @@ -804,6 +861,7 @@ static struct pmu paicrypt = { .stop = paicrypt_stop, .read = paicrypt_read, .sched_task = paicrypt_sched_task, + .check_period = pai_check_period, .attr_groups = paicrypt_attr_groups }; @@ -1015,6 +1073,7 @@ static struct pmu paiext = { .stop = paiext_stop, .read = paiext_read, .sched_task = paiext_sched_task, + .check_period = pai_check_period, .attr_groups = paiext_attr_groups, }; @@ -1218,23 +1277,61 @@ static int __init paipmu_setup(void) return install_ok; } +static int pai_online_cpu(unsigned int cpu) +{ + int rc; + + mutex_lock(&pai_reserve_mutex); + rc = pai_alloc_cpu(PAI_PMU_CRYPTO, cpu, true); + if (rc) + goto out; + rc = pai_alloc_cpu(PAI_PMU_EXT, cpu, true); + if (rc) + pai_event_destroy_cpu(PAI_PMU_CRYPTO, cpu, true); +out: + mutex_unlock(&pai_reserve_mutex); + return rc; +} + +static int pai_offline_cpu(unsigned int cpu) +{ + mutex_lock(&pai_reserve_mutex); + pai_event_destroy_cpu(PAI_PMU_CRYPTO, cpu, true); + pai_event_destroy_cpu(PAI_PMU_EXT, cpu, true); + mutex_unlock(&pai_reserve_mutex); + return 0; +} + static int __init pai_init(void) { + int state, rc; + /* Setup s390dbf facility */ - paidbg = debug_register("pai", 32, 256, 128); + paidbg = debug_register("pai", 1, 1, 128); if (!paidbg) { pr_err("Registration of s390dbf pai failed\n"); return -ENOMEM; } debug_register_view(paidbg, &debug_sprintf_view); - if (!paipmu_setup()) { - /* No PMU registration, no need for debug buffer */ - debug_unregister_view(paidbg, &debug_sprintf_view); - debug_unregister(paidbg); - return -ENODEV; - } + /* CPUHP_BP_PREPARE_DYN --> before CPU is brought online */ + state = cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "perf/pai:prepare", + pai_online_cpu, pai_offline_cpu); + rc = state < 0 ? state : 0; + if (rc < 0) + goto out_debug; + + rc = -ENODEV; + if (!paipmu_setup()) + goto out_cpuhp; return 0; + +out_cpuhp: + cpuhp_remove_state(state); +out_debug: + debug_unregister_view(paidbg, &debug_sprintf_view); + debug_unregister(paidbg); + return rc; } device_initcall(pai_init); diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 0ba7f89b8161..32499cad86f0 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -659,23 +659,13 @@ int smp_cpu_get_polarization(int cpu) return per_cpu(pcpu_devices, cpu).polarization; } -void smp_cpu_set_capacity(int cpu, unsigned long val) -{ - per_cpu(pcpu_devices, cpu).capacity = val; -} - -unsigned long smp_cpu_get_capacity(int cpu) -{ - return per_cpu(pcpu_devices, cpu).capacity; -} - void smp_set_core_capacity(int cpu, unsigned long val) { int i; cpu = smp_get_base_cpu(cpu); for (i = cpu; (i <= cpu + smp_cpu_mtid) && (i < nr_cpu_ids); i++) - smp_cpu_set_capacity(i, val); + topology_set_cpu_scale(i, val); } int smp_cpu_get_cpu_address(int cpu) @@ -727,7 +717,7 @@ static int smp_add_core(struct sclp_core_entry *core, cpumask_t *avail, else pcpu->state = CPU_STATE_STANDBY; smp_cpu_set_polarization(cpu, POLARIZATION_UNKNOWN); - smp_cpu_set_capacity(cpu, CPU_CAPACITY_HIGH); + topology_set_cpu_scale(cpu, CPU_CAPACITY_HIGH); set_cpu_present(cpu, true); if (!early && arch_register_cpu(cpu)) set_cpu_present(cpu, false); @@ -909,7 +899,6 @@ int __cpu_disable(void) cregs[6].val &= ~0xff000000UL; /* disable all I/O interrupts */ cregs[14].val &= ~0x1f000000UL; /* disable most machine checks */ __local_ctl_load(0, 15, cregs); - clear_cpu_flag(CIF_NOHZ_DELAY); return 0; } @@ -968,7 +957,7 @@ void __init smp_prepare_boot_cpu(void) ipl_pcpu->state = CPU_STATE_CONFIGURED; lc->pcpu = (unsigned long)ipl_pcpu; smp_cpu_set_polarization(0, POLARIZATION_UNKNOWN); - smp_cpu_set_capacity(0, CPU_CAPACITY_HIGH); + topology_set_cpu_scale(0, CPU_CAPACITY_HIGH); } void __init smp_setup_processor_id(void) @@ -1039,6 +1028,7 @@ static ssize_t cpu_configure_store(struct device *dev, per_cpu(pcpu_devices, cpu + i).state = CPU_STATE_STANDBY; smp_cpu_set_polarization(cpu + i, POLARIZATION_UNKNOWN); + set_cpu_enabled(cpu + i, false); } topology_expect_change(); break; @@ -1054,6 +1044,7 @@ static ssize_t cpu_configure_store(struct device *dev, per_cpu(pcpu_devices, cpu + i).state = CPU_STATE_CONFIGURED; smp_cpu_set_polarization(cpu + i, POLARIZATION_UNKNOWN); + set_cpu_enabled(cpu + i, true); } topology_expect_change(); break; @@ -1091,6 +1082,7 @@ bool arch_cpu_is_hotpluggable(int cpu) int arch_register_cpu(int cpu) { + struct pcpu *pcpu = per_cpu_ptr(&pcpu_devices, cpu); struct cpu *c = per_cpu_ptr(&cpu_devices, cpu); int rc; @@ -1104,6 +1096,8 @@ int arch_register_cpu(int cpu) rc = topology_cpu_init(c); if (rc) goto out_topology; + if (pcpu->state != CPU_STATE_CONFIGURED) + set_cpu_enabled(cpu, false); return 0; out_topology: diff --git a/arch/s390/kernel/syscall.c b/arch/s390/kernel/syscall.c index 75d5a3cab14e..bcc0b76bd868 100644 --- a/arch/s390/kernel/syscall.c +++ b/arch/s390/kernel/syscall.c @@ -93,12 +93,13 @@ SYSCALL_DEFINE0(ni_syscall) return -ENOSYS; } -void noinstr __do_syscall(struct pt_regs *regs, int per_trap) +void noinstr __do_syscall(struct pt_regs *regs, unsigned long flags) { unsigned long nr; + bool permit; + + enter_from_user_mode_randomize_stack(regs); - enter_from_user_mode(regs); - add_random_kstack_offset(); regs->psw = get_lowcore()->svc_old_psw; regs->int_code = get_lowcore()->svc_int_code; update_timer_sys(); @@ -106,7 +107,7 @@ void noinstr __do_syscall(struct pt_regs *regs, int per_trap) current->thread.last_break = regs->last_break; local_irq_enable(); regs->orig_gpr2 = regs->gprs[2]; - if (unlikely(per_trap)) + if (unlikely(flags & SYSCALL_FLAG_PER_TRAP)) set_thread_flag(TIF_PER_TRAP); regs->flags = 0; set_pt_regs_flag(regs, PIF_SYSCALL); @@ -121,7 +122,9 @@ void noinstr __do_syscall(struct pt_regs *regs, int per_trap) regs->psw.addr = current->restart_block.arch_data; current->restart_block.arch_data = 1; } - nr = syscall_enter_from_user_mode_work(regs, nr); + + permit = syscall_enter_from_user_mode_work(regs, &nr); + /* * In the s390 ptrace ABI, both the syscall number and the return value * use gpr2. However, userspace puts the syscall number either in the @@ -129,7 +132,7 @@ void noinstr __do_syscall(struct pt_regs *regs, int per_trap) * work, the ptrace code sets PIF_SYSCALL_RET_SET, which is checked here * and if set, the syscall will be skipped. */ - if (unlikely(test_and_clear_pt_regs_flag(regs, PIF_SYSCALL_RET_SET))) + if (unlikely(test_and_clear_pt_regs_flag(regs, PIF_SYSCALL_RET_SET) || !permit)) goto out; regs->gprs[2] = -ENOSYS; if (likely(nr < NR_syscalls)) { diff --git a/arch/s390/kernel/syscalls/syscall.tbl b/arch/s390/kernel/syscalls/syscall.tbl index 09a7ef04d979..1b45e68a217b 100644 --- a/arch/s390/kernel/syscalls/syscall.tbl +++ b/arch/s390/kernel/syscalls/syscall.tbl @@ -398,3 +398,4 @@ 469 common file_setattr sys_file_setattr 470 common listns sys_listns 471 common rseq_slice_yield sys_rseq_slice_yield +472 common fchroot sys_fchroot diff --git a/arch/s390/kernel/sysinfo.c b/arch/s390/kernel/sysinfo.c index 33ca3e47a0e6..45b4f448fe3d 100644 --- a/arch/s390/kernel/sysinfo.c +++ b/arch/s390/kernel/sysinfo.c @@ -325,6 +325,7 @@ int unregister_service_level(struct service_level *slr) EXPORT_SYMBOL(unregister_service_level); static void *service_level_start(struct seq_file *m, loff_t *pos) +__acquires_shared(service_level_sem) { down_read(&service_level_sem); return seq_list_start(&service_level_list, *pos); @@ -336,6 +337,7 @@ static void *service_level_next(struct seq_file *m, void *p, loff_t *pos) } static void service_level_stop(struct seq_file *m, void *p) +__releases_shared(service_level_sem) { up_read(&service_level_sem); } diff --git a/arch/s390/kernel/text_amode31.S b/arch/s390/kernel/text_amode31.S index 26f2981aa09e..f007d892d0c5 100644 --- a/arch/s390/kernel/text_amode31.S +++ b/arch/s390/kernel/text_amode31.S @@ -5,6 +5,7 @@ * Copyright IBM Corp. 2019 */ +#include <linux/cfi_types.h> #include <linux/linkage.h> #include <asm/asm-extable.h> #include <asm/errno.h> @@ -26,7 +27,7 @@ /* * int _diag14_amode31(unsigned long rx, unsigned long ry1, unsigned long subcode) */ -SYM_FUNC_START(_diag14_amode31) +SYM_TYPED_FUNC_START(_diag14_amode31) lgr %r1,%r2 lgr %r2,%r3 lgr %r3,%r4 @@ -46,7 +47,7 @@ SYM_FUNC_END(_diag14_amode31) /* * int _diag210_amode31(struct diag210 *addr) */ -SYM_FUNC_START(_diag210_amode31) +SYM_TYPED_FUNC_START(_diag210_amode31) lgr %r1,%r2 lhi %r2,-1 sam31 @@ -64,7 +65,7 @@ SYM_FUNC_END(_diag210_amode31) /* * int diag8c(struct diag8c *addr, struct ccw_dev_id *devno, size_t len) */ -SYM_FUNC_START(_diag8c_amode31) +SYM_TYPED_FUNC_START(_diag8c_amode31) llgf %r3,0(%r3) sam31 diag %r2,%r4,0x8c @@ -77,7 +78,7 @@ SYM_FUNC_END(_diag8c_amode31) /* * int _diag26c_amode31(void *req, void *resp, enum diag26c_sc subcode) */ -SYM_FUNC_START(_diag26c_amode31) +SYM_TYPED_FUNC_START(_diag26c_amode31) lghi %r5,-EOPNOTSUPP sam31 diag %r2,%r4,0x26c @@ -91,7 +92,7 @@ SYM_FUNC_END(_diag26c_amode31) /* * void _diag0c_amode31(unsigned long rx) */ -SYM_FUNC_START(_diag0c_amode31) +SYM_TYPED_FUNC_START(_diag0c_amode31) sam31 diag %r2,%r2,0x0c sam64 @@ -103,7 +104,7 @@ SYM_FUNC_END(_diag0c_amode31) * * Calls diag 308 subcode 1 and continues execution */ -SYM_FUNC_START(_diag308_reset_amode31) +SYM_TYPED_FUNC_START(_diag308_reset_amode31) larl %r4,ctlregs # Save control registers stctg %c0,%c15,0(%r4) lg %r2,0(%r4) # Disable lowcore protection diff --git a/arch/s390/kernel/topology.c b/arch/s390/kernel/topology.c index 1377c6f3f670..42fc0294f543 100644 --- a/arch/s390/kernel/topology.c +++ b/arch/s390/kernel/topology.c @@ -147,7 +147,7 @@ static void add_cpus_to_mask(struct topology_core *tl_core, cpumask_set_cpu(cpu, &book->mask); cpumask_set_cpu(cpu, &socket->mask); smp_cpu_set_polarization(cpu, tl_core->pp); - smp_cpu_set_capacity(cpu, CPU_CAPACITY_HIGH); + topology_set_cpu_scale(cpu, CPU_CAPACITY_HIGH); } } } diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index 564403496a7c..b6ba4465f59d 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -9,7 +9,9 @@ * Copyright (C) 1991, 1992 Linus Torvalds */ +#include <linux/capability.h> #include <linux/cpufeature.h> +#include <linux/debugfs.h> #include <linux/kprobes.h> #include <linux/kdebug.h> #include <linux/randomize_kstack.h> @@ -33,6 +35,12 @@ #include <asm/fault.h> #include "entry.h" +struct pgm_stat { + unsigned int count[128]; +}; + +static DEFINE_PER_CPU_SHARED_ALIGNED(struct pgm_stat, pgm_stat); + static inline void __user *get_trap_ip(struct pt_regs *regs) { unsigned long address; @@ -327,11 +335,12 @@ void __init trap_init(void) static void (*pgm_check_table[128])(struct pt_regs *regs); -void noinstr __do_pgm_check(struct pt_regs *regs) +void noinstr __do_pgm_check(struct pt_regs *regs, unsigned long flags) { struct lowcore *lc = get_lowcore(); bool percpu_needs_fixup; irqentry_state_t state; + struct pgm_stat *stat; unsigned int trapnr; union teid teid; @@ -339,6 +348,10 @@ void noinstr __do_pgm_check(struct pt_regs *regs) regs->int_code = lc->pgm_int_code; regs->int_parm_long = teid.val; regs->monitor_code = lc->monitor_code; + + trapnr = regs->int_code & PGM_INT_CODE_MASK; + stat = this_cpu_ptr(&pgm_stat); + stat->count[trapnr]++; /* * In case of a guest fault, short-circuit the fault handler and return. * This way the sie64a() function will return 0; fault address and @@ -346,7 +359,7 @@ void noinstr __do_pgm_check(struct pt_regs *regs) * the fault number in current->thread.gmap_int_code. KVM will be * able to use this information to handle the fault. */ - if (test_pt_regs_flag(regs, PIF_GUEST_FAULT)) { + if (flags & PGM_FLAG_GUEST_FAULT) { current->thread.gmap_teid.val = regs->int_parm_long; current->thread.gmap_int_code = regs->int_code & 0xffff; return; @@ -383,7 +396,6 @@ void noinstr __do_pgm_check(struct pt_regs *regs) if (!irqs_disabled_flags(regs->psw.mask)) trace_hardirqs_on(); __arch_local_irq_ssm(regs->psw.mask & ~PSW_MASK_PER); - trapnr = regs->int_code & PGM_INT_CODE_MASK; if (trapnr) pgm_check_table[trapnr](regs); out: @@ -393,6 +405,33 @@ out: percpu_exit(regs, percpu_needs_fixup); } +static int pgm_check_stat_show(struct seq_file *p, void *v) +{ + int i, cpu; + + cpus_read_lock(); + seq_puts(p, " "); + for_each_online_cpu(cpu) + seq_printf(p, "CPU%-8d", cpu); + seq_putc(p, '\n'); + for (i = 0; i < 128; i++) { + seq_printf(p, "%02x: ", i); + for_each_online_cpu(cpu) + seq_printf(p, "%10u ", per_cpu(pgm_stat, cpu).count[i]); + seq_putc(p, '\n'); + } + cpus_read_unlock(); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(pgm_check_stat); + +static int __init debugfs_pgm_check_init(void) +{ + debugfs_create_file("exceptions", 0400, arch_debugfs_dir, NULL, &pgm_check_stat_fops); + return 0; +} +late_initcall(debugfs_pgm_check_init); + /* * The program check table contains exactly 128 (0x00-0x7f) entries. Each * line defines the function to be called corresponding to the program check diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c index a284f98d9716..dc14ebc0105b 100644 --- a/arch/s390/kernel/uv.c +++ b/arch/s390/kernel/uv.c @@ -16,6 +16,7 @@ #include <linux/swap.h> #include <linux/pagewalk.h> #include <linux/backing-dev.h> +#include <linux/vmalloc.h> #include <asm/facility.h> #include <asm/sections.h> #include <asm/uv.h> @@ -209,6 +210,70 @@ int uv_convert_from_secure_pte(pte_t pte) return uv_convert_from_secure_folio(pfn_folio(pte_pfn(pte))); } +static int uv_free_range_cb(pte_t *ptep, unsigned long addr, void *data) +{ + pte_t pte = ptep_get(ptep); + + if (!pte_present(pte)) + return 0; + /* + * Note: do not update the pte here, since there is no code which + * accesses the memory range, besides bugs. The invalidation of ptes + * and TLB flushing is deferred like for regular vfree() calls. + */ + __free_page(pte_page(pte)); + return 0; +} + +void uv_free_stor_var(void *stor_var) +{ + unsigned long addr, size; + struct vm_struct *area; + + if (!stor_var) + return; + area = find_vm_area(stor_var); + if (WARN_ON_ONCE(!area || !(area->flags & VM_SPARSE))) + return; + size = get_vm_area_size(area); + addr = (unsigned long)area->addr; + apply_to_existing_page_range(&init_mm, addr, size, uv_free_range_cb, NULL); + free_vm_area(area); +} +EXPORT_SYMBOL_FOR_MODULES(uv_free_stor_var, "kvm"); + +static int uv_alloc_range_cb(pte_t *ptep, unsigned long addr, void *data) +{ + struct page *page; + pte_t pte; + + page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!page) + return -ENOMEM; + pte = __pte(page_to_phys(page) | pgprot_val(PAGE_KERNEL)); + set_pte(ptep, pte); + return 0; +} + +void *uv_alloc_stor_var(unsigned long size) +{ + struct vm_struct *area; + unsigned long addr; + + size = PAGE_ALIGN(size); + area = get_vm_area(size, VM_SPARSE); + if (!area) + return NULL; + addr = (unsigned long)area->addr; + if (apply_to_page_range(&init_mm, addr, size, uv_alloc_range_cb, NULL)) + goto out; + return area->addr; +out: + uv_free_stor_var(area->addr); + return NULL; +} +EXPORT_SYMBOL_FOR_MODULES(uv_alloc_stor_var, "kvm"); + /* * Calculate the expected ref_count for a folio that would otherwise have no * further pins. This was cribbed from similar functions in other places in @@ -831,7 +896,7 @@ int uv_retrieve_secret(u16 secret_idx, u8 *buf, size_t buf_size) .buf_size = buf_size, }; - uv_call_sched(0, (u64)&uvcb); + uv_call(0, (u64)&uvcb); switch (uvcb.header.rc) { case UVC_RC_EXECUTED: diff --git a/arch/s390/kernel/vdso/Makefile b/arch/s390/kernel/vdso/Makefile index fece5d975eaf..35c834b895ec 100644 --- a/arch/s390/kernel/vdso/Makefile +++ b/arch/s390/kernel/vdso/Makefile @@ -30,7 +30,8 @@ KBUILD_CFLAGS_VDSO := $(filter-out -fno-asynchronous-unwind-tables,$(KBUILD_CFLA KBUILD_CFLAGS_VDSO += -fPIC -fno-common -fno-builtin -fasynchronous-unwind-tables KBUILD_CFLAGS_VDSO += -fno-stack-protector $(DISABLE_KSTACK_ERASE) ldflags-y := -shared -soname=linux-vdso.so.1 \ - --hash-style=both --build-id=sha1 -T + --hash-style=both --build-id=sha1 \ + $(call ld-option, --eh-frame-hdr) -T $(targets:%=$(obj)/%.dbg): KBUILD_CFLAGS = $(KBUILD_CFLAGS_VDSO) $(targets:%=$(obj)/%.dbg): KBUILD_AFLAGS = $(KBUILD_AFLAGS_VDSO) diff --git a/arch/s390/kernel/vdso/vdso.lds.S b/arch/s390/kernel/vdso/vdso.lds.S index 7bec4de0e8e0..841daeec4be2 100644 --- a/arch/s390/kernel/vdso/vdso.lds.S +++ b/arch/s390/kernel/vdso/vdso.lds.S @@ -82,12 +82,15 @@ SECTIONS * We must supply the ELF program headers explicitly to get just one * PT_LOAD segment, and set the flags explicitly to make segments read-only. */ +#define PF_R FLAGS(4) +#define PF_RX FLAGS(5) + PHDRS { - text PT_LOAD FILEHDR PHDRS FLAGS(5); /* PF_R|PF_X */ - dynamic PT_DYNAMIC FLAGS(4); /* PF_R */ - note PT_NOTE FLAGS(4); /* PF_R */ - eh_frame_hdr PT_GNU_EH_FRAME; + text PT_LOAD PF_RX FILEHDR PHDRS; + dynamic PT_DYNAMIC PF_R; + note PT_NOTE PF_R; + eh_frame_hdr PT_GNU_EH_FRAME PF_R; } /* diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c index d804e1140c2e..efcbf406f03e 100644 --- a/arch/s390/kernel/vtime.c +++ b/arch/s390/kernel/vtime.c @@ -32,7 +32,7 @@ static atomic64_t virt_timer_elapsed; DEFINE_PER_CPU(u64, mt_cycles[8]); static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 }; static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 }; -static DEFINE_PER_CPU(u64, mt_scaling_jiffies); +static DEFINE_PER_CPU(unsigned long, mt_scaling_jiffies); static inline void set_vtimer(u64 expires) { @@ -81,7 +81,7 @@ static void update_mt_scaling(void) memcpy(cycles_old, cycles_new, sizeof(u64) * (smp_cpu_mtid + 1)); } - __this_cpu_write(mt_scaling_jiffies, jiffies_64); + __this_cpu_write(mt_scaling_jiffies, jiffies); } static inline u64 update_tsk_timer(unsigned long *tsk_vtime, u64 new) @@ -144,7 +144,7 @@ static int do_account_vtime(struct task_struct *tsk) lc->system_timer += timer; /* Update MT utilization calculation */ - if (smp_cpu_mtid && time_after64(jiffies_64, __this_cpu_read(mt_scaling_jiffies))) + if (smp_cpu_mtid && time_after(jiffies, __this_cpu_read(mt_scaling_jiffies))) update_mt_scaling(); /* Calculate cputime delta */ diff --git a/arch/s390/kvm/Makefile b/arch/s390/kvm/Makefile index dac9d53b23d8..a4e3875b5bdd 100644 --- a/arch/s390/kvm/Makefile +++ b/arch/s390/kvm/Makefile @@ -3,13 +3,4 @@ # # Copyright IBM Corp. 2008 -include $(srctree)/virt/kvm/Makefile.kvm - -ccflags-y := -Ivirt/kvm -Iarch/s390/kvm - -kvm-y += kvm-s390.o intercept.o interrupt.o priv.o sigp.o -kvm-y += diag.o gaccess.o guestdbg.o vsie.o pv.o -kvm-y += dat.o gmap.o faultin.o - -kvm-$(CONFIG_VFIO_PCI_ZDEV_KVM) += pci.o -obj-$(CONFIG_KVM) += kvm.o +obj-$(CONFIG_KVM) += s390/ diff --git a/arch/s390/kvm/gmap/Makefile b/arch/s390/kvm/gmap/Makefile new file mode 100644 index 000000000000..dd4ef062c536 --- /dev/null +++ b/arch/s390/kvm/gmap/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 + +GMAP ?= ../gmap + +gmap-y += $(GMAP)/dat.o $(GMAP)/gmap.o $(GMAP)/faultin.o $(GMAP)/kvm_mmu.o diff --git a/arch/s390/kvm/dat.c b/arch/s390/kvm/gmap/dat.c index 3f2d6e8902d7..24547e39fab2 100644 --- a/arch/s390/kvm/dat.c +++ b/arch/s390/kvm/gmap/dat.c @@ -613,6 +613,7 @@ long _dat_walk_gfn_range(gfn_t start, gfn_t end, union asce asce, return dat_crste_walk_range(start, min(end, asce_end(asce)), table, &walk); } +#if KVM_S390_MANAGES_S390_GUEST int dat_get_storage_key(union asce asce, gfn_t gfn, union skey *skey) { union crste *crstep; @@ -722,9 +723,12 @@ int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gf if (rc) return rc; - if (!ptep) + if (!ptep) { + if (!oldkey) + oldkey = &prev; return page_cond_set_storage_key(large_crste_to_phys(*crstep, gfn), skey, oldkey, nq, mr, mc); + } old = pgste_get_lock(ptep); pgste = old; @@ -734,6 +738,7 @@ int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gf pgste.fp = skey.fp; pgste.gc = skey.c; pgste.gr = skey.r; + prev.skey = 0; if (!ptep->h.i) { rc = page_cond_set_storage_key(pte_origin(*ptep), skey, &prev, nq, mr, mc); @@ -839,6 +844,7 @@ long dat_reset_skeys(union asce asce, gfn_t start) return _dat_walk_gfn_range(start, asce_end(asce), asce, &ops, DAT_WALK_IGN_HOLES, NULL); } +#endif /* KVM_S390_MANAGES_S390_GUEST */ struct slot_priv { unsigned long token; @@ -920,11 +926,8 @@ static void pgste_set_unlock_multiple(union pte *first, int n, union pgste *pgst { int i; - for (i = 0; i < n; i++) { - if (!pgstes[i].pcl) - break; + for (i = 0; i < n; i++) pgste_set_unlock(first + i, pgstes[i]); - } } static bool pgste_get_trylock_multiple(union pte *first, int n, union pgste *pgstes) @@ -937,7 +940,7 @@ static bool pgste_get_trylock_multiple(union pte *first, int n, union pgste *pgs } if (i == n) return true; - pgste_set_unlock_multiple(first, n, pgstes); + pgste_set_unlock_multiple(first, i, pgstes); return false; } @@ -1009,6 +1012,7 @@ bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end) return _dat_walk_gfn_range(start, end, asce, &test_age_ops, 0, NULL) > 0; } +#if KVM_S390_MANAGES_S390_GUEST static long dat_set_pn_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) { union crste newcrste, oldcrste; @@ -1334,3 +1338,4 @@ int dat_set_cmma_bits(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn, } return _dat_walk_gfn_range(gfn, gfn + count, asce, &ops, DAT_WALK_IGN_HOLES, &state); } +#endif /* KVM_S390_MANAGES_S390_GUEST */ diff --git a/arch/s390/kvm/dat.h b/arch/s390/kvm/gmap/dat.h index 141ee7b9f019..e452c141b841 100644 --- a/arch/s390/kvm/dat.h +++ b/arch/s390/kvm/gmap/dat.h @@ -6,9 +6,10 @@ * Author(s): Claudio Imbrenda <imbrenda@linux.ibm.com> */ -#ifndef __KVM_S390_DAT_H -#define __KVM_S390_DAT_H +#ifndef ARCH_KVM_GMAP_DAT_H +#define ARCH_KVM_GMAP_DAT_H +#include <linux/kvm_host.h> #include <linux/radix-tree.h> #include <linux/refcount.h> #include <linux/io.h> @@ -532,6 +533,8 @@ int dat_entry_walk(struct kvm_s390_mmu_cache *mc, gfn_t gfn, union asce asce, in void dat_free_level(struct crst_table *table, bool owns_ptes); struct crst_table *dat_alloc_crst_sleepable(unsigned long init); int dat_set_asce_limit(struct kvm_s390_mmu_cache *mc, union asce *asce, int newtype); + +#if KVM_S390_MANAGES_S390_GUEST int dat_get_storage_key(union asce asce, gfn_t gfn, union skey *skey); int dat_set_storage_key(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn, union skey skey, bool nq); @@ -539,21 +542,33 @@ int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gf union skey skey, union skey *oldkey, bool nq, bool mr, bool mc); int dat_reset_reference_bit(union asce asce, gfn_t gfn, union skey *skey); long dat_reset_skeys(union asce asce, gfn_t start); +#endif /* KVM_S390_MANAGES_S390_GUEST */ unsigned long dat_get_ptval(struct page_table *table, struct ptval_param param); void dat_set_ptval(struct page_table *table, struct ptval_param param, unsigned long val); int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end, u16 type, u16 param); + +#if KVM_S390_MANAGES_S390_GUEST int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn); +#else +static inline int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn) +{ + return 0; +} +#endif /* KVM_S390_MANAGES_S390_GUEST */ + bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end); +#if KVM_S390_MANAGES_S390_GUEST int dat_perform_essa(union asce asce, gfn_t gfn, int orc, union essa_state *state, bool *dirty); long dat_reset_cmma(union asce asce, gfn_t start_gfn); int dat_peek_cmma(gfn_t start, union asce asce, unsigned int *count, u8 *values); int dat_get_cmma(union asce asce, gfn_t *start, unsigned int *count, u8 *values, atomic64_t *rem); int dat_set_cmma_bits(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn, unsigned long count, unsigned long mask, const uint8_t *bits); +#endif /* KVM_S390_MANAGES_S390_GUEST */ int kvm_s390_mmu_cache_topup(struct kvm_s390_mmu_cache *mc); @@ -975,4 +990,4 @@ static inline bool crste_is_ucas(union crste crste) return is_pmd(crste) && crste.h.i && crste.h.fc0.tl == 1 && crste.h.fc == 0; } -#endif /* __KVM_S390_DAT_H */ +#endif /* ARCH_KVM_GMAP_DAT_H */ diff --git a/arch/s390/kvm/faultin.c b/arch/s390/kvm/gmap/faultin.c index 3cc45f7f5b2d..1dc79807012c 100644 --- a/arch/s390/kvm/faultin.c +++ b/arch/s390/kvm/gmap/faultin.c @@ -9,10 +9,11 @@ #include <linux/kvm_host.h> #include "gmap.h" -#include "trace.h" #include "faultin.h" bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu); +#define CREATE_TRACE_POINTS +#include "trace_gmap.h" /* * kvm_s390_faultin_gfn() - handle a dat fault. diff --git a/arch/s390/kvm/faultin.h b/arch/s390/kvm/gmap/faultin.h index f86176d2769c..f343b6fb6f16 100644 --- a/arch/s390/kvm/faultin.h +++ b/arch/s390/kvm/gmap/faultin.h @@ -6,8 +6,8 @@ * Author(s): Claudio Imbrenda <imbrenda@linux.ibm.com> */ -#ifndef __KVM_S390_FAULTIN_H -#define __KVM_S390_FAULTIN_H +#ifndef ARCH_KVM_GMAP_FAULTIN_H +#define ARCH_KVM_GMAP_FAULTIN_H #include <linux/kvm_host.h> @@ -89,4 +89,4 @@ static inline int kvm_s390_get_guest_pages(struct kvm *kvm, struct guest_fault * #define kvm_s390_array_needs_retry_safe(kvm, seq, array) \ kvm_s390_multiple_faults_need_retry(kvm, seq, array, ARRAY_SIZE(array), false) -#endif /* __KVM_S390_FAULTIN_H */ +#endif /* ARCH_KVM_GMAP_FAULTIN_H */ diff --git a/arch/s390/kvm/gmap.c b/arch/s390/kvm/gmap/gmap.c index 8abb4f55b306..4968330e9553 100644 --- a/arch/s390/kvm/gmap.c +++ b/arch/s390/kvm/gmap/gmap.c @@ -21,14 +21,9 @@ #include "dat.h" #include "gmap.h" -#include "kvm-s390.h" +#include "s390.h" #include "faultin.h" -static inline bool kvm_s390_is_in_sie(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.sie_block->prog0c & PROG_IN_SIE; -} - static int gmap_limit_to_type(gfn_t limit) { if (!limit) @@ -256,6 +251,12 @@ int s390_replace_asce(struct gmap *gmap) return 0; } +#if KVM_S390_MANAGES_S390_GUEST +static inline bool kvm_s390_is_in_sie(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.sie_block->prog0c & PROG_IN_SIE; +} + bool _gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end, bool hint) { struct kvm *kvm = gmap->kvm; @@ -278,6 +279,7 @@ bool _gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end, bool hint) } return true; } +#endif /* KVM_S390_MANAGES_S390_GUEST */ struct clear_young_pte_priv { struct gmap *gmap; @@ -941,10 +943,11 @@ void gmap_split_huge_pages(struct gmap *gmap) scoped_guard(read_lock, &gmap->kvm->mmu_lock) start = _dat_walk_gfn_range(start, asce_end(gmap->asce), gmap->asce, &ops, DAT_WALK_IGN_HOLES, gmap); - cond_resched(); } while (start); } +#if KVM_S390_MANAGES_S390_GUEST + static int _gmap_enable_skeys(struct gmap *gmap) { gfn_t start = 0; @@ -963,7 +966,6 @@ static int _gmap_enable_skeys(struct gmap *gmap) do { scoped_guard(write_lock, &gmap->kvm->mmu_lock) start = dat_reset_skeys(gmap->asce, start); - cond_resched(); } while (start); return 0; } @@ -977,6 +979,7 @@ int gmap_enable_skeys(struct gmap *gmap) mmap_write_unlock(gmap->kvm->mm); return rc; } +#endif /* KVM_S390_MANAGES_S390_GUEST */ static long _destroy_pages_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) { @@ -1019,7 +1022,6 @@ int gmap_pv_destroy_range(struct gmap *gmap, gfn_t start, gfn_t end, bool interr DAT_WALK_IGN_HOLES, NULL); if (interruptible && fatal_signal_pending(current)) return -EINTR; - cond_resched(); } while (start && start < end); return 0; } @@ -1098,6 +1100,7 @@ int gmap_protect_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gf return 0; } +#if KVM_S390_MANAGES_S390_GUEST static long __set_cmma_clean_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) { union pgste pgste; @@ -1138,9 +1141,9 @@ void _gmap_set_cmma_all(struct gmap *gmap, bool dirty) gfn = _dat_walk_gfn_range(gfn, asce_end(gmap->asce), gmap->asce, &ops, DAT_WALK_IGN_HOLES, &gmap->kvm->arch.cmma_dirty_pages); - cond_resched(); } while (gfn); } +#endif /* KVM_S390_MANAGES_S390_GUEST */ static void gmap_unshadow_level(struct gmap *sg, gfn_t r_gfn, int level) { diff --git a/arch/s390/kvm/gmap.h b/arch/s390/kvm/gmap/gmap.h index 39938d363ec9..8f47f29c89fd 100644 --- a/arch/s390/kvm/gmap.h +++ b/arch/s390/kvm/gmap/gmap.h @@ -7,8 +7,10 @@ * Claudio Imbrenda <imbrenda@linux.ibm.com> */ -#ifndef ARCH_KVM_S390_GMAP_H -#define ARCH_KVM_S390_GMAP_H +#ifndef ARCH_KVM_GMAP_GMAP_H +#define ARCH_KVM_GMAP_GMAP_H + +#include <linux/kvm_host.h> #include "dat.h" @@ -83,7 +85,6 @@ struct gmap_cache { for (pos = (head); n = pos ? pos->next : NULL, pos; pos = n) int s390_replace_asce(struct gmap *gmap); -bool _gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end, bool hint); bool gmap_age_gfn(struct gmap *gmap, gfn_t start, gfn_t end); bool gmap_unmap_gfn_range(struct gmap *gmap, struct kvm_memory_slot *slot, gfn_t start, gfn_t end); int gmap_try_fixup_minor(struct gmap *gmap, struct guest_fault *fault); @@ -98,13 +99,16 @@ int gmap_set_limit(struct gmap *gmap, gfn_t limit); int gmap_ucas_translate(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, gpa_t *gaddr); int gmap_ucas_map(struct gmap *gmap, gfn_t p_gfn, gfn_t c_gfn, unsigned long count); void gmap_ucas_unmap(struct gmap *gmap, gfn_t c_gfn, unsigned long count); + +#if KVM_S390_MANAGES_S390_GUEST int gmap_enable_skeys(struct gmap *gmap); +#endif /* KVM_S390_MANAGES_S390_GUEST */ + int gmap_pv_destroy_range(struct gmap *gmap, gfn_t start, gfn_t end, bool interruptible); int gmap_insert_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn, int level); int gmap_protect_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn, kvm_pfn_t pfn, int level, bool wr); -void _gmap_set_cmma_all(struct gmap *gmap, bool dirty); void _gmap_handle_vsie_unshadow_event(struct gmap *parent, gfn_t gfn); struct gmap *gmap_create_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, union asce asce, int edat_level); @@ -158,6 +162,14 @@ static inline void gmap_handle_vsie_unshadow_event(struct gmap *parent, gfn_t gf _gmap_handle_vsie_unshadow_event(parent, gfn); } +#if KVM_S390_MANAGES_S390_GUEST +bool _gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end, bool hint); +#else +static inline bool _gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end, bool hint) +{ + return true; +} +#endif /* KVM_S390_MANAGES_S390_GUEST */ static inline bool gmap_mkold_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end) { return _gmap_unmap_prefix(gmap, gfn, end, true); @@ -198,6 +210,8 @@ static inline bool pte_needs_unshadow(union pte oldpte, union pte newpte, union return !newpte.h.p || !newpte.s.pr; } +#if KVM_S390_MANAGES_S390_GUEST +void _gmap_set_cmma_all(struct gmap *gmap, bool dirty); static inline void gmap_set_cmma_all_dirty(struct gmap *gmap) { _gmap_set_cmma_all(gmap, true); @@ -207,6 +221,7 @@ static inline void gmap_set_cmma_all_clean(struct gmap *gmap) { _gmap_set_cmma_all(gmap, false); } +#endif /* KVM_S390_MANAGES_S390_GUEST */ static inline union pgste _gmap_ptep_xchg(struct gmap *gmap, union pte *ptep, union pte newpte, union pgste pgste, gfn_t gfn, bool needs_lock) @@ -330,4 +345,4 @@ static inline bool gmap_is_shadow_valid(struct gmap *sg, union asce asce, int ed return sg->guest_asce.val == asce.val && sg->edat_level == edat_level; } -#endif /* ARCH_KVM_S390_GMAP_H */ +#endif /* ARCH_KVM_GMAP_GMAP_H */ diff --git a/arch/s390/kvm/gmap/kvm_mmu.c b/arch/s390/kvm/gmap/kvm_mmu.c new file mode 100644 index 000000000000..b08b8229bb6f --- /dev/null +++ b/arch/s390/kvm/gmap/kvm_mmu.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/kvm_types.h> +#include <linux/kvm_host.h> + +#include "s390.h" +#include "gmap.h" +#include "dat.h" +#include "kvm_mmu.h" + +/* + * Get (and clear) the dirty memory log for a memory slot. + */ +int s390_kvm_mmu_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log) +{ + int r; + unsigned long n; + struct kvm_memory_slot *memslot; + int is_dirty; + + if (kvm_is_ucontrol(kvm)) + return -EINVAL; + + mutex_lock(&kvm->slots_lock); + + r = -EINVAL; + if (log->slot >= KVM_USER_MEM_SLOTS) + goto out; + + r = kvm_get_dirty_log(kvm, log, &is_dirty, &memslot); + if (r) + goto out; + + /* Clear the dirty log */ + if (is_dirty) { + n = kvm_dirty_bitmap_bytes(memslot); + memset(memslot->dirty_bitmap, 0, n); + } + r = 0; +out: + mutex_unlock(&kvm->slots_lock); + return r; +} + +int s390_kvm_mmu_prepare_memory_region(struct kvm *kvm, + const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ + if (kvm_is_ucontrol(kvm) && new && new->id < KVM_USER_MEM_SLOTS) + return -EINVAL; + + /* When we are protected, we should not change the memory slots */ + if (kvm_s390_pv_get_handle(kvm)) + return -EINVAL; + + if (change != KVM_MR_DELETE && change != KVM_MR_FLAGS_ONLY) { + /* + * A few sanity checks. The memory in userland is ok to be + * fragmented into various different vmas. It is okay to mmap() + * and munmap() stuff in this slot after doing this call at any + * time. + */ + if (new->userspace_addr & ~PAGE_MASK) + return -EINVAL; + if ((new->base_gfn + new->npages) * PAGE_SIZE > kvm->arch.mem_limit) + return -EINVAL; + if (!asce_contains_gfn(kvm->arch.gmap->asce, new->base_gfn + new->npages - 1)) + return -EINVAL; + } + + if (!kvm_s390_is_migration_mode(kvm)) + return 0; + + /* + * Turn off migration mode when: + * - userspace creates a new memslot with dirty logging off, + * - userspace modifies an existing memslot (MOVE or FLAGS_ONLY) and + * dirty logging is turned off. + * Migration mode expects dirty page logging being enabled to store + * its dirty bitmap. + */ + if (change != KVM_MR_DELETE && + !(new->flags & KVM_MEM_LOG_DIRTY_PAGES)) + WARN(kvm_s390_vm_stop_migration(kvm), + "Failed to stop migration mode"); + + return 0; +} + +void s390_kvm_mmu_commit_memory_region(struct kvm *kvm, + struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ + struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL; + int rc = 0; + + guard(mutex)(&kvm->slots_arch_lock); + + if (change == KVM_MR_FLAGS_ONLY) + return; + + mc = kvm_s390_new_mmu_cache(); + if (!mc) { + rc = -ENOMEM; + goto out; + } + + scoped_guard(write_lock, &kvm->mmu_lock) { + kvm_s390_update_cmma_dirty(kvm, old); + switch (change) { + case KVM_MR_DELETE: + rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages); + break; + case KVM_MR_MOVE: + rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages); + if (rc) + break; + fallthrough; + case KVM_MR_CREATE: + rc = dat_create_slot(mc, kvm->arch.gmap->asce, new->base_gfn, new->npages); + break; + case KVM_MR_FLAGS_ONLY: + break; + default: + WARN(1, "Unknown KVM MR CHANGE: %d\n", change); + } + } +out: + if (rc) + pr_warn("failed to commit memory region\n"); +} diff --git a/arch/s390/kvm/gmap/kvm_mmu.h b/arch/s390/kvm/gmap/kvm_mmu.h new file mode 100644 index 000000000000..cdbd390bd33c --- /dev/null +++ b/arch/s390/kvm/gmap/kvm_mmu.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef ARCH_KVM_GMAP_KVM_MMU_H +#define ARCH_KVM_GMAP_KVM_MMU_H + +#include <linux/kvm_host.h> + +int s390_kvm_mmu_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log); +int s390_kvm_mmu_prepare_memory_region(struct kvm *kvm, + const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, + enum kvm_mr_change change); +void s390_kvm_mmu_commit_memory_region(struct kvm *kvm, + struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change); + +#endif /* ARCH_KVM_GMAP_KVM_MMU_H */ diff --git a/arch/s390/kvm/gmap/trace_gmap.h b/arch/s390/kvm/gmap/trace_gmap.h new file mode 100644 index 000000000000..15ec88bdce55 --- /dev/null +++ b/arch/s390/kvm/gmap/trace_gmap.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#if !defined(GMAP_TRACE_KVM_H) || defined(TRACE_HEADER_MULTI_READ) +#define GMAP_TRACE_KVM_H + +#include <linux/tracepoint.h> + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM kvm +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH ../gmap +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace_gmap + +#define __KVM_FIELDS \ + __field(unsigned long, pswmask) \ + __field(unsigned long, pswaddr) +#define __KVM_ASSIGN ({\ + __entry->pswmask = vcpu->arch.sie_block->gpsw.mask; \ + __entry->pswaddr = vcpu->arch.sie_block->gpsw.addr; \ + }) +#define __KVM_PRINT \ + __entry->pswmask,\ + __entry->pswaddr + +TRACE_EVENT(kvm_s390_major_guest_pfault, + TP_PROTO(struct kvm_vcpu *vcpu), + TP_ARGS(vcpu), + + TP_STRUCT__entry( + __field(int, id) + __KVM_FIELDS + ), + + TP_fast_assign( + __entry->id = vcpu->vcpu_id; + __KVM_ASSIGN + ), + TP_printk("%02d[%016lx-%016lx]: major fault, maybe applicable for pfault", + __entry->id, + __KVM_PRINT + ) + ); + +#endif /* GMAP_TRACE_KVM_H */ + +/* This part must be outside protection */ +#include <trace/define_trace.h> diff --git a/arch/s390/kvm/s390/Makefile b/arch/s390/kvm/s390/Makefile new file mode 100644 index 000000000000..762a63826423 --- /dev/null +++ b/arch/s390/kvm/s390/Makefile @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0 + +KVM := ../../../../virt/kvm +include $(srctree)/virt/kvm/Makefile.kvm +include $(srctree)/arch/s390/kvm/gmap/Makefile + +ccflags-y := -I$(src) -I$(srctree)/arch/s390/kvm/gmap + +kvm-y += s390.o intercept.o interrupt.o priv.o sigp.o +kvm-y += diag.o gaccess.o guestdbg.o vsie.o pv.o +kvm-y += $(gmap-y) + +kvm-$(CONFIG_VFIO_PCI_ZDEV_KVM) += pci.o +obj-$(CONFIG_KVM) += kvm.o diff --git a/arch/s390/kvm/diag.c b/arch/s390/kvm/s390/diag.c index d89d1c381522..09033c8ff1bb 100644 --- a/arch/s390/kvm/diag.c +++ b/arch/s390/kvm/s390/diag.c @@ -12,7 +12,7 @@ #include <linux/kvm_host.h> #include <asm/gmap_helpers.h> #include <asm/virtio-ccw.h> -#include "kvm-s390.h" +#include "s390.h" #include "trace.h" #include "trace-s390.h" #include "gaccess.h" @@ -186,7 +186,8 @@ static int diag9c_forwarding_overrun(void) static int __diag_time_slice_end_directed(struct kvm_vcpu *vcpu) { struct kvm_vcpu *tcpu; - int tcpu_cpu; + const char *result; + int tcpu_cpu = -1; int tid; tid = vcpu->run->s.regs.gprs[(vcpu->arch.sie_block->ipa & 0xf0) >> 4]; @@ -211,21 +212,23 @@ static int __diag_time_slice_end_directed(struct kvm_vcpu *vcpu) if (!vcpu_is_preempted(tcpu_cpu)) goto no_yield; smp_yield_cpu(tcpu_cpu); - VCPU_EVENT(vcpu, 5, - "diag time slice end directed to %d: yield forwarded", - tid); vcpu->stat.diag_9c_forward++; - return 0; + result = "yield forwarded"; + goto out; } if (kvm_vcpu_yield_to(tcpu) <= 0) goto no_yield; - VCPU_EVENT(vcpu, 5, "diag time slice end directed to %d: done", tid); - return 0; + result = "done"; + goto out; no_yield: - VCPU_EVENT(vcpu, 5, "diag time slice end directed to %d: ignored", tid); vcpu->stat.diag_9c_ignored++; + result = "ignored"; +out: + VCPU_EVENT(vcpu, 5, "diag time slice end directed to %d: %s", tid, + result); + trace_kvm_s390_diag_9c(vcpu, tid, tcpu_cpu, result); return 0; } diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/s390/gaccess.c index 36102b2727fb..e5c064f263df 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/s390/gaccess.c @@ -17,7 +17,7 @@ #include <asm/access-regs.h> #include <asm/fault.h> #include <asm/dat-bits.h> -#include "kvm-s390.h" +#include "s390.h" #include "dat.h" #include "gmap.h" #include "gaccess.h" @@ -151,10 +151,8 @@ retry: ic = &kvm->arch.sca->ipte_control; old = READ_ONCE(*ic); do { - if (old.k) { - cond_resched(); + if (old.k) goto retry; - } new = old; new.k = 1; } while (!try_cmpxchg(&ic->val, &old.val, new.val)); @@ -189,10 +187,8 @@ retry: ic = &kvm->arch.sca->ipte_control; old = READ_ONCE(*ic); do { - if (old.kg) { - cond_resched(); + if (old.kg) goto retry; - } new = old; new.k = 1; new.kh++; diff --git a/arch/s390/kvm/gaccess.h b/arch/s390/kvm/s390/gaccess.h index b5385cec60f4..ef922b3b4990 100644 --- a/arch/s390/kvm/gaccess.h +++ b/arch/s390/kvm/s390/gaccess.h @@ -14,7 +14,7 @@ #include <linux/kvm_host.h> #include <linux/uaccess.h> #include <linux/ptrace.h> -#include "kvm-s390.h" +#include "s390.h" /** * kvm_s390_real_to_abs - convert guest real address to guest absolute address diff --git a/arch/s390/kvm/guestdbg.c b/arch/s390/kvm/s390/guestdbg.c index 69835e1d4f20..1bf7e91b7e61 100644 --- a/arch/s390/kvm/guestdbg.c +++ b/arch/s390/kvm/s390/guestdbg.c @@ -8,7 +8,7 @@ */ #include <linux/kvm_host.h> #include <linux/errno.h> -#include "kvm-s390.h" +#include "s390.h" #include "gaccess.h" /* @@ -184,7 +184,7 @@ static int __import_wp_info(struct kvm_vcpu *vcpu, if (wp_info->len < 0 || wp_info->len > MAX_WP_SIZE) return -EINVAL; - wp_info->old_data = kmalloc(bp_data->len, GFP_KERNEL_ACCOUNT); + wp_info->old_data = kmalloc(wp_info->len, GFP_KERNEL_ACCOUNT); if (!wp_info->old_data) return -ENOMEM; /* try to backup the original value */ @@ -252,7 +252,7 @@ int kvm_s390_import_bp_data(struct kvm_vcpu *vcpu, ret = __import_wp_info(vcpu, &bp_data[i], &wp_info[nr_wp]); if (ret) - goto error; + goto error_wp; nr_wp++; break; case KVM_HW_BP: @@ -267,7 +267,12 @@ int kvm_s390_import_bp_data(struct kvm_vcpu *vcpu, vcpu->arch.guestdbg.hw_bp_info = bp_info; vcpu->arch.guestdbg.nr_hw_wp = nr_wp; vcpu->arch.guestdbg.hw_wp_info = wp_info; + kfree(bp_data); return 0; + +error_wp: + while (nr_wp--) + kfree(wp_info[nr_wp].old_data); error: kfree(bp_data); kfree(wp_info); diff --git a/arch/s390/kvm/intercept.c b/arch/s390/kvm/s390/intercept.c index 1980df61ef30..ca1205dfac8b 100644 --- a/arch/s390/kvm/intercept.c +++ b/arch/s390/kvm/s390/intercept.c @@ -17,7 +17,7 @@ #include <asm/sysinfo.h> #include <asm/uv.h> -#include "kvm-s390.h" +#include "s390.h" #include "gaccess.h" #include "trace.h" #include "trace-s390.h" diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/s390/interrupt.c index 8f24bcd1a6d3..0381ae981703 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/s390/interrupt.c @@ -29,7 +29,7 @@ #include <asm/nmi.h> #include <asm/airq.h> #include <asm/tpi.h> -#include "kvm-s390.h" +#include "s390.h" #include "gaccess.h" #include "trace-s390.h" #include "pci.h" @@ -273,6 +273,11 @@ static inline int gisa_tac_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc) return test_and_clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa); } +static inline int gisa_test_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc) +{ + return test_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *)gisa); +} + static inline unsigned long pending_irqs_no_gisa(struct kvm_vcpu *vcpu) { unsigned long pending = vcpu->kvm->arch.float_int.pending_irqs | @@ -1550,23 +1555,21 @@ static int __inject_set_prefix(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) } #define KVM_S390_STOP_SUPP_FLAGS (KVM_S390_STOP_FLAG_STORE_STATUS) -static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) +static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq, bool *storestatus) { struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; struct kvm_s390_stop_info *stop = &li->irq.stop; - int rc = 0; vcpu->stat.inject_stop_signal++; trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_STOP, 0, 0); if (irq->u.stop.flags & ~KVM_S390_STOP_SUPP_FLAGS) return -EINVAL; - if (is_vcpu_stopped(vcpu)) { - if (irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS) - rc = kvm_s390_store_status_unloaded(vcpu, - KVM_S390_STORE_STATUS_NOADDR); - return rc; + if (!(irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS)) + return 0; + *storestatus = true; + return -EWOULDBLOCK; } if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs)) @@ -2102,7 +2105,7 @@ void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu) spin_unlock(&li->lock); } -static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) +static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq, bool *storestatus) { int rc; @@ -2114,7 +2117,7 @@ static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) rc = __inject_set_prefix(vcpu, irq); break; case KVM_S390_SIGP_STOP: - rc = __inject_sigp_stop(vcpu, irq); + rc = __inject_sigp_stop(vcpu, irq, storestatus); break; case KVM_S390_RESTART: rc = __inject_sigp_restart(vcpu); @@ -2150,11 +2153,16 @@ static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) { struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; + bool storestatus = false; int rc; spin_lock(&li->lock); - rc = do_inject_vcpu(vcpu, irq); + rc = do_inject_vcpu(vcpu, irq, &storestatus); spin_unlock(&li->lock); + + if (rc == -EWOULDBLOCK && storestatus) + rc = kvm_s390_store_status_unloaded(vcpu, KVM_S390_STORE_STATUS_NOADDR); + if (!rc) kvm_s390_vcpu_wakeup(vcpu); return rc; @@ -2242,7 +2250,7 @@ static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) ret = -ENOMEM; goto out_nolock; } - if (gisa_tac_ipm_gisc(gi->origin, i)) { + if (gisa_test_ipm_gisc(gi->origin, i)) { irq = (struct kvm_s390_irq *) &buf[n]; irq->type = KVM_S390_INT_IO(1, 0, 0, 0); irq->u.io.io_int_word = isc_to_int_word(i); @@ -3013,7 +3021,7 @@ static int adapter_indicators_set(struct kvm *kvm, if (!summary_info) { spin_unlock_irqrestore(&adapter->maps_lock, flags); summary_page = pin_map_page(kvm, adapter_int->summary_addr, 0); - if (WARN_ON_ONCE(!summary_page)) + if (!summary_page) return -1; idx = srcu_read_lock(&kvm->srcu); map = page_address(summary_page); @@ -3108,9 +3116,7 @@ static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e, void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu, struct mcck_volatile_info *mcck_info) { - struct kvm_s390_interrupt_info inti; - struct kvm_s390_irq irq; - struct kvm_s390_mchk_info *mchk; + struct kvm_s390_irq irq = {}; union mci mci; __u64 cr14 = 0; /* upper bits are not used */ int rc; @@ -3129,20 +3135,14 @@ void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu, if (mci.w) cr14 |= CR14_WARNING_SUBMASK; - mchk = mci.ck ? &inti.mchk : &irq.u.mchk; - mchk->cr14 = cr14; - mchk->mcic = mcck_info->mcic; - mchk->ext_damage_code = mcck_info->ext_damage_code; - mchk->failing_storage_address = mcck_info->failing_storage_address; - if (mci.ck) { - /* Inject the floating machine check */ - inti.type = KVM_S390_MCHK; - rc = __inject_vm(vcpu->kvm, &inti); - } else { - /* Inject the machine check to specified vcpu */ - irq.type = KVM_S390_MCHK; - rc = kvm_s390_inject_vcpu(vcpu, &irq); - } + irq.u.mchk.cr14 = cr14; + irq.u.mchk.mcic = mcck_info->mcic; + irq.u.mchk.ext_damage_code = mcck_info->ext_damage_code; + irq.u.mchk.failing_storage_address = mcck_info->failing_storage_address; + + /* Inject the machine check to specified vcpu */ + irq.type = KVM_S390_MCHK; + rc = kvm_s390_inject_vcpu(vcpu, &irq); WARN_ON_ONCE(rc); } @@ -3197,7 +3197,8 @@ int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len) { struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; - struct kvm_s390_irq *buf; + struct kvm_s390_irq *buf __free(kvfree) = NULL; + bool tmp, storestatus = false; int r = 0; int n; @@ -3205,31 +3206,33 @@ int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len if (!buf) return -ENOMEM; - if (copy_from_user((void *) buf, irqstate, len)) { - r = -EFAULT; - goto out_free; - } + if (copy_from_user((void *)buf, irqstate, len)) + return -EFAULT; - /* - * Don't allow setting the interrupt state - * when there are already interrupts pending - */ - spin_lock(&li->lock); - if (li->pending_irqs) { - r = -EBUSY; - goto out_unlock; - } + scoped_guard(spinlock, &li->lock) { + /* + * Don't allow setting the interrupt state + * when there are already interrupts pending + */ + if (li->pending_irqs) + return -EBUSY; - for (n = 0; n < len / sizeof(*buf); n++) { - r = do_inject_vcpu(vcpu, &buf[n]); - if (r) - break; + for (n = 0; n < len / sizeof(*buf); n++) { + tmp = false; + r = do_inject_vcpu(vcpu, &buf[n], &tmp); + if (r == -EWOULDBLOCK && tmp) { + storestatus = true; + r = 0; + } + if (r) + break; + } } -out_unlock: - spin_unlock(&li->lock); -out_free: - vfree(buf); + if (storestatus) { + n = kvm_s390_store_status_unloaded(vcpu, KVM_S390_STORE_STATUS_NOADDR); + return r ? r : n; + } return r; } diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/s390/pci.c index 50f5ec79600e..82892e1e03d9 100644 --- a/arch/s390/kvm/pci.c +++ b/arch/s390/kvm/s390/pci.c @@ -14,7 +14,7 @@ #include <asm/pci_io.h> #include <asm/sclp.h> #include "pci.h" -#include "kvm-s390.h" +#include "s390.h" struct zpci_aift *aift; diff --git a/arch/s390/kvm/pci.h b/arch/s390/kvm/s390/pci.h index fdf8c7bf4ed0..fdf8c7bf4ed0 100644 --- a/arch/s390/kvm/pci.h +++ b/arch/s390/kvm/s390/pci.h diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/s390/priv.c index b1ba24c346ef..b3cb2c2c3aa7 100644 --- a/arch/s390/kvm/priv.c +++ b/arch/s390/kvm/s390/priv.c @@ -26,7 +26,7 @@ #include <asm/ap.h> #include <asm/gmap_helpers.h> #include "gaccess.h" -#include "kvm-s390.h" +#include "s390.h" #include "trace.h" #include "gmap.h" diff --git a/arch/s390/kvm/pv.c b/arch/s390/kvm/s390/pv.c index b02e0159d3cd..b18abd0e29ef 100644 --- a/arch/s390/kvm/pv.c +++ b/arch/s390/kvm/s390/pv.c @@ -18,7 +18,7 @@ #include <linux/sched/mm.h> #include <linux/mmu_notifier.h> #include <asm/gmap_helpers.h> -#include "kvm-s390.h" +#include "s390.h" #include "dat.h" #include "gaccess.h" #include "gmap.h" @@ -242,6 +242,10 @@ static void kvm_s390_clear_pv_state(struct kvm *kvm) kvm->arch.pv.guest_len = 0; kvm->arch.pv.stor_base = 0; kvm->arch.pv.stor_var = NULL; + if (kvm->arch.pv.dumping) { + kvm_s390_vcpu_unblock_all(kvm); + kvm->arch.pv.dumping = false; + } } static void kvm_s390_pv_dispose_cpu(struct kvm_vcpu *vcpu, bool free_stor_base) @@ -338,7 +342,7 @@ int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) /* only free resources when the destroy was successful */ static void kvm_s390_pv_dealloc_vm(struct kvm *kvm) { - vfree(kvm->arch.pv.stor_var); + uv_free_stor_var(kvm->arch.pv.stor_var); free_pages(kvm->arch.pv.stor_base, get_order(uv_info.guest_base_stor_len)); kvm_s390_clear_pv_state(kvm); @@ -370,7 +374,7 @@ static int kvm_s390_pv_alloc_vm(struct kvm *kvm) /* Allocate variable storage */ vlen = ALIGN(virt * ((npages * PAGE_SIZE) / HPAGE_SIZE), PAGE_SIZE); vlen += uv_info.guest_virt_base_stor_len; - kvm->arch.pv.stor_var = vzalloc(vlen); + kvm->arch.pv.stor_var = uv_alloc_stor_var(vlen); if (!kvm->arch.pv.stor_var) goto out_err; return 0; @@ -415,7 +419,7 @@ static int kvm_s390_pv_dispose_one_leftover(struct kvm *kvm, */ free_pages(leftover->stor_base, get_order(uv_info.guest_base_stor_len)); free_pages(leftover->old_gmap_table, CRST_ALLOC_ORDER); - vfree(leftover->stor_var); + uv_free_stor_var(leftover->stor_var); done_fast: atomic_dec(&kvm->mm->context.protected_count); return 0; @@ -430,7 +434,7 @@ static int kvm_s390_pv_deinit_vm_fast(struct kvm *kvm, u16 *rc, u16 *rrc) }; int cc; - cc = uv_call_sched(0, (u64)&uvcb); + cc = uv_call(0, (u64)&uvcb); if (rc) *rc = uvcb.header.rc; if (rrc) @@ -747,7 +751,7 @@ int kvm_s390_pv_init_vm(struct kvm *kvm, u16 *rc, u16 *rrc) } gmap_split_huge_pages(kvm->arch.gmap); - cc = uv_call_sched(0, (u64)&uvcb); + cc = uv_call(0, (u64)&uvcb); *rc = uvcb.header.rc; *rrc = uvcb.header.rrc; KVM_UV_EVENT(kvm, 3, "PROTVIRT CREATE VM: handle %llx len %llx rc %x rrc %x flags %04x", @@ -833,7 +837,6 @@ int kvm_s390_pv_unpack(struct kvm *kvm, unsigned long addr, unsigned long size, while (offset < size) { ret = unpack_one(kvm, addr, tweak, offset, rc, rrc); if (ret == -EAGAIN) { - cond_resched(); if (fatal_signal_pending(current)) break; continue; @@ -876,7 +879,7 @@ int kvm_s390_pv_dump_cpu(struct kvm_vcpu *vcpu, void *buff, u16 *rc, u16 *rrc) }; int cc; - cc = uv_call_sched(0, (u64)&uvcb); + cc = uv_call(0, (u64)&uvcb); *rc = uvcb.header.rc; *rrc = uvcb.header.rrc; return cc; @@ -960,7 +963,7 @@ int kvm_s390_pv_dump_stor_state(struct kvm *kvm, void __user *buff_user, /* We will loop until the user buffer is filled or an error occurs */ do { /* Get 1MB worth of guest storage state data */ - cc = uv_call_sched(0, (u64)&uvcb); + cc = uv_call(0, (u64)&uvcb); /* All or nothing */ if (cc) { @@ -1038,7 +1041,7 @@ int kvm_s390_pv_dump_complete(struct kvm *kvm, void __user *buff_user, return -ENOMEM; complete.dump_area_origin = (u64)compl_data; - ret = uv_call_sched(0, (u64)&complete); + ret = uv_call(0, (u64)&complete); *rc = complete.header.rc; *rrc = complete.header.rrc; KVM_UV_EVENT(kvm, 3, "PROTVIRT DUMP COMPLETE: rc %x rrc %x", diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/s390/s390.c index 518a69c55e85..5c73f43782a7 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/s390/s390.c @@ -50,11 +50,12 @@ #include <asm/fpu.h> #include <asm/ap.h> #include <asm/uv.h> -#include "kvm-s390.h" +#include "s390.h" #include "gaccess.h" #include "gmap.h" #include "faultin.h" #include "pci.h" +#include "kvm_mmu.h" #define CREATE_TRACE_POINTS #include "trace.h" @@ -455,8 +456,7 @@ static void __init kvm_s390_cpu_feat_init(void) !test_facility(3) || !nested) return; allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2); - if (sclp.has_64bscao) - allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO); + allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO); if (sclp.has_siif) allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF); if (sclp.has_gpere) @@ -562,11 +562,12 @@ static void __kvm_s390_exit(void) static int kvm_s390_keyop(struct kvm_s390_mmu_cache *mc, struct kvm *kvm, int op, unsigned long addr, union skey skey) { - union asce asce = kvm->arch.gmap->asce; gfn_t gfn = gpa_to_gfn(addr); + union asce asce; int r; guard(read_lock)(&kvm->mmu_lock); + asce = kvm->arch.gmap->asce; switch (op) { case KVM_S390_KEYOP_SSKE: @@ -746,33 +747,7 @@ static void sca_del_vcpu(struct kvm_vcpu *vcpu); int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log) { - int r; - unsigned long n; - struct kvm_memory_slot *memslot; - int is_dirty; - - if (kvm_is_ucontrol(kvm)) - return -EINVAL; - - mutex_lock(&kvm->slots_lock); - - r = -EINVAL; - if (log->slot >= KVM_USER_MEM_SLOTS) - goto out; - - r = kvm_get_dirty_log(kvm, log, &is_dirty, &memslot); - if (r) - goto out; - - /* Clear the dirty log */ - if (is_dirty) { - n = kvm_dirty_bitmap_bytes(memslot); - memset(memslot->dirty_bitmap, 0, n); - } - r = 0; -out: - mutex_unlock(&kvm->slots_lock); - return r; + return s390_kvm_mmu_get_dirty_log(kvm, log); } static void icpt_operexc_on_all_vcpus(struct kvm *kvm) @@ -1027,7 +1002,6 @@ static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *att do { scoped_guard(read_lock, &kvm->mmu_lock) start_gfn = dat_reset_cmma(kvm->arch.gmap->asce, start_gfn); - cond_resched(); } while (start_gfn); ret = 0; break; @@ -1268,7 +1242,7 @@ static int kvm_s390_vm_start_migration(struct kvm *kvm) * Must be called with kvm->slots_arch_lock to avoid races with ourselves, * kvm_s390_vm_start_migration() and kvm_s390_get_cmma_bits(). */ -static int kvm_s390_vm_stop_migration(struct kvm *kvm) +int kvm_s390_vm_stop_migration(struct kvm *kvm) { /* migration mode already disabled */ if (!kvm->arch.migration_mode) @@ -2194,7 +2168,7 @@ static int kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX) return -EINVAL; - keys = kvmalloc_array(args->count, sizeof(*keys), GFP_KERNEL_ACCOUNT); + keys = kvmalloc_objs(*keys, args->count, GFP_KERNEL_ACCOUNT); if (!keys) return -ENOMEM; @@ -2231,7 +2205,7 @@ static int kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX) return -EINVAL; - keys = kvmalloc_array(args->count, sizeof(*keys), GFP_KERNEL_ACCOUNT); + keys = kvmalloc_objs(*keys, args->count, GFP_KERNEL_ACCOUNT); if (!keys) return -ENOMEM; @@ -3410,6 +3384,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id); kvm_s390_clear_local_irqs(vcpu); kvm_clear_async_pf_completion_queue(vcpu); + kvm_s390_clear_bp_data(vcpu); if (!kvm_is_ucontrol(vcpu->kvm)) sca_del_vcpu(vcpu); kvm_s390_update_topology_change_report(vcpu->kvm, 1); @@ -4248,8 +4223,10 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, /* enforce guest PER */ kvm_s390_set_cpuflags(vcpu, CPUSTAT_P); - if (dbg->control & KVM_GUESTDBG_USE_HW_BP) - rc = kvm_s390_import_bp_data(vcpu, dbg); + if (dbg->control & KVM_GUESTDBG_USE_HW_BP) { + scoped_guard(srcu, &vcpu->kvm->srcu) + rc = kvm_s390_import_bp_data(vcpu, dbg); + } } else { kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P); vcpu->arch.guestdbg.last_bp = 0; @@ -4474,8 +4451,8 @@ int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clo static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token, unsigned long token) { - struct kvm_s390_interrupt inti; - struct kvm_s390_irq irq; + struct kvm_s390_interrupt inti = {}; + struct kvm_s390_irq irq = {}; struct kvm_s390_interrupt_info *inti_mem = NULL; int ret = 0; @@ -5069,7 +5046,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) pr_err_ratelimited("can't run stopped vcpu %d\n", vcpu->vcpu_id); rc = -EINVAL; - goto out; + goto out_sigset; } kernel_fpu_begin(&fpu, KERNEL_FPC | KERNEL_VXR); @@ -5099,9 +5076,11 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) store_regs(vcpu); kernel_fpu_end(&fpu, KERNEL_FPC | KERNEL_VXR); + vcpu->stat.exit_userspace++; + +out_sigset: kvm_sigset_deactivate(vcpu); - vcpu->stat.exit_userspace++; out: vcpu_put(vcpu); return rc; @@ -5450,7 +5429,8 @@ long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl, if (copy_from_user(&s390irq, argp, sizeof(s390irq))) return -EFAULT; - rc = kvm_s390_inject_vcpu(vcpu, &s390irq); + scoped_guard(srcu, &vcpu->kvm->srcu) + rc = kvm_s390_inject_vcpu(vcpu, &s390irq); break; } case KVM_S390_INTERRUPT: { @@ -5463,7 +5443,8 @@ long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl, return -EFAULT; if (s390int_to_s390irq(&s390int, &s390irq)) return -EINVAL; - rc = kvm_s390_inject_vcpu(vcpu, &s390irq); + scoped_guard(srcu, &vcpu->kvm->srcu) + rc = kvm_s390_inject_vcpu(vcpu, &s390irq); break; } default: @@ -5732,7 +5713,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, r = kvm_s390_handle_pv_vcpu_dump(vcpu, &cmd); /* Always copy over UV rc / rrc data */ - if (copy_to_user((__u8 __user *)argp, &cmd.rc, + if (copy_to_user(argp + offsetof(struct kvm_pv_cmd, rc), &cmd.rc, sizeof(cmd.rc) + sizeof(cmd.rrc))) r = -EFAULT; break; @@ -5769,45 +5750,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *new, enum kvm_mr_change change) { - if (kvm_is_ucontrol(kvm) && new && new->id < KVM_USER_MEM_SLOTS) - return -EINVAL; - - /* When we are protected, we should not change the memory slots */ - if (kvm_s390_pv_get_handle(kvm)) - return -EINVAL; - - if (change != KVM_MR_DELETE && change != KVM_MR_FLAGS_ONLY) { - /* - * A few sanity checks. The memory in userland is ok to be - * fragmented into various different vmas. It is okay to mmap() - * and munmap() stuff in this slot after doing this call at any - * time. - */ - if (new->userspace_addr & ~PAGE_MASK) - return -EINVAL; - if ((new->base_gfn + new->npages) * PAGE_SIZE > kvm->arch.mem_limit) - return -EINVAL; - if (!asce_contains_gfn(kvm->arch.gmap->asce, new->base_gfn + new->npages - 1)) - return -EINVAL; - } - - if (!kvm->arch.migration_mode) - return 0; - - /* - * Turn off migration mode when: - * - userspace creates a new memslot with dirty logging off, - * - userspace modifies an existing memslot (MOVE or FLAGS_ONLY) and - * dirty logging is turned off. - * Migration mode expects dirty page logging being enabled to store - * its dirty bitmap. - */ - if (change != KVM_MR_DELETE && - !(new->flags & KVM_MEM_LOG_DIRTY_PAGES)) - WARN(kvm_s390_vm_stop_migration(kvm), - "Failed to stop migration mode"); - - return 0; + return s390_kvm_mmu_prepare_memory_region(kvm, old, new, change); } static long cmma_d_count_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) @@ -5823,55 +5766,22 @@ static long cmma_d_count_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_ return 0; } -void kvm_arch_commit_memory_region(struct kvm *kvm, - struct kvm_memory_slot *old, - const struct kvm_memory_slot *new, - enum kvm_mr_change change) +void kvm_s390_update_cmma_dirty(struct kvm *kvm, struct kvm_memory_slot *old) { const struct dat_walk_ops ops = { .pte_entry = cmma_d_count_pte, }; - struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL; - int rc = 0; - - guard(mutex)(&kvm->slots_arch_lock); - if (change == KVM_MR_FLAGS_ONLY) - return; - - mc = kvm_s390_new_mmu_cache(); - if (!mc) { - rc = -ENOMEM; - goto out; + if (kvm->arch.migration_mode && kvm->arch.use_cmma && old) { + _dat_walk_gfn_range(old->base_gfn, old->base_gfn + old->npages, + kvm->arch.gmap->asce, &ops, DAT_WALK_IGN_HOLES, + &kvm->arch.cmma_dirty_pages); } +} - scoped_guard(write_lock, &kvm->mmu_lock) { - if (kvm->arch.migration_mode && kvm->arch.use_cmma && old) { - _dat_walk_gfn_range(old->base_gfn, old->base_gfn + old->npages, - kvm->arch.gmap->asce, &ops, DAT_WALK_IGN_HOLES, - &kvm->arch.cmma_dirty_pages); - } - - switch (change) { - case KVM_MR_DELETE: - rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages); - break; - case KVM_MR_MOVE: - rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages); - if (rc) - break; - fallthrough; - case KVM_MR_CREATE: - rc = dat_create_slot(mc, kvm->arch.gmap->asce, new->base_gfn, new->npages); - break; - case KVM_MR_FLAGS_ONLY: - break; - default: - WARN(1, "Unknown KVM MR CHANGE: %d\n", change); - } - } -out: - if (rc) - pr_warn("failed to commit memory region\n"); - return; +void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ + s390_kvm_mmu_commit_memory_region(kvm, old, new, change); } /** diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/s390/s390.h index 6d2842fb71a3..d284a263ba70 100644 --- a/arch/s390/kvm/kvm-s390.h +++ b/arch/s390/kvm/s390/s390.h @@ -452,7 +452,7 @@ void kvm_s390_vsie_destroy(struct kvm *kvm); int kvm_s390_handle_sigp(struct kvm_vcpu *vcpu); int kvm_s390_handle_sigp_pei(struct kvm_vcpu *vcpu); -/* implemented in kvm-s390.c */ +/* implemented in s390.c */ int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod); int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long addr); int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr); @@ -472,6 +472,9 @@ int __kvm_s390_mprotect_many(struct gmap *gmap, gpa_t gpa, u8 npages, unsigned i unsigned long bits); bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu); +void kvm_s390_update_cmma_dirty(struct kvm *kvm, struct kvm_memory_slot *old); +int kvm_s390_vm_stop_migration(struct kvm *kvm); + /* implemented in diag.c */ int kvm_s390_handle_diag(struct kvm_vcpu *vcpu); @@ -594,6 +597,11 @@ static inline bool kvm_s390_cur_gmap_fault_is_write(void) return test_facility(75) && (current->thread.gmap_teid.fsi == TEID_FSI_STORE); } +static __always_inline int kvm_s390_is_migration_mode(struct kvm *kvm) +{ + return kvm->arch.migration_mode; +} + /** * kvm_s390_vcpu_crypto_reset_all * diff --git a/arch/s390/kvm/sigp.c b/arch/s390/kvm/s390/sigp.c index 55c34cb35428..131b3371ef4f 100644 --- a/arch/s390/kvm/sigp.c +++ b/arch/s390/kvm/s390/sigp.c @@ -14,7 +14,7 @@ #include <linux/slab.h> #include <asm/sigp.h> #include "gaccess.h" -#include "kvm-s390.h" +#include "s390.h" #include "trace.h" static int __sigp_sense(struct kvm_vcpu *vcpu, struct kvm_vcpu *dst_vcpu, diff --git a/arch/s390/kvm/trace-s390.h b/arch/s390/kvm/s390/trace-s390.h index 9e28f165c114..9e28f165c114 100644 --- a/arch/s390/kvm/trace-s390.h +++ b/arch/s390/kvm/s390/trace-s390.h diff --git a/arch/s390/kvm/trace.h b/arch/s390/kvm/s390/trace.h index aa419eb6a0c8..3d2fffb1555a 100644 --- a/arch/s390/kvm/trace.h +++ b/arch/s390/kvm/s390/trace.h @@ -45,20 +45,6 @@ TRACE_EVENT(kvm_s390_skey_related_inst, VCPU_TP_PRINTK("%s", "storage key related instruction") ); -TRACE_EVENT(kvm_s390_major_guest_pfault, - TP_PROTO(VCPU_PROTO_COMMON), - TP_ARGS(VCPU_ARGS_COMMON), - - TP_STRUCT__entry( - VCPU_FIELD_COMMON - ), - - TP_fast_assign( - VCPU_ASSIGN_COMMON - ), - VCPU_TP_PRINTK("%s", "major fault, maybe applicable for pfault") - ); - TRACE_EVENT(kvm_s390_pfault_init, TP_PROTO(VCPU_PROTO_COMMON, long pfault_token), TP_ARGS(VCPU_ARGS_COMMON, pfault_token), @@ -283,6 +269,32 @@ TRACE_EVENT(kvm_s390_handle_diag, __print_symbolic(__entry->code, diagnose_codes)) ); +TRACE_EVENT(kvm_s390_diag_9c, + TP_PROTO(VCPU_PROTO_COMMON, int target_vcpu, int target_cpu, + const char *result), + TP_ARGS(VCPU_ARGS_COMMON, target_vcpu, target_cpu, result), + + TP_STRUCT__entry( + VCPU_FIELD_COMMON + __field(int, target_vcpu) + __field(int, target_cpu) + __string(result, result) + ), + + TP_fast_assign( + VCPU_ASSIGN_COMMON + __entry->target_vcpu = target_vcpu; + __entry->target_cpu = target_cpu; + __assign_str(result); + ), + + VCPU_TP_PRINTK( + "diag=9c target_vcpu=%d target_pcpu=%d result=%s", + __entry->target_vcpu, + __entry->target_cpu, + __get_str(result)) + ); + TRACE_EVENT(kvm_s390_handle_lctl, TP_PROTO(VCPU_PROTO_COMMON, int g, int reg1, int reg3, u64 addr), TP_ARGS(VCPU_ARGS_COMMON, g, reg1, reg3, addr), diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/s390/vsie.c index eea24562e7db..5926689fa226 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/s390/vsie.c @@ -23,7 +23,7 @@ #include <asm/nmi.h> #include <asm/dis.h> #include <asm/facility.h> -#include "kvm-s390.h" +#include "s390.h" #include "gaccess.h" #include "gmap.h" @@ -33,10 +33,7 @@ enum vsie_page_flags { struct vsie_page { struct kvm_s390_sie_block scb_s; /* 0x0000 */ - /* - * the backup info for machine check. ensure it's at - * the same offset as that in struct sie_page! - */ + /* backup info for machine check */ struct mcck_volatile_info mcck_info; /* 0x0200 */ /* * The pinned original scb. Be aware that other VCPUs can modify @@ -71,6 +68,8 @@ struct vsie_page { }; static_assert(sizeof(struct vsie_page) == PAGE_SIZE); +static_assert(offsetof(struct vsie_page, mcck_info) == offsetof(struct sie_page, mcck_info)); +static_assert(IS_ALIGNED(offsetof(struct vsie_page, crycb), 8)); /* trigger a validity icpt for the given scb */ static int set_validity_icpt(struct kvm_s390_sie_block *scb, @@ -173,6 +172,7 @@ static int setup_apcb10(struct kvm_vcpu *vcpu, struct kvm_s390_apcb1 *apcb_s, sizeof(struct kvm_s390_apcb0))) return -EFAULT; + memset(apcb_s, 0, sizeof(*apcb_s)); apcb_s->apm[0] = apcb_h->apm[0] & tmp.apm[0]; apcb_s->aqm[0] = apcb_h->aqm[0] & tmp.aqm[0] & 0xffff000000000000UL; apcb_s->adm[0] = apcb_h->adm[0] & tmp.adm[0] & 0xffff000000000000UL; @@ -701,7 +701,7 @@ static int pin_guest_page(struct kvm *kvm, gpa_t gpa, hpa_t *hpa) /* Unpins a page previously pinned via pin_guest_page, marking it as dirty. */ static void unpin_guest_page(struct kvm *kvm, gpa_t gpa, hpa_t hpa) { - kvm_release_page_dirty(pfn_to_page(hpa >> PAGE_SHIFT)); + kvm_release_page_dirty(pfn_to_page(phys_to_pfn(hpa))); /* mark the page always as dirty for migration */ mark_page_dirty(kvm, gpa_to_gfn(gpa)); } @@ -1434,7 +1434,6 @@ static int vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) } if (sg) sg = gmap_put(sg); - cond_resched(); } if (sg) sg = gmap_put(sg); @@ -1486,7 +1485,7 @@ static struct vsie_page *get_vsie_page(struct kvm *kvm, unsigned long addr) int nr_vcpus; rcu_read_lock(); - vsie_page = radix_tree_lookup(&kvm->arch.vsie.addr_to_page, addr >> 9); + vsie_page = radix_tree_lookup(&kvm->arch.vsie.addr_to_page, addr >> SCB_ALIGNMENT_SHIFT); rcu_read_unlock(); if (vsie_page) { if (try_get_vsie_page(vsie_page)) { @@ -1527,13 +1526,14 @@ static struct vsie_page *get_vsie_page(struct kvm *kvm, unsigned long addr) } if (vsie_page->scb_gpa != ULONG_MAX) radix_tree_delete(&kvm->arch.vsie.addr_to_page, - vsie_page->scb_gpa >> 9); + vsie_page->scb_gpa >> SCB_ALIGNMENT_SHIFT); } /* Mark it as invalid until it resides in the tree. */ vsie_page->scb_gpa = ULONG_MAX; /* Double use of the same address or allocation failure. */ - if (radix_tree_insert(&kvm->arch.vsie.addr_to_page, addr >> 9, vsie_page)) { + if (radix_tree_insert(&kvm->arch.vsie.addr_to_page, addr >> SCB_ALIGNMENT_SHIFT, + vsie_page)) { put_vsie_page(vsie_page); mutex_unlock(&kvm->arch.vsie.mutex); return NULL; @@ -1565,7 +1565,6 @@ int kvm_s390_handle_vsie(struct kvm_vcpu *vcpu) if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); - BUILD_BUG_ON(sizeof(struct vsie_page) != PAGE_SIZE); scb_addr = kvm_s390_get_base_disp_s(vcpu, NULL); /* 512 byte alignment */ @@ -1632,7 +1631,7 @@ void kvm_s390_vsie_destroy(struct kvm *kvm) /* free the radix tree entry */ if (vsie_page->scb_gpa != ULONG_MAX) radix_tree_delete(&kvm->arch.vsie.addr_to_page, - vsie_page->scb_gpa >> 9); + vsie_page->scb_gpa >> SCB_ALIGNMENT_SHIFT); free_page((unsigned long)vsie_page); } kvm->arch.vsie.page_count = 0; diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile index aa6cc6a1fe88..bb707700d7af 100644 --- a/arch/s390/lib/Makefile +++ b/arch/s390/lib/Makefile @@ -3,6 +3,8 @@ # Makefile for s390-specific library files.. # +CONTEXT_ANALYSIS := y + # string.o implements standard library functions like memset/memcpy etc. # Use -ffreestanding to ensure that the compiler does not try to "optimize" # them into calls to themselves. diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c index 10db1e56a811..dbabca35c008 100644 --- a/arch/s390/lib/spinlock.c +++ b/arch/s390/lib/spinlock.c @@ -18,6 +18,7 @@ #include <asm/alternative.h> #include <asm/machine.h> #include <asm/asm.h> +#include <trace/events/lock.h> int spin_retry = -1; @@ -281,10 +282,12 @@ static inline void arch_spin_lock_classic(arch_spinlock_t *lp) void arch_spin_lock_wait(arch_spinlock_t *lp) { + trace_contention_begin(lp, LCB_F_SPIN); if (test_cpu_flag(CIF_DEDICATED_CPU)) arch_spin_lock_queued(lp); else arch_spin_lock_classic(lp); + trace_contention_end(lp, 0); } EXPORT_SYMBOL(arch_spin_lock_wait); diff --git a/arch/s390/mm/Makefile b/arch/s390/mm/Makefile index 193899c39ca7..7dea37a5ad3b 100644 --- a/arch/s390/mm/Makefile +++ b/arch/s390/mm/Makefile @@ -3,6 +3,8 @@ # Makefile for the linux s390-specific parts of the memory manager. # +CONTEXT_ANALYSIS := y + obj-y := init.o fault.o extmem.o mmap.o vmem.o maccess.o obj-y += page-states.o pageattr.o pgtable.o pgalloc.o extable.o diff --git a/arch/s390/mm/cmm.c b/arch/s390/mm/cmm.c index eb7ef63fab1e..d342b68ee9be 100644 --- a/arch/s390/mm/cmm.c +++ b/arch/s390/mm/cmm.c @@ -95,7 +95,6 @@ static long cmm_alloc_pages(long nr, long *counter, (*counter)++; spin_unlock(&cmm_lock); nr--; - cond_resched(); } return nr; } @@ -134,7 +133,6 @@ static long cmm_free_pages(long nr, long *counter, struct cmm_page_array **list) inc = __cmm_free_pages(inc, counter, list); if (inc) break; - cond_resched(); } return nr + inc; } diff --git a/arch/s390/mm/extable.c b/arch/s390/mm/extable.c index 063b4346742d..7498e858c401 100644 --- a/arch/s390/mm/extable.c +++ b/arch/s390/mm/extable.c @@ -50,6 +50,22 @@ static bool ex_handler_ua_load_reg(const struct exception_table_entry *ex, return true; } +static bool ex_handler_zeropad(const struct exception_table_entry *ex, struct pt_regs *regs) +{ + unsigned int reg_addr = FIELD_GET(EX_DATA_REG_ADDR, ex->data); + unsigned int reg_data = FIELD_GET(EX_DATA_REG_ERR, ex->data); + unsigned long data, addr, offset; + + addr = regs->gprs[reg_addr]; + offset = addr & (sizeof(unsigned long) - 1); + addr &= ~(sizeof(unsigned long) - 1); + data = *(unsigned long *)addr; + data <<= BITS_PER_BYTE * offset; + regs->gprs[reg_data] = data; + regs->psw.addr = extable_fixup(ex); + return true; +} + static bool ex_handler_fpc(const struct exception_table_entry *ex, struct pt_regs *regs) { fpu_sfpc(0); @@ -118,6 +134,8 @@ bool fixup_exception(struct pt_regs *regs) return ex_handler_ua_load_reg(ex, false, regs); case EX_TYPE_UA_LOAD_REGPAIR: return ex_handler_ua_load_reg(ex, true, regs); + case EX_TYPE_ZEROPAD: + return ex_handler_zeropad(ex, regs); case EX_TYPE_FPC: return ex_handler_fpc(ex, regs); case EX_TYPE_UA_MVCOS_TO: diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 028aeb9c48d6..46d828926009 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -406,12 +406,13 @@ NOKPROBE_SYMBOL(do_dat_exception); #if IS_ENABLED(CONFIG_KVM) void do_secure_storage_access(struct pt_regs *regs) +__context_unsafe(/* folio_walk_end() not instrumented */) { union teid teid = { .val = regs->int_parm_long }; unsigned long addr = get_fault_address(regs); + struct mm_struct *mm = current->mm; struct vm_area_struct *vma; struct folio_walk fw; - struct mm_struct *mm; struct folio *folio; int rc; @@ -427,10 +428,8 @@ void do_secure_storage_access(struct pt_regs *regs) * was not supposed to do, e.g. branching into secure * memory. Trigger a segmentation fault. */ - if (user_mode(regs)) { - send_sig(SIGSEGV, current, 0); - return; - } + if (user_mode(regs)) + return handle_fault_error_nolock(regs, SEGV_ACCERR); /* * The kernel should never run into this case and * there is no way out of this situation. @@ -438,13 +437,9 @@ void do_secure_storage_access(struct pt_regs *regs) panic("Unexpected PGM 0x3d with TEID bit 61=0"); } if (is_kernel_fault(regs)) { - folio = virt_to_folio((void *)addr); - if (unlikely(!folio_try_get(folio))) - return; - rc = uv_convert_from_secure(folio_to_phys(folio)); - if (!rc) - clear_bit(PG_arch_1, &folio->flags.f); - folio_put(folio); + if (is_vmalloc_addr((void *)addr)) + return handle_fault_error_nolock(regs, 0); + rc = uv_convert_from_secure(__pa(addr)); /* * There are some valid fixup types for kernel * accesses to donated secure memory. zeropad is one @@ -453,25 +448,22 @@ void do_secure_storage_access(struct pt_regs *regs) if (rc) return handle_fault_error_nolock(regs, 0); } else { - if (faulthandler_disabled()) + if (faulthandler_disabled() || !mm) return handle_fault_error_nolock(regs, 0); - mm = current->mm; - mmap_read_lock(mm); - vma = find_vma(mm, addr); + vma = lock_mm_and_find_vma(mm, addr, regs); if (!vma) - return handle_fault_error(regs, SEGV_MAPERR); + return handle_fault_error_nolock(regs, SEGV_MAPERR); folio = folio_walk_start(&fw, vma, addr, 0); - if (!folio) { - mmap_read_unlock(mm); - return; - } + if (!folio) + goto out; /* arch_make_folio_accessible() needs a raised refcount. */ folio_get(folio); rc = arch_make_folio_accessible(folio); folio_put(folio); folio_walk_end(&fw, vma); if (rc) - send_sig(SIGSEGV, current, 0); + return handle_fault_error(regs, SEGV_ACCERR); +out: mmap_read_unlock(mm); } } diff --git a/arch/s390/mm/gmap_helpers.c b/arch/s390/mm/gmap_helpers.c index 4bf7c9012feb..ff63ffb1dbd2 100644 --- a/arch/s390/mm/gmap_helpers.c +++ b/arch/s390/mm/gmap_helpers.c @@ -40,6 +40,7 @@ * and locked. */ pte_t *try_get_locked_pte(struct mm_struct *mm, unsigned long vmaddr, spinlock_t **ptl) +__context_unsafe(/* Returns nonnull if lock taken or not taken */) { pmd_t *pmdp, pmd, pmdval; pud_t *pudp, pud; @@ -90,6 +91,7 @@ EXPORT_SYMBOL_GPL(try_get_locked_pte); * Context: needs to be called while holding the mmap lock. */ void gmap_helper_zap_one_page(struct mm_struct *mm, unsigned long vmaddr) +__context_unsafe(/* pte_unmap_unlock() not instrumented */) { struct vm_area_struct *vma; spinlock_t *ptl; /* Lock for the host (userspace) page table */ @@ -161,6 +163,7 @@ EXPORT_SYMBOL_GPL(gmap_helper_discard); * disabled. */ void gmap_helper_try_set_pte_unused(struct mm_struct *mm, unsigned long vmaddr) +__context_unsafe(/* pte_unmap_unlock() not instrumented */) { spinlock_t *ptl; /* Lock for the host (userspace) page table */ pte_t *ptep; @@ -200,7 +203,7 @@ static int find_zeropage_pte_entry(pte_t *pte, unsigned long addr, * currently only works in COW mappings, which is also where * mm_forbids_zeropage() is checked. */ - if (!is_cow_mapping(walk->vma->vm_flags)) + if (!vma_is_cow_mapping(walk->vma)) return -EFAULT; *found_addr = addr; @@ -272,7 +275,6 @@ retry: * truncation. In that case, the shared zeropage would be gone * and we can simply retry and make progress. */ - cond_resched(); goto retry; } diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c index db35d8fe8609..f84aa9265430 100644 --- a/arch/s390/mm/hugetlbpage.c +++ b/arch/s390/mm/hugetlbpage.c @@ -147,10 +147,12 @@ void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr, if (likely(pte_present(pte))) rste |= _REGION3_ENTRY_LARGE; rste |= _REGION_ENTRY_TYPE_R3; - } else if (likely(pte_present(pte))) - rste |= _SEGMENT_ENTRY_LARGE; - - set_pte(ptep, __pte(rste)); + set_pud((pud_t *)ptep, __pud(rste)); + } else { + if (likely(pte_present(pte))) + rste |= _SEGMENT_ENTRY_LARGE; + set_pmd((pmd_t *)ptep, __pmd(rste)); + } } void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index f07168a0d3dd..be7e009e7b59 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -50,6 +50,7 @@ #include <linux/virtio_anchor.h> #include <linux/virtio_config.h> #include <linux/execmem.h> +#include <linux/cc_platform.h> pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(".bss..swapper_pg_dir"); pgd_t invalid_pg_dir[PTRS_PER_PGD] __section(".bss..invalid_pg_dir"); @@ -142,6 +143,20 @@ bool force_dma_unencrypted(struct device *dev) return is_prot_virt_guest(); } + +bool cc_platform_has(enum cc_attr attr) +{ + switch (attr) { + case CC_ATTR_MEM_ENCRYPT: + case CC_ATTR_GUEST_MEM_ENCRYPT: + return is_prot_virt_guest(); + + default: + return false; + } +} +EXPORT_SYMBOL_GPL(cc_platform_has); + /* protected virtualization */ static void __init pv_init(void) { @@ -151,7 +166,7 @@ static void __init pv_init(void) virtio_set_mem_acc_cb(virtio_require_restricted_mem_acc); /* make sure bounce buffers are shared */ - swiotlb_init(true, SWIOTLB_FORCE | SWIOTLB_VERBOSE); + swiotlb_init(true, SWIOTLB_VERBOSE | SWIOTLB_ANY); swiotlb_update_mem_attributes(); } diff --git a/arch/s390/mm/maccess.c b/arch/s390/mm/maccess.c index cfd219fe495c..f39968dd8063 100644 --- a/arch/s390/mm/maccess.c +++ b/arch/s390/mm/maccess.c @@ -96,7 +96,7 @@ size_t memcpy_real_iter(struct iov_iter *iter, unsigned long src, size_t count) pte = mk_pte_phys(phys, PAGE_KERNEL_RO); mutex_lock(&memcpy_real_mutex); - if (pte_val(pte) != pte_val(*memcpy_real_ptep)) { + if (pte_val(pte) != pte_val(ptep_get(memcpy_real_ptep))) { __ptep_ipte(__memcpy_real_area, memcpy_real_ptep, 0, 0, IPTE_GLOBAL); set_pte(memcpy_real_ptep, pte); } diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c index e6f788696dd1..1e202e3d08e7 100644 --- a/arch/s390/mm/pageattr.c +++ b/arch/s390/mm/pageattr.c @@ -105,7 +105,6 @@ static int walk_pte_level(pmd_t *pmdp, unsigned long addr, unsigned long end, pgt_set((unsigned long *)ptep, pte_val(new), addr, CRDTE_DTT_PAGE); ptep++; addr += PAGE_SIZE; - cond_resched(); } while (addr < end); return 0; } @@ -194,7 +193,6 @@ static int walk_pmd_level(pud_t *pudp, unsigned long addr, unsigned long end, } pmdp++; addr = next; - cond_resched(); } while (addr < end); return rc; } @@ -281,7 +279,6 @@ static int walk_pud_level(p4d_t *p4d, unsigned long addr, unsigned long end, } pudp++; addr = next; - cond_resched(); } while (addr < end && !rc); return rc; } @@ -301,7 +298,6 @@ static int walk_p4d_level(pgd_t *pgd, unsigned long addr, unsigned long end, rc = walk_pud_level(p4dp, addr, next, flags); p4dp++; addr = next; - cond_resched(); } while (addr < end && !rc); return rc; } @@ -323,7 +319,6 @@ static int change_page_attr(unsigned long addr, unsigned long end, rc = walk_p4d_level(pgdp, addr, next, flags); if (rc) break; - cond_resched(); } while (pgdp++, addr = next, addr < end && !rc); return rc; } diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index 7ac44543e051..4b160eedc5a0 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -55,63 +55,46 @@ static void __crst_table_upgrade(void *arg) int crst_table_upgrade(struct mm_struct *mm, unsigned long end) { - unsigned long *pgd = NULL, *p4d = NULL, *__pgd; - unsigned long asce_limit = mm->context.asce_limit; + unsigned long *table, *pgd; + int rc, notify; mmap_assert_write_locked(mm); - /* upgrade should only happen from 3 to 4, 3 to 5, or 4 to 5 levels */ - VM_BUG_ON(asce_limit < _REGION2_SIZE); - - if (end <= asce_limit) - return 0; - - if (asce_limit == _REGION2_SIZE) { - p4d = crst_table_alloc(mm); - if (unlikely(!p4d)) - goto err_p4d; - crst_table_init(p4d, _REGION2_ENTRY_EMPTY); - pagetable_p4d_ctor(virt_to_ptdesc(p4d)); - } - if (end > _REGION1_SIZE) { - pgd = crst_table_alloc(mm); - if (unlikely(!pgd)) - goto err_pgd; - crst_table_init(pgd, _REGION1_ENTRY_EMPTY); - pagetable_pgd_ctor(virt_to_ptdesc(pgd)); - } - - spin_lock_bh(&mm->page_table_lock); - - if (p4d) { - __pgd = (unsigned long *) mm->pgd; - p4d_populate(mm, (p4d_t *) p4d, (pud_t *) __pgd); - mm->pgd = (pgd_t *) p4d; - mm->context.asce_limit = _REGION1_SIZE; - mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | - _ASCE_USER_BITS | _ASCE_TYPE_REGION2; - mm_inc_nr_puds(mm); - } - if (pgd) { - __pgd = (unsigned long *) mm->pgd; - pgd_populate(mm, (pgd_t *) pgd, (p4d_t *) __pgd); - mm->pgd = (pgd_t *) pgd; - mm->context.asce_limit = TASK_SIZE_MAX; - mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | - _ASCE_USER_BITS | _ASCE_TYPE_REGION1; + VM_BUG_ON(mm->context.asce_limit < _REGION2_SIZE); + rc = 0; + notify = 0; + while (mm->context.asce_limit < end) { + table = crst_table_alloc(mm); + if (!table) { + rc = -ENOMEM; + break; + } + spin_lock_bh(&mm->page_table_lock); + pgd = (unsigned long *)mm->pgd; + if (mm->context.asce_limit == _REGION2_SIZE) { + crst_table_init(table, _REGION2_ENTRY_EMPTY); + p4d_populate(mm, (p4d_t *)table, (pud_t *)pgd); + pagetable_p4d_ctor(virt_to_ptdesc(table)); + mm->pgd = (pgd_t *)table; + mm->context.asce_limit = _REGION1_SIZE; + mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | + _ASCE_USER_BITS | _ASCE_TYPE_REGION2; + mm_inc_nr_puds(mm); + } else { + crst_table_init(table, _REGION1_ENTRY_EMPTY); + pgd_populate(mm, (pgd_t *)table, (p4d_t *)pgd); + pagetable_pgd_ctor(virt_to_ptdesc(table)); + mm->pgd = (pgd_t *)table; + mm->context.asce_limit = TASK_SIZE_MAX; + mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | + _ASCE_USER_BITS | _ASCE_TYPE_REGION1; + } + notify = 1; + spin_unlock_bh(&mm->page_table_lock); } - - spin_unlock_bh(&mm->page_table_lock); - - on_each_cpu(__crst_table_upgrade, mm, 0); - - return 0; - -err_pgd: - pagetable_dtor(virt_to_ptdesc(p4d)); - crst_table_free(mm, p4d); -err_p4d: - return -ENOMEM; + if (notify) + on_each_cpu(__crst_table_upgrade, mm, 0); + return rc; } unsigned long *page_table_alloc_noprof(struct mm_struct *mm) @@ -268,7 +251,6 @@ static int base_segment_walk(unsigned long *origin, unsigned long addr, return rc; if (!alloc) base_pgt_free(table); - cond_resched(); } while (ste++, addr = next, addr < end); return 0; } diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 94ee06ae8913..d2879ce860a1 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -39,7 +39,6 @@ static void __ref *vmem_alloc_pages(unsigned int order) static void vmem_free_pages(unsigned long addr, int order, struct vmem_altmap *altmap) { - unsigned int nr_pages = 1 << order; struct page *page; if (altmap) { @@ -49,8 +48,7 @@ static void vmem_free_pages(unsigned long addr, int order, struct vmem_altmap *a page = virt_to_page((void *)addr); if (PageReserved(page)) { /* allocated from memblock */ - while (nr_pages--) - free_reserved_page(page++); + free_reserved_pages(page, order); } else { free_pages(addr, order); } diff --git a/arch/s390/net/Makefile b/arch/s390/net/Makefile index 9275cf63192a..0a7323c9b372 100644 --- a/arch/s390/net/Makefile +++ b/arch/s390/net/Makefile @@ -2,5 +2,8 @@ # # Arch-specific network modules # + +CONTEXT_ANALYSIS := y + obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o bpf_timed_may_goto.o obj-$(CONFIG_HAVE_PNETID) += pnet.o diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index b60877478b45..c4b47070bb59 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -21,6 +21,7 @@ #include <linux/filter.h> #include <linux/init.h> #include <linux/bpf.h> +#include <linux/cfi.h> #include <linux/mm.h> #include <linux/kernel.h> #include <asm/cacheflush.h> @@ -356,6 +357,19 @@ static void emit6_pcrel_rilc(struct bpf_jit *jit, u32 op, u8 mask, s64 pcrel) } \ }) +static inline void emit_u32_data(const u32 data, struct bpf_jit *jit) +{ + if (jit->prg_buf) + *(u32 *)(jit->prg_buf + jit->prg) = data; + jit->prg += 4; +} + +static inline void emit_kcfi(u32 hash, struct bpf_jit *jit) +{ + if (IS_ENABLED(CONFIG_CFI)) + emit_u32_data(hash, jit); +} + /* * Return whether this is the first pass. The first pass is special, since we * don't know any sizes yet, and thus must be conservative. @@ -597,6 +611,8 @@ static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp) { BUILD_BUG_ON(sizeof(struct prog_frame) != STACK_FRAME_OVERHEAD); + emit_kcfi(bpf_is_subprog(fp) ? cfi_bpf_subprog_hash : cfi_bpf_hash, jit); + /* No-op for hotpatching */ /* brcl 0,prologue_plt */ EMIT6_PCREL_RILC(0xc0040000, 0, jit->prologue_plt); @@ -616,7 +632,7 @@ static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp) bpf_skip(jit, 6); } /* Tail calls have to skip above initialization */ - jit->tail_call_start = jit->prg; + jit->tail_call_start = jit->prg - cfi_get_offset(); if (fp->aux->exception_cb) { /* * Switch stack, the new address is in the 2nd parameter. @@ -774,6 +790,8 @@ static void bpf_jit_probe_atomic_pre(struct bpf_jit *jit, struct bpf_insn *insn, struct bpf_jit_probe *probe) { + int load_reg; + if (BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) return; @@ -783,6 +801,14 @@ static void bpf_jit_probe_atomic_pre(struct bpf_jit *jit, EMIT4(0xb9080000, REG_W1, insn->dst_reg); probe->arena_reg = REG_W1; probe->prg = jit->prg; + /* + * A read-modify-write carrying BPF_FETCH reads the old value into + * src_reg, or into r0 for a BPF_CMPXCHG. Clear that register on + * fault, the remaining atomics only write memory. + */ + load_reg = bpf_atomic_load_reg(insn); + if (load_reg >= 0) + probe->reg = reg2hex[load_reg]; } static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp, @@ -1684,6 +1710,7 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, if (load_probe.prg != -1) { probe.prg = jit->prg; probe.arena_reg = load_probe.arena_reg; + probe.reg = load_probe.reg; } loop_start = jit->prg; /* 0: {csy|csg} %w0,%src,off(%arena) */ @@ -2420,11 +2447,13 @@ skip_init_ctx: jit_data->ctx = jit; jit_data->pass = pass; } - fp->bpf_func = (void *) jit.prg_buf; + fp->bpf_func = (void *)jit.prg_buf + cfi_get_offset(); fp->jited = 1; - fp->jited_len = jit.size; + fp->jited_len = jit.size - cfi_get_offset(); if (!fp->is_func || extra_pass) { + for (int i = 0; i < fp->len; i++) + jit.addrs[i] -= cfi_get_offset(); bpf_prog_fill_jited_linfo(fp, jit.addrs + 1); free_addrs: kvfree(jit.addrs); @@ -2690,8 +2719,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, return -ENOTSUPP; /* Return to %r14 in the struct_ops case. */ - if (flags & BPF_TRAMP_F_INDIRECT) + if (flags & BPF_TRAMP_F_INDIRECT) { flags |= BPF_TRAMP_F_SKIP_FRAME; + emit_kcfi(cfi_get_func_hash(func_addr), jit); + } /* * Compute how many arguments we need to pass to BPF programs. diff --git a/arch/s390/pci/Makefile b/arch/s390/pci/Makefile index 1810e0944a4e..b8be1316491b 100644 --- a/arch/s390/pci/Makefile +++ b/arch/s390/pci/Makefile @@ -3,6 +3,8 @@ # Makefile for the s390 PCI subsystem. # +CONTEXT_ANALYSIS := y + obj-$(CONFIG_PCI) += pci.o pci_irq.o pci_clp.o \ pci_event.o pci_debug.o pci_insn.o pci_mmio.o \ pci_bus.o pci_kvm_hook.o pci_report.o pci_fixup.o diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index 39bd2adfc240..d64c544b32b0 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -842,6 +842,7 @@ struct zpci_dev *zpci_create_device(u32 fid, u32 fh, enum zpci_state state) mutex_init(&zdev->state_lock); mutex_init(&zdev->fmb_lock); mutex_init(&zdev->kzdev_lock); + mutex_init(&zdev->pending_errs_lock); return zdev; @@ -1173,10 +1174,8 @@ int zpci_scan_devices(void) return rc; zpci_add_devices(&scan_list); - zpci_bus_for_each(zbus) { + zpci_bus_for_each(zbus) zpci_bus_scan_bus(zbus); - cond_resched(); - } return 0; } diff --git a/arch/s390/pci/pci_event.c b/arch/s390/pci/pci_event.c index 839bd91c056e..f317a1465dad 100644 --- a/arch/s390/pci/pci_event.c +++ b/arch/s390/pci/pci_event.c @@ -17,23 +17,6 @@ #include "pci_bus.h" #include "pci_report.h" -/* Content Code Description for PCI Function Error */ -struct zpci_ccdf_err { - u32 reserved1; - u32 fh; /* function handle */ - u32 fid; /* function id */ - u32 ett : 4; /* expected table type */ - u32 mvn : 12; /* MSI vector number */ - u32 dmaas : 8; /* DMA address space */ - u32 : 6; - u32 q : 1; /* event qualifier */ - u32 rw : 1; /* read/write */ - u64 faddr; /* failing address */ - u32 reserved3; - u16 reserved4; - u16 pec; /* PCI event code */ -} __packed; - /* Content Code Description for PCI Function Availability */ struct zpci_ccdf_avail { u32 reserved1; @@ -60,18 +43,6 @@ static inline bool ers_result_indicates_abort(pci_ers_result_t ers_res) } } -static bool is_passed_through(struct pci_dev *pdev) -{ - struct zpci_dev *zdev = to_zpci(pdev); - bool ret; - - mutex_lock(&zdev->kzdev_lock); - ret = !!zdev->kzdev; - mutex_unlock(&zdev->kzdev_lock); - - return ret; -} - static bool is_driver_supported(struct pci_driver *driver) { if (!driver || !driver->err_handler) @@ -81,6 +52,68 @@ static bool is_driver_supported(struct pci_driver *driver) return true; } +static int zpci_store_pci_error(struct pci_dev *pdev, + struct zpci_ccdf_err *ccdf) +{ + struct zpci_dev *zdev = to_zpci(pdev); + int i; + + guard(mutex)(&zdev->pending_errs_lock); + if (!zdev->pending_errs.mediated_recovery) + return -EINVAL; + + if (zdev->pending_errs.count >= ZPCI_ERR_PENDING_MAX) { + dev_warn_ratelimited(&pdev->dev, + "%s: Maximum number (%d) of pending error events queued\n", + pci_name(pdev), + ZPCI_ERR_PENDING_MAX); + return -ENOMEM; + } + + i = zdev->pending_errs.tail % ZPCI_ERR_PENDING_MAX; + memcpy(&zdev->pending_errs.err[i], ccdf, sizeof(struct zpci_ccdf_err)); + zdev->pending_errs.tail++; + zdev->pending_errs.count++; + return 0; +} + +int zpci_get_pending_error(struct zpci_dev *zdev, + struct zpci_ccdf_err *ccdf) +{ + int head; + + guard(mutex)(&zdev->pending_errs_lock); + + if (!zdev->pending_errs.count) + return -ENOMSG; + + head = zdev->pending_errs.head % ZPCI_ERR_PENDING_MAX; + memcpy(ccdf, &zdev->pending_errs.err[head], + sizeof(struct zpci_ccdf_err)); + zdev->pending_errs.head++; + zdev->pending_errs.count--; + return 0; +} +EXPORT_SYMBOL_GPL(zpci_get_pending_error); + +void zpci_start_mediated_recovery(struct zpci_dev *zdev) +{ + guard(mutex)(&zdev->pending_errs_lock); + zdev->pending_errs.mediated_recovery = true; +} +EXPORT_SYMBOL_GPL(zpci_start_mediated_recovery); + +void zpci_stop_mediated_recovery(struct zpci_dev *zdev) +{ + guard(mutex)(&zdev->pending_errs_lock); + zdev->pending_errs.mediated_recovery = false; + if (zdev->pending_errs.count) + pr_info("Unhandled PCI error events count=%d for PCI function 0x%x\n", + zdev->pending_errs.count, zdev->fid); + memset(&zdev->pending_errs, 0, sizeof(struct zpci_ccdf_pending)); +} +EXPORT_SYMBOL_GPL(zpci_stop_mediated_recovery); + static pci_ers_result_t zpci_event_notify_error_detected(struct pci_dev *pdev, struct pci_driver *driver) { @@ -175,12 +208,15 @@ static pci_ers_result_t zpci_event_do_reset(struct pci_dev *pdev, * and the platform determines which functions are affected for * multi-function devices. */ -static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev) +static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev, + struct zpci_ccdf_err *ccdf) { pci_ers_result_t ers_res = PCI_ERS_RESULT_DISCONNECT; struct zpci_dev *zdev = to_zpci(pdev); + bool mediated_recovery = false; char *status_str = "success"; struct pci_driver *driver; + int rc; /* * Ensure that the PCI function is not removed concurrently, no driver @@ -194,13 +230,6 @@ static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev) } pdev->error_state = pci_channel_io_frozen; - if (is_passed_through(pdev)) { - pr_info("%s: Cannot be recovered in the host because it is a pass-through device\n", - pci_name(pdev)); - status_str = "failed (pass-through)"; - goto out_unlock; - } - driver = to_pci_driver(pdev->dev.driver); if (!is_driver_supported(driver)) { if (!driver) { @@ -216,12 +245,24 @@ static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev) goto out_unlock; } + rc = zpci_store_pci_error(pdev, ccdf); + if (!rc || rc == -ENOMEM) + mediated_recovery = true; + ers_res = zpci_event_notify_error_detected(pdev, driver); if (ers_result_indicates_abort(ers_res)) { status_str = "failed (abort on detection)"; goto out_unlock; } + if (mediated_recovery) { + pr_info("%s: Leaving recovery of pass-through device to user-space\n", + pci_name(pdev)); + ers_res = PCI_ERS_RESULT_RECOVERED; + status_str = "in progress"; + goto out_unlock; + } + if (ers_res != PCI_ERS_RESULT_NEED_RESET) { ers_res = zpci_event_do_error_state_clear(pdev, driver); if (ers_result_indicates_abort(ers_res)) { @@ -266,28 +307,28 @@ out_unlock: * @pdev: PCI function for which to report * @es: PCI channel failure state to report */ -static void zpci_event_io_failure(struct pci_dev *pdev, pci_channel_state_t es) +static void zpci_event_io_failure(struct pci_dev *pdev, pci_channel_state_t es, + struct zpci_ccdf_err *ccdf) { struct pci_driver *driver; pci_dev_lock(pdev); pdev->error_state = es; - /** - * While vfio-pci's error_detected callback notifies user-space QEMU - * reacts to this by freezing the guest. In an s390 environment PCI - * errors are rarely fatal so this is overkill. Instead in the future - * we will inject the error event and let the guest recover the device - * itself. - */ - if (is_passed_through(pdev)) - goto out; + + zpci_store_pci_error(pdev, ccdf); driver = to_pci_driver(pdev->dev.driver); if (driver && driver->err_handler && driver->err_handler->error_detected) driver->err_handler->error_detected(pdev, pdev->error_state); -out: + pci_dev_unlock(pdev); } +static void __zpci_event_print_error(struct pci_dev *pdev, struct zpci_ccdf_err *ccdf) +{ + pr_err("%s: Event 0x%x reports an error for PCI function 0x%x\n", + pdev ? pci_name(pdev) : "n/a", ccdf->pec, ccdf->fid); +} + static void __zpci_event_error(struct zpci_ccdf_err *ccdf) { struct zpci_dev *zdev = get_zdev_by_fid(ccdf->fid); @@ -301,24 +342,24 @@ static void __zpci_event_error(struct zpci_ccdf_err *ccdf) zpci_err("error CCDF:\n"); zpci_err_hex(ccdf, sizeof(*ccdf)); - if (zdev) { - mutex_lock(&zdev->state_lock); - rc = clp_refresh_fh(zdev->fid, &fh); - if (rc) - goto no_pdev; - if (!fh || ccdf->fh != fh) { - /* Ignore events with stale handles */ - zpci_dbg(3, "err fid:%x, fh:%x (stale %x)\n", - ccdf->fid, fh, ccdf->fh); - goto no_pdev; - } - zpci_update_fh(zdev, ccdf->fh); - if (zdev->zbus->bus) - pdev = pci_get_slot(zdev->zbus->bus, zdev->devfn); + if (!zdev) + return __zpci_event_print_error(NULL, ccdf); + + mutex_lock(&zdev->state_lock); + rc = clp_refresh_fh(zdev->fid, &fh); + if (rc) + goto no_pdev; + if (!fh || ccdf->fh != fh) { + /* Ignore events with stale handles */ + zpci_dbg(3, "err fid:%x, fh:%x (stale %x)\n", + ccdf->fid, fh, ccdf->fh); + goto no_pdev; } + zpci_update_fh(zdev, ccdf->fh); + if (zdev->zbus->bus) + pdev = pci_get_slot(zdev->zbus->bus, zdev->devfn); - pr_err("%s: Event 0x%x reports an error for PCI function 0x%x\n", - pdev ? pci_name(pdev) : "n/a", ccdf->pec, ccdf->fid); + __zpci_event_print_error(pdev, ccdf); if (!pdev) goto no_pdev; @@ -330,18 +371,17 @@ static void __zpci_event_error(struct zpci_ccdf_err *ccdf) break; case 0x0040: /* Service Action or Error Recovery Failed */ case 0x003b: - zpci_event_io_failure(pdev, pci_channel_io_perm_failure); + zpci_event_io_failure(pdev, pci_channel_io_perm_failure, ccdf); break; default: /* PCI function left in the error state attempt to recover */ - ers_res = zpci_event_attempt_error_recovery(pdev); + ers_res = zpci_event_attempt_error_recovery(pdev, ccdf); if (ers_res != PCI_ERS_RESULT_RECOVERED) - zpci_event_io_failure(pdev, pci_channel_io_perm_failure); + zpci_event_io_failure(pdev, pci_channel_io_perm_failure, ccdf); break; } pci_dev_put(pdev); no_pdev: - if (zdev) - mutex_unlock(&zdev->state_lock); + mutex_unlock(&zdev->state_lock); zpci_zdev_put(zdev); } @@ -382,98 +422,106 @@ static void zpci_event_reappear(struct zpci_dev *zdev) zpci_dbg(1, "rea fid:%x, fh:%x\n", zdev->fid, zdev->fh); } -static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf) +static bool zpci_event_avail_any_device(struct zpci_ccdf_avail *ccdf) { - struct zpci_dev *zdev = get_zdev_by_fid(ccdf->fid); - bool existing_zdev = !!zdev; - enum zpci_state state; - - zpci_dbg(3, "avl fid:%x, fh:%x, pec:%x\n", - ccdf->fid, ccdf->fh, ccdf->pec); + /* 0x0306 - No handle or fid stored */ + if (ccdf->pec != 0x0306) + return false; + /* 0x308 or 0x302 for multiple devices */ + zpci_remove_reserved_devices(); + zpci_scan_devices(); + return true; +} - if (existing_zdev) - mutex_lock(&zdev->state_lock); +static void zpci_event_avail_new_device(struct zpci_ccdf_avail *ccdf) +{ + struct zpci_dev *zdev; switch (ccdf->pec) { case 0x0301: /* Reserved|Standby -> Configured */ - if (!zdev) { - zdev = zpci_create_device(ccdf->fid, ccdf->fh, ZPCI_FN_STATE_CONFIGURED); - if (IS_ERR(zdev)) - break; - if (zpci_add_device(zdev)) { - kfree(zdev); - break; - } - } else { - if (zdev->state == ZPCI_FN_STATE_RESERVED) - zpci_event_reappear(zdev); - /* the configuration request may be stale */ - else if (zdev->state != ZPCI_FN_STATE_STANDBY) - break; - zdev->state = ZPCI_FN_STATE_CONFIGURED; + zdev = zpci_create_device(ccdf->fid, ccdf->fh, ZPCI_FN_STATE_CONFIGURED); + if (IS_ERR(zdev)) + break; + if (zpci_add_device(zdev)) { + kfree(zdev); + break; } zpci_scan_configured_device(zdev, ccdf->fh); break; case 0x0302: /* Reserved -> Standby */ - if (!zdev) { - zdev = zpci_create_device(ccdf->fid, ccdf->fh, ZPCI_FN_STATE_STANDBY); - if (IS_ERR(zdev)) - break; - if (zpci_add_device(zdev)) { - kfree(zdev); - break; - } - } else { - if (zdev->state == ZPCI_FN_STATE_RESERVED) - zpci_event_reappear(zdev); - zpci_update_fh(zdev, ccdf->fh); + zdev = zpci_create_device(ccdf->fid, ccdf->fh, ZPCI_FN_STATE_STANDBY); + if (IS_ERR(zdev)) + break; + if (zpci_add_device(zdev)) { + kfree(zdev); + break; } break; + } +} + +static void zpci_event_avail_existing_device(struct zpci_dev *zdev, struct zpci_ccdf_avail *ccdf) +{ + enum zpci_state state; + + switch (ccdf->pec) { + case 0x0301: /* Reserved|Standby -> Configured */ + if (zdev->state == ZPCI_FN_STATE_RESERVED) + zpci_event_reappear(zdev); + /* the configuration request may be stale */ + else if (zdev->state != ZPCI_FN_STATE_STANDBY) + break; + zdev->state = ZPCI_FN_STATE_CONFIGURED; + zpci_scan_configured_device(zdev, ccdf->fh); + break; + case 0x0302: /* Reserved -> Standby */ + if (zdev->state == ZPCI_FN_STATE_RESERVED) + zpci_event_reappear(zdev); + zpci_update_fh(zdev, ccdf->fh); + break; case 0x0303: /* Deconfiguration requested */ - if (zdev) { - /* The event may have been queued before we configured - * the device. - */ - if (zdev->state != ZPCI_FN_STATE_CONFIGURED) - break; - zpci_update_fh(zdev, ccdf->fh); - zpci_deconfigure_device(zdev); - } + /* The event may have been queued before we configured + * the device. + */ + if (zdev->state != ZPCI_FN_STATE_CONFIGURED) + break; + zpci_update_fh(zdev, ccdf->fh); + zpci_deconfigure_device(zdev); break; case 0x0304: /* Configured -> Standby|Reserved */ - if (zdev) { - /* The event may have been queued before we configured - * the device.: - */ - if (zdev->state == ZPCI_FN_STATE_CONFIGURED) - zpci_event_hard_deconfigured(zdev, ccdf->fh); - /* The 0x0304 event may immediately reserve the device */ - if (!clp_get_state(zdev->fid, &state) && - state == ZPCI_FN_STATE_RESERVED) { - zpci_device_reserved(zdev); - } + /* The event may have been queued before we configured + * the device.: + */ + if (zdev->state == ZPCI_FN_STATE_CONFIGURED) + zpci_event_hard_deconfigured(zdev, ccdf->fh); + /* The 0x0304 event may immediately reserve the device */ + if (!clp_get_state(zdev->fid, &state) && + state == ZPCI_FN_STATE_RESERVED) { + zpci_device_reserved(zdev); } break; - case 0x0306: /* 0x308 or 0x302 for multiple devices */ - zpci_remove_reserved_devices(); - zpci_scan_devices(); - break; case 0x0308: /* Standby -> Reserved */ - if (!zdev) - break; zpci_device_reserved(zdev); break; - default: - break; - } - if (existing_zdev) { - mutex_unlock(&zdev->state_lock); - zpci_zdev_put(zdev); } } void zpci_event_availability(void *data) { - if (zpci_is_enabled()) - __zpci_event_availability(data); + struct zpci_ccdf_avail *ccdf = data; + struct zpci_dev *zdev; + + if (!zpci_is_enabled()) + return; + zpci_dbg(3, "avl fid:%x, fh:%x, pec:%x\n", + ccdf->fid, ccdf->fh, ccdf->pec); + if (zpci_event_avail_any_device(ccdf)) + return; + zdev = get_zdev_by_fid(ccdf->fid); + if (!zdev) + return zpci_event_avail_new_device(ccdf); + mutex_lock(&zdev->state_lock); + zpci_event_avail_existing_device(zdev, ccdf); + mutex_unlock(&zdev->state_lock); + zpci_zdev_put(zdev); } diff --git a/arch/s390/pci/pci_sysfs.c b/arch/s390/pci/pci_sysfs.c index d98d97df792a..bbb76113a4d0 100644 --- a/arch/s390/pci/pci_sysfs.c +++ b/arch/s390/pci/pci_sysfs.c @@ -153,6 +153,9 @@ static ssize_t report_error_write(struct file *filp, struct kobject *kobj, if (off || (count < sizeof(*report))) return -EINVAL; + if (count < (report->length + sizeof(*report))) + return -EINVAL; + ret = sclp_pci_report(report, zdev->fh, zdev->fid); return ret ? ret : count; diff --git a/arch/s390/purgatory/Makefile b/arch/s390/purgatory/Makefile index e74410bb1b88..b2f17e0d7fd6 100644 --- a/arch/s390/purgatory/Makefile +++ b/arch/s390/purgatory/Makefile @@ -1,5 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 +CONTEXT_ANALYSIS := y + purgatory-y := head.o purgatory.o string.o sha256.o targets += $(purgatory-y) purgatory.lds purgatory purgatory.chk purgatory.ro diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c index 2d28a569f793..32dd5a57240d 100644 --- a/arch/s390/tools/gen_facilities.c +++ b/arch/s390/tools/gen_facilities.c @@ -96,6 +96,13 @@ static struct facility_def facility_defs[] = { 150, /* enhanced sort */ 151, /* deflate conversion */ 155, /* msa extension 9 */ + 165, /* nnpa facility */ + 170, /* ineffective-nonconstrained-transaction facility */ + 193, /* bear enhancement facility */ + 194, /* rdp enhancement facility */ + 196, /* processor activity instrumentation facility */ + 197, /* processor activity instrumentation extension 1 */ + 201, /* concurrent-functions facility */ -1 /* END */ } }, @@ -112,13 +119,6 @@ static struct facility_def facility_defs[] = { 12, /* AP Query Configuration Information */ 15, /* AP Facilities Test */ 156, /* etoken facility */ - 165, /* nnpa facility */ - 170, /* ineffective-nonconstrained-transaction facility */ - 193, /* bear enhancement facility */ - 194, /* rdp enhancement facility */ - 196, /* processor activity instrumentation facility */ - 197, /* processor activity instrumentation extension 1 */ - 201, /* concurrent-functions facility */ -1 /* END */ } }, diff --git a/arch/s390/tools/relocs.c b/arch/s390/tools/relocs.c index 30a732c808f3..72178dc90a4c 100644 --- a/arch/s390/tools/relocs.c +++ b/arch/s390/tools/relocs.c @@ -41,6 +41,10 @@ static Elf_Ehdr ehdr; static unsigned long shnum; static unsigned int shstrndx; +static unsigned int shsymtabndx; +static unsigned int shxsymtabndx; + +static int sym_index(Elf_Sym *sym); struct relocs { uint32_t *offset; @@ -54,11 +58,40 @@ static struct relocs relocs64; struct section { Elf_Shdr shdr; struct section *link; + Elf_Sym *symtab; + Elf32_Word *xsymtab; Elf_Rel *reltab; + char *strtab; }; static struct section *secs; +static const char *sec_name(unsigned shndx) +{ + const char *sec_strtab; + const char *name = "<noname>"; + sec_strtab = secs[shstrndx].strtab; + + if (shndx < shnum) + name = sec_strtab + secs[shndx].shdr.sh_name; + else if (shndx == SHN_ABS) + name = "ABSOLUTE"; + else if (shndx == SHN_COMMON) + name = "COMMON"; + return name; +} + +static const char *sym_name(const char *sym_strtab, Elf_Sym *sym) +{ + const char *name; + + if (sym->st_name) + name = sym_strtab + sym->st_name; + else + name = sec_name(sym_index(sym)); + return name; +} + #if BYTE_ORDER == LITTLE_ENDIAN #define le16_to_cpu(val) (val) #define le32_to_cpu(val) (val) @@ -105,6 +138,23 @@ static uint64_t elf64_to_cpu(uint64_t val) #define elf_off_to_cpu(x) elf64_to_cpu(x) #define elf_xword_to_cpu(x) elf64_to_cpu(x) +static int sym_index(Elf_Sym *sym) +{ + Elf_Sym *symtab = secs[shsymtabndx].symtab; + Elf32_Word *xsymtab = secs[shxsymtabndx].xsymtab; + unsigned long offset; + int index; + + if (sym->st_shndx != SHN_XINDEX) + return sym->st_shndx; + + /* calculate offset of sym from head of table. */ + offset = (unsigned long)sym - (unsigned long)symtab; + index = offset / sizeof(*sym); + + return elf32_to_cpu(xsymtab[index]); +} + static void die(char *fmt, ...) { va_list ap; @@ -216,6 +266,81 @@ static void read_shdrs(FILE *fp) } +static void read_strtabs(FILE *fp) +{ + int i; + + for (i = 0; i < shnum; i++) { + struct section *sec = &secs[i]; + + if (sec->shdr.sh_type != SHT_STRTAB) + continue; + + sec->strtab = malloc(sec->shdr.sh_size); + if (!sec->strtab) + die("malloc of %" FMT " bytes for strtab failed\n", sec->shdr.sh_size); + + if (fseek(fp, sec->shdr.sh_offset, SEEK_SET) < 0) + die("Seek to %" FMT " failed: %s\n", sec->shdr.sh_offset, strerror(errno)); + + if (fread(sec->strtab, 1, sec->shdr.sh_size, fp) != sec->shdr.sh_size) + die("Cannot read symbol table: %s\n", strerror(errno)); + } +} + +static void read_symtabs(FILE *fp) +{ + int i, j; + + for (i = 0; i < shnum; i++) { + struct section *sec = &secs[i]; + int num_syms; + + switch (sec->shdr.sh_type) { + case SHT_SYMTAB_SHNDX: + sec->xsymtab = malloc(sec->shdr.sh_size); + if (!sec->xsymtab) + die("malloc of %" FMT " bytes for xsymtab failed\n", sec->shdr.sh_size); + + if (fseek(fp, sec->shdr.sh_offset, SEEK_SET) < 0) + die("Seek to %" FMT " failed: %s\n", sec->shdr.sh_offset, strerror(errno)); + + if (fread(sec->xsymtab, 1, sec->shdr.sh_size, fp) != sec->shdr.sh_size) + die("Cannot read extended symbol table: %s\n", strerror(errno)); + + shxsymtabndx = i; + continue; + + case SHT_SYMTAB: + num_syms = sec->shdr.sh_size / sizeof(Elf_Sym); + + sec->symtab = malloc(sec->shdr.sh_size); + if (!sec->symtab) + die("malloc of %" FMT " bytes for symtab failed\n", sec->shdr.sh_size); + + if (fseek(fp, sec->shdr.sh_offset, SEEK_SET) < 0) + die("Seek to %" FMT " failed: %s\n", sec->shdr.sh_offset, strerror(errno)); + + if (fread(sec->symtab, 1, sec->shdr.sh_size, fp) != sec->shdr.sh_size) + die("Cannot read symbol table: %s\n", strerror(errno)); + + for (j = 0; j < num_syms; j++) { + Elf_Sym *sym = &sec->symtab[j]; + + sym->st_name = elf_word_to_cpu(sym->st_name); + sym->st_value = elf_addr_to_cpu(sym->st_value); + sym->st_size = elf_xword_to_cpu(sym->st_size); + sym->st_shndx = elf_half_to_cpu(sym->st_shndx); + } + shsymtabndx = i; + continue; + + default: + continue; + } + } +} + static void read_relocs(FILE *fp) { int i, j; @@ -263,7 +388,8 @@ static void add_reloc(struct relocs *r, uint32_t offset) r->offset[r->count++] = offset; } -static int do_reloc(struct section *sec, Elf_Rel *rel) +static int do_reloc(struct section *sec, Elf_Rel *rel, ElfW(Sym) *sym, + const char *symname) { unsigned int r_type = ELF64_R_TYPE(rel->r_info); ElfW(Addr) offset = rel->r_offset; @@ -279,6 +405,22 @@ static int do_reloc(struct section *sec, Elf_Rel *rel) case R_390_GOTPCDBL: case R_390_GOTOFF64: break; + case R_390_32: { + static const char kcfipfx[] = "__kcfi_typeid_"; + + if (sym->st_shndx != SHN_ABS) + die("Unsupported relocation type: %d\n", r_type); + /* + * Symbols with __kcfi_typeid_ prefix have constant values, + * which do not change if bzImage is loaded at a different + * physical address than the address for which it has been + * compiled. + */ + if (!strncmp(kcfipfx, symname, sizeof(kcfipfx) - 1)) + break; + die("Invalid absolute R_390_32 relocation: %s\n", symname); + break; + } case R_390_64: add_reloc(&relocs64, offset); break; @@ -296,21 +438,29 @@ static void walk_relocs(void) /* Walk through the relocations */ for (i = 0; i < shnum; i++) { - struct section *sec_applies; + char *sym_strtab; + Elf_Sym *sh_symtab; + struct section *sec_applies, *sec_symtab; int j; struct section *sec = &secs[i]; if (sec->shdr.sh_type != SHT_REL_TYPE) continue; + sec_symtab = sec->link; sec_applies = &secs[sec->shdr.sh_info]; if (!(sec_applies->shdr.sh_flags & SHF_ALLOC)) continue; + sh_symtab = sec_symtab->symtab; + sym_strtab = sec_symtab->link->strtab; + for (j = 0; j < sec->shdr.sh_size / sizeof(Elf_Rel); j++) { Elf_Rel *rel = &sec->reltab[j]; + Elf_Sym *sym = &sh_symtab[ELF_R_SYM(rel->r_info)]; + const char *symname = sym_name(sym_strtab, sym); - do_reloc(sec, rel); + do_reloc(sec, rel, sym, symname); } } } @@ -349,6 +499,8 @@ static void process(FILE *fp) { read_ehdr(fp); read_shdrs(fp); + read_strtabs(fp); + read_symtabs(fp); read_relocs(fp); emit_relocs(); } |