summaryrefslogtreecommitdiff
path: root/arch/s390/include
diff options
context:
space:
mode:
Diffstat (limited to 'arch/s390/include')
-rw-r--r--arch/s390/include/asm/cpacf.h178
-rw-r--r--arch/s390/include/asm/facilities_src.h24
-rw-r--r--arch/s390/include/asm/fpu/api.h32
-rw-r--r--arch/s390/include/asm/lowcore.h3
-rw-r--r--arch/s390/include/asm/mmu.h1
-rw-r--r--arch/s390/include/asm/mmu_context.h1
-rw-r--r--arch/s390/include/asm/pci.h5
-rw-r--r--arch/s390/include/asm/pgtable.h74
-rw-r--r--arch/s390/include/asm/tlbflush.h72
-rw-r--r--arch/s390/include/asm/vx-insn.h148
-rw-r--r--arch/s390/include/uapi/asm/Kbuild1
11 files changed, 293 insertions, 246 deletions
diff --git a/arch/s390/include/asm/cpacf.h b/arch/s390/include/asm/cpacf.h
index d28621de8e0b..2c680db7e5c1 100644
--- a/arch/s390/include/asm/cpacf.h
+++ b/arch/s390/include/asm/cpacf.h
@@ -28,67 +28,51 @@
#define CPACF_PPNO 0xb93c /* MSA5 */
/*
- * Function codes for the KM (CIPHER MESSAGE)
- * instruction (0x80 is the decipher modifier bit)
+ * Decryption modifier bit
+ */
+#define CPACF_DECRYPT 0x80
+
+/*
+ * Function codes for the KM (CIPHER MESSAGE) instruction
*/
#define CPACF_KM_QUERY 0x00
-#define CPACF_KM_DEA_ENC 0x01
-#define CPACF_KM_DEA_DEC 0x81
-#define CPACF_KM_TDEA_128_ENC 0x02
-#define CPACF_KM_TDEA_128_DEC 0x82
-#define CPACF_KM_TDEA_192_ENC 0x03
-#define CPACF_KM_TDEA_192_DEC 0x83
-#define CPACF_KM_AES_128_ENC 0x12
-#define CPACF_KM_AES_128_DEC 0x92
-#define CPACF_KM_AES_192_ENC 0x13
-#define CPACF_KM_AES_192_DEC 0x93
-#define CPACF_KM_AES_256_ENC 0x14
-#define CPACF_KM_AES_256_DEC 0x94
-#define CPACF_KM_XTS_128_ENC 0x32
-#define CPACF_KM_XTS_128_DEC 0xb2
-#define CPACF_KM_XTS_256_ENC 0x34
-#define CPACF_KM_XTS_256_DEC 0xb4
+#define CPACF_KM_DEA 0x01
+#define CPACF_KM_TDEA_128 0x02
+#define CPACF_KM_TDEA_192 0x03
+#define CPACF_KM_AES_128 0x12
+#define CPACF_KM_AES_192 0x13
+#define CPACF_KM_AES_256 0x14
+#define CPACF_KM_XTS_128 0x32
+#define CPACF_KM_XTS_256 0x34
/*
* Function codes for the KMC (CIPHER MESSAGE WITH CHAINING)
- * instruction (0x80 is the decipher modifier bit)
+ * instruction
*/
#define CPACF_KMC_QUERY 0x00
-#define CPACF_KMC_DEA_ENC 0x01
-#define CPACF_KMC_DEA_DEC 0x81
-#define CPACF_KMC_TDEA_128_ENC 0x02
-#define CPACF_KMC_TDEA_128_DEC 0x82
-#define CPACF_KMC_TDEA_192_ENC 0x03
-#define CPACF_KMC_TDEA_192_DEC 0x83
-#define CPACF_KMC_AES_128_ENC 0x12
-#define CPACF_KMC_AES_128_DEC 0x92
-#define CPACF_KMC_AES_192_ENC 0x13
-#define CPACF_KMC_AES_192_DEC 0x93
-#define CPACF_KMC_AES_256_ENC 0x14
-#define CPACF_KMC_AES_256_DEC 0x94
+#define CPACF_KMC_DEA 0x01
+#define CPACF_KMC_TDEA_128 0x02
+#define CPACF_KMC_TDEA_192 0x03
+#define CPACF_KMC_AES_128 0x12
+#define CPACF_KMC_AES_192 0x13
+#define CPACF_KMC_AES_256 0x14
#define CPACF_KMC_PRNG 0x43
/*
* Function codes for the KMCTR (CIPHER MESSAGE WITH COUNTER)
- * instruction (0x80 is the decipher modifier bit)
+ * instruction
*/
-#define CPACF_KMCTR_QUERY 0x00
-#define CPACF_KMCTR_DEA_ENC 0x01
-#define CPACF_KMCTR_DEA_DEC 0x81
-#define CPACF_KMCTR_TDEA_128_ENC 0x02
-#define CPACF_KMCTR_TDEA_128_DEC 0x82
-#define CPACF_KMCTR_TDEA_192_ENC 0x03
-#define CPACF_KMCTR_TDEA_192_DEC 0x83
-#define CPACF_KMCTR_AES_128_ENC 0x12
-#define CPACF_KMCTR_AES_128_DEC 0x92
-#define CPACF_KMCTR_AES_192_ENC 0x13
-#define CPACF_KMCTR_AES_192_DEC 0x93
-#define CPACF_KMCTR_AES_256_ENC 0x14
-#define CPACF_KMCTR_AES_256_DEC 0x94
+#define CPACF_KMCTR_QUERY 0x00
+#define CPACF_KMCTR_DEA 0x01
+#define CPACF_KMCTR_TDEA_128 0x02
+#define CPACF_KMCTR_TDEA_192 0x03
+#define CPACF_KMCTR_AES_128 0x12
+#define CPACF_KMCTR_AES_192 0x13
+#define CPACF_KMCTR_AES_256 0x14
/*
* Function codes for the KIMD (COMPUTE INTERMEDIATE MESSAGE DIGEST)
- * instruction (0x80 is the decipher modifier bit)
+ * instruction
*/
#define CPACF_KIMD_QUERY 0x00
#define CPACF_KIMD_SHA_1 0x01
@@ -98,7 +82,7 @@
/*
* Function codes for the KLMD (COMPUTE LAST MESSAGE DIGEST)
- * instruction (0x80 is the decipher modifier bit)
+ * instruction
*/
#define CPACF_KLMD_QUERY 0x00
#define CPACF_KLMD_SHA_1 0x01
@@ -107,7 +91,7 @@
/*
* function codes for the KMAC (COMPUTE MESSAGE AUTHENTICATION CODE)
- * instruction (0x80 is the decipher modifier bit)
+ * instruction
*/
#define CPACF_KMAC_QUERY 0x00
#define CPACF_KMAC_DEA 0x01
@@ -116,12 +100,14 @@
/*
* Function codes for the PPNO (PERFORM PSEUDORANDOM NUMBER OPERATION)
- * instruction (0x80 is the decipher modifier bit)
+ * instruction
*/
#define CPACF_PPNO_QUERY 0x00
#define CPACF_PPNO_SHA512_DRNG_GEN 0x03
#define CPACF_PPNO_SHA512_DRNG_SEED 0x83
+typedef struct { unsigned char bytes[16]; } cpacf_mask_t;
+
/**
* cpacf_query() - check if a specific CPACF function is available
* @opcode: the opcode of the crypto instruction
@@ -132,55 +118,66 @@
*
* Returns 1 if @func is available for @opcode, 0 otherwise
*/
-static inline void __cpacf_query(unsigned int opcode, unsigned char *status)
+static inline void __cpacf_query(unsigned int opcode, cpacf_mask_t *mask)
{
- typedef struct { unsigned char _[16]; } status_type;
register unsigned long r0 asm("0") = 0; /* query function */
- register unsigned long r1 asm("1") = (unsigned long) status;
+ register unsigned long r1 asm("1") = (unsigned long) mask;
asm volatile(
" spm 0\n" /* pckmo doesn't change the cc */
/* Parameter registers are ignored, but may not be 0 */
"0: .insn rrf,%[opc] << 16,2,2,2,0\n"
" brc 1,0b\n" /* handle partial completion */
- : "=m" (*(status_type *) status)
+ : "=m" (*mask)
: [fc] "d" (r0), [pba] "a" (r1), [opc] "i" (opcode)
: "cc");
}
-static inline int cpacf_query(unsigned int opcode, unsigned int func)
+static inline int __cpacf_check_opcode(unsigned int opcode)
{
- unsigned char status[16];
-
switch (opcode) {
case CPACF_KMAC:
case CPACF_KM:
case CPACF_KMC:
case CPACF_KIMD:
case CPACF_KLMD:
- if (!test_facility(17)) /* check for MSA */
- return 0;
- break;
+ return test_facility(17); /* check for MSA */
case CPACF_PCKMO:
- if (!test_facility(76)) /* check for MSA3 */
- return 0;
- break;
+ return test_facility(76); /* check for MSA3 */
case CPACF_KMF:
case CPACF_KMO:
case CPACF_PCC:
case CPACF_KMCTR:
- if (!test_facility(77)) /* check for MSA4 */
- return 0;
- break;
+ return test_facility(77); /* check for MSA4 */
case CPACF_PPNO:
- if (!test_facility(57)) /* check for MSA5 */
- return 0;
- break;
+ return test_facility(57); /* check for MSA5 */
default:
BUG();
}
- __cpacf_query(opcode, status);
- return (status[func >> 3] & (0x80 >> (func & 7))) != 0;
+}
+
+static inline int cpacf_query(unsigned int opcode, cpacf_mask_t *mask)
+{
+ if (__cpacf_check_opcode(opcode)) {
+ __cpacf_query(opcode, mask);
+ return 1;
+ }
+ memset(mask, 0, sizeof(*mask));
+ return 0;
+}
+
+static inline int cpacf_test_func(cpacf_mask_t *mask, unsigned int func)
+{
+ return (mask->bytes[func >> 3] & (0x80 >> (func & 7))) != 0;
+}
+
+static inline int cpacf_query_func(unsigned int opcode, unsigned int func)
+{
+ cpacf_mask_t mask;
+
+ if (cpacf_query(opcode, &mask))
+ return cpacf_test_func(&mask, func);
+ return 0;
}
/**
@@ -194,7 +191,7 @@ static inline int cpacf_query(unsigned int opcode, unsigned int func)
* Returns 0 for the query func, number of processed bytes for
* encryption/decryption funcs
*/
-static inline int cpacf_km(long func, void *param,
+static inline int cpacf_km(unsigned long func, void *param,
u8 *dest, const u8 *src, long src_len)
{
register unsigned long r0 asm("0") = (unsigned long) func;
@@ -224,7 +221,7 @@ static inline int cpacf_km(long func, void *param,
* Returns 0 for the query func, number of processed bytes for
* encryption/decryption funcs
*/
-static inline int cpacf_kmc(long func, void *param,
+static inline int cpacf_kmc(unsigned long func, void *param,
u8 *dest, const u8 *src, long src_len)
{
register unsigned long r0 asm("0") = (unsigned long) func;
@@ -250,11 +247,9 @@ static inline int cpacf_kmc(long func, void *param,
* @param: address of parameter block; see POP for details on each func
* @src: address of source memory area
* @src_len: length of src operand in bytes
- *
- * Returns 0 for the query func, number of processed bytes for digest funcs
*/
-static inline int cpacf_kimd(long func, void *param,
- const u8 *src, long src_len)
+static inline void cpacf_kimd(unsigned long func, void *param,
+ const u8 *src, long src_len)
{
register unsigned long r0 asm("0") = (unsigned long) func;
register unsigned long r1 asm("1") = (unsigned long) param;
@@ -267,8 +262,6 @@ static inline int cpacf_kimd(long func, void *param,
: [src] "+a" (r2), [len] "+d" (r3)
: [fc] "d" (r0), [pba] "a" (r1), [opc] "i" (CPACF_KIMD)
: "cc", "memory");
-
- return src_len - r3;
}
/**
@@ -277,11 +270,9 @@ static inline int cpacf_kimd(long func, void *param,
* @param: address of parameter block; see POP for details on each func
* @src: address of source memory area
* @src_len: length of src operand in bytes
- *
- * Returns 0 for the query func, number of processed bytes for digest funcs
*/
-static inline int cpacf_klmd(long func, void *param,
- const u8 *src, long src_len)
+static inline void cpacf_klmd(unsigned long func, void *param,
+ const u8 *src, long src_len)
{
register unsigned long r0 asm("0") = (unsigned long) func;
register unsigned long r1 asm("1") = (unsigned long) param;
@@ -294,8 +285,6 @@ static inline int cpacf_klmd(long func, void *param,
: [src] "+a" (r2), [len] "+d" (r3)
: [fc] "d" (r0), [pba] "a" (r1), [opc] "i" (CPACF_KLMD)
: "cc", "memory");
-
- return src_len - r3;
}
/**
@@ -308,7 +297,7 @@ static inline int cpacf_klmd(long func, void *param,
*
* Returns 0 for the query func, number of processed bytes for digest funcs
*/
-static inline int cpacf_kmac(long func, void *param,
+static inline int cpacf_kmac(unsigned long func, void *param,
const u8 *src, long src_len)
{
register unsigned long r0 asm("0") = (unsigned long) func;
@@ -338,7 +327,7 @@ static inline int cpacf_kmac(long func, void *param,
* Returns 0 for the query func, number of processed bytes for
* encryption/decryption funcs
*/
-static inline int cpacf_kmctr(long func, void *param, u8 *dest,
+static inline int cpacf_kmctr(unsigned long func, void *param, u8 *dest,
const u8 *src, long src_len, u8 *counter)
{
register unsigned long r0 asm("0") = (unsigned long) func;
@@ -368,13 +357,10 @@ static inline int cpacf_kmctr(long func, void *param, u8 *dest,
* @dest_len: size of destination memory area in bytes
* @seed: address of seed data
* @seed_len: size of seed data in bytes
- *
- * Returns 0 for the query func, number of random bytes stored in
- * dest buffer for generate function
*/
-static inline int cpacf_ppno(long func, void *param,
- u8 *dest, long dest_len,
- const u8 *seed, long seed_len)
+static inline void cpacf_ppno(unsigned long func, void *param,
+ u8 *dest, long dest_len,
+ const u8 *seed, long seed_len)
{
register unsigned long r0 asm("0") = (unsigned long) func;
register unsigned long r1 asm("1") = (unsigned long) param;
@@ -390,8 +376,6 @@ static inline int cpacf_ppno(long func, void *param,
: [fc] "d" (r0), [pba] "a" (r1),
[seed] "a" (r4), [slen] "d" (r5), [opc] "i" (CPACF_PPNO)
: "cc", "memory");
-
- return dest_len - r3;
}
/**
@@ -399,10 +383,8 @@ static inline int cpacf_ppno(long func, void *param,
* instruction
* @func: the function code passed to PCC; see CPACF_KM_xxx defines
* @param: address of parameter block; see POP for details on each func
- *
- * Returns 0.
*/
-static inline int cpacf_pcc(long func, void *param)
+static inline void cpacf_pcc(unsigned long func, void *param)
{
register unsigned long r0 asm("0") = (unsigned long) func;
register unsigned long r1 asm("1") = (unsigned long) param;
@@ -413,8 +395,6 @@ static inline int cpacf_pcc(long func, void *param)
:
: [fc] "d" (r0), [pba] "a" (r1), [opc] "i" (CPACF_PCC)
: "cc", "memory");
-
- return 0;
}
#endif /* _ASM_S390_CPACF_H */
diff --git a/arch/s390/include/asm/facilities_src.h b/arch/s390/include/asm/facilities_src.h
index 4917728e5828..3b758f66e48b 100644
--- a/arch/s390/include/asm/facilities_src.h
+++ b/arch/s390/include/asm/facilities_src.h
@@ -55,4 +55,28 @@ static struct facility_def facility_defs[] = {
-1 /* END */
}
},
+ {
+ .name = "FACILITIES_KVM",
+ .bits = (int[]){
+ 0, /* N3 instructions */
+ 1, /* z/Arch mode installed */
+ 2, /* z/Arch mode active */
+ 3, /* DAT-enhancement */
+ 4, /* idte segment table */
+ 5, /* idte region table */
+ 6, /* ASN-and-LX reuse */
+ 7, /* stfle */
+ 8, /* enhanced-DAT 1 */
+ 9, /* sense-running-status */
+ 10, /* conditional sske */
+ 13, /* ipte-range */
+ 14, /* nonquiescing key-setting */
+ 73, /* transactional execution */
+ 75, /* access-exception-fetch/store indication */
+ 76, /* msa extension 3 */
+ 77, /* msa extension 4 */
+ 78, /* enhanced-DAT 2 */
+ -1 /* END */
+ }
+ },
};
diff --git a/arch/s390/include/asm/fpu/api.h b/arch/s390/include/asm/fpu/api.h
index 6aba6fc406ad..02124d66bfb5 100644
--- a/arch/s390/include/asm/fpu/api.h
+++ b/arch/s390/include/asm/fpu/api.h
@@ -64,18 +64,18 @@ static inline int test_fp_ctl(u32 fpc)
return rc;
}
-#define KERNEL_VXR_V0V7 1
-#define KERNEL_VXR_V8V15 2
-#define KERNEL_VXR_V16V23 4
-#define KERNEL_VXR_V24V31 8
-#define KERNEL_FPR 16
-#define KERNEL_FPC 256
+#define KERNEL_FPC 1
+#define KERNEL_VXR_V0V7 2
+#define KERNEL_VXR_V8V15 4
+#define KERNEL_VXR_V16V23 8
+#define KERNEL_VXR_V24V31 16
#define KERNEL_VXR_LOW (KERNEL_VXR_V0V7|KERNEL_VXR_V8V15)
#define KERNEL_VXR_MID (KERNEL_VXR_V8V15|KERNEL_VXR_V16V23)
#define KERNEL_VXR_HIGH (KERNEL_VXR_V16V23|KERNEL_VXR_V24V31)
-#define KERNEL_FPU_MASK (KERNEL_VXR_LOW|KERNEL_VXR_HIGH|KERNEL_FPR)
+#define KERNEL_VXR (KERNEL_VXR_LOW|KERNEL_VXR_HIGH)
+#define KERNEL_FPR (KERNEL_FPC|KERNEL_VXR_V0V7)
struct kernel_fpu;
@@ -87,18 +87,28 @@ struct kernel_fpu;
* Prefer using the kernel_fpu_begin()/kernel_fpu_end() pair of functions.
*/
void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags);
-void __kernel_fpu_end(struct kernel_fpu *state);
+void __kernel_fpu_end(struct kernel_fpu *state, u32 flags);
static inline void kernel_fpu_begin(struct kernel_fpu *state, u32 flags)
{
preempt_disable();
- __kernel_fpu_begin(state, flags);
+ state->mask = S390_lowcore.fpu_flags;
+ if (!test_cpu_flag(CIF_FPU))
+ /* Save user space FPU state and register contents */
+ save_fpu_regs();
+ else if (state->mask & flags)
+ /* Save FPU/vector register in-use by the kernel */
+ __kernel_fpu_begin(state, flags);
+ S390_lowcore.fpu_flags |= flags;
}
-static inline void kernel_fpu_end(struct kernel_fpu *state)
+static inline void kernel_fpu_end(struct kernel_fpu *state, u32 flags)
{
- __kernel_fpu_end(state);
+ S390_lowcore.fpu_flags = state->mask;
+ if (state->mask & flags)
+ /* Restore FPU/vector register in-use by the kernel */
+ __kernel_fpu_end(state, flags);
preempt_enable();
}
diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h
index d79ba7cf75b0..7b93b78f423c 100644
--- a/arch/s390/include/asm/lowcore.h
+++ b/arch/s390/include/asm/lowcore.h
@@ -129,7 +129,8 @@ struct lowcore {
__u8 pad_0x0390[0x0398-0x0390]; /* 0x0390 */
__u64 gmap; /* 0x0398 */
__u32 spinlock_lockval; /* 0x03a0 */
- __u8 pad_0x03a0[0x0400-0x03a4]; /* 0x03a4 */
+ __u32 fpu_flags; /* 0x03a4 */
+ __u8 pad_0x03a8[0x0400-0x03a8]; /* 0x03a8 */
/* Per cpu primary space access list */
__u32 paste[16]; /* 0x0400 */
diff --git a/arch/s390/include/asm/mmu.h b/arch/s390/include/asm/mmu.h
index 6d39329c894b..bea785d7f853 100644
--- a/arch/s390/include/asm/mmu.h
+++ b/arch/s390/include/asm/mmu.h
@@ -12,6 +12,7 @@ typedef struct {
struct list_head pgtable_list;
spinlock_t gmap_lock;
struct list_head gmap_list;
+ unsigned long gmap_asce;
unsigned long asce;
unsigned long asce_limit;
unsigned long vdso_base;
diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h
index c6a088c91aee..515fea5a3fc4 100644
--- a/arch/s390/include/asm/mmu_context.h
+++ b/arch/s390/include/asm/mmu_context.h
@@ -21,6 +21,7 @@ static inline int init_new_context(struct task_struct *tsk,
INIT_LIST_HEAD(&mm->context.gmap_list);
cpumask_clear(&mm->context.cpu_attach_mask);
atomic_set(&mm->context.flush_count, 0);
+ mm->context.gmap_asce = 0;
mm->context.flush_mm = 0;
#ifdef CONFIG_PGSTE
mm->context.alloc_pgste = page_table_allocate_pgste;
diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h
index 0da91c4d30fd..6611f798d2be 100644
--- a/arch/s390/include/asm/pci.h
+++ b/arch/s390/include/asm/pci.h
@@ -11,6 +11,7 @@
#include <asm-generic/pci.h>
#include <asm/pci_clp.h>
#include <asm/pci_debug.h>
+#include <asm/sclp.h>
#define PCIBIOS_MIN_IO 0x1000
#define PCIBIOS_MIN_MEM 0x10000000
@@ -117,6 +118,7 @@ struct zpci_dev {
spinlock_t iommu_bitmap_lock;
unsigned long *iommu_bitmap;
+ unsigned long *lazy_bitmap;
unsigned long iommu_size;
unsigned long iommu_pages;
unsigned int next_bit;
@@ -216,6 +218,9 @@ void zpci_debug_init_device(struct zpci_dev *, const char *);
void zpci_debug_exit_device(struct zpci_dev *);
void zpci_debug_info(struct zpci_dev *, struct seq_file *);
+/* Error reporting */
+int zpci_report_error(struct pci_dev *, struct zpci_report_error_header *);
+
#ifdef CONFIG_NUMA
/* Returns the node based on PCI bus */
diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index 72c7f60bfe83..0362cd5fa187 100644
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -874,35 +874,31 @@ static inline pte_t pte_mkhuge(pte_t pte)
}
#endif
-static inline void __ptep_ipte(unsigned long address, pte_t *ptep)
-{
- unsigned long pto = (unsigned long) ptep;
-
- /* Invalidation + global TLB flush for the pte */
- asm volatile(
- " ipte %2,%3"
- : "=m" (*ptep) : "m" (*ptep), "a" (pto), "a" (address));
-}
+#define IPTE_GLOBAL 0
+#define IPTE_LOCAL 1
-static inline void __ptep_ipte_local(unsigned long address, pte_t *ptep)
+static inline void __ptep_ipte(unsigned long address, pte_t *ptep, int local)
{
unsigned long pto = (unsigned long) ptep;
- /* Invalidation + local TLB flush for the pte */
+ /* Invalidation + TLB flush for the pte */
asm volatile(
- " .insn rrf,0xb2210000,%2,%3,0,1"
- : "=m" (*ptep) : "m" (*ptep), "a" (pto), "a" (address));
+ " .insn rrf,0xb2210000,%[r1],%[r2],0,%[m4]"
+ : "+m" (*ptep) : [r1] "a" (pto), [r2] "a" (address),
+ [m4] "i" (local));
}
-static inline void __ptep_ipte_range(unsigned long address, int nr, pte_t *ptep)
+static inline void __ptep_ipte_range(unsigned long address, int nr,
+ pte_t *ptep, int local)
{
unsigned long pto = (unsigned long) ptep;
- /* Invalidate a range of ptes + global TLB flush of the ptes */
+ /* Invalidate a range of ptes + TLB flush of the ptes */
do {
asm volatile(
- " .insn rrf,0xb2210000,%2,%0,%1,0"
- : "+a" (address), "+a" (nr) : "a" (pto) : "memory");
+ " .insn rrf,0xb2210000,%[r1],%[r2],%[r3],%[m4]"
+ : [r2] "+a" (address), [r3] "+a" (nr)
+ : [r1] "a" (pto), [m4] "i" (local) : "memory");
} while (nr != 255);
}
@@ -1239,53 +1235,33 @@ static inline void __pmdp_csp(pmd_t *pmdp)
pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID);
}
-static inline void __pmdp_idte(unsigned long address, pmd_t *pmdp)
-{
- unsigned long sto;
-
- sto = (unsigned long) pmdp - pmd_index(address) * sizeof(pmd_t);
- asm volatile(
- " .insn rrf,0xb98e0000,%2,%3,0,0"
- : "=m" (*pmdp)
- : "m" (*pmdp), "a" (sto), "a" ((address & HPAGE_MASK))
- : "cc" );
-}
-
-static inline void __pudp_idte(unsigned long address, pud_t *pudp)
-{
- unsigned long r3o;
-
- r3o = (unsigned long) pudp - pud_index(address) * sizeof(pud_t);
- r3o |= _ASCE_TYPE_REGION3;
- asm volatile(
- " .insn rrf,0xb98e0000,%2,%3,0,0"
- : "=m" (*pudp)
- : "m" (*pudp), "a" (r3o), "a" ((address & PUD_MASK))
- : "cc");
-}
+#define IDTE_GLOBAL 0
+#define IDTE_LOCAL 1
-static inline void __pmdp_idte_local(unsigned long address, pmd_t *pmdp)
+static inline void __pmdp_idte(unsigned long address, pmd_t *pmdp, int local)
{
unsigned long sto;
sto = (unsigned long) pmdp - pmd_index(address) * sizeof(pmd_t);
asm volatile(
- " .insn rrf,0xb98e0000,%2,%3,0,1"
- : "=m" (*pmdp)
- : "m" (*pmdp), "a" (sto), "a" ((address & HPAGE_MASK))
+ " .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"
+ : "+m" (*pmdp)
+ : [r1] "a" (sto), [r2] "a" ((address & HPAGE_MASK)),
+ [m4] "i" (local)
: "cc" );
}
-static inline void __pudp_idte_local(unsigned long address, pud_t *pudp)
+static inline void __pudp_idte(unsigned long address, pud_t *pudp, int local)
{
unsigned long r3o;
r3o = (unsigned long) pudp - pud_index(address) * sizeof(pud_t);
r3o |= _ASCE_TYPE_REGION3;
asm volatile(
- " .insn rrf,0xb98e0000,%2,%3,0,1"
- : "=m" (*pudp)
- : "m" (*pudp), "a" (r3o), "a" ((address & PUD_MASK))
+ " .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"
+ : "+m" (*pudp)
+ : [r1] "a" (r3o), [r2] "a" ((address & PUD_MASK)),
+ [m4] "i" (local)
: "cc");
}
diff --git a/arch/s390/include/asm/tlbflush.h b/arch/s390/include/asm/tlbflush.h
index 1a691ef740cf..39846100682a 100644
--- a/arch/s390/include/asm/tlbflush.h
+++ b/arch/s390/include/asm/tlbflush.h
@@ -26,17 +26,6 @@ static inline void __tlb_flush_idte(unsigned long asce)
: : "a" (2048), "a" (asce) : "cc");
}
-/*
- * Flush TLB entries for a specific ASCE on the local CPU
- */
-static inline void __tlb_flush_idte_local(unsigned long asce)
-{
- /* Local TLB flush for the mm */
- asm volatile(
- " .insn rrf,0xb98e0000,0,%0,%1,1"
- : : "a" (2048), "a" (asce) : "cc");
-}
-
#ifdef CONFIG_SMP
void smp_ptlb_all(void);
@@ -65,35 +54,33 @@ static inline void __tlb_flush_full(struct mm_struct *mm)
/* Global TLB flush */
__tlb_flush_global();
/* Reset TLB flush mask */
- if (MACHINE_HAS_TLB_LC)
- cpumask_copy(mm_cpumask(mm),
- &mm->context.cpu_attach_mask);
+ cpumask_copy(mm_cpumask(mm), &mm->context.cpu_attach_mask);
}
atomic_dec(&mm->context.flush_count);
preempt_enable();
}
-/*
- * Flush TLB entries for a specific ASCE on all CPUs. Should never be used
- * when more than one asce (e.g. gmap) ran on this mm.
- */
-static inline void __tlb_flush_asce(struct mm_struct *mm, unsigned long asce)
+static inline void __tlb_flush_mm(struct mm_struct *mm)
{
+ unsigned long gmap_asce;
+
+ /*
+ * If the machine has IDTE we prefer to do a per mm flush
+ * on all cpus instead of doing a local flush if the mm
+ * only ran on the local cpu.
+ */
preempt_disable();
atomic_inc(&mm->context.flush_count);
- if (MACHINE_HAS_TLB_LC &&
- cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) {
- __tlb_flush_idte_local(asce);
+ gmap_asce = READ_ONCE(mm->context.gmap_asce);
+ if (MACHINE_HAS_IDTE && gmap_asce != -1UL) {
+ if (gmap_asce)
+ __tlb_flush_idte(gmap_asce);
+ __tlb_flush_idte(mm->context.asce);
} else {
- if (MACHINE_HAS_IDTE)
- __tlb_flush_idte(asce);
- else
- __tlb_flush_global();
- /* Reset TLB flush mask */
- if (MACHINE_HAS_TLB_LC)
- cpumask_copy(mm_cpumask(mm),
- &mm->context.cpu_attach_mask);
+ __tlb_flush_full(mm);
}
+ /* Reset TLB flush mask */
+ cpumask_copy(mm_cpumask(mm), &mm->context.cpu_attach_mask);
atomic_dec(&mm->context.flush_count);
preempt_enable();
}
@@ -112,36 +99,17 @@ static inline void __tlb_flush_kernel(void)
/*
* Flush TLB entries for a specific ASCE on all CPUs.
*/
-static inline void __tlb_flush_asce(struct mm_struct *mm, unsigned long asce)
+static inline void __tlb_flush_mm(struct mm_struct *mm)
{
- if (MACHINE_HAS_TLB_LC)
- __tlb_flush_idte_local(asce);
- else
- __tlb_flush_local();
+ __tlb_flush_local();
}
static inline void __tlb_flush_kernel(void)
{
- if (MACHINE_HAS_TLB_LC)
- __tlb_flush_idte_local(init_mm.context.asce);
- else
- __tlb_flush_local();
+ __tlb_flush_local();
}
#endif
-static inline void __tlb_flush_mm(struct mm_struct * mm)
-{
- /*
- * If the machine has IDTE we prefer to do a per mm flush
- * on all cpus instead of doing a local flush if the mm
- * only ran on the local cpu.
- */
- if (MACHINE_HAS_IDTE && list_empty(&mm->context.gmap_list))
- __tlb_flush_asce(mm, mm->context.asce);
- else
- __tlb_flush_full(mm);
-}
-
static inline void __tlb_flush_mm_lazy(struct mm_struct * mm)
{
if (mm->context.flush_mm) {
diff --git a/arch/s390/include/asm/vx-insn.h b/arch/s390/include/asm/vx-insn.h
index 4a3135620f5e..49c24a2afce0 100644
--- a/arch/s390/include/asm/vx-insn.h
+++ b/arch/s390/include/asm/vx-insn.h
@@ -16,15 +16,13 @@
/* Macros to generate vector instruction byte code */
-#define REG_NUM_INVALID 255
-
/* GR_NUM - Retrieve general-purpose register number
*
* @opd: Operand to store register number
* @r64: String designation register in the format "%rN"
*/
.macro GR_NUM opd gr
- \opd = REG_NUM_INVALID
+ \opd = 255
.ifc \gr,%r0
\opd = 0
.endif
@@ -73,14 +71,11 @@
.ifc \gr,%r15
\opd = 15
.endif
- .if \opd == REG_NUM_INVALID
- .error "Invalid general-purpose register designation: \gr"
+ .if \opd == 255
+ \opd = \gr
.endif
.endm
-/* VX_R() - Macro to encode the VX_NUM into the instruction */
-#define VX_R(v) (v & 0x0F)
-
/* VX_NUM - Retrieve vector register number
*
* @opd: Operand to store register number
@@ -88,11 +83,10 @@
*
* The vector register number is used for as input number to the
* instruction and, as well as, to compute the RXB field of the
- * instruction. To encode the particular vector register number,
- * use the VX_R(v) macro to extract the instruction opcode.
+ * instruction.
*/
.macro VX_NUM opd vxr
- \opd = REG_NUM_INVALID
+ \opd = 255
.ifc \vxr,%v0
\opd = 0
.endif
@@ -189,8 +183,8 @@
.ifc \vxr,%v31
\opd = 31
.endif
- .if \opd == REG_NUM_INVALID
- .error "Invalid vector register designation: \vxr"
+ .if \opd == 255
+ \opd = \vxr
.endif
.endm
@@ -251,7 +245,7 @@
/* VECTOR GENERATE BYTE MASK */
.macro VGBM vr imm2
VX_NUM v1, \vr
- .word (0xE700 | (VX_R(v1) << 4))
+ .word (0xE700 | ((v1&15) << 4))
.word \imm2
MRXBOPC 0, 0x44, v1
.endm
@@ -267,7 +261,7 @@
VX_NUM v1, \v
GR_NUM b2, "%r0"
GR_NUM r3, \gr
- .word 0xE700 | (VX_R(v1) << 4) | r3
+ .word 0xE700 | ((v1&15) << 4) | r3
.word (b2 << 12) | (\disp)
MRXBOPC \m, 0x22, v1
.endm
@@ -284,12 +278,21 @@
VLVG \v, \gr, \index, 3
.endm
+/* VECTOR LOAD REGISTER */
+.macro VLR v1, v2
+ VX_NUM v1, \v1
+ VX_NUM v2, \v2
+ .word 0xE700 | ((v1&15) << 4) | (v2&15)
+ .word 0
+ MRXBOPC 0, 0x56, v1, v2
+.endm
+
/* VECTOR LOAD */
.macro VL v, disp, index="%r0", base
VX_NUM v1, \v
GR_NUM x2, \index
GR_NUM b2, \base
- .word 0xE700 | (VX_R(v1) << 4) | x2
+ .word 0xE700 | ((v1&15) << 4) | x2
.word (b2 << 12) | (\disp)
MRXBOPC 0, 0x06, v1
.endm
@@ -299,7 +302,7 @@
VX_NUM v1, \vr1
GR_NUM x2, \index
GR_NUM b2, \base
- .word 0xE700 | (VX_R(v1) << 4) | x2
+ .word 0xE700 | ((v1&15) << 4) | x2
.word (b2 << 12) | (\disp)
MRXBOPC \m3, \opc, v1
.endm
@@ -319,7 +322,7 @@
/* VECTOR LOAD ELEMENT IMMEDIATE */
.macro VLEIx vr1, imm2, m3, opc
VX_NUM v1, \vr1
- .word 0xE700 | (VX_R(v1) << 4)
+ .word 0xE700 | ((v1&15) << 4)
.word \imm2
MRXBOPC \m3, \opc, v1
.endm
@@ -341,7 +344,7 @@
GR_NUM r1, \gr
GR_NUM b2, \base
VX_NUM v3, \vr
- .word 0xE700 | (r1 << 4) | VX_R(v3)
+ .word 0xE700 | (r1 << 4) | (v3&15)
.word (b2 << 12) | (\disp)
MRXBOPC \m, 0x21, v3
.endm
@@ -363,7 +366,7 @@
VX_NUM v1, \vfrom
VX_NUM v3, \vto
GR_NUM b2, \base /* Base register */
- .word 0xE700 | (VX_R(v1) << 4) | VX_R(v3)
+ .word 0xE700 | ((v1&15) << 4) | (v3&15)
.word (b2 << 12) | (\disp)
MRXBOPC 0, 0x36, v1, v3
.endm
@@ -373,7 +376,7 @@
VX_NUM v1, \vfrom
VX_NUM v3, \vto
GR_NUM b2, \base /* Base register */
- .word 0xE700 | (VX_R(v1) << 4) | VX_R(v3)
+ .word 0xE700 | ((v1&15) << 4) | (v3&15)
.word (b2 << 12) | (\disp)
MRXBOPC 0, 0x3E, v1, v3
.endm
@@ -384,16 +387,16 @@
VX_NUM v2, \vr2
VX_NUM v3, \vr3
VX_NUM v4, \vr4
- .word 0xE700 | (VX_R(v1) << 4) | VX_R(v2)
- .word (VX_R(v3) << 12)
- MRXBOPC VX_R(v4), 0x8C, v1, v2, v3, v4
+ .word 0xE700 | ((v1&15) << 4) | (v2&15)
+ .word ((v3&15) << 12)
+ MRXBOPC (v4&15), 0x8C, v1, v2, v3, v4
.endm
/* VECTOR UNPACK LOGICAL LOW */
.macro VUPLL vr1, vr2, m3
VX_NUM v1, \vr1
VX_NUM v2, \vr2
- .word 0xE700 | (VX_R(v1) << 4) | VX_R(v2)
+ .word 0xE700 | ((v1&15) << 4) | (v2&15)
.word 0x0000
MRXBOPC \m3, 0xD4, v1, v2
.endm
@@ -410,13 +413,23 @@
/* Vector integer instructions */
+/* VECTOR AND */
+.macro VN vr1, vr2, vr3
+ VX_NUM v1, \vr1
+ VX_NUM v2, \vr2
+ VX_NUM v3, \vr3
+ .word 0xE700 | ((v1&15) << 4) | (v2&15)
+ .word ((v3&15) << 12)
+ MRXBOPC 0, 0x68, v1, v2, v3
+.endm
+
/* VECTOR EXCLUSIVE OR */
.macro VX vr1, vr2, vr3
VX_NUM v1, \vr1
VX_NUM v2, \vr2
VX_NUM v3, \vr3
- .word 0xE700 | (VX_R(v1) << 4) | VX_R(v2)
- .word (VX_R(v3) << 12)
+ .word 0xE700 | ((v1&15) << 4) | (v2&15)
+ .word ((v3&15) << 12)
MRXBOPC 0, 0x6D, v1, v2, v3
.endm
@@ -425,8 +438,8 @@
VX_NUM v1, \vr1
VX_NUM v2, \vr2
VX_NUM v3, \vr3
- .word 0xE700 | (VX_R(v1) << 4) | VX_R(v2)
- .word (VX_R(v3) << 12)
+ .word 0xE700 | ((v1&15) << 4) | (v2&15)
+ .word ((v3&15) << 12)
MRXBOPC \m4, 0xB4, v1, v2, v3
.endm
.macro VGFMB vr1, vr2, vr3
@@ -448,9 +461,9 @@
VX_NUM v2, \vr2
VX_NUM v3, \vr3
VX_NUM v4, \vr4
- .word 0xE700 | (VX_R(v1) << 4) | VX_R(v2)
- .word (VX_R(v3) << 12) | (\m5 << 8)
- MRXBOPC VX_R(v4), 0xBC, v1, v2, v3, v4
+ .word 0xE700 | ((v1&15) << 4) | (v2&15)
+ .word ((v3&15) << 12) | (\m5 << 8)
+ MRXBOPC (v4&15), 0xBC, v1, v2, v3, v4
.endm
.macro VGFMAB vr1, vr2, vr3, vr4
VGFMA \vr1, \vr2, \vr3, \vr4, 0
@@ -470,11 +483,78 @@
VX_NUM v1, \vr1
VX_NUM v2, \vr2
VX_NUM v3, \vr3
- .word 0xE700 | (VX_R(v1) << 4) | VX_R(v2)
- .word (VX_R(v3) << 12)
+ .word 0xE700 | ((v1&15) << 4) | (v2&15)
+ .word ((v3&15) << 12)
MRXBOPC 0, 0x7D, v1, v2, v3
.endm
+/* VECTOR REPLICATE IMMEDIATE */
+.macro VREPI vr1, imm2, m3
+ VX_NUM v1, \vr1
+ .word 0xE700 | ((v1&15) << 4)
+ .word \imm2
+ MRXBOPC \m3, 0x45, v1
+.endm
+.macro VREPIB vr1, imm2
+ VREPI \vr1, \imm2, 0
+.endm
+.macro VREPIH vr1, imm2
+ VREPI \vr1, \imm2, 1
+.endm
+.macro VREPIF vr1, imm2
+ VREPI \vr1, \imm2, 2
+.endm
+.macro VREPIG vr1, imm2
+ VREP \vr1, \imm2, 3
+.endm
+
+/* VECTOR ADD */
+.macro VA vr1, vr2, vr3, m4
+ VX_NUM v1, \vr1
+ VX_NUM v2, \vr2
+ VX_NUM v3, \vr3
+ .word 0xE700 | ((v1&15) << 4) | (v2&15)
+ .word ((v3&15) << 12)
+ MRXBOPC \m4, 0xF3, v1, v2, v3
+.endm
+.macro VAB vr1, vr2, vr3
+ VA \vr1, \vr2, \vr3, 0
+.endm
+.macro VAH vr1, vr2, vr3
+ VA \vr1, \vr2, \vr3, 1
+.endm
+.macro VAF vr1, vr2, vr3
+ VA \vr1, \vr2, \vr3, 2
+.endm
+.macro VAG vr1, vr2, vr3
+ VA \vr1, \vr2, \vr3, 3
+.endm
+.macro VAQ vr1, vr2, vr3
+ VA \vr1, \vr2, \vr3, 4
+.endm
+
+/* VECTOR ELEMENT SHIFT RIGHT ARITHMETIC */
+.macro VESRAV vr1, vr2, vr3, m4
+ VX_NUM v1, \vr1
+ VX_NUM v2, \vr2
+ VX_NUM v3, \vr3
+ .word 0xE700 | ((v1&15) << 4) | (v2&15)
+ .word ((v3&15) << 12)
+ MRXBOPC \m4, 0x7A, v1, v2, v3
+.endm
+
+.macro VESRAVB vr1, vr2, vr3
+ VESRAV \vr1, \vr2, \vr3, 0
+.endm
+.macro VESRAVH vr1, vr2, vr3
+ VESRAV \vr1, \vr2, \vr3, 1
+.endm
+.macro VESRAVF vr1, vr2, vr3
+ VESRAV \vr1, \vr2, \vr3, 2
+.endm
+.macro VESRAVG vr1, vr2, vr3
+ VESRAV \vr1, \vr2, \vr3, 3
+.endm
#endif /* __ASSEMBLY__ */
#endif /* __ASM_S390_VX_INSN_H */
diff --git a/arch/s390/include/uapi/asm/Kbuild b/arch/s390/include/uapi/asm/Kbuild
index 08fe6dad9026..cc44b09c25fc 100644
--- a/arch/s390/include/uapi/asm/Kbuild
+++ b/arch/s390/include/uapi/asm/Kbuild
@@ -6,6 +6,7 @@ header-y += bitsperlong.h
header-y += byteorder.h
header-y += chpid.h
header-y += chsc.h
+header-y += clp.h
header-y += cmb.h
header-y += dasd.h
header-y += debug.h