diff options
| author | Heiko Carstens <hca@linux.ibm.com> | 2026-07-20 10:58:34 +0200 |
|---|---|---|
| committer | Vasily Gorbik <gor@linux.ibm.com> | 2026-07-30 00:29:11 +0200 |
| commit | 656db3022083ac2239f9f75ddfbd8f598ca64f5f (patch) | |
| tree | 5535b598136511a525bc6635665a3fbc4a0f4ba1 /arch/s390/include | |
| parent | e462200886fb56cc318700b2fbbb2206ac28a656 (diff) | |
s390: Add support for DCACHE_WORD_ACCESS (again)
Implement load_unaligned_zeropad() and enable DCACHE_WORD_ACCESS to
speed up string operations in fs/dcache.c and fs/namei.c.
With the secure storage access exception cases addressed, add support
for DCACHE_WORD_ACCESS again.
Tested-by: Christian Borntraeger <borntraeger@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Reviewed-by: Sven Schnelle <svens@linux.ibm.com>
Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>
Diffstat (limited to 'arch/s390/include')
| -rw-r--r-- | arch/s390/include/asm/asm-extable.h | 4 | ||||
| -rw-r--r-- | arch/s390/include/asm/word-at-a-time.h | 22 |
2 files changed, 26 insertions, 0 deletions
diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h index 99748c20e767..d23ea0c94e4e 100644 --- a/arch/s390/include/asm/asm-extable.h +++ b/arch/s390/include/asm/asm-extable.h @@ -12,6 +12,7 @@ #define EX_TYPE_UA_FAULT 3 #define EX_TYPE_UA_LOAD_REG 5 #define EX_TYPE_UA_LOAD_REGPAIR 6 +#define EX_TYPE_ZEROPAD 7 #define EX_TYPE_FPC 8 #define EX_TYPE_UA_MVCOS_TO 9 #define EX_TYPE_UA_MVCOS_FROM 10 @@ -79,6 +80,9 @@ #define EX_TABLE_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0) +#define EX_TABLE_ZEROPAD(_fault, _target, _regdata, _regaddr) \ + __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_ZEROPAD, _regdata, _regaddr, 0) + #define EX_TABLE_FPC(_fault, _target) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FPC, __stringify(%%r0), __stringify(%%r0), 0) diff --git a/arch/s390/include/asm/word-at-a-time.h b/arch/s390/include/asm/word-at-a-time.h index e9287036392d..eaa19dee7699 100644 --- a/arch/s390/include/asm/word-at-a-time.h +++ b/arch/s390/include/asm/word-at-a-time.h @@ -4,6 +4,7 @@ #include <linux/bitops.h> #include <linux/wordpart.h> +#include <asm/asm-extable.h> #include <asm/bitsperlong.h> struct word_at_a_time { @@ -40,4 +41,25 @@ static inline unsigned long zero_bytemask(unsigned long data) return ~1UL << data; } +/* + * Load an unaligned word from kernel space. + * + * In the (very unlikely) case of the word being a page-crosser + * and the next page not being mapped, take the exception and + * return zeroes in the non-existing part. + */ +static inline unsigned long load_unaligned_zeropad(const void *addr) +{ + unsigned long data; + + asm_inline volatile( + "0: lg %[data],0(%[addr])\n" + "1: nopr %%r7\n" + EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr]) + EX_TABLE_ZEROPAD(1b, 1b, %[data], %[addr]) + : [data] "=d" (data) + : [addr] "a" (addr), "m" (*(unsigned long *)addr)); + return data; +} + #endif /* _ASM_WORD_AT_A_TIME_H */ |